EKS addons module

terraform-kubernetes-addons:aws is a custom module maintained here and provides:

• helm v3 charts
• manifests
• operators

For commonly used addons one Kubernetes and most specifically with EKS.

The configuration is curated to be tightly integrated with AWS and EKS.

Customization

All the configuration is done in eks-addons/terragrunt.hcl.

dependencies {
  paths = ["../eks-addons-critical"]
}

include "root" {
  path           = find_in_parent_folders()
  expose         = true
  merge_strategy = "deep"
}

include "vpc" {
  path           = "../../../../../../dependency-blocks/vpc.hcl"
  expose         = true
  merge_strategy = "deep"
}

include "eks" {
  path           = "../../../../../../dependency-blocks/eks.hcl"
  expose         = true
  merge_strategy = "deep"
}

terraform {
  source = "github.com/particuleio/terraform-kubernetes-addons.git//modules/aws?ref=v12.8.0"
}

generate "provider-local" {
  path      = "provider-local.tf"
  if_exists = "overwrite"
  contents  = file("../../../../../../provider-config/eks-addons/eks-addons.tf")
}

generate "provider-github" {
  path      = "provider-github.tf"
  if_exists = "overwrite_terragrunt"
  contents  = <<-EOF
    provider "github" {
      owner = "${include.root.locals.merged.github_owner}"
    }
  EOF
}

inputs = {

  priority-class = {
    name = basename(get_terragrunt_dir())
  }

  priority-class-ds = {
    name = "${basename(get_terragrunt_dir())}-ds"
  }

  cluster-name = dependency.eks.outputs.cluster_name

  tags = merge(
    include.root.locals.custom_tags
  )

  eks = {
    "cluster_oidc_issuer_url" = dependency.eks.outputs.cluster_oidc_issuer_url
    "oidc_provider_arn"       = dependency.eks.outputs.oidc_provider_arn
    "cluster_endpoint"        = dependency.eks.outputs.cluster_endpoint
  }

  cert-manager = {
    enabled                   = true
    acme_http01_enabled       = true
    acme_dns01_enabled        = true
    acme_http01_ingress_class = "nginx"
    extra_values              = <<-EXTRA_VALUES
      ingressShim:
        defaultIssuerName: letsencrypt
        defaultIssuerKind: ClusterIssuer
        defaultIssuerGroup: cert-manager.io
      EXTRA_VALUES
  }

  cluster-autoscaler = {
    enabled      = false
    version      = "v1.24.0"
    extra_values = <<-EXTRA_VALUES
    extraArgs:
      scale-down-utilization-threshold: 0.7
    EXTRA_VALUES
  }

  # For this to work:
  # * GITHUB_TOKEN should be set
  flux2 = {
    enabled               = false
    target_path           = "gitops/clusters/${include.root.locals.merged.env}/${include.root.locals.merged.name}"
    github_url            = "ssh://git@github.com/particuleio/teks"
    repository            = "teks"
    branch                = "flux"
    repository_visibility = "public"
    version               = "v0.37.0"
    auto_image_update     = true
  }

  kube-prometheus-stack = {
    enabled                           = true
    allowed_cidrs                     = dependency.vpc.outputs.intra_subnets_cidr_blocks
    thanos_sidecar_enabled            = true
    thanos_bucket_force_destroy       = true
    grafana_create_iam_resources_irsa = true
    extra_values                      = <<-EXTRA_VALUES
      grafana:
        image:
          tag: 9.3.1
        deploymentStrategy:
          type: Recreate
        ingress:
          annotations:
            kubernetes.io/tls-acme: "true"
          ingressClassName: nginx
          enabled: true
          hosts:
            - telemetry.${include.root.locals.merged.default_domain_name}
          tls:
            - secretName: ${include.root.locals.merged.default_domain_name}
              hosts:
                - telemetry.${include.root.locals.merged.default_domain_name}
        persistence:
          enabled: true
          accessModes:
            - ReadWriteOnce
          size: 1Gi
      prometheus:
        prometheusSpec:
          nodeSelector:
            kubernetes.io/arch: amd64
          scrapeInterval: 60s
          retention: 2d
          retentionSize: "10GB"
          ruleSelectorNilUsesHelmValues: false
          serviceMonitorSelectorNilUsesHelmValues: false
          podMonitorSelectorNilUsesHelmValues: false
          probeSelectorNilUsesHelmValues: false
          storageSpec:
            volumeClaimTemplate:
              spec:
                accessModes: ["ReadWriteOnce"]
                resources:
                  requests:
                    storage: 10Gi
          resources:
            requests:
              cpu: 1
              memory: 2Gi
            limits:
              cpu: 2
              memory: 2Gi
      EXTRA_VALUES
  }

  loki-stack = {
    enabled              = true
    bucket_force_destroy = true
    extra_values         = <<-VALUES
      resources:
        requests:
          cpu: 1
          memory: 2Gi
        limits:
          cpu: 2
          memory: 4Gi
      loki:
        limits_config:
          ingestion_rate_mb: 320
          ingestion_burst_size_mb: 512
          max_streams_per_user: 100000
        chunk_store_config:
          max_look_back_period: 2160h
        table_manager:
          retention_deletes_enabled: true
          retention_period: 2160h
      ingress:
        enabled: true
        annotations:
          kubernetes.io/tls-acme: "true"
          nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
          nginx.ingress.kubernetes.io/auth-tls-secret: "telemetry/loki-ca"
        hosts:
          - logz.${include.root.locals.merged.default_domain_name}
        tls:
          - secretName: logz.${include.root.locals.merged.default_domain_name}
            hosts:
              - logz.${include.root.locals.merged.default_domain_name}
        VALUES
    bucket_lifecycle_rule = [
      {
        id      = "log"
        enabled = true
        transition = [
          {
            days          = 14
            storage_class = "INTELLIGENT_TIERING"
          },
        ]
        expiration = {
          days = 365
        }
      },
    ]
  }

  ingress-nginx = {
    enabled       = true
    use_nlb_ip    = true
    allowed_cidrs = dependency.vpc.outputs.intra_subnets_cidr_blocks
    extra_values  = <<-EXTRA_VALUES
      controller:
        ingressClassResource:
          enabled: true
          default: true
        replicaCount: 2
        minAvailable: 1
        kind: "Deployment"
        resources:
          requests:
            cpu: 300m
            memory: 128Mi
        service:
          annotations:
            service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true
      defaultBackend:
        enabled: true
        replicaCount: 2
        minAvailable: 0
        nodeSelector:
          kubernetes.io/arch: amd64
      EXTRA_VALUES
  }

  karpenter = {
    enabled      = true
    iam_role_arn = dependency.eks.outputs.eks_managed_node_groups["initial"].iam_role_arn
  }

  promtail = {
    enabled = true
    wait    = false
  }

  thanos = {
    enabled              = true
    bucket_force_destroy = true
    # Waiting for ARM support https://github.com/bitnami/charts/issues/7305
    extra_values = <<-EXTRA_VALUES
      query:
        nodeSelector:
          kubernetes.io/arch: amd64
      queryFrontend:
        nodeSelector:
          kubernetes.io/arch: amd64
      bucketweb:
        nodeSelector:
          kubernetes.io/arch: amd64
      compactor:
        nodeSelector:
          kubernetes.io/arch: amd64
      storegateway:
        nodeSelector:
          kubernetes.io/arch: amd64
      ruler:
        nodeSelector:
          kubernetes.io/arch: amd64
      receive:
        nodeSelector:
          kubernetes.io/arch: amd64
      receiveDistributor:
        nodeSelector:
          kubernetes.io/arch: amd64
      EXTRA_VALUES
  }
}

Default charts values

Some values are defined by default directly into the module. These can off course be overridden and or merged/replaced. You can find the defaults values in the upstream module. Eg. default values for cluster-autoscaler are in cluster-autoscaler.tf.

Overriding Helm provider values

Helm provider have defaults values defined here:

  helm_defaults_defaults = {
    atomic                = false
    cleanup_on_fail       = false
    dependency_update     = false
    disable_crd_hooks     = false
    disable_webhooks      = false
    force_update          = false
    recreate_pods         = false
    render_subchart_notes = true
    replace               = false
    reset_values          = false
    reuse_values          = false
    skip_crds             = false
    timeout               = 3600
    verify                = false
    wait                  = true
    extra_values          = ""
  }

These can be overridden globally with the helm_defaults input variable or can be overridden per chart in terragrunt.hcl:

  helm_defaults = {
    replace = true
    verify  = true
    timeout = 300
  }


  cluster_autoscaler = {
    create_iam_resources_irsa = true
    iam_policy_override       = ""
    version                   = "v1.14.7"
    chart_version             = "6.4.0"
    enabled                   = true
    default_network_policy    = true
    cluster_name              = dependency.eks.outputs.cluster_id
    timeout                   = 3600 <= here you can add any helm provider override
  }

Overriding charts values.yaml

It is possible to add or override values per charts. Helm provider use the same merge logic as Helm so you can basically rewrite the whole values.yaml if needed.

Each chart has a extra_values variable where you can specify custom values.

flux = {
    create_iam_resources_irsa = true
    version                   = "1.18.0"
    chart_version             = "1.2.0"
    enabled                   = false
    default_network_policy    = true

    extra_values = <<EXTRA_VALUES
git:
  url: "ssh://git@gitlab.com/myrepo/gitops-${local.env}.git"
  pollInterval: "2m"
rbac:
  create: false
registry:
  automationInterval: "2m"
EXTRA_VALUES
}

There are some examples in the terragrunt.hcl file. Not all the variables available are present. If you want a full list of variable, you can find them in the upstream module. For example for cluster-autoscaler you can see the default here.