Integrate Hashi Corp Vault, with GCS Backend Using Terraform and Helm in k8s cluster.

In this setup, we will integrate HashiCorp Vault within our Kubernetes cluster, utilizing a Google Cloud Storage (GCS) bucket as a storage backend for secure secret management. Vault will retrieve secrets stored in GCS and expose them as environment variables within our Kubernetes deployment YAML files, enabling secure and centralized secret handling for applications in the cluster.

Configuring GCS Bucket for Vault Storage with Terraform

a. main.tf
# Enable required APIs
resource "google_project_service" "kms_api" {
  project = "PROJECT-ID"
  service = "cloudkms.googleapis.com"
  disable_on_destroy = false
}

# GCS Bucket for Vault
resource "google_storage_bucket" "vault_bucket" {
  name          = "BUCKET-NAME"
  location      = "asia-south1"
  force_destroy = true
  storage_class = "STANDARD"
  versioning {
    enabled = true
  }
}
# Service Account for Vault
resource "google_service_account" "vault_sa" {
  project      = "PROJECT-ID"
  account_id   = "vault-sa"
  display_name = "Vault Service Account"
}
# Assign Storage Admin IAM Role to the Service Account for the GCS bucket
resource "google_storage_bucket_iam_member" "vault_sa_storage_admin" {
  bucket = google_storage_bucket.vault_bucket.name
  role   = "roles/storage.admin"
  member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# Assign Object Admin IAM Role to the Service Account for the GCS bucket
resource "google_storage_bucket_iam_member" "vault_sa_storage_object_admin" {
  bucket = google_storage_bucket.vault_bucket.name
  role   = "roles/storage.objectAdmin"
  member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# Generate a key for the Service Account
resource "google_service_account_key" "vault_sa_key" {
  service_account_id = google_service_account.vault_sa.name
  private_key_type   = "TYPE_GOOGLE_CREDENTIALS_FILE"
  key_algorithm      = "KEY_ALG_RSA_2048"
}
# Output the private key (sensitive)
output "vault_sa_private_key" {
  value     = google_service_account_key.vault_sa_key.private_key
  sensitive = true
}
# KMS Key Ring for Vault Encryption
resource "google_kms_key_ring" "my_key_ring" {
  project   = "PROJECT-ID"
  name      = "DUMMY-KEY-RING-NAME"
  location  = "asia-south1"
  depends_on = [google_project_service.kms_api]
}
# KMS Crypto Key for Vault Encryption/Decryption
resource "google_kms_crypto_key" "my_crypto_key" {
  name     = "DUMMY-KEY-CRYPTO-NAME"
  key_ring = google_kms_key_ring.my_key_ring.id
  lifecycle {
    prevent_destroy = true
  }
  rotation_period = "100000s"
  purpose         = "ENCRYPT_DECRYPT"
}
# IAM Role: Allow Vault Service Account to view KMS Key Ring
resource "google_project_iam_member" "kms_key_ring_viewer" {
  project = "PROJECT-ID"
  role    = "roles/cloudkms.viewer"
  member  = "serviceAccount:${google_service_account.vault_sa.email}"
}
# IAM Role: Allow Vault Service Account to administer KMS Key Ring
resource "google_project_iam_member" "kms_key_ring_admin" {
  project = "PROJECT-ID"
  role    = "roles/cloudkms.admin"
  member  = "serviceAccount:${google_service_account.vault_sa.email}"
}
# IAM Role: Allow Vault Service Account to encrypt and decrypt with KMS Crypto Key
resource "google_project_iam_member" "kms_crypto_key_user" {
  project = "PROJECT-ID"
  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
  member  = "serviceAccount:${google_service_account.vault_sa.email}"
}
# KMS crypto key IAM binding for encryption/decryption
resource "google_kms_crypto_key_iam_member" "crypto_key_iam" {
  crypto_key_id = google_kms_crypto_key.my_crypto_key.id
  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
  member        = "serviceAccount:${google_service_account.vault_sa.email}"
}

GCS for Vault storage using Terraform. It sets up a GCS bucket, a service account with necessary permissions, and a KMS key for Vault encryption.

b. provider.tf

provider "google" {
  project = "PROJECT-ID" 
  region  = "asia-south1"
}

c. Apply above HCL config’s

terraform init
terraform plan
terraform apply
terraform output vault_sa_private_key ## Keep this 
output key safely
## (this will be used in your vault-credentials as credentials.json)

Vault Setup Helm a. GCP Credentials Secret

# vault_credentials.yaml

apiVersion: v1
kind: Secret
metadata:
  name: vault-credentials
  namespace: vault
type: Opaque
data:
  credentials.json: <This base64 encoded one from your terraform config>

  #### NOTE: if you want OIDC enabled go for below one ###
  client_id: <Id>
  client_secret: <secret>
b. Vault Helm Values — (Which you will be overriding with your custom configs)
### vault_values.yaml
global:
  enabled: true
  tlsDisable: false
  namespace: vault
server:
  extraEnvironmentVars:
    GOOGLE_REGION: <Put yours - 😂😂>
    GOOGLE_PROJECT: <Your GCP Project - ofc i am not putting mine - 😂😂 >
    GOOGLE_APPLICATION_CREDENTIALS: /vault/userconfig/vault-credentials/credentials.json

  extraSecretMounts:
    - name: cognito-client-details
      secretName: vault-credentials
      mountPath: /etc/secrets/vault-credentials
  extraVolumes:
    - type: "secret"
      name: "vault-credentials"
  ha:
    enabled: true
    replicas: 3
    config: |
      ui = true

      listener "tcp" {
        tls_disable = "false"
        address = "[::]:8200"
        cluster_address = "[::]:8201"
      }
      storage "gcs" {
        bucket     = "<Bucket Name>"
        ha_enabled = true
      }
      seal "gcpckms" {
        project    = "<your gcp project name>"
        region     = "<Ofcourse the region>"
        key_ring   = "<the key ring name defined in your terraform setup>"
        crypto_key = "<the crypto key name defined in your terraform setup>"
      }

      ## Some colorful steps above haha!!
      ## If you want OIDC enable go for below steps, Please make sure you have ingress added
      auth_config "oidc" {
        type        = "oidc"
        description = "DUMMY-DESCRIPTION"

        config = {
          oidc_discovery_url    = "<Your Discovery URL>"
          oidc_client_id        = "$__file{/etc/secrets/vault-credentials/client_id}"
          oidc_client_secret    = "$__file{/etc/secrets/vault-credentials/client_secret}"
          default_role          = "default"
          bound_audiences       = ["$__file{/etc/secrets/vault-credentials/client_id}"]
          allowed_redirect_uris = [
            "https://vault.<Whatever your domain name>/ui/vault/auth/oidc/oidc/callback",
            "<http://localhost:8250/oidc/callback>"
          ]
          claim_mappings = {
            "cognito:groups" = "groups"
            "email"         = "email"
          }
          groups_claim = "cognito:groups"
          scopes       = ["email", "openid"]
        }
      }
      service_registration "kubernetes" {}
  service:
    enabled: true
    type: ClusterIP
    port: 8200
    targetPort: 8200
ui:
  enabled: true
  serviceType: LoadBalancer
  ports:
    - name: https
      port: 443
      targetPort: 8200
      protocol: TCP
injector:
  enabled: true
  certs:
    secretName: null
    # caBundle is a base64-encoded PEM-encoded certificate bundle for the CA
    # that signed the TLS certificate that the webhook serves. This must be set
    # if secretName is non-null unless an external service like cert-manager is
    # keeping the caBundle updated.
    caBundle: "<Put the Ca Bundle in case you are having difference>"

c. Vault Unseal Process

d. MakeFile

SHELL=/bin/bash
SELF = $(MAKE)
TOPLEVEL = $(shell git rev-parse --show-toplevel)
include $(TOPLEVEL)/Makefile.helpers

export KUBECONFIG = $(HOME)/.kube/config
REPO_PATH ?= $(shell git rev-parse --show-toplevel)
APPLICATION_CONFIG_PATH = $(REPO_PATH)/MY-PATH-OF-APPLICATION
## Apply config and initialize Vault
apply_config:
 kubectl --kubeconfig $(KUBECONFIG) create namespace NAMESPACE
 kubectl --kubeconfig $(KUBECONFIG) apply -f $(APPLICATION_CONFIG_PATH)/vault/vault_gcp_credentials.yaml
 helm install vault hashicorp/vault -f vault_values.yaml

 sleep 30
 $(SELF) vault_init
 $(SELF) vault_auth_k8s
## Initialize Vault and unseal it
vault_init:
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- vault operator init > vault_init_output.txt ## Please save this file content somewhere also dont push to github Hehe !!
 export UNSEAL_KEY_1=$$(grep 'Unseal Key 1:' vault_init_output.txt | awk '{print $$NF}') && \\
 export UNSEAL_KEY_2=$$(grep 'Unseal Key 2:' vault_init_output.txt | awk '{print $$NF}') && \\
 export UNSEAL_KEY_3=$$(grep 'Unseal Key 3:' vault_init_output.txt | awk '{print $$NF}') && \\
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- vault operator unseal $$UNSEAL_KEY_1 && \\
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-1 -- vault operator unseal $$UNSEAL_KEY_2 && \\
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-2 -- vault operator unseal $$UNSEAL_KEY_3
## Configure Vault for Kubernetes authentication and set policies/roles
vault_auth_k8s:
 @read -p "Enter the Vault Token: " VAULT_TOKEN && \\
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault login $$VAULT_TOKEN"

 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault auth enable kubernetes"
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault write auth/kubernetes/config \\
  kubernetes_host=\\"https://\\$${KUBERNETES_PORT_443_TCP_ADDR}:443\\" \\
  token_reviewer_jwt=\\"\\$$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\\" \\
  kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \\
  issuer=\\"<https://kubernetes.default.svc.cluster.local>\\""
 kubectl --kubeconfig $(KUBECONFIG) exec -n vault vault-values-0 -- vault secrets enable -path=streaming_service kv-v2
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "echo 'path \\"stream_service/*\\" { capabilities = [\\"read\\"] }' | vault policy write vault-policy-streaming -"
 kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault write auth/kubernetes/role/vault-role-stream \\
  bound_service_account_names=service-account-name \\
  bound_service_account_namespaces=Namespace-Name \\
  policies=vault-policy-stream \\
  ttl=1h"

To Apply above makefile

make apply_config

d. Make File Sequence Diagram

To put data in your path you can either do post request, or use vault command, i prefer Curl

curl --location --request GET 'http://<Vault-Svc-IP>/v1/stream_service/data/dummy_service/config' \\
--header 'X-Vault-Token: <$Your_TOKEN>' \\
--header 'Content-Type: application/json' \\
--data '{
    "data": {
        "DUMMY_API_KEY": "<$PUT-YOURS-😂😂>",
    }
}'

Setup Agent Injector + Load Env Vars

a. Deployment.yaml

## Your-service-deployment.yaml

replicaCount: 1
fullnameOverride: "dummy-service"
# Source Your Config.txt to get env vars
command: ["/bin/bash"]
args: ['-c', 'source /vault/secrets/config.txt && /init'] ## /init to call your docker entrypoint
envVars: []
ports:
  - name: application
    containerPort: 9000
    protocol: TCP
  - name: metrics
    containerPort: 9002
    protocol: TCP
probes:
  readiness:
    enabled: true
    initialDelaySeconds: 5
    path: /api/v1/dummy/health
    port: application
image:
  repository: <Image> ## you can use busy box image for dummy purpose
  pullPolicy: IfNotPresent
  tag: "<Tag>"
imagePullSecrets:
  - name: ecr-credentials
secret:
  enabled: false
serviceAccount:
  create: true
  annotations: {}
  name: ""
# Annotate your pod
podAnnotations:
  vault.hashicorp.com/agent-inject: 'true'
  vault.hashicorp.com/agent-inject-status: 'update'
  vault.hashicorp.com/role: 'vault-role-stream' //The role we define earlier
  vault.hashicorp.com/agent-inject-secret-config.txt: 'stream_service/data/dummy_service/config'
  # The Path where we have store our secrets
  *'stream_service/data/dummy_service/config' in here /data has to be put after our secrets initial path to so that vault can read it*

  vault.hashicorp.com/agent-inject-template-config.txt: |
      {{- with secret "stream_service/data/dummy_service/config" -}}
        {{- range $key, $value := .Data.data }}
          export {{ $key }}="{{ $value }}"
        {{- end }}
      {{- end -}}
service:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: application
      protocol: TCP
    - name: metrics
      port: 9002
      targetPort: metrics
      protocol: TCP
resources:
  limits:
    memory: 200Mi
    cpu: 200m
  requests:
    memory: 200Mi
    cpu: 200m
autoscaling:
  enabled: false
  minReplicas: 1
  maxReplicas: 3
  targetCPUUtilizationPercentage: 80
nodeSelector:
  alpha.eksctl.io/nodegroup-name: "<Dummy-nodegroup>"

Check if secrets is being export or not in above config.txt

kubectl exec -it POD -n NAMESPACE -- cat /vault/secrets/config.txt

References

https://developer.hashicorp.com/vault/docs

Blog

Integrate Hashi Corp Vault, with GCS Backend Using Terraform and Helm in k8s cluster.

Gaurav Poudel

Join Our Newsletter. No Spam, Only the good stuff.

Related