diff --git a/k3s_platform_addon.yml b/k3s_platform_addon.yml new file mode 100644 index 0000000..c1df3d7 --- /dev/null +++ b/k3s_platform_addon.yml @@ -0,0 +1,8 @@ +- name: Install single-node k3s platform addons + hosts: k3s + user: root + become: yes + gather_facts: yes + tasks: + - include_role: + name: vhosts/k3s_platform_addon diff --git a/init_k3s_single_node_gitops.yml b/k3s_platform_bootstrap_with_gitops.yml similarity index 75% rename from init_k3s_single_node_gitops.yml rename to k3s_platform_bootstrap_with_gitops.yml index 127cd1c..40e0bcc 100644 --- a/init_k3s_single_node_gitops.yml +++ b/k3s_platform_bootstrap_with_gitops.yml @@ -6,3 +6,5 @@ tasks: - include_role: name: vhosts/k3s_platform_bootstrap + - include_role: + name: vhosts/k3s_platform_addon diff --git a/roles/vhosts/k3s_platform_addon/defaults/main.yml b/roles/vhosts/k3s_platform_addon/defaults/main.yml new file mode 100644 index 0000000..6774b00 --- /dev/null +++ b/roles/vhosts/k3s_platform_addon/defaults/main.yml @@ -0,0 +1,205 @@ +k3s_platform_k3s_version: "v1.30.6+k3s1" +k3s_platform_cluster_name: "svc-plus" +k3s_platform_cluster_cidr: "10.42.0.0/16" +k3s_platform_service_cidr: "10.43.0.0/16" +k3s_platform_disable_components: + - traefik +k3s_platform_write_kubeconfig_mode: "0644" +k3s_platform_tls_sans: [] +k3s_platform_kubeconfig_path: /etc/rancher/k3s/k3s.yaml +k3s_platform_k3s_config_path: /etc/rancher/k3s/config.yaml +k3s_platform_install_dir: /opt/rancher/k3s +k3s_platform_k3s_install_script_url: https://get.k3s.io +k3s_platform_k3s_install_script_path: /tmp/k3s-install.sh +k3s_platform_k3s_install_log_path: /var/log/k3s-install.log +k3s_platform_k3s_install_timeout_seconds: 1800 +k3s_platform_deployment_mode: k3s_platform + +k3s_platform_git_branch: "main" +k3s_platform_flux_namespace: "flux-system" +k3s_platform_flux_source_name: "platform-config" +k3s_platform_flux_root_path: "{{ lookup('env', 'K3S_PLATFORM_FLUX_ROOT_PATH') | default('./environments/clusters/prod', true) }}" +k3s_platform_git_url: "{{ lookup('env', 'GITOPS_REPO') | default('https://github.com/x-evor/gitops.git', true) }}" +k3s_platform_git_auth_mode: "{{ lookup('env', 'GITOPS_AUTH_MODE') | default('public', true) }}" + +k3s_platform_git_private_key: "{{ lookup('env', 'GITOPS_FLUX_DEPLOY_KEY') }}" +k3s_platform_git_public_key: "{{ lookup('env', 'GITOPS_FLUX_DEPLOY_KEY_PUB') }}" +k3s_platform_git_private_key_path: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa" +k3s_platform_git_http_username: "{{ lookup('env', 'GITOPS_FLUX_HTTP_USERNAME') | default('x-access-token', true) }}" +k3s_platform_git_http_password: "{{ lookup('env', 'GITOPS_FLUX_HTTP_PASSWORD') | default(lookup('env', 'GITOPS_FLUX_TOKEN'), true) }}" +k3s_platform_git_bearer_token: "{{ lookup('env', 'GITOPS_FLUX_BEARER_TOKEN') }}" + +k3s_platform_namespaces: + - flux-system + - platform + - database + - observability + - core-prod + - core-pre +k3s_platform_bootstrap_namespaces: + - flux-system +k3s_platform_manage_extsvc: false +k3s_platform_vault_release_name: vault +k3s_platform_vault_chart_version: "0.28.0" +k3s_platform_vault_bootstrap_mode: "{{ lookup('env', 'K3S_PLATFORM_VAULT_BOOTSTRAP_MODE') | default('init', true) }}" +k3s_platform_vault_init_phase: "{{ lookup('env', 'K3S_PLATFORM_VAULT_INIT_PHASE') | default('post_flux', true) }}" +k3s_platform_vault_admin_username: admin +k3s_platform_vault_init_json_path: /tmp/vault-init.json +k3s_platform_vault_init_stdout_path: /tmp/vault-init.stdout +k3s_platform_vault_init_stderr_path: /tmp/vault-init.stderr +k3s_platform_vault_allow_sensitive_output: "{{ lookup('env', 'K3S_PLATFORM_VAULT_ALLOW_SENSITIVE_OUTPUT') | default('false', true) | bool }}" + +bootstrap_vault_url: "{{ lookup('env', 'VAULT_URL') | default('', true) }}" +bootstrap_vault_namespace: "{{ lookup('env', 'VAULT_NAMESPACE') | default('', true) }}" +bootstrap_vault_token: "{{ lookup('env', 'VAULT_TOKEN') | default('', true) }}" +bootstrap_gitops_repo: "{{ k3s_platform_git_url }}" +bootstrap_gitops_auth_mode: "{{ k3s_platform_git_auth_mode }}" +k3s_platform_vault_server_url: "{{ bootstrap_vault_url }}" +k3s_platform_vault_namespace: "{{ bootstrap_vault_namespace }}" +k3s_platform_vault_token: "{{ bootstrap_vault_token }}" + +cloudflare_api_token: "{{ lookup('env', 'CLOUDFLARE_API_TOKEN') | default(lookup('env', 'CLOUDFLARE_DNS_API_TOKEN'), true) }}" +cloudflare_zone_id: "{{ lookup('env', 'CLOUDFLARE_ZONE_ID') | default('', true) }}" +vault_root_token: "{{ lookup('env', 'VAULT_ROOT_TOKEN') | default('', true) }}" +vault_init_json: "{{ lookup('env', 'VAULT_INIT_JSON') | default('', true) }}" + +k3s_platform_flux_env: + GIT_URL: "{{ k3s_platform_git_url }}" + GIT_BRANCH: "{{ k3s_platform_git_branch }}" + GIT_SOURCE_NAME: "{{ k3s_platform_flux_source_name }}" + FLUX_NAMESPACE: "{{ k3s_platform_flux_namespace }}" + FLUX_ROOT_PATH: "{{ k3s_platform_flux_root_path }}" + +k3s_platform_postgres_user: "{{ lookup('env', 'POSTGRES_USER') | default('postgres', true) }}" +k3s_platform_postgres_password: "{{ lookup('env', 'POSTGRES_PASSWORD') | default('', true) }}" +k3s_platform_internal_service_token: "{{ lookup('env', 'INTERNAL_SERVICE_TOKEN') | default('', true) }}" +k3s_platform_ghcr_username: "{{ lookup('env', 'GHCR_USERNAME') | default('svc-design', true) }}" +k3s_platform_ghcr_token: "{{ lookup('env', 'GHCR_TOKEN') | default('', true) }}" + +k3s_platform_helm_repos: + - { name: "external-secrets", url: "https://charts.external-secrets.io" } + - { name: "stakater", url: "https://stakater.github.io/stakater-charts" } + - { name: "caddy-ingress", url: "https://caddyserver.github.io/ingress/" } + - { name: "apisix", url: "https://charts.apiseven.com" } + - { name: "external-dns", url: "https://kubernetes-sigs.github.io/external-dns/" } + +k3s_platform_external_secrets_chart_version: "0.20.4" +k3s_platform_reloader_chart_version: "1.3.0" +k3s_platform_caddy_chart_version: "1.3.0" +k3s_platform_apisix_chart_version: "2.13.0" +k3s_platform_external_dns_chart_version: "1.20.0" + +k3s_platform_values: + namespaces: + platform: platform + vault: extsvc + components: + externalSecrets: + enabled: true + releaseName: external-secrets + reloader: + enabled: true + releaseName: reloader + caddy: + enabled: true + releaseName: caddy + values: + ingressController: + enabled: true + ingressClass: + create: true + name: caddy + default: false + service: + type: LoadBalancer + apisix: + enabled: true + releaseName: apisix + values: + etcd: + enabled: false + externalEtcd: + user: "" + ingress-controller: + enabled: false + dashboard: + enabled: false + gateway: + enabled: true + type: ClusterIP + control: + enabled: false + apisix: + admin: + enabled: false + deployment: + mode: standalone + role: data_plane + standalone: + config: | + routes: + - id: 1 + uri: "/*" + hosts: + - api.svc.plus + name: api-gateway-prod + upstream: + type: roundrobin + nodes: + "accounts.core-prod.svc.cluster.local:80": 1 + - id: 2 + uri: "/*" + hosts: + - api-pre.svc.plus + name: api-gateway-pre + upstream: + type: roundrobin + nodes: + "accounts.core-pre.svc.cluster.local:80": 1 + externalDns: + enabled: true + releaseName: external-dns + values: + provider: cloudflare + policy: sync + registry: txt + txtOwnerId: svc-plus-k3s + sources: + - ingress + domainFilters: + - svc.plus + env: + - name: CF_API_TOKEN + valueFrom: + secretKeyRef: + name: cloudflare-api-token + key: api-token + sharedTlsSecretSync: + enabled: true + name: postgresql-vultr-tls-sync + sourceNamespace: platform + sourceSecretName: postgresql-vultr-tls + targetNamespace: database + targetSecretName: postgresql-vultr-tls + refreshSchedule: "*/5 * * * *" + apisixIngress: + enabled: true + name: apisix-gateway + namespace: platform + className: caddy + annotations: + external-dns.alpha.kubernetes.io/hostname: api.svc.plus,api-pre.svc.plus + tls: + secretName: apisix-gateway-tls + hosts: + - api.svc.plus + - api-pre.svc.plus + hosts: + - host: api.svc.plus + serviceName: apisix-gateway + servicePort: 80 + - host: api-pre.svc.plus + serviceName: apisix-gateway + servicePort: 80 + postgresqlTlsIngress: + enabled: false diff --git a/roles/vhosts/k3s_platform_addon/tasks/main.yml b/roles/vhosts/k3s_platform_addon/tasks/main.yml new file mode 100644 index 0000000..559142d --- /dev/null +++ b/roles/vhosts/k3s_platform_addon/tasks/main.yml @@ -0,0 +1,123 @@ +- name: Ensure platform namespace exists + ansible.builtin.shell: | + set -euo pipefail + export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" + kubectl get namespace platform >/dev/null 2>&1 || kubectl create namespace platform + args: + executable: /bin/bash + +- name: Ensure platform Helm repositories are configured + ansible.builtin.shell: | + set -euo pipefail + if helm repo list | awk '{print $1}' | grep -qx '{{ item.name }}'; then + helm repo add --force-update '{{ item.name }}' '{{ item.url }}' >/dev/null + else + helm repo add '{{ item.name }}' '{{ item.url }}' >/dev/null + fi + args: + executable: /bin/bash + loop: "{{ k3s_platform_helm_repos }}" + +- name: Refresh platform Helm repositories + ansible.builtin.shell: | + set -euo pipefail + helm repo update >/dev/null + args: + executable: /bin/bash + +- name: Render direct Helm values for caddy + ansible.builtin.copy: + content: "{{ (k3s_platform_values.components.caddy['values'] | default({})) | to_nice_yaml(indent=2) }}" + dest: /tmp/platform-caddy-values.yaml + mode: "0600" + +- name: Render direct Helm values for apisix + ansible.builtin.copy: + content: "{{ (k3s_platform_values.components.apisix['values'] | default({})) | to_nice_yaml(indent=2) }}" + dest: /tmp/platform-apisix-values.yaml + mode: "0600" + +- name: Render direct Helm values for external-dns + ansible.builtin.copy: + content: "{{ (k3s_platform_values.components.externalDns['values'] | default({})) | to_nice_yaml(indent=2) }}" + dest: /tmp/platform-external-dns-values.yaml + mode: "0600" + +- name: Install external-secrets directly with Helm + ansible.builtin.shell: | + set -euo pipefail + export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" + helm upgrade --install external-secrets external-secrets/external-secrets \ + --namespace platform \ + --create-namespace \ + --version "{{ k3s_platform_external_secrets_chart_version }}" \ + --set installCRDs=true \ + --wait \ + --timeout 10m + args: + executable: /bin/bash + when: + - k3s_platform_values.components.externalSecrets.enabled | default(true) + +- name: Install reloader directly with Helm + ansible.builtin.shell: | + set -euo pipefail + export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" + helm upgrade --install reloader stakater/reloader \ + --namespace platform \ + --create-namespace \ + --version "{{ k3s_platform_reloader_chart_version }}" \ + --wait \ + --timeout 10m + args: + executable: /bin/bash + when: + - k3s_platform_values.components.reloader.enabled | default(true) + +- name: Install caddy directly with Helm + ansible.builtin.shell: | + set -euo pipefail + export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" + helm upgrade --install "{{ k3s_platform_values.components.caddy.releaseName }}" caddy-ingress/caddy-ingress-controller \ + --namespace platform \ + --create-namespace \ + --version "{{ k3s_platform_caddy_chart_version }}" \ + -f /tmp/platform-caddy-values.yaml \ + --wait \ + --timeout 10m + args: + executable: /bin/bash + when: + - k3s_platform_values.components.caddy.enabled | default(false) + +- name: Install apisix directly with Helm + ansible.builtin.shell: | + set -euo pipefail + export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" + helm upgrade --install "{{ k3s_platform_values.components.apisix.releaseName }}" apisix/apisix \ + --namespace platform \ + --create-namespace \ + --version "{{ k3s_platform_apisix_chart_version }}" \ + -f /tmp/platform-apisix-values.yaml \ + --wait \ + --timeout 10m + args: + executable: /bin/bash + when: + - k3s_platform_values.components.apisix.enabled | default(false) + +- name: Install external-dns directly with Helm + ansible.builtin.shell: | + set -euo pipefail + export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" + helm upgrade --install "{{ k3s_platform_values.components.externalDns.releaseName }}" external-dns/external-dns \ + --namespace platform \ + --create-namespace \ + --version "{{ k3s_platform_external_dns_chart_version }}" \ + -f /tmp/platform-external-dns-values.yaml \ + --wait \ + --timeout 10m + args: + executable: /bin/bash + when: + - k3s_platform_values.components.externalDns.enabled | default(false) diff --git a/roles/vhosts/k3s_platform_bootstrap/tasks/main.yml b/roles/vhosts/k3s_platform_bootstrap/tasks/main.yml index 370300e..eb09d18 100644 --- a/roles/vhosts/k3s_platform_bootstrap/tasks/main.yml +++ b/roles/vhosts/k3s_platform_bootstrap/tasks/main.yml @@ -888,194 +888,3 @@ kubectl -n "{{ k3s_platform_flux_namespace }}" get gitrepositories,kustomizations args: executable: /bin/bash - -- name: Verify platform namespaces exist after Flux bootstrap - ansible.builtin.shell: | - set -euo pipefail - export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" - kubectl get namespace "{{ item }}" - args: - executable: /bin/bash - loop: "{{ k3s_platform_namespaces }}" - -- name: Wait for Vault server rollout after Flux bootstrap - ansible.builtin.shell: | - set -euo pipefail - export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" - kubectl -n extsvc wait --for=condition=Ready pod \ - -l app.kubernetes.io/name={{ k3s_platform_vault_release_name }} \ - --timeout=300s - args: - executable: /bin/bash - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - -- name: Discover Vault pod name after Flux bootstrap - ansible.builtin.shell: | - set -euo pipefail - export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" - kubectl -n extsvc get pod \ - -l app.kubernetes.io/name={{ k3s_platform_vault_release_name }} \ - -o jsonpath='{.items[0].metadata.name}' - register: k3s_platform_vault_pod_name_result - changed_when: false - args: - executable: /bin/bash - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - -- name: Capture Vault status after Flux bootstrap - ansible.builtin.command: - argv: - - kubectl - - -n - - extsvc - - exec - - "{{ k3s_platform_vault_pod_name_result.stdout }}" - - -- - - /bin/vault - - status - - -format=json - register: k3s_platform_vault_status_result - changed_when: false - failed_when: false - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - -- name: Parse Vault status JSON after Flux bootstrap - ansible.builtin.set_fact: - k3s_platform_vault_status: >- - {{ - ( - k3s_platform_vault_status_result.stdout - if (k3s_platform_vault_status_result.stdout | default('') | trim | length) > 0 - else '{}' - ) | from_json - }} - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - -- name: Assert Vault bootstrap mode is valid after Flux bootstrap - ansible.builtin.assert: - that: - - k3s_platform_vault_bootstrap_mode in ['init', 'migrate'] - fail_msg: "k3s_platform_vault_bootstrap_mode must be init or migrate." - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - -- name: Assert migrate mode has inputs when Vault is already initialized after Flux bootstrap - ansible.builtin.assert: - that: - - (vault_root_token | default('')) | length > 0 - - (vault_init_json | default('')) | length > 0 - fail_msg: "Vault migrate mode requires vault_root_token and vault_init_json." - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_bootstrap_mode == 'migrate' - - k3s_platform_vault_status.initialized | default(false) - -- name: Assert migrate mode targets an initialized Vault after Flux bootstrap - ansible.builtin.assert: - that: - - k3s_platform_vault_status.initialized | default(false) - fail_msg: "Vault migrate mode requires an already initialized Vault instance." - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_bootstrap_mode == 'migrate' - -- name: Initialize Vault when requested and not yet initialized after Flux bootstrap - ansible.builtin.command: - argv: - - kubectl - - -n - - extsvc - - exec - - "{{ k3s_platform_vault_pod_name_result.stdout }}" - - -- - - /bin/vault - - operator - - init - - -format=json - - -key-shares=1 - - -key-threshold=1 - register: k3s_platform_vault_init_result - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_bootstrap_mode == 'init' - - not (k3s_platform_vault_status.initialized | default(false)) - -- name: Store Vault init JSON on disk after Flux bootstrap - ansible.builtin.copy: - content: "{{ k3s_platform_vault_init_result.stdout }}" - dest: "{{ k3s_platform_vault_init_json_path }}" - mode: "0600" - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_init_result is defined - -- name: Parse Vault init JSON after Flux bootstrap - ansible.builtin.set_fact: - k3s_platform_vault_init_data: "{{ k3s_platform_vault_init_result.stdout | from_json }}" - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_init_result is defined - -- name: Unseal Vault after init and Flux bootstrap - ansible.builtin.command: - argv: - - kubectl - - -n - - extsvc - - exec - - "{{ k3s_platform_vault_pod_name_result.stdout }}" - - -- - - /bin/vault - - operator - - unseal - - "{{ k3s_platform_vault_init_data.unseal_keys_b64[0] }}" - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_init_result is defined - -- name: Sync Vault bootstrap secret with generated root token after Flux bootstrap - ansible.builtin.shell: | - set -euo pipefail - export KUBECONFIG="{{ k3s_platform_kubeconfig_path }}" - kubectl -n extsvc create secret generic vault-bootstrap \ - --from-literal=rootToken="{{ k3s_platform_vault_init_data.root_token }}" \ - --from-literal=cloudflareApiToken="{{ cloudflare_api_token | default('') }}" \ - --dry-run=client -o yaml | kubectl apply -f - - args: - executable: /bin/bash - no_log: true - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_init_result is defined - -- name: Emit Vault bootstrap outputs after Flux bootstrap - ansible.builtin.debug: - msg: - mode: "{{ k3s_platform_vault_bootstrap_mode }}" - vault_initialized: "{{ k3s_platform_vault_status.initialized | default(false) }}" - vault_sealed: "{{ k3s_platform_vault_status.sealed | default(true) }}" - admin_username: "{{ k3s_platform_vault_admin_username }}" - init_json_path: "{{ k3s_platform_vault_init_json_path if k3s_platform_vault_init_result is defined else omit }}" - root_token: "{{ k3s_platform_vault_init_data.root_token if (k3s_platform_vault_init_result is defined and k3s_platform_vault_allow_sensitive_output) else omit }}" - init_json: "{{ k3s_platform_vault_init_result.stdout if (k3s_platform_vault_init_result is defined and k3s_platform_vault_allow_sensitive_output) else omit }}" - when: - - k3s_platform_manage_extsvc | bool - - k3s_platform_vault_init_phase == 'post_flux' - - k3s_platform_vault_bootstrap_mode == 'init' - - k3s_platform_vault_init_result is defined