Merge pull request #160 from cloud-neutral-toolkit/codex/templatize-provider.tf-and-backend.tf-files
Remove shared provider_backend template
This commit is contained in:
GitHub
commit
b5372e0018
3
.gitignore
vendored
3
.gitignore
vendored
@ -39,6 +39,9 @@ coverage.xml
|
||||
.terraform/
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
# Generated Terraform provider/backend files for AWS cloud envs
|
||||
iac-template/terraform-hcl-standard/aws-cloud/envs/*/provider.tf
|
||||
iac-template/terraform-hcl-standard/aws-cloud/envs/*/backend.tf
|
||||
|
||||
# Ansible
|
||||
*.retry
|
||||
|
||||
@ -0,0 +1,60 @@
|
||||
defaults:
|
||||
terraform_required_version: ">= 1.2"
|
||||
aws_provider_version: "~> 5.92.0"
|
||||
session_name: "TerraformDevSession"
|
||||
|
||||
modules:
|
||||
dev:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/core/terraform.tfstate"
|
||||
|
||||
dev-alb:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/alb/terraform.tfstate"
|
||||
|
||||
dev-ec2:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/ec2/terraform.tfstate"
|
||||
|
||||
dev-kafka:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/kafka/terraform.tfstate"
|
||||
|
||||
dev-landingzone:
|
||||
account: dev
|
||||
backend:
|
||||
key: "bootstrap/dev-landingzone/terraform.tfstate"
|
||||
|
||||
dev-nlb:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/nlb/terraform.tfstate"
|
||||
|
||||
dev-object:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/s3/terraform.tfstate"
|
||||
|
||||
dev-rds:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/rds/terraform.tfstate"
|
||||
|
||||
dev-redis:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/redis/terraform.tfstate"
|
||||
|
||||
dev-role:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/iam/terraform.tfstate"
|
||||
|
||||
dev-vpc:
|
||||
account: dev
|
||||
backend:
|
||||
key: "account/dev/vpc/terraform.tfstate"
|
||||
@ -1,8 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/alb/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/ec2/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/kafka/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "bootstrap/dev-landingzone/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/nlb/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/s3/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/iam/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/iam/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "svc-plus-iac-state"
|
||||
key = "account/dev/vpc/terraform.tfstate"
|
||||
region = "ap-northeast-1"
|
||||
dynamodb_table = "svc-plus-iac-state-dynamodb-lock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.2"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.92.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = local.account.region
|
||||
|
||||
assume_role {
|
||||
role_arn = "arn:aws:iam::730335654753:role/TerraformDeployRole-Dev"
|
||||
session_name = "TerraformDevSession"
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
CURRENT_DIR = Path(__file__).resolve().parent
|
||||
PROJECT_ROOT = CURRENT_DIR.parent
|
||||
CONFIG_DIR = Path(
|
||||
os.environ.get("AWS_CLOUD_CONFIG_PATH", PROJECT_ROOT / "aws-cloud" / "config")
|
||||
)
|
||||
TEMPLATE_DIR = CURRENT_DIR / "templates"
|
||||
ENVS_DIR = CURRENT_DIR / "envs"
|
||||
|
||||
sys.path.append(str(PROJECT_ROOT / "utils"))
|
||||
from config_loader import load_merged_config # noqa: E402
|
||||
|
||||
|
||||
def build_provider_config(module_name: str, module_config: Dict, account_config: Dict, defaults: Dict) -> Dict:
|
||||
region = module_config.get("region") or account_config.get("region")
|
||||
if not region:
|
||||
raise ValueError(f"Region is required for module {module_name}")
|
||||
|
||||
return {
|
||||
"terraform": {
|
||||
"required_version": module_config.get("terraform_required_version")
|
||||
or defaults.get("terraform_required_version", ">= 1.2"),
|
||||
"aws_provider_version": module_config.get("aws_provider_version")
|
||||
or defaults.get("aws_provider_version", "~> 5.92.0"),
|
||||
},
|
||||
"region": region,
|
||||
"assume_role_arn": module_config.get("assume_role_arn")
|
||||
or account_config.get("role_to_assume"),
|
||||
"session_name": module_config.get("session_name")
|
||||
or defaults.get("session_name", "TerraformSession"),
|
||||
}
|
||||
|
||||
|
||||
def build_backend_config(module_name: str, module_config: Dict, account_config: Dict) -> Dict:
|
||||
backend_overrides = module_config.get("backend", {})
|
||||
backend_bucket = backend_overrides.get("bucket") or account_config.get("backend", {}).get(
|
||||
"bucket"
|
||||
)
|
||||
dynamodb_table = backend_overrides.get("dynamodb_table") or account_config.get(
|
||||
"backend", {}
|
||||
).get("dynamodb_table")
|
||||
backend_key = backend_overrides.get("key")
|
||||
backend_region = backend_overrides.get("region") or account_config.get("region")
|
||||
|
||||
if not backend_bucket:
|
||||
raise ValueError(f"Backend bucket is required for module {module_name}")
|
||||
if not backend_key:
|
||||
raise ValueError(f"Backend key is required for module {module_name}")
|
||||
if not backend_region:
|
||||
raise ValueError(f"Backend region is required for module {module_name}")
|
||||
|
||||
return {
|
||||
"bucket": backend_bucket,
|
||||
"key": backend_key,
|
||||
"region": backend_region,
|
||||
"dynamodb_table": dynamodb_table,
|
||||
}
|
||||
|
||||
|
||||
def load_account_config(account_name: str, additional_inputs: list[str] | None = None) -> Dict:
|
||||
account_config_path = CONFIG_DIR / "accounts" / f"{account_name}.yaml"
|
||||
config_inputs = [str(account_config_path)] + [str(CONFIG_DIR / path) for path in additional_inputs or []]
|
||||
return load_merged_config(config_inputs)
|
||||
|
||||
|
||||
def render_templates():
|
||||
provider_backend_cfg = load_merged_config(CONFIG_DIR / "provider_backend.yaml")
|
||||
defaults = provider_backend_cfg.get("defaults", {})
|
||||
modules = provider_backend_cfg.get("modules", {})
|
||||
|
||||
env = Environment(loader=FileSystemLoader(TEMPLATE_DIR), keep_trailing_newline=True)
|
||||
provider_template = env.get_template("provider.tf.j2")
|
||||
backend_template = env.get_template("backend.tf.j2")
|
||||
|
||||
for module_name, module_config in modules.items():
|
||||
module_dir = ENVS_DIR / module_name
|
||||
if not module_dir.exists():
|
||||
print(f"⚠️ Skipping {module_name}: {module_dir} not found")
|
||||
continue
|
||||
|
||||
account_name = module_config.get("account")
|
||||
if not account_name:
|
||||
raise ValueError(f"Account is required for module {module_name}")
|
||||
|
||||
account_config = load_account_config(account_name, module_config.get("config_inputs"))
|
||||
|
||||
provider_config = build_provider_config(module_name, module_config, account_config, defaults)
|
||||
backend_config = build_backend_config(module_name, module_config, account_config)
|
||||
|
||||
provider_content = provider_template.render(provider=provider_config)
|
||||
backend_content = backend_template.render(backend=backend_config)
|
||||
|
||||
(module_dir / "provider.tf").write_text(provider_content, encoding="utf-8")
|
||||
(module_dir / "backend.tf").write_text(backend_content, encoding="utf-8")
|
||||
print(f"✅ Rendered provider/backend for {module_name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
render_templates()
|
||||
@ -0,0 +1,8 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "{{ backend.bucket }}"
|
||||
key = "{{ backend.key }}"
|
||||
region = "{{ backend.region }}"{% if backend.dynamodb_table %}
|
||||
dynamodb_table = "{{ backend.dynamodb_table }}"{% endif %}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
{% set terraform = provider.terraform %}
|
||||
terraform {
|
||||
required_version = "{{ terraform.required_version }}"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "{{ terraform.aws_provider_version }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = "{{ provider.region }}"{% if provider.assume_role_arn %}
|
||||
assume_role {
|
||||
role_arn = "{{ provider.assume_role_arn }}"
|
||||
session_name = "{{ provider.session_name }}"
|
||||
}
|
||||
{% endif %}
|
||||
}
|
||||
86
iac-template/terraform-hcl-standard/utils/config_loader.py
Normal file
86
iac-template/terraform-hcl-standard/utils/config_loader.py
Normal file
@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
import yaml
|
||||
|
||||
DEFAULT_IGNORE_FILES = {"vpn-keys.yaml"}
|
||||
|
||||
|
||||
def deep_merge(dict1: dict, dict2: Mapping) -> dict:
|
||||
"""Recursively merge ``dict2`` into ``dict1`` and return a new dict."""
|
||||
result = dict1.copy()
|
||||
for key, value in dict2.items():
|
||||
if key in result and isinstance(result[key], dict) and isinstance(value, Mapping):
|
||||
result[key] = deep_merge(result[key], value)
|
||||
elif key in result and isinstance(result[key], list) and isinstance(value, list):
|
||||
result[key] = result[key] + value
|
||||
else:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _iter_yaml_files(path: Path, ignore_files: set[str]) -> Iterable[Path]:
|
||||
if path.is_file():
|
||||
if path.suffix in {".yaml", ".yml"} and path.name not in ignore_files:
|
||||
yield path
|
||||
return
|
||||
|
||||
patterns = ["**/*.yaml", "**/*.yml"]
|
||||
seen: set[Path] = set()
|
||||
for pattern in patterns:
|
||||
for file_path in sorted(path.glob(pattern)):
|
||||
if file_path.name in ignore_files or file_path in seen:
|
||||
continue
|
||||
seen.add(file_path)
|
||||
yield file_path
|
||||
|
||||
|
||||
def _normalize_inputs(config_inputs: list[str] | str | Path | None) -> list[str]:
|
||||
if config_inputs is None:
|
||||
env_paths = os.environ.get("CONFIG_PATHS") or os.environ.get("CONFIG_PATH")
|
||||
config_inputs = env_paths.split(os.pathsep) if env_paths else ["config"]
|
||||
|
||||
if isinstance(config_inputs, (Path, os.PathLike)):
|
||||
config_inputs = [config_inputs]
|
||||
|
||||
if isinstance(config_inputs, str):
|
||||
config_inputs = [value for value in config_inputs.split(os.pathsep) if value]
|
||||
|
||||
return [str(Path(path).expanduser()) for path in config_inputs]
|
||||
|
||||
|
||||
def load_merged_config(config_inputs: list[str] | str | Path | None = None, ignore_files: list[str] | None = None) -> dict:
|
||||
"""
|
||||
Load and deep-merge YAML content from multiple files or directories.
|
||||
|
||||
``config_inputs`` accepts:
|
||||
- A single path string or Path-like
|
||||
- A list of path strings
|
||||
- ``None`` (defaults to environment variable ``CONFIG_PATHS`` / ``CONFIG_PATH`` or ``config``)
|
||||
"""
|
||||
|
||||
ignore = DEFAULT_IGNORE_FILES | set(ignore_files or [])
|
||||
merged: dict = {}
|
||||
|
||||
resolved_inputs = _normalize_inputs(config_inputs)
|
||||
if not resolved_inputs:
|
||||
raise ValueError("No configuration inputs provided")
|
||||
|
||||
loaded_paths: list[str] = []
|
||||
for raw_path in resolved_inputs:
|
||||
path = Path(raw_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"❌ 配置路径不存在: {path}")
|
||||
|
||||
loaded_paths.append(str(path))
|
||||
for file_path in _iter_yaml_files(path, ignore):
|
||||
with open(file_path, "r", encoding="utf-8") as handle:
|
||||
content = yaml.safe_load(handle) or {}
|
||||
merged = deep_merge(merged, content)
|
||||
|
||||
merged["__config_paths__"] = loaded_paths
|
||||
return merged
|
||||
Loading…
Reference in New Issue
Block a user