diff --git a/.gitmessage.txt b/.gitmessage.txt new file mode 100644 index 00000000..2b1593c3 --- /dev/null +++ b/.gitmessage.txt @@ -0,0 +1,52 @@ +# 💡 提交说明模板 (Git Commit Message Template) +# +# 📌 标准格式： +# (): <简要描述> +# +# 📖 示例： +# feat(iac): 新增支持多环境配置加载 +# feat(deploy): Add support for multi-environment config loading +# +# 可选结构： +# - 中文描述（团队成员理解方便） +# - 英文描述（CI/CD / PR 审阅更规范） +# +# 🧱 支持类型： +# feat 💡 新功能 / Feature +# fix 🐛 修复 bug / Bug fix +# docs 📚 文档变更 / Documentation +# style 🎨 代码格式 / Style only +# refactor 🔨 重构 / Refactor (无功能变更) +# perf 🚀 性能优化 / Performance +# test 🧪 测试相关 / Add or update tests +# chore 🔧 构建、工具、依赖更新 / Chores +# +# ⏱️ 每次提交只关注一类改动 + +# ---------------------- COMMIT MESSAGE START ---------------------- + +feat(iac): 重构目录结构并支持多环境配置加载 +feat(iac): Refactor structure and support multi-environment config loading + +- 新增 config/sit 等多环境配置目录结构 +- Add config/sit and other environment-specific config directories + +- 重构 deploy.py 适配 CONFIG_PATH 环境变量 +- Refactor deploy.py to support CONFIG_PATH environment variable + +- 支持自动合并 config/*/*.yaml 配置 +- Enable automatic merging of config/*/*.yaml files + +- 增强 run.sh 脚本，集成 Pulumi/Ansible/Terraform 初始化检查 +- Enhance run.sh with Pulumi/Ansible/Terraform initialization checks + +- 新增 inventory.py 动态生成 Ansible 主机列表 +- Add inventory.py to dynamically generate Ansible hosts + +- 整理 base.yaml、vpc.yaml 等配置文件 +- Organize base.yaml, vpc.yaml and related config files + +# ----------------------- COMMIT MESSAGE END ----------------------- + +# 📝 注意：提交时只保留实际变更部分，其余注释会被 Git 忽略 + diff --git a/README.md b/README.md index cea70463..86193212 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,41 @@ # Modern Container Application Reference Architecture -Welcome to the repository for the Modern Container Application Reference Architecture. This repository contains a comprehensive guide and reference architecture for building scalable, portable, resilient, and agile containerized applications. +Welcome to the repository for the Modern Container Application Reference Architecture. This repository contains a comprehensive guide and reference architecture for building scalable, portable, resilient, and agile containerized applications. 一个基于 Pulumi + Ansible 的基础设施自动化项目模板，支持多环境部署（dev / staging / prod），实现从基础设施创建到主机配置的全流程自动化管理。 +--- ## Overview The project aims to create a multi-cloud environment that leverages containers for deploying modern applications. The key objective is to set up a unified authentication system using **OIDC** via **Keycloak** for **AWS**, **GCP**, **Azure**, **GitHub**, **Harbor ** and **Grafana **. +## 🚀 项目功能 + +- 使用 Pulumi（Python）创建 AWS 基础设施（VPC、子网、安全组、EC2） +- 配置结构模块化：`base.yaml`, `vpc.yaml`, `firewall.yaml`, `instances.yaml` +- 支持 Spot / On-Demand 实例，支持 TTL 标签 +- 自动输出 EC2 IP，动态生成 Ansible Inventory +- 使用 Ansible Playbook 远程安装软件或部署服务 +- 支持多环境 stack（dev/staging/prod） + +## 项目结构 + +├── config/ # 多环境配置 +│ ├── base.yaml +│ ├── vpc.yaml +│ ├── firewall.yaml +│ └── instances.yaml +├── iac_modules/ +│ └── pulumi/ +│ ├── deploy.py # Pulumi 主入口 +│ ├── modules/ # VPC/SG/EC2 模块 +│ ├── utils/config_loader.py +│ └── requirements.txt +├── scripts/ +│ ├── infra.sh # 一键部署脚本 +│ └── inventory.py # 动态 Ansible inventory +├── ansible/ +│ └── playbooks/ +│ └── setup.yml # 应用部署 playbook + ## Phase 1: Implementing OIDC Login In this first phase, we focus on implementing OpenID Connect (OIDC) login functionality for the following platforms: diff --git a/config/sit/base.yaml b/config/sit/base.yaml new file mode 100644 index 00000000..d85b4c04 --- /dev/null +++ b/config/sit/base.yaml @@ -0,0 +1,7 @@ +aws: + access_key: YOUR_ACCESS_KEY + secret_key: YOUR_SECRET_KEY + region: us-east-1 + key_pairs: + - name: dev_key + key_file: keys/dev_ssh.pub diff --git a/config/sit/firewall.yaml b/config/sit/firewall.yaml new file mode 100644 index 00000000..12e59652 --- /dev/null +++ b/config/sit/firewall.yaml @@ -0,0 +1,6 @@ +firewall_rules: + - name: allow-ssh-web + allow: + - protocol: tcp + ports: ["22", "80", "443"] + source_ranges: ["0.0.0.0/0"] diff --git a/config/sit/instances.yaml b/config/sit/instances.yaml new file mode 100644 index 00000000..0e6f8fde --- /dev/null +++ b/config/sit/instances.yaml @@ -0,0 +1,32 @@ +instances: + - name: master-1 + ami: ami-0c2b8ca1dad447f8a + type: t3.micro + disk_size_gb: 20 + subnet: public-subnet-1 + lifecycle: spot # 可选: ondemand（默认）或 spot + ttl: 1h # 可选: 自动标记 TTL，仅作为标识，不自动销毁 + + - name: slave-1 + ami: ami-0c2b8ca1dad447f8a + type: t3.micro + disk_size_gb: 20 + subnet: private-subnet-1 + lifecycle: spot + ttl: 1h + + - name: agent-1 + ami: ami-0c2b8ca1dad447f8a + type: t3.micro + disk_size_gb: 20 + subnet: private-subnet-1 + lifecycle: spot + ttl: 1h + + - name: agent-2 + ami: ami-0c2b8ca1dad447f8a + type: t3.micro + disk_size_gb: 20 + subnet: private-subnet-1 + lifecycle: spot + ttl: 1h diff --git a/config/sit/vpc.yaml b/config/sit/vpc.yaml new file mode 100644 index 00000000..c5d1e933 --- /dev/null +++ b/config/sit/vpc.yaml @@ -0,0 +1,24 @@ +vpc: + name: dev-vpc + cidr_block: 10.0.0.0/16 + subnets: + - name: public-subnet-1 + cidr_block: 10.0.1.0/24 + availability_zone: us-east-1a + type: public + - name: private-subnet-1 + cidr_block: 10.0.101.0/24 + availability_zone: us-east-1a + type: private + + routes: + - name: public-route + destination_cidr_block: 0.0.0.0/0 + subnet_type: public + gateway: internet_gateway + + peering: + enabled: false + peer_vpc_id: null + peer_region: null + auto_accept: false diff --git a/iac_modules/pulumi/deploy.py b/iac_modules/pulumi/deploy.py new file mode 100644 index 00000000..e6757d7d --- /dev/null +++ b/iac_modules/pulumi/deploy.py @@ -0,0 +1,63 @@ +import os +import pulumi +import pulumi_aws as aws +from utils.config_loader import load_merged_config +from modules.vpc.vpc import create_vpc +from modules.security_group.sg import create_security_group +from modules.ec2.ec2_instance import create_instances + +# ✅ 自动从环境变量获取配置路径，默认为 "config/" +config_dir = os.environ.get("CONFIG_PATH", "config") +config = load_merged_config(config_dir) + +# ✅ 提取配置项（如为空跳过） +aws_conf = config.get("aws") +vpc_conf = config.get("vpc") +instances_conf = config.get("instances", []) +firewall_rules = config.get("firewall_rules", []) + +if not aws_conf or not vpc_conf: + pulumi.log.warn(f"❌ 配置不完整，缺少 aws 或 vpc 段，终止部署。CONFIG_PATH={config_dir}") + exit(0) + +# ✅ 配置 AWS 凭据 +aws.config.region = aws_conf["region"] +aws.config.access_key = aws_conf["access_key"] +aws.config.secret_key = aws_conf["secret_key"] + +# ✅ 创建 VPC 与子网 +vpc_result = create_vpc(vpc_conf, aws_conf["region"]) +vpc = vpc_result["vpc"] +subnets = vpc_result["subnets"] + +# ✅ 创建安全组（取第一组规则） +if not firewall_rules: + pulumi.log.warn("⚠️ 未定义 firewall_rules，默认跳过安全组配置") + sg_id = None +else: + sg = create_security_group(vpc.id, firewall_rules[0]) + sg_id = sg.id + +# ✅ SSH 密钥对 +key_cfg = aws_conf["key_pairs"][0] +public_key_path = key_cfg["key_file"] +if not os.path.exists(public_key_path): + raise FileNotFoundError(f"❌ SSH 公钥文件不存在: {public_key_path}") +with open(public_key_path, "r") as f: + public_key = f.read().strip() + +key_pair = aws.ec2.KeyPair("main-key", + key_name=key_cfg["name"], + public_key=public_key +) + +# ✅ 创建实例（自动匹配子网） +if not instances_conf: + pulumi.log.warn("⚠️ 未配置任何 EC2 实例，跳过实例部署") + outputs = {} +else: + outputs = create_instances(instances_conf, subnets, sg_id, key_pair.key_name) + +# ✅ 导出所有实例的公网 IP +for name, ip in outputs.items(): + pulumi.export(f"{name}_ip", ip) diff --git a/iac_modules/pulumi/modules/ec2/ec2_instance.py b/iac_modules/pulumi/modules/ec2/ec2_instance.py new file mode 100644 index 00000000..6077b9cd --- /dev/null +++ b/iac_modules/pulumi/modules/ec2/ec2_instance.py @@ -0,0 +1,54 @@ +import pulumi_aws as aws + +def create_instances(instances_config, subnets_dict, sg_id, key_name): + outputs = {} + + for instance_cfg in instances_config: + name = instance_cfg["name"] + subnet_name = instance_cfg["subnet"] + subnet_id = subnets_dict[subnet_name].id + ami = instance_cfg["ami"] + instance_type = instance_cfg["type"] + disk_size = instance_cfg["disk_size_gb"] + + # 读取可选字段 + lifecycle = instance_cfg.get("lifecycle", "ondemand") # 默认按需 + ttl = instance_cfg.get("ttl", "none") # 默认无 TTL + + # 设置 EC2 标签 + tags = { + "Name": name, + "Lifecycle": lifecycle, + "TTL": ttl, + } + + # 如果是 Spot 实例，设置市场选项（不设 max_price → 自动出价） + instance_market_options = None + if lifecycle == "spot": + instance_market_options = aws.ec2.InstanceInstanceMarketOptionsArgs( + market_type="spot", + spot_options=aws.ec2.InstanceInstanceMarketOptionsSpotOptionsArgs( + instance_interruption_behavior="terminate", + spot_instance_type="one-time" + ) + ) + + # 创建 EC2 实例 + ec2 = aws.ec2.Instance(name, + ami=ami, + instance_type=instance_type, + key_name=key_name, + subnet_id=subnet_id, + vpc_security_group_ids=[sg_id], + associate_public_ip_address=True, + root_block_device={ + "volume_size": disk_size, + "volume_type": "gp2" + }, + instance_market_options=instance_market_options, + tags=tags + ) + + outputs[name] = ec2.public_ip + + return outputs diff --git a/iac_modules/pulumi/requirements.txt b/iac_modules/pulumi/requirements.txt new file mode 100644 index 00000000..2d212c01 --- /dev/null +++ b/iac_modules/pulumi/requirements.txt @@ -0,0 +1,3 @@ +pulumi +pulumi-aws +PyYAML diff --git a/iac_modules/pulumi/utils/config_loader.py b/iac_modules/pulumi/utils/config_loader.py new file mode 100644 index 00000000..b622be6a --- /dev/null +++ b/iac_modules/pulumi/utils/config_loader.py @@ -0,0 +1,35 @@ +import os +import glob +import yaml +from collections.abc import Mapping + +def deep_merge(dict1, dict2): + result = dict1.copy() + for k, v in dict2.items(): + if k in result and isinstance(result[k], dict) and isinstance(v, Mapping): + result[k] = deep_merge(result[k], v) + elif k in result and isinstance(result[k], list) and isinstance(v, list): + result[k] += v + else: + result[k] = v + return result + +def load_merged_config(config_dir=None): + config_dir = config_dir or os.environ.get("CONFIG_PATH", "config") + + if not os.path.isdir(config_dir): + raise FileNotFoundError(f"❌ 配置目录不存在: {config_dir}") + + merged = {} + files = sorted(glob.glob(os.path.join(config_dir, "*.yaml")) + glob.glob(os.path.join(config_dir, "*.yml"))) + + if not files: + raise FileNotFoundError(f"⚠️ 未找到任何 YAML 配置文件于: {config_dir}") + + for file in files: + with open(file) as f: + part = yaml.safe_load(f) or {} + merged = deep_merge(merged, part) + + merged["__config_path__"] = config_dir # 可选调试字段 + return merged diff --git a/scripts/inventory.py b/scripts/inventory.py new file mode 100644 index 00000000..7a634481 --- /dev/null +++ b/scripts/inventory.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import os +import yaml +from collections import defaultdict + +def get_pulumi_outputs(): + output = subprocess.check_output(["pulumi", "stack", "output", "--json"]) + return json.loads(output) + +def merge_instance_config(config_dir="config"): + merged = {} + for fname in os.listdir(config_dir): + if fname.endswith(".yaml"): + with open(os.path.join(config_dir, fname)) as f: + data = yaml.safe_load(f) + if isinstance(data, dict): + merged.update(data) + return merged.get("instances", []) + +def build_inventory(pulumi_outputs, instance_cfgs): + inventory = {"_meta": {"hostvars": {}}} + groups = defaultdict(list) + + for inst in instance_cfgs: + name = inst["name"] + public_ip = pulumi_outputs.get(f"{name}_ip") + + if not public_ip: + continue # skip not created instances + + # 默认分组：all + groups["all"].append(name) + + # 根据 subnet 或 lifecycle 添加分组 + if "subnet" in inst: + groups[inst["subnet"]].append(name) + if "lifecycle" in inst: + groups[inst["lifecycle"]].append(name) + + # hostvars + inventory["_meta"]["hostvars"][name] = { + "ansible_host": public_ip, + "ansible_user": "ubuntu", + "instance_type": inst.get("type"), + "ttl": inst.get("ttl", "none"), + "lifecycle": inst.get("lifecycle", "ondemand"), + } + + # 将分组注入 inventory + for group, hosts in groups.items(): + inventory[group] = {"hosts": hosts} + + return inventory + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--list', action='store_true') + args = parser.parse_args() + + if args.list: + pulumi_data = get_pulumi_outputs() + instance_cfgs = merge_instance_config() + inventory = build_inventory(pulumi_data, instance_cfgs) + print(json.dumps(inventory, indent=2)) + else: + print(json.dumps({})) + +if __name__ == "__main__": + main() diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100644 index 00000000..4cbe28db --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,207 @@ +#!/bin/bash +set -e + +# 项目根目录（从任意位置运行都有效） +cd "$(dirname "$0")/.." + +# ========== 参数解析 ========== +DEFAULT_ENV="dev" +DEFAULT_CONFIG="config" + +if [[ -n "$1" && "$1" != up && "$1" != down && "$1" != delete && "$1" != export && "$1" != import && "$1" != init && "$1" != ansible && "$1" != help ]]; then + STACK_ENV="$1" + ACTION="${2:-up}" +else + STACK_ENV="${STACK_ENV:-$DEFAULT_ENV}" + ACTION="${1:-help}" +fi + +STACK_NAME="${STACK_NAME:-$STACK_ENV}" +CONFIG_PATH="${CONFIG_PATH:-config/$STACK_ENV}" + +# ========== 模块目录 ========== +PULUMI_DIR="iac_modules/pulumi" +TERRAFORM_DIR="iac_modules/terraform" +ANSIBLE_DIR="ansible" + +# ========== 帮助信息 ========== +print_help() { + + echo "" + echo "🧰 iac_cli - 多环境自动化管理器 (IaC + Ansible + GitOps)" + echo "" + + echo "用法:" + echo " ./scripts/run.sh [env] [命令]" + echo " STACK_ENV=prod CONFIG_PATH=config/prod ./scripts/run.sh up" + echo "" + echo "🌍 当前环境: $STACK_ENV" + echo "📁 当前配置路径: $CONFIG_PATH" + echo "" + echo "支持命令:" + echo " up 🚀 部署资源" + echo " down 🔥 销毁资源" + echo " delete 🗑️ 删除 stack" + echo " export 📤 导出 stack 状态" + echo " import 📥 导入 stack 状态" + echo " init ⚙️ 初始化依赖" + echo " ansible 🧪 执行 ansible-playbook" + echo " help 📖 显示帮助" + echo "" +} + +# ========== 检查 Pulumi ========== +ensure_pulumi() { + if ! command -v pulumi &> /dev/null; then + echo "📦 未检测到 Pulumi，正在自动安装..." + case "$(uname | tr '[:upper:]' '[:lower:]')" in + linux) + curl -fsSL https://get.pulumi.com | sh + export PATH="$HOME/.pulumi/bin:$PATH" + ;; + darwin) + brew install pulumi || (curl -fsSL https://get.pulumi.com | sh && export PATH="$HOME/.pulumi/bin:$PATH") + ;; + msys*|mingw*|cygwin*) + echo "👉 Windows 用户请手动安装 Pulumi：https://www.pulumi.com/docs/get-started/install/" + exit 1 + ;; + *) + echo "❌ 当前平台不支持自动安装 Pulumi" + exit 1 + ;; + esac + fi + echo "✅ Pulumi 版本: $(pulumi version)" +} + +# ========== 检查 Ansible ========== +ensure_ansible() { + if ! command -v ansible &> /dev/null; then + echo "❌ 未检测到 Ansible，请手动安装：" + case "$(uname | tr '[:upper:]' '[:lower:]')" in + linux) + echo "👉 Ubuntu/Debian: sudo apt install ansible" + echo "👉 RHEL/CentOS: sudo yum install ansible" + ;; + darwin) + echo "👉 macOS: brew install ansible" + ;; + msys*|mingw*|cygwin*) + echo "👉 Windows 用户请参考官方安装指南：https://docs.ansible.com/" + ;; + *) + echo "👉 其他平台请参考：https://docs.ansible.com/" + ;; + esac + exit 1 + else + echo "✅ Ansible 已安装: $(ansible --version | head -n 1)" + fi +} + +# ========== 检查 Terraform ========== +ensure_terraform() { + if ! command -v terraform &> /dev/null; then + echo "❌ 未检测到 Terraform，请手动安装：" + echo "👉 https://developer.hashicorp.com/terraform/install" + exit 1 + fi + echo "✅ Terraform 已安装: $(terraform version | head -n1)" +} + +# ========== 环境初始化检查 ========== +init_env() { + echo "⚙️ 初始化 Pulumi + Ansible 环境..." + + # 1️⃣ 检查 Pulumi + ensure_pulumi + + # 2️⃣ 安装 Python 依赖 + if [ -f "$PULUMI_DIR/requirements.txt" ]; then + echo "📦 安装 Python 依赖..." + pip3 install -r "$PULUMI_DIR/requirements.txt" + fi + + # 3️⃣ 检查 Ansible + ensure_ansible + + # 4️⃣ 检查 Terraform（可选） + if [ -d "$TERRAFORM_DIR" ]; then + ensure_terraform + fi + + # 5️⃣ 初始化 Pulumi Stack + cd "$PULUMI_DIR" + pulumi login --local > /dev/null + if ! pulumi stack ls | grep -q "$STACK_NAME"; then + echo "📂 创建 Pulumi Stack: $STACK_NAME" + pulumi stack init "$STACK_NAME" + else + echo "✅ Stack 已存在：$STACK_NAME" + fi + + echo "✅ 初始化完成 ✅" +} + +# ========== 执行 Pulumi ========== +pulumi_run() { + cd "$PULUMI_DIR" + case "$ACTION" in + up) + if [ ! -d "$CONFIG_PATH" ] || [ -z "$(ls -A $CONFIG_PATH/*.yaml 2>/dev/null)" ]; then + echo "⚠️ 配置目录为空：$CONFIG_PATH，跳过部署" + exit 0 + fi + echo "🚀 正在部署 stack: $STACK_NAME" + pulumi up --yes + ;; + down) + echo "🔥 正在销毁 stack: $STACK_NAME" + pulumi destroy --yes + ;; + delete) + echo "🗑️ 删除 Stack: $STACK_NAME" + pulumi stack rm "$STACK_NAME" --yes + ;; + export) + echo "📤 导出 stack 状态" + pulumi stack export --file stack-export.json + ;; + import) + echo "📥 导入 stack 状态" + pulumi stack import --file stack-export.json + ;; + init) + init_env + ;; + *) + print_help + ;; + esac +} + +# ========== 执行 Ansible ========== +run_ansible() { + if [ ! -f scripts/inventory.py ]; then + echo "❌ 未找到 scripts/inventory.py" + exit 1 + fi + echo "🧪 执行 Ansible Playbook" + ansible-playbook -i scripts/inventory.py "$ANSIBLE_DIR/playbooks/setup.yml" +} + +# ========== 分发 ========== +case "$ACTION" in + up|down|delete|export|import|init) + export CONFIG_PATH + export STACK_ENV + pulumi_run + ;; + ansible) + run_ansible + ;; + help|*) + print_help + ;; +esac