feat(iac): 完善 EC2/VPC/SG 模块,支持 AMI 自动解析与资源依赖控制

- 💡 支持 config/ 中通过关键词定义 EC2 实例的 AMI(如 'Ubuntu 22.04')
- ⚙️ 安全组规则支持 source_ranges/egress_ranges 配置化控制
-  增强 create_instances 函数,支持 user_data、spot/ondemand、TTL、owner 等标签
- 🔗 自动构建依赖关系,确保 VPC/Subnet/SG 完成后再部署 EC2
- ☁️ 使用 boto3 检查 AWS credentials,有效支持 ~/.aws/config profile 管理
- 🛠️ 支持 config 中动态启用/禁用模块(vpc/security_group/ec2)
- 🌐 完整验证 pulumi up/destroy/refresh 流程,确保干净状态

This commit enables modular, dynamic provisioning of AWS VPC, EC2 and Security Groups with Pulumi.
Supports keyword-based AMI resolution, secure profile-based credential loading, and full lifecycle control.
This commit is contained in:
Haitao Pan 2025-03-30 20:06:07 +08:00
parent 30779f819d
commit af8a848faa
16 changed files with 348 additions and 93 deletions

View File

@ -1,7 +1,6 @@
aws:
access_key: YOUR_ACCESS_KEY
secret_key: YOUR_SECRET_KEY
region: us-east-1
profile: default
region: ap-northeast-1
key_pairs:
- name: dev_key
key_file: keys/dev_ssh.pub
key_file: ~/.ssh/id_rsa.pub

View File

@ -1,6 +1,7 @@
firewall_rules:
- name: allow-ssh-web
source_ranges: ["0.0.0.0/0"]
egress_ranges: ["10.0.0.0/16"]
allow:
- protocol: tcp
ports: ["22", "80", "443"]
source_ranges: ["0.0.0.0/0"]

View File

@ -1,32 +1,45 @@
instances:
- name: master-1
ami: ami-0c2b8ca1dad447f8a
ami: ubuntu-22.04 # ✅ 可用 ami-xxx 或关键词(如 ubuntu-22.04
type: t3.micro
disk_size_gb: 20
subnet: public-subnet-1
lifecycle: spot # 可选: ondemand默认或 spot
ttl: 1h # 可选: 自动标记 TTL仅作为标识不自动销毁
lifecycle: spot # 可选: ondemand默认或 spot
ttl: 1h # 可选: 标记生命周期(不会自动销毁)
env: sit # 可选: dev/sit/prod 等环境标签
owner: devops # 可选: 资源责任人标签
associate_public_ip: true # ✅ 明确配置是否需要公网 IP
- name: slave-1
ami: ami-0c2b8ca1dad447f8a
ami: ubuntu-22.04
type: t3.micro
disk_size_gb: 20
subnet: private-subnet-1
lifecycle: spot
ttl: 1h
env: sit
owner: devops
associate_public_ip: false
- name: agent-1
ami: ami-0c2b8ca1dad447f8a
ami: ubuntu-22.04
type: t3.micro
disk_size_gb: 20
subnet: private-subnet-1
lifecycle: spot
ttl: 1h
env: sit
owner: devops
associate_public_ip: false
- name: agent-2
ami: ami-0c2b8ca1dad447f8a
ami: ubuntu-22.04
type: t3.micro
disk_size_gb: 20
subnet: private-subnet-1
lifecycle: spot
ttl: 1h
env: sit
owner: devops
associate_public_ip: false

View File

@ -1,14 +1,14 @@
vpc:
name: dev-vpc
cidr_block: 10.0.0.0/16
cidr_block: 10.1.0.0/16
subnets:
- name: public-subnet-1
cidr_block: 10.0.1.0/24
availability_zone: us-east-1a
cidr_block: 10.1.1.0/24
availability_zone: ap-northeast-1a
type: public
- name: private-subnet-1
cidr_block: 10.0.101.0/24
availability_zone: us-east-1a
cidr_block: 10.1.101.0/24
availability_zone: ap-northeast-1c
type: private
routes:

View File

@ -0,0 +1,112 @@
import os
import sys
import pulumi
import pulumi_aws as aws
import boto3
from botocore.exceptions import ProfileNotFound, NoCredentialsError
from utils.config_loader import load_merged_config
from modules.vpc.vpc import create_vpc
from modules.security_group.sg import create_security_group
from modules.ec2.ec2_instance import create_instances
# ✅ 加载配置
config_dir = os.environ.get("CONFIG_PATH", "config")
config = load_merged_config(config_dir)
aws_conf = config.get("aws", {})
region = aws_conf.get("region", "us-east-1")
profile = aws_conf.get("profile", "default")
key_pairs = aws_conf.get("key_pairs", [])
# ✅ 设置 AWS 配置
aws.config.region = region
aws.config.profile = profile
pulumi.runtime.set_config("aws:region", region)
# ✅ 检查 AWS 凭证
try:
session = boto3.Session(profile_name=profile)
credentials = session.get_credentials()
if not credentials:
raise NoCredentialsError()
except (ProfileNotFound, NoCredentialsError):
pulumi.log.error(f"❌ AWS profile '{profile}' 无效或找不到凭证")
sys.exit(1)
else:
pulumi.log.info(f"✅ AWS credentials loaded (profile: {profile}, region: {region})")
# ✅ 初始化资源容器
global_dependencies = []
vpc = None
subnets = {}
sg = None
key_pair = None
# ========================
# ✅ [模块] VPC + Subnets
# ========================
vpc_conf = config.get("vpc", {})
if vpc_conf.get("enabled", True):
vpc_result = create_vpc(vpc_conf, region)
vpc = vpc_result["vpc"]
subnets = vpc_result["subnets"]
global_dependencies.append(vpc)
global_dependencies.extend(subnets.values())
pulumi.log.info("✅ VPC/Subnet 已创建")
else:
pulumi.log.warn("⏭️ 跳过 VPC 创建")
# ========================
# ✅ [模块] Security Group
# ========================
firewall_rules = config.get("firewall_rules", [])
if firewall_rules and vpc and config.get("security_group", {}).get("enabled", True):
sg = create_security_group(vpc.id, firewall_rules[0])
global_dependencies.append(sg)
pulumi.log.info("✅ Security Group 已创建")
else:
pulumi.log.warn("⏭️ 跳过 Security Group 创建")
# ========================
# ✅ [模块] SSH Key Pair
# ========================
if key_pairs:
key_cfg = key_pairs[0]
public_key_path = os.path.expanduser(key_cfg["key_file"])
if not os.path.exists(public_key_path):
raise FileNotFoundError(f"❌ SSH 公钥文件不存在: {public_key_path}")
with open(public_key_path) as f:
public_key = f.read().strip()
key_pair = aws.ec2.KeyPair("main-key",
key_name=key_cfg["name"],
public_key=public_key
)
global_dependencies.append(key_pair)
pulumi.log.info("✅ SSH KeyPair 已创建")
else:
pulumi.log.warn("⏭️ 跳过 KeyPair 创建")
# ========================
# ✅ [模块] EC2 实例部署
# ========================
instances_conf = config.get("instances", [])
ec2_outputs = {}
if instances_conf and config.get("ec2", {}).get("enabled", True):
ec2_outputs = create_instances(
instances_conf,
subnets,
sg, # ✅ 注意这里传的是资源对象
key_pair.key_name if key_pair else None,
depends_on=global_dependencies
)
pulumi.log.info("✅ EC2 实例已创建")
else:
pulumi.log.warn("⏭️ 跳过 EC2 实例部署")
# ========================
# ✅ 导出所有实例信息
# ========================
for name, ip in ec2_outputs.items():
pulumi.export(f"{name}", ip)

View File

@ -1,63 +0,0 @@
import os
import pulumi
import pulumi_aws as aws
from utils.config_loader import load_merged_config
from modules.vpc.vpc import create_vpc
from modules.security_group.sg import create_security_group
from modules.ec2.ec2_instance import create_instances
# ✅ 自动从环境变量获取配置路径,默认为 "config/"
config_dir = os.environ.get("CONFIG_PATH", "config")
config = load_merged_config(config_dir)
# ✅ 提取配置项(如为空跳过)
aws_conf = config.get("aws")
vpc_conf = config.get("vpc")
instances_conf = config.get("instances", [])
firewall_rules = config.get("firewall_rules", [])
if not aws_conf or not vpc_conf:
pulumi.log.warn(f"❌ 配置不完整,缺少 aws 或 vpc 段终止部署。CONFIG_PATH={config_dir}")
exit(0)
# ✅ 配置 AWS 凭据
aws.config.region = aws_conf["region"]
aws.config.access_key = aws_conf["access_key"]
aws.config.secret_key = aws_conf["secret_key"]
# ✅ 创建 VPC 与子网
vpc_result = create_vpc(vpc_conf, aws_conf["region"])
vpc = vpc_result["vpc"]
subnets = vpc_result["subnets"]
# ✅ 创建安全组(取第一组规则)
if not firewall_rules:
pulumi.log.warn("⚠️ 未定义 firewall_rules默认跳过安全组配置")
sg_id = None
else:
sg = create_security_group(vpc.id, firewall_rules[0])
sg_id = sg.id
# ✅ SSH 密钥对
key_cfg = aws_conf["key_pairs"][0]
public_key_path = key_cfg["key_file"]
if not os.path.exists(public_key_path):
raise FileNotFoundError(f"❌ SSH 公钥文件不存在: {public_key_path}")
with open(public_key_path, "r") as f:
public_key = f.read().strip()
key_pair = aws.ec2.KeyPair("main-key",
key_name=key_cfg["name"],
public_key=public_key
)
# ✅ 创建实例(自动匹配子网)
if not instances_conf:
pulumi.log.warn("⚠️ 未配置任何 EC2 实例,跳过实例部署")
outputs = {}
else:
outputs = create_instances(instances_conf, subnets, sg_id, key_pair.key_name)
# ✅ 导出所有实例的公网 IP
for name, ip in outputs.items():
pulumi.export(f"{name}_ip", ip)

View File

View File

@ -1,28 +1,53 @@
import os
import pulumi
import pulumi_aws as aws
from .utils import resolve_ami
def create_instances(instances_config, subnets_dict, sg_id, key_name):
def create_instances(instances_config, subnets_dict, sg_resource, key_name, depends_on=None):
outputs = {}
for instance_cfg in instances_config:
name = instance_cfg["name"]
subnet_name = instance_cfg["subnet"]
subnet_id = subnets_dict[subnet_name].id
ami = instance_cfg["ami"]
subnet = subnets_dict[subnet_name]
subnet_id = subnet.id
# ✅ 自动解析 AMI关键词或 AMI ID
region = aws.config.region
ami = resolve_ami(instance_cfg["ami"], region)
instance_type = instance_cfg["type"]
disk_size = instance_cfg["disk_size_gb"]
# 读取可选字段
lifecycle = instance_cfg.get("lifecycle", "ondemand") # 默认按需
ttl = instance_cfg.get("ttl", "none") # 默认无 TTL
# ✅ 可选字段解析
lifecycle = instance_cfg.get("lifecycle", "ondemand")
ttl = instance_cfg.get("ttl", "none")
env = instance_cfg.get("env", "dev")
owner = instance_cfg.get("owner", "unknown")
user_data_path = instance_cfg.get("user_data")
private_ip = instance_cfg.get("private_ip", None)
associate_public_ip = instance_cfg.get("associate_public_ip", True)
# 设置 EC2 标签
# ✅ User data 读取(可选)
user_data = None
if user_data_path:
expanded_path = os.path.expanduser(user_data_path)
if os.path.exists(expanded_path):
with open(expanded_path, "r") as f:
user_data = f.read()
else:
pulumi.log.warn(f"⚠️ user_data 文件不存在: {expanded_path}")
# ✅ 标签定义
tags = {
"Name": name,
"Lifecycle": lifecycle,
"TTL": ttl,
"Environment": env,
"Owner": owner,
}
# 如果是 Spot 实例,设置市场选项(不设 max_price → 自动出价)
# ✅ Spot 实例配置
instance_market_options = None
if lifecycle == "spot":
instance_market_options = aws.ec2.InstanceInstanceMarketOptionsArgs(
@ -33,22 +58,35 @@ def create_instances(instances_config, subnets_dict, sg_id, key_name):
)
)
# 创建 EC2 实例
# ✅ 构建依赖项(必须是 Resource 对象)
resource_dependencies = [subnet]
if isinstance(sg_resource, pulumi.Resource):
resource_dependencies.append(sg_resource)
if depends_on:
resource_dependencies.extend(depends_on)
# ✅ 创建 EC2 实例
ec2 = aws.ec2.Instance(name,
ami=ami,
instance_type=instance_type,
key_name=key_name,
subnet_id=subnet_id,
vpc_security_group_ids=[sg_id],
associate_public_ip_address=True,
private_ip=private_ip,
associate_public_ip_address=associate_public_ip,
vpc_security_group_ids=[sg_resource.id] if sg_resource else [],
user_data=user_data,
root_block_device={
"volume_size": disk_size,
"volume_type": "gp2"
},
instance_market_options=instance_market_options,
tags=tags
tags=tags,
opts=pulumi.ResourceOptions(depends_on=resource_dependencies)
)
outputs[name] = ec2.public_ip
# ✅ 输出信息收集
outputs[name + "_id"] = ec2.id
outputs[name + "_public_ip"] = ec2.public_ip
outputs[name + "_private_ip"] = ec2.private_ip
return outputs

View File

@ -0,0 +1,37 @@
import pulumi_aws as aws
def resolve_ami(ami_keyword: str, region: str) -> str:
"""
根据关键词解析 AMI ID如果已是 AMI ID则直接返回
"""
if not aws.config.region:
raise ValueError("❌ AWS region is not set. Please set aws.config.region before calling resolve_ami")
if ami_keyword.startswith("ami-"):
return ami_keyword
keyword = ami_keyword.lower()
if keyword in ["ubuntu-22.04", "ubuntu22.04"]:
result = aws.ec2.get_ami(
most_recent=True,
owners=["099720109477"], # Canonical
filters=[
{"name": "name", "values": ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]},
{"name": "virtualization-type", "values": ["hvm"]},
],
)
return result.id
if keyword in ["rocky-8.10", "rockylinux-8.10", "rocky8.10"]:
result = aws.ec2.get_ami(
most_recent=True,
owners=["792107900819"], # Rocky Linux
filters=[
{"name": "name", "values": ["Rocky-8-ec2-8.10*x86_64"]},
{"name": "architecture", "values": ["x86_64"]},
],
)
return result.id
raise ValueError(f"❌ Unsupported AMI keyword: {ami_keyword}")

View File

@ -0,0 +1,54 @@
import pulumi_aws as aws
from pulumi_aws.ec2 import SecurityGroup, SecurityGroupIngressArgs, SecurityGroupEgressArgs
def create_security_group(vpc_id: str, rule_config: dict) -> SecurityGroup:
"""
创建 Security Group支持 ingress/egress 配置
:param vpc_id: 目标 VPC ID
:param rule_config: 单个 firewall_rules 的字典配置
:return: 创建的 SecurityGroup 资源对象
"""
ingress_rules = []
source_ranges = rule_config.get("source_ranges", ["0.0.0.0/0"])
egress_ranges = rule_config.get("egress_ranges", ["0.0.0.0/0"])
for allow_rule in rule_config.get("allow", []):
protocol = allow_rule.get("protocol", "tcp")
for port in allow_rule.get("ports", []):
if isinstance(port, str) and port in ["*", "any", "all"]:
from_port = 0
to_port = 65535
else:
port = int(port)
from_port = port
to_port = port
ingress_rules.append(
SecurityGroupIngressArgs(
protocol=protocol,
from_port=from_port,
to_port=to_port,
cidr_blocks=source_ranges
)
)
sg = aws.ec2.SecurityGroup(
rule_config.get("name", "default-sg"),
vpc_id=vpc_id,
description=f"Security Group: {rule_config.get('name', 'N/A')}",
ingress=ingress_rules,
egress=[
SecurityGroupEgressArgs(
protocol="-1",
from_port=0,
to_port=0,
cidr_blocks=egress_ranges
)
],
tags={"Name": rule_config.get("name", "default-sg")}
)
return sg

View File

@ -0,0 +1,53 @@
import pulumi_aws as aws
import pulumi
def create_vpc(vpc_conf, region):
# 1. VPC
vpc = aws.ec2.Vpc(vpc_conf['name'],
cidr_block=vpc_conf['cidr_block'],
tags={"Name": vpc_conf['name']}
)
# 2. Internet Gateway若有 public 子网)
has_public = any(subnet["type"] == "public" for subnet in vpc_conf["subnets"])
igw = aws.ec2.InternetGateway("main-igw", vpc_id=vpc.id) if has_public else None
# 3. 子网
subnets = {}
for subnet_cfg in vpc_conf["subnets"]:
subnet = aws.ec2.Subnet(subnet_cfg["name"],
vpc_id=vpc.id,
cidr_block=subnet_cfg["cidr_block"],
map_public_ip_on_launch=subnet_cfg["type"] == "public",
availability_zone=subnet_cfg["availability_zone"],
tags={"Name": subnet_cfg["name"]}
)
subnets[subnet_cfg["name"]] = subnet
# 4. 路由表(仅 public 支持)
if has_public:
rt = aws.ec2.RouteTable("public-route-table",
vpc_id=vpc.id,
routes=[{
"cidr_block": r["destination_cidr_block"],
"gateway_id": igw.id
} for r in vpc_conf.get("routes", []) if r["subnet_type"] == "public"]
)
# 关联 public 子网
for subnet_cfg in vpc_conf["subnets"]:
if subnet_cfg["type"] == "public":
aws.ec2.RouteTableAssociation(f"{subnet_cfg['name']}-assoc",
subnet_id=subnets[subnet_cfg["name"]].id,
route_table_id=rt.id
)
# 5. TODO: peering 支持(预留接口)
# if vpc_conf.get("peering", {}).get("enabled"):
# ...
return {
"vpc": vpc,
"subnets": subnets,
"igw": igw
}

View File

@ -1,4 +1,5 @@
pulumi
boto3
pulumi-aws
pulumi-gcp
pulumi-azure-native

View File

@ -138,7 +138,7 @@ EOF
# 2. 激活虚拟环境zsh/bash
source .venv/bin/activate
# 3. 安装依赖
python3 -m pip install -r requirements.txt
"$PYTHON_BIN" -m pip install -r "$PROJECT_ROOT/requirements.txt"
fi
# 3⃣ 检查 Ansible
@ -201,6 +201,16 @@ pulumi_run() {
# ✅ 明确选择 stack若不存在则创建避免交互式提示
pulumi stack select "$STACK_NAME" 2>/dev/null || pulumi stack init "$STACK_NAME"
# ✅ 自动从 config 读取 region 并设置 pulumi config防止 provider 报错)
if [ -f "$CONFIG_PATH/base.yaml" ]; then
region=$(grep '^ *region:' "$CONFIG_PATH/base.yaml" | awk '{print $2}')
if [ -n "$region" ]; then
pulumi config set aws:region "$region" --stack "$STACK_NAME" --non-interactive
echo "✅ Pulumi config 中设置 aws:region=$region"
fi
fi
if [ ! -d "$CONFIG_PATH" ] || [ -z "$(find "$CONFIG_PATH" -maxdepth 1 -name '*.yml' -o -name '*.yaml')" ]; then
echo "⚠️ 配置目录为空:$CONFIG_PATH,跳过部署"
exit 0