diff --git a/config/sit/base.yaml b/config/sit/base.yaml index d85b4c04..f4e2dea4 100644 --- a/config/sit/base.yaml +++ b/config/sit/base.yaml @@ -1,7 +1,6 @@ aws: - access_key: YOUR_ACCESS_KEY - secret_key: YOUR_SECRET_KEY - region: us-east-1 + profile: default + region: ap-northeast-1 key_pairs: - name: dev_key - key_file: keys/dev_ssh.pub + key_file: ~/.ssh/id_rsa.pub diff --git a/config/sit/firewall.yaml b/config/sit/firewall.yaml index 12e59652..30c29134 100644 --- a/config/sit/firewall.yaml +++ b/config/sit/firewall.yaml @@ -1,6 +1,7 @@ firewall_rules: - name: allow-ssh-web + source_ranges: ["0.0.0.0/0"] + egress_ranges: ["10.0.0.0/16"] allow: - protocol: tcp ports: ["22", "80", "443"] - source_ranges: ["0.0.0.0/0"] diff --git a/config/sit/instances.yaml b/config/sit/instances.yaml index 0e6f8fde..e5730adb 100644 --- a/config/sit/instances.yaml +++ b/config/sit/instances.yaml @@ -1,32 +1,45 @@ instances: - name: master-1 - ami: ami-0c2b8ca1dad447f8a + ami: ubuntu-22.04 # ✅ 可用 ami-xxx 或关键词(如 ubuntu-22.04) type: t3.micro disk_size_gb: 20 subnet: public-subnet-1 - lifecycle: spot # 可选: ondemand(默认)或 spot - ttl: 1h # 可选: 自动标记 TTL,仅作为标识,不自动销毁 + lifecycle: spot # 可选: ondemand(默认)或 spot + ttl: 1h # 可选: 标记生命周期(不会自动销毁) + env: sit # 可选: dev/sit/prod 等环境标签 + owner: devops # 可选: 资源责任人标签 + associate_public_ip: true # ✅ 明确配置是否需要公网 IP - name: slave-1 - ami: ami-0c2b8ca1dad447f8a + ami: ubuntu-22.04 type: t3.micro disk_size_gb: 20 subnet: private-subnet-1 lifecycle: spot ttl: 1h + env: sit + owner: devops + associate_public_ip: false - name: agent-1 - ami: ami-0c2b8ca1dad447f8a + ami: ubuntu-22.04 type: t3.micro disk_size_gb: 20 subnet: private-subnet-1 lifecycle: spot ttl: 1h + env: sit + owner: devops + associate_public_ip: false - name: agent-2 - ami: ami-0c2b8ca1dad447f8a + ami: ubuntu-22.04 type: t3.micro disk_size_gb: 20 subnet: private-subnet-1 lifecycle: spot ttl: 1h + env: sit + owner: devops + associate_public_ip: false + diff --git a/config/sit/vpc.yaml b/config/sit/vpc.yaml index c5d1e933..85064bf9 100644 --- a/config/sit/vpc.yaml +++ b/config/sit/vpc.yaml @@ -1,14 +1,14 @@ vpc: name: dev-vpc - cidr_block: 10.0.0.0/16 + cidr_block: 10.1.0.0/16 subnets: - name: public-subnet-1 - cidr_block: 10.0.1.0/24 - availability_zone: us-east-1a + cidr_block: 10.1.1.0/24 + availability_zone: ap-northeast-1a type: public - name: private-subnet-1 - cidr_block: 10.0.101.0/24 - availability_zone: us-east-1a + cidr_block: 10.1.101.0/24 + availability_zone: ap-northeast-1c type: private routes: diff --git a/iac_modules/pulumi/__main__.py b/iac_modules/pulumi/__main__.py new file mode 100644 index 00000000..8ea77903 --- /dev/null +++ b/iac_modules/pulumi/__main__.py @@ -0,0 +1,112 @@ +import os +import sys +import pulumi +import pulumi_aws as aws +import boto3 +from botocore.exceptions import ProfileNotFound, NoCredentialsError + +from utils.config_loader import load_merged_config +from modules.vpc.vpc import create_vpc +from modules.security_group.sg import create_security_group +from modules.ec2.ec2_instance import create_instances + +# ✅ 加载配置 +config_dir = os.environ.get("CONFIG_PATH", "config") +config = load_merged_config(config_dir) + +aws_conf = config.get("aws", {}) +region = aws_conf.get("region", "us-east-1") +profile = aws_conf.get("profile", "default") +key_pairs = aws_conf.get("key_pairs", []) + +# ✅ 设置 AWS 配置 +aws.config.region = region +aws.config.profile = profile +pulumi.runtime.set_config("aws:region", region) + +# ✅ 检查 AWS 凭证 +try: + session = boto3.Session(profile_name=profile) + credentials = session.get_credentials() + if not credentials: + raise NoCredentialsError() +except (ProfileNotFound, NoCredentialsError): + pulumi.log.error(f"❌ AWS profile '{profile}' 无效或找不到凭证") + sys.exit(1) +else: + pulumi.log.info(f"✅ AWS credentials loaded (profile: {profile}, region: {region})") + +# ✅ 初始化资源容器 +global_dependencies = [] +vpc = None +subnets = {} +sg = None +key_pair = None + +# ======================== +# ✅ [模块] VPC + Subnets +# ======================== +vpc_conf = config.get("vpc", {}) +if vpc_conf.get("enabled", True): + vpc_result = create_vpc(vpc_conf, region) + vpc = vpc_result["vpc"] + subnets = vpc_result["subnets"] + global_dependencies.append(vpc) + global_dependencies.extend(subnets.values()) + pulumi.log.info("✅ VPC/Subnet 已创建") +else: + pulumi.log.warn("⏭️ 跳过 VPC 创建") + +# ======================== +# ✅ [模块] Security Group +# ======================== +firewall_rules = config.get("firewall_rules", []) +if firewall_rules and vpc and config.get("security_group", {}).get("enabled", True): + sg = create_security_group(vpc.id, firewall_rules[0]) + global_dependencies.append(sg) + pulumi.log.info("✅ Security Group 已创建") +else: + pulumi.log.warn("⏭️ 跳过 Security Group 创建") + +# ======================== +# ✅ [模块] SSH Key Pair +# ======================== +if key_pairs: + key_cfg = key_pairs[0] + public_key_path = os.path.expanduser(key_cfg["key_file"]) + if not os.path.exists(public_key_path): + raise FileNotFoundError(f"❌ SSH 公钥文件不存在: {public_key_path}") + with open(public_key_path) as f: + public_key = f.read().strip() + key_pair = aws.ec2.KeyPair("main-key", + key_name=key_cfg["name"], + public_key=public_key + ) + global_dependencies.append(key_pair) + pulumi.log.info("✅ SSH KeyPair 已创建") +else: + pulumi.log.warn("⏭️ 跳过 KeyPair 创建") + +# ======================== +# ✅ [模块] EC2 实例部署 +# ======================== +instances_conf = config.get("instances", []) +ec2_outputs = {} + +if instances_conf and config.get("ec2", {}).get("enabled", True): + ec2_outputs = create_instances( + instances_conf, + subnets, + sg, # ✅ 注意这里传的是资源对象 + key_pair.key_name if key_pair else None, + depends_on=global_dependencies + ) + pulumi.log.info("✅ EC2 实例已创建") +else: + pulumi.log.warn("⏭️ 跳过 EC2 实例部署") + +# ======================== +# ✅ 导出所有实例信息 +# ======================== +for name, ip in ec2_outputs.items(): + pulumi.export(f"{name}", ip) diff --git a/iac_modules/pulumi/deploy.py b/iac_modules/pulumi/deploy.py deleted file mode 100644 index e6757d7d..00000000 --- a/iac_modules/pulumi/deploy.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import pulumi -import pulumi_aws as aws -from utils.config_loader import load_merged_config -from modules.vpc.vpc import create_vpc -from modules.security_group.sg import create_security_group -from modules.ec2.ec2_instance import create_instances - -# ✅ 自动从环境变量获取配置路径,默认为 "config/" -config_dir = os.environ.get("CONFIG_PATH", "config") -config = load_merged_config(config_dir) - -# ✅ 提取配置项(如为空跳过) -aws_conf = config.get("aws") -vpc_conf = config.get("vpc") -instances_conf = config.get("instances", []) -firewall_rules = config.get("firewall_rules", []) - -if not aws_conf or not vpc_conf: - pulumi.log.warn(f"❌ 配置不完整,缺少 aws 或 vpc 段,终止部署。CONFIG_PATH={config_dir}") - exit(0) - -# ✅ 配置 AWS 凭据 -aws.config.region = aws_conf["region"] -aws.config.access_key = aws_conf["access_key"] -aws.config.secret_key = aws_conf["secret_key"] - -# ✅ 创建 VPC 与子网 -vpc_result = create_vpc(vpc_conf, aws_conf["region"]) -vpc = vpc_result["vpc"] -subnets = vpc_result["subnets"] - -# ✅ 创建安全组(取第一组规则) -if not firewall_rules: - pulumi.log.warn("⚠️ 未定义 firewall_rules,默认跳过安全组配置") - sg_id = None -else: - sg = create_security_group(vpc.id, firewall_rules[0]) - sg_id = sg.id - -# ✅ SSH 密钥对 -key_cfg = aws_conf["key_pairs"][0] -public_key_path = key_cfg["key_file"] -if not os.path.exists(public_key_path): - raise FileNotFoundError(f"❌ SSH 公钥文件不存在: {public_key_path}") -with open(public_key_path, "r") as f: - public_key = f.read().strip() - -key_pair = aws.ec2.KeyPair("main-key", - key_name=key_cfg["name"], - public_key=public_key -) - -# ✅ 创建实例(自动匹配子网) -if not instances_conf: - pulumi.log.warn("⚠️ 未配置任何 EC2 实例,跳过实例部署") - outputs = {} -else: - outputs = create_instances(instances_conf, subnets, sg_id, key_pair.key_name) - -# ✅ 导出所有实例的公网 IP -for name, ip in outputs.items(): - pulumi.export(f"{name}_ip", ip) diff --git a/iac_modules/pulumi/modules/__init__.py b/iac_modules/pulumi/modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/iac_modules/pulumi/modules/ec2/__init__.py b/iac_modules/pulumi/modules/ec2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/iac_modules/pulumi/modules/ec2/ec2_instance.py b/iac_modules/pulumi/modules/ec2/ec2_instance.py index 6077b9cd..8675636e 100644 --- a/iac_modules/pulumi/modules/ec2/ec2_instance.py +++ b/iac_modules/pulumi/modules/ec2/ec2_instance.py @@ -1,28 +1,53 @@ +import os +import pulumi import pulumi_aws as aws +from .utils import resolve_ami -def create_instances(instances_config, subnets_dict, sg_id, key_name): +def create_instances(instances_config, subnets_dict, sg_resource, key_name, depends_on=None): outputs = {} for instance_cfg in instances_config: name = instance_cfg["name"] subnet_name = instance_cfg["subnet"] - subnet_id = subnets_dict[subnet_name].id - ami = instance_cfg["ami"] + subnet = subnets_dict[subnet_name] + subnet_id = subnet.id + + # ✅ 自动解析 AMI(关键词或 AMI ID) + region = aws.config.region + ami = resolve_ami(instance_cfg["ami"], region) + instance_type = instance_cfg["type"] disk_size = instance_cfg["disk_size_gb"] - # 读取可选字段 - lifecycle = instance_cfg.get("lifecycle", "ondemand") # 默认按需 - ttl = instance_cfg.get("ttl", "none") # 默认无 TTL + # ✅ 可选字段解析 + lifecycle = instance_cfg.get("lifecycle", "ondemand") + ttl = instance_cfg.get("ttl", "none") + env = instance_cfg.get("env", "dev") + owner = instance_cfg.get("owner", "unknown") + user_data_path = instance_cfg.get("user_data") + private_ip = instance_cfg.get("private_ip", None) + associate_public_ip = instance_cfg.get("associate_public_ip", True) - # 设置 EC2 标签 + # ✅ User data 读取(可选) + user_data = None + if user_data_path: + expanded_path = os.path.expanduser(user_data_path) + if os.path.exists(expanded_path): + with open(expanded_path, "r") as f: + user_data = f.read() + else: + pulumi.log.warn(f"⚠️ user_data 文件不存在: {expanded_path}") + + # ✅ 标签定义 tags = { "Name": name, "Lifecycle": lifecycle, "TTL": ttl, + "Environment": env, + "Owner": owner, } - # 如果是 Spot 实例,设置市场选项(不设 max_price → 自动出价) + # ✅ Spot 实例配置 instance_market_options = None if lifecycle == "spot": instance_market_options = aws.ec2.InstanceInstanceMarketOptionsArgs( @@ -33,22 +58,35 @@ def create_instances(instances_config, subnets_dict, sg_id, key_name): ) ) - # 创建 EC2 实例 + # ✅ 构建依赖项(必须是 Resource 对象) + resource_dependencies = [subnet] + if isinstance(sg_resource, pulumi.Resource): + resource_dependencies.append(sg_resource) + if depends_on: + resource_dependencies.extend(depends_on) + + # ✅ 创建 EC2 实例 ec2 = aws.ec2.Instance(name, ami=ami, instance_type=instance_type, key_name=key_name, subnet_id=subnet_id, - vpc_security_group_ids=[sg_id], - associate_public_ip_address=True, + private_ip=private_ip, + associate_public_ip_address=associate_public_ip, + vpc_security_group_ids=[sg_resource.id] if sg_resource else [], + user_data=user_data, root_block_device={ "volume_size": disk_size, "volume_type": "gp2" }, instance_market_options=instance_market_options, - tags=tags + tags=tags, + opts=pulumi.ResourceOptions(depends_on=resource_dependencies) ) - outputs[name] = ec2.public_ip + # ✅ 输出信息收集 + outputs[name + "_id"] = ec2.id + outputs[name + "_public_ip"] = ec2.public_ip + outputs[name + "_private_ip"] = ec2.private_ip return outputs diff --git a/iac_modules/pulumi/modules/ec2/utils.py b/iac_modules/pulumi/modules/ec2/utils.py new file mode 100644 index 00000000..4ae0d2ed --- /dev/null +++ b/iac_modules/pulumi/modules/ec2/utils.py @@ -0,0 +1,37 @@ +import pulumi_aws as aws + +def resolve_ami(ami_keyword: str, region: str) -> str: + """ + 根据关键词解析 AMI ID。如果已是 AMI ID,则直接返回。 + """ + if not aws.config.region: + raise ValueError("❌ AWS region is not set. Please set aws.config.region before calling resolve_ami") + + if ami_keyword.startswith("ami-"): + return ami_keyword + + keyword = ami_keyword.lower() + + if keyword in ["ubuntu-22.04", "ubuntu22.04"]: + result = aws.ec2.get_ami( + most_recent=True, + owners=["099720109477"], # Canonical + filters=[ + {"name": "name", "values": ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]}, + {"name": "virtualization-type", "values": ["hvm"]}, + ], + ) + return result.id + + if keyword in ["rocky-8.10", "rockylinux-8.10", "rocky8.10"]: + result = aws.ec2.get_ami( + most_recent=True, + owners=["792107900819"], # Rocky Linux + filters=[ + {"name": "name", "values": ["Rocky-8-ec2-8.10*x86_64"]}, + {"name": "architecture", "values": ["x86_64"]}, + ], + ) + return result.id + + raise ValueError(f"❌ Unsupported AMI keyword: {ami_keyword}") diff --git a/iac_modules/pulumi/modules/security_group/__init__.py b/iac_modules/pulumi/modules/security_group/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/iac_modules/pulumi/modules/security_group/sg.py b/iac_modules/pulumi/modules/security_group/sg.py new file mode 100644 index 00000000..0dbe290a --- /dev/null +++ b/iac_modules/pulumi/modules/security_group/sg.py @@ -0,0 +1,54 @@ +import pulumi_aws as aws +from pulumi_aws.ec2 import SecurityGroup, SecurityGroupIngressArgs, SecurityGroupEgressArgs + +def create_security_group(vpc_id: str, rule_config: dict) -> SecurityGroup: + """ + 创建 Security Group,支持 ingress/egress 配置 + :param vpc_id: 目标 VPC ID + :param rule_config: 单个 firewall_rules 的字典配置 + :return: 创建的 SecurityGroup 资源对象 + """ + ingress_rules = [] + + source_ranges = rule_config.get("source_ranges", ["0.0.0.0/0"]) + egress_ranges = rule_config.get("egress_ranges", ["0.0.0.0/0"]) + + for allow_rule in rule_config.get("allow", []): + protocol = allow_rule.get("protocol", "tcp") + + for port in allow_rule.get("ports", []): + if isinstance(port, str) and port in ["*", "any", "all"]: + from_port = 0 + to_port = 65535 + else: + port = int(port) + from_port = port + to_port = port + + ingress_rules.append( + SecurityGroupIngressArgs( + protocol=protocol, + from_port=from_port, + to_port=to_port, + cidr_blocks=source_ranges + ) + ) + + sg = aws.ec2.SecurityGroup( + rule_config.get("name", "default-sg"), + vpc_id=vpc_id, + description=f"Security Group: {rule_config.get('name', 'N/A')}", + ingress=ingress_rules, + egress=[ + SecurityGroupEgressArgs( + protocol="-1", + from_port=0, + to_port=0, + cidr_blocks=egress_ranges + ) + ], + tags={"Name": rule_config.get("name", "default-sg")} + ) + + return sg + diff --git a/iac_modules/pulumi/modules/vpc/__init__.py b/iac_modules/pulumi/modules/vpc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/iac_modules/pulumi/modules/vpc/vpc.py b/iac_modules/pulumi/modules/vpc/vpc.py new file mode 100644 index 00000000..7fa93903 --- /dev/null +++ b/iac_modules/pulumi/modules/vpc/vpc.py @@ -0,0 +1,53 @@ +import pulumi_aws as aws +import pulumi + +def create_vpc(vpc_conf, region): + # 1. VPC + vpc = aws.ec2.Vpc(vpc_conf['name'], + cidr_block=vpc_conf['cidr_block'], + tags={"Name": vpc_conf['name']} + ) + + # 2. Internet Gateway(若有 public 子网) + has_public = any(subnet["type"] == "public" for subnet in vpc_conf["subnets"]) + igw = aws.ec2.InternetGateway("main-igw", vpc_id=vpc.id) if has_public else None + + # 3. 子网 + subnets = {} + for subnet_cfg in vpc_conf["subnets"]: + subnet = aws.ec2.Subnet(subnet_cfg["name"], + vpc_id=vpc.id, + cidr_block=subnet_cfg["cidr_block"], + map_public_ip_on_launch=subnet_cfg["type"] == "public", + availability_zone=subnet_cfg["availability_zone"], + tags={"Name": subnet_cfg["name"]} + ) + subnets[subnet_cfg["name"]] = subnet + + # 4. 路由表(仅 public 支持) + if has_public: + rt = aws.ec2.RouteTable("public-route-table", + vpc_id=vpc.id, + routes=[{ + "cidr_block": r["destination_cidr_block"], + "gateway_id": igw.id + } for r in vpc_conf.get("routes", []) if r["subnet_type"] == "public"] + ) + + # 关联 public 子网 + for subnet_cfg in vpc_conf["subnets"]: + if subnet_cfg["type"] == "public": + aws.ec2.RouteTableAssociation(f"{subnet_cfg['name']}-assoc", + subnet_id=subnets[subnet_cfg["name"]].id, + route_table_id=rt.id + ) + + # 5. TODO: peering 支持(预留接口) + # if vpc_conf.get("peering", {}).get("enabled"): + # ... + + return { + "vpc": vpc, + "subnets": subnets, + "igw": igw + } diff --git a/requirements.txt b/requirements.txt index b60a3a3f..1fc49d7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pulumi +boto3 pulumi-aws pulumi-gcp pulumi-azure-native diff --git a/scripts/run.sh b/scripts/run.sh index f7cc0599..3a9733f5 100644 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -138,7 +138,7 @@ EOF # 2. 激活虚拟环境(zsh/bash) source .venv/bin/activate # 3. 安装依赖 - python3 -m pip install -r requirements.txt + "$PYTHON_BIN" -m pip install -r "$PROJECT_ROOT/requirements.txt" fi # 3️⃣ 检查 Ansible @@ -201,6 +201,16 @@ pulumi_run() { # ✅ 明确选择 stack,若不存在则创建,避免交互式提示 pulumi stack select "$STACK_NAME" 2>/dev/null || pulumi stack init "$STACK_NAME" + # ✅ 自动从 config 读取 region 并设置 pulumi config(防止 provider 报错) + if [ -f "$CONFIG_PATH/base.yaml" ]; then + region=$(grep '^ *region:' "$CONFIG_PATH/base.yaml" | awk '{print $2}') + if [ -n "$region" ]; then + pulumi config set aws:region "$region" --stack "$STACK_NAME" --non-interactive + echo "✅ Pulumi config 中设置 aws:region=$region" + fi + fi + + if [ ! -d "$CONFIG_PATH" ] || [ -z "$(find "$CONFIG_PATH" -maxdepth 1 -name '*.yml' -o -name '*.yaml')" ]; then echo "⚠️ 配置目录为空:$CONFIG_PATH,跳过部署" exit 0