我最近通过 CDK 在 AWS 上设置了一个应用程序。该应用程序由一个 Dockerized nodejs 应用程序组成,该应用程序连接到一个 RDS 实例,并且还具有一个 Redis 缓存层。在部署应用程序几天后,即使流量很小,成本也比我预期的要高得多。通过成本浏览器查看后,看起来一半的成本来自 NAT 网关。
在我当前的设置中,我创建了两个 VPC。一个用于应用程序堆栈,另一个用于 CodePipeline。我需要为管道添加一个,因为如果没有它,我在 CodeBuildAction 步骤中尝试拉取 Docker 映像时会遇到速率限制。
我对网络位不是很满意,但我觉得涉及到额外的资源。管道 VPC 具有三个 NAT 网关和三个 EIP。这些最终只是坐在那里等待下一次部署,这似乎是一种巨大的浪费。似乎为在 CDK 中附加 VPC 的每个构造分配了一个新的网关 + EIP。我可以让它重复使用同一个吗?有没有一种替代方法可以完全添加 VPC 并且不受 Docker 的速率限制?
我还发现 NAT 网关与我当前的 Fargate 任务成本一样昂贵,这非常令人惊讶(我可能只是天真)。是否有替代方案可以满足我的目的,但成本要低一些?
无论如何,这是我的两个堆栈:
// pipeline-stack.ts
import { SecretValue, Stack, StackProps } from "aws-cdk-lib";
import { Construct } from "constructs";
import { Artifact, IStage, Pipeline } from "aws-cdk-lib/aws-codepipeline";
import {
CloudFormationCreateUpdateStackAction,
CodeBuildAction,
CodeBuildActionType,
GitHubSourceAction,
} from "aws-cdk-lib/aws-codepipeline-actions";
import {
BuildEnvironmentVariableType,
BuildSpec,
LinuxBuildImage,
PipelineProject,
} from "aws-cdk-lib/aws-codebuild";
import { SnsTopic } from "aws-cdk-lib/aws-events-targets";
import { Topic } from "aws-cdk-lib/aws-sns";
import { EventField, RuleTargetInput } from "aws-cdk-lib/aws-events";
import { EmailSubscription, SmsSubscription } from "aws-cdk-lib/aws-sns-subscriptions";
import ApiStack from "./stacks/api-stack";
import { ManagedPolicy, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam";
import { SecurityGroup, SubnetType, Vpc } from "aws-cdk-lib/aws-ec2";
import { Secret } from "aws-cdk-lib/aws-ecs";
import { BuildEnvironmentVariable } from "aws-cdk-lib/aws-codebuild/lib/project";
import * as SecretsManager from "aws-cdk-lib/aws-secretsmanager";
import { getApplicationEnvironment, getApplicationSecrets } from "./secrets-helper";
const capFirst = (str: string): string => {
return str.charAt(0).toUpperCase() + str.slice(1);
};
interface PipelineStackProps extends StackProps {
environment: string;
emailAddress: string;
phoneNumber: string;
branch: string;
secrets: {
arn: string;
};
repo: {
uri: string;
name: string;
};
}
export class PipelineStack extends Stack {
private readonly envName: string;
private readonly pipeline: Pipeline;
// source outputs
private cdkSourceOutput: Artifact;
private applicationSourceOutput: Artifact;
// code source actions
private cdkSourceAction: GitHubSourceAction;
private applicationSourceAction: GitHubSourceAction;
// build outputs
private cdkBuildOutput: Artifact;
private applicationBuildOutput: Artifact;
// notifications
private pipelineNotificationsTopic: Topic;
private readonly codeBuildVpc: Vpc;
private readonly codeBuildSecurityGroup: SecurityGroup;
private readonly secrets: SecretsManager.ISecret;
private readonly ecrCodeBuildRole: Role;
// stages
private sourceStage: IStage;
private selfMutateStage: IStage;
private buildStage: IStage;
private apiTestsStage: IStage;
constructor(scope: Construct, id: string, props: PipelineStackProps) {
super(scope, id, props);
this.envName = props.environment;
this.addNotifications(props);
this.ecrCodeBuildRole = new Role(this, "application-build-project-role", {
assumedBy: new ServicePrincipal("codebuild.amazonaws.com"),
managedPolicies: [
ManagedPolicy.fromAwsManagedPolicyName("AmazonEC2ContainerRegistryPowerUser"),
],
});
this.codeBuildVpc = new Vpc(this, "codebuild-vpc", {
vpcName: "codebuild-vpc",
enableDnsSupport: true,
});
this.codeBuildSecurityGroup = new SecurityGroup(this, "codebuild-vpc-security-group", {
vpc: this.codeBuildVpc,
allowAllOutbound: true,
});
this.secrets = SecretsManager.Secret.fromSecretCompleteArn(this, "secrets", props.secrets.arn);
this.pipeline = new Pipeline(this, "pipeline", {
pipelineName: `${capFirst(this.envName)}Pipeline`,
crossAccountKeys: false,
restartExecutionOnUpdate: true,
});
// STAGE 1 - Source Stage
this.addSourceStage(props);
// STAGE 2 - Build Stage
this.addBuildStage(props);
// STAGE 3: SelfMutate Stage
this.addSelfMutateStage();
// STAGE 4: Testing
this.addTestStage();
}
addNotifications(props: PipelineStackProps) {
this.pipelineNotificationsTopic = new Topic(this, "pipeline-notifications-topic", {
topicName: `PipelineNotifications${capFirst(props.environment)}`,
});
this.pipelineNotificationsTopic.addSubscription(new EmailSubscription(props.emailAddress));
this.pipelineNotificationsTopic.addSubscription(new SmsSubscription(props.phoneNumber));
}
/**
* Stage 1
*/
addSourceStage(props: PipelineStackProps) {
this.cdkSourceOutput = new Artifact("cdk-source-output");
this.cdkSourceAction = new GitHubSourceAction({
actionName: "CdkSource",
owner: "my-org",
repo: "my-cdk-repo",
branch: "main",
oauthToken: SecretValue.secretsManager("/connections/github/access-token"),
output: this.cdkSourceOutput,
});
this.applicationSourceOutput = new Artifact("ApplicationSourceOutput");
this.applicationSourceAction = new GitHubSourceAction({
actionName: "ApplicationSource",
owner: "my-org",
repo: "my-application-repo",
branch: props.branch,
oauthToken: SecretValue.secretsManager("/connections/github/access-token"),
output: this.applicationSourceOutput,
});
this.sourceStage = this.pipeline.addStage({
stageName: "Source",
actions: [this.cdkSourceAction, this.applicationSourceAction],
});
}
/**
* stage 2
*/
addBuildStage(props: PipelineStackProps) {
const cdkBuildAction = this.createCdkBuildAction();
const applicationBuildAction = this.createApplicationBuildAction(props);
this.buildStage = this.pipeline.addStage({
stageName: "Build",
actions: [cdkBuildAction, applicationBuildAction],
});
}
/**
* stage 3
*/
addSelfMutateStage() {
this.selfMutateStage = this.pipeline.addStage({
stageName: "PipelineUpdate",
actions: [
new CloudFormationCreateUpdateStackAction({
actionName: "PipelineCreateUpdateStackAction",
stackName: this.stackName,
templatePath: this.cdkBuildOutput.atPath(`${this.stackName}.template.json`),
adminPermissions: true,
}),
],
});
}
/**
* stage 4
*/
addTestStage() {
const testAction = new CodeBuildAction({
actionName: "RunApiTests",
type: CodeBuildActionType.TEST,
input: this.applicationSourceOutput,
project: new PipelineProject(this, "api-tests-project", {
vpc: this.codeBuildVpc,
securityGroups: [this.codeBuildSecurityGroup],
environment: {
buildImage: LinuxBuildImage.STANDARD_5_0,
privileged: true,
},
buildSpec: BuildSpec.fromObject({
version: "0.2",
phases: {
install: {
commands: ["cp .env.testing .env"],
},
build: {
commands: [
"ls",
"docker-compose -f docker-compose.staging.yml run -e NODE_ENV=testing --rm api node ace test",
],
},
},
}),
}),
runOrder: 1,
});
this.apiTestsStage = this.pipeline.addStage({
stageName: "RunApiTests",
actions: [testAction],
});
}
createCdkBuildAction() {
this.cdkBuildOutput = new Artifact("CdkBuildOutput");
return new CodeBuildAction({
actionName: "CdkBuildAction",
input: this.cdkSourceOutput,
outputs: [this.cdkBuildOutput],
project: new PipelineProject(this, "cdk-build-project", {
environment: {
buildImage: LinuxBuildImage.STANDARD_5_0,
},
buildSpec: BuildSpec.fromSourceFilename("build-specs/cdk-build-spec.yml"),
}),
});
}
createApplicationBuildAction(props: PipelineStackProps) {
this.applicationBuildOutput = new Artifact("ApplicationBuildOutput");
const project = new PipelineProject(this, "application-build-project", {
vpc: this.codeBuildVpc,
securityGroups: [this.codeBuildSecurityGroup],
environment: {
buildImage: LinuxBuildImage.STANDARD_5_0,
privileged: true,
},
environmentVariables: {
ENV: {
value: this.envName,
},
ECR_REPO_URI: {
value: props.repo.uri,
},
ECR_REPO_NAME: {
value: props.repo.name,
},
AWS_REGION: {
value: props.env!.region,
},
},
buildSpec: BuildSpec.fromObject({
version: "0.2",
phases: {
pre_build: {
commands: [
"echo 'Logging into Amazon ECR...'",
"aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $ECR_REPO_URI",
'COMMIT_HASH=$(echo "$CODEBUILD_RESOLVED_SOURCE_VERSION" | head -c 8)',
],
},
build: {
commands: ["docker build -t $ECR_REPO_NAME:latest ."],
},
post_build: {
commands: [
"docker tag $ECR_REPO_NAME:latest $ECR_REPO_URI/$ECR_REPO_NAME:latest",
"docker tag $ECR_REPO_NAME:latest $ECR_REPO_URI/$ECR_REPO_NAME:$ENV-$COMMIT_HASH",
"docker push $ECR_REPO_URI/$ECR_REPO_NAME:latest",
"docker push $ECR_REPO_URI/$ECR_REPO_NAME:$ENV-$COMMIT_HASH",
],
},
},
}),
role: this.ecrCodeBuildRole,
});
return new CodeBuildAction({
actionName: "ApplicationBuildAction",
input: this.applicationSourceOutput,
outputs: [this.applicationBuildOutput],
project: project,
});
}
public addDatabaseMigrationStage(apiStack: ApiStack, stageName: string): IStage {
let buildEnv: { [name: string]: BuildEnvironmentVariable } = {
ENV: {
value: this.envName,
},
ECR_REPO_URI: {
type: BuildEnvironmentVariableType.PLAINTEXT,
value: apiStack.repoUri,
},
ECR_REPO_NAME: {
type: BuildEnvironmentVariableType.PLAINTEXT,
value: apiStack.repoName,
},
AWS_REGION: {
type: BuildEnvironmentVariableType.PLAINTEXT,
value: this.region,
},
};
buildEnv = this.getBuildEnvAppSecrets(getApplicationSecrets(this.secrets), buildEnv);
buildEnv = this.getBuildEnvAppEnvVars(
getApplicationEnvironment({
REDIS_HOST: apiStack.redisHost.importValue,
REDIS_PORT: apiStack.redisPort.importValue,
}),
buildEnv,
);
let envVarNames = Object.keys(buildEnv);
const envFileCommand = `printenv | grep '${envVarNames.join("\\|")}' >> .env`;
return this.pipeline.addStage({
stageName: stageName,
actions: [
new CodeBuildAction({
actionName: "DatabaseMigrations",
input: this.applicationSourceOutput,
project: new PipelineProject(this, "database-migrations-project", {
description: "Run database migrations against RDS database",
environment: {
buildImage: LinuxBuildImage.STANDARD_5_0,
privileged: true,
},
environmentVariables: buildEnv,
buildSpec: BuildSpec.fromObject({
version: "0.2",
phases: {
pre_build: {
commands: [
"echo 'Logging into Amazon ECR...'",
"aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $ECR_REPO_URI",
'COMMIT_HASH=$(echo "$CODEBUILD_RESOLVED_SOURCE_VERSION" | head -c 8)',
envFileCommand,
"cat .env",
],
},
build: {
commands: [
`docker run --env-file .env --name api $ECR_REPO_URI/$ECR_REPO_NAME:$ENV-$COMMIT_HASH node ace migration:run --force`,
": > .env",
],
},
},
}),
role: this.ecrCodeBuildRole,
}),
}),
],
});
}
private getBuildEnvAppSecrets(
secrets: { [key: string]: Secret },
buildEnv: { [name: string]: BuildEnvironmentVariable },
): { [name: string]: BuildEnvironmentVariable } {
for (let key in secrets) {
buildEnv[key] = {
type: BuildEnvironmentVariableType.SECRETS_MANAGER,
value: `${this.secrets.secretArn}:${key}`,
};
}
return buildEnv;
}
private getBuildEnvAppEnvVars(
vars: { [key: string]: string },
buildEnv: { [name: string]: BuildEnvironmentVariable },
): { [name: string]: BuildEnvironmentVariable } {
for (let key in vars) {
buildEnv[key] = {
value: vars[key],
};
}
return buildEnv;
}
public addApplicationStage(apiStack: ApiStack, stageName: string): IStage {
return this.pipeline.addStage({
stageName: stageName,
actions: [
new CloudFormationCreateUpdateStackAction({
actionName: "ApplicationUpdate",
stackName: apiStack.stackName,
templatePath: this.cdkBuildOutput.atPath(`${apiStack.stackName}.template.json`),
adminPermissions: true,
}),
],
});
}
}
// api-stack.ts
import { CfnOutput, CfnResource, Lazy, Stack, StackProps } from "aws-cdk-lib";
import * as EC2 from "aws-cdk-lib/aws-ec2";
import { ISubnet } from "aws-cdk-lib/aws-ec2";
import * as ECS from "aws-cdk-lib/aws-ecs";
import { DeploymentControllerType, ScalableTaskCount } from "aws-cdk-lib/aws-ecs";
import * as EcsPatterns from "aws-cdk-lib/aws-ecs-patterns";
import * as RDS from "aws-cdk-lib/aws-rds";
import { Credentials } from "aws-cdk-lib/aws-rds";
import * as Route53 from "aws-cdk-lib/aws-route53";
import * as Route53Targets from "aws-cdk-lib/aws-route53-targets";
import * as ECR from "aws-cdk-lib/aws-ecr";
import * as CertificateManager from "aws-cdk-lib/aws-certificatemanager";
import * as SecretsManager from "aws-cdk-lib/aws-secretsmanager";
import * as ElasticCache from "aws-cdk-lib/aws-elasticache";
import { Construct } from "constructs";
import { getApplicationEnvironment, getApplicationSecrets } from "../secrets-helper";
export type ApiStackProps = StackProps & {
environment: string;
hostedZone: {
id: string;
name: string;
};
domainName: string;
scaling: {
desiredCount: number;
maxCount: number;
cpuPercentage: number;
memoryPercentage: number;
};
repository: {
uri: string;
arn: string;
name: string;
};
secrets: { arn: string };
};
export default class ApiStack extends Stack {
vpc: EC2.Vpc;
cluster: ECS.Cluster;
ecsService: EcsPatterns.ApplicationLoadBalancedFargateService;
certificate: CertificateManager.ICertificate;
repository: ECR.IRepository;
database: RDS.IDatabaseInstance;
databaseCredentials: Credentials;
hostedZone: Route53.IHostedZone;
aliasRecord: Route53.ARecord;
redis: ElasticCache.CfnReplicationGroup;
repoUri: string;
repoName: string;
applicationEnvVariables: {
[key: string]: string;
};
redisHost: CfnOutput;
redisPort: CfnOutput;
gatewayUrl: CfnOutput;
constructor(scope: Construct, id: string, props: ApiStackProps) {
super(scope, id, props);
this.repoUri = props.repository.uri;
this.repoName = props.repository.name;
this.setUpVpc(props);
this.setUpRedisCluster(props);
this.setUpDatabase(props);
this.setUpCluster(props);
this.setUpHostedZone(props);
this.setUpCertificate(props);
this.setUpRepository(props);
this.setUpEcsService(props);
this.setUpAliasRecord(props);
}
private resourceName(props: ApiStackProps, resourceType: string): string {
return `twibs-api-${resourceType}-${props.environment}`;
}
private setUpVpc(props: ApiStackProps) {
this.vpc = new EC2.Vpc(this, this.resourceName(props, "vpc"), {
maxAzs: 3, // Default is all AZs in region
});
}
private setUpRedisCluster(props: ApiStackProps) {
const subnetGroup = new ElasticCache.CfnSubnetGroup(this, "cache-subnet-group", {
cacheSubnetGroupName: "redis-cache-subnet-group",
subnetIds: this.vpc.privateSubnets.map((subnet: ISubnet) => subnet.subnetId),
description: "Subnet group for Redis Cache cluster",
});
const securityGroup = new EC2.SecurityGroup(this, "redis-security-group", {
vpc: this.vpc,
description: `SecurityGroup associated with RedisDB Cluster - ${props.environment}`,
allowAllOutbound: false,
});
securityGroup.addIngressRule(
EC2.Peer.ipv4(this.vpc.vpcCidrBlock),
EC2.Port.tcp(6379),
"Allow from VPC on port 6379",
);
this.redis = new ElasticCache.CfnReplicationGroup(this, "redis", {
numNodeGroups: 1,
cacheNodeType: "cache.t2.small",
engine: "redis",
multiAzEnabled: false,
autoMinorVersionUpgrade: false,
cacheParameterGroupName: "default.redis6.x.cluster.on",
engineVersion: "6.x",
cacheSubnetGroupName: subnetGroup.ref,
securityGroupIds: [securityGroup.securityGroupId],
replicationGroupDescription: "RedisDB setup by CDK",
replicasPerNodeGroup: 0,
port: 6379,
});
}
private setUpDatabase(props: ApiStackProps) {
if (["production", "staging", "develop"].includes(props.environment)) {
return;
}
this.databaseCredentials = Credentials.fromUsername("my_db_username");
this.database = new RDS.DatabaseInstance(this, "database", {
vpc: this.vpc,
engine: RDS.DatabaseInstanceEngine.postgres({
version: RDS.PostgresEngineVersion.VER_13_4,
}),
credentials: this.databaseCredentials,
databaseName: `my_app_${props.environment}`,
deletionProtection: true,
});
}
private setUpCluster(props: ApiStackProps) {
this.cluster = new ECS.Cluster(this, this.resourceName(props, "cluster"), {
vpc: this.vpc,
capacity: {
instanceType: EC2.InstanceType.of(EC2.InstanceClass.T3, EC2.InstanceSize.`SMALL`),
},
});
}
private setUpHostedZone(props: ApiStackProps) {
this.hostedZone = Route53.HostedZone.fromHostedZoneAttributes(
this,
this.resourceName(props, "hosted-zone"),
{
hostedZoneId: props.hostedZone.id,
zoneName: props.hostedZone.name,
},
);
}
private setUpCertificate(props: ApiStackProps) {
this.certificate = new CertificateManager.Certificate(this, "certificate", {
domainName: props.domainName,
validation: CertificateManager.CertificateValidation.fromDns(this.hostedZone),
});
}
private setUpRepository(props: ApiStackProps) {
this.repository = ECR.Repository.fromRepositoryAttributes(
this,
this.resourceName(props, "repository"),
{
repositoryArn: props.repository.arn,
repositoryName: props.repository.name,
},
);
}
private setUpEcsService(props: ApiStackProps) {
const secrets = SecretsManager.Secret.fromSecretCompleteArn(this, "secrets", props.secrets.arn);
this.redisHost = new CfnOutput(this, "redis-host-output", {
value: this.redis.attrConfigurationEndPointAddress,
exportName: "redis-host-output",
});
this.redisPort = new CfnOutput(this, "redis-port-output", {
value: this.redis.attrConfigurationEndPointPort,
exportName: "redis-port-output",
});
// Create a load-balanced ecs-service service and make it public
this.ecsService = new EcsPatterns.ApplicationLoadBalancedFargateService(
this,
this.resourceName(props, "ecs-service"),
{
serviceName: `${props.environment}-api-service`,
cluster: this.cluster, // Required
cpu: 256, // Default is 256
desiredCount: props.scaling.desiredCount, // Default is 1
taskImageOptions: {
image: ECS.ContainerImage.fromEcrRepository(this.repository),
environment: getApplicationEnvironment({
REDIS_HOST: this.redis.attrConfigurationEndPointAddress,
REDIS_PORT: this.redis.attrConfigurationEndPointPort,
}),
secrets: getApplicationSecrets(secrets),
},
memoryLimitMiB: 512, // Default is 512
publicLoadBalancer: true, // Default is false
domainZone: this.hostedZone,
certificate: this.certificate,
},
);
const scalableTarget = this.ecsService.service.autoScaleTaskCount({
minCapacity: props.scaling.desiredCount,
maxCapacity: props.scaling.maxCount,
});
scalableTarget.scaleOnCpuUtilization("cpu-scaling", {
targetUtilizationPercent: props.scaling.cpuPercentage,
});
scalableTarget.scaleOnMemoryUtilization("memory-scaling", {
targetUtilizationPercent: props.scaling.memoryPercentage,
});
secrets.grantRead(this.ecsService.taskDefinition.taskRole);
}
private setUpAliasRecord(props: ApiStackProps) {
this.gatewayUrl = new CfnOutput(this, "gateway-url-output", {
value: this.ecsService.loadBalancer.loadBalancerDnsName,
});
this.aliasRecord = new Route53.ARecord(this, "alias-record", {
zone: this.hostedZone,
recordName: props.domainName,
target: Route53.RecordTarget.fromAlias(
new Route53Targets.LoadBalancerTarget(this.ecsService.loadBalancer),
),
});
const shouldCreateWWW = props.domainName.split(".").length === 2;
if (shouldCreateWWW) {
new Route53.ARecord(this, "alias-record-www", {
zone: this.hostedZone,
recordName: `www.${props.domainName}`,
target: Route53.RecordTarget.fromAlias(
new Route53Targets.LoadBalancerTarget(this.ecsService.loadBalancer),
),
});
}
}
}
任何意见是极大的赞赏。