Skip to content

Commit f29840a

Browse files
drewhan90drewhan90
authored
feat: cleanup idle stages by project tag (#35)
1 parent e7c2133 commit f29840a

File tree

7 files changed

+110
-14
lines changed

7 files changed

+110
-14
lines changed

‎cdk/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ PUBLISH ?= false
1010
STACK ?= UGC-$(STAGE)
1111
COGNITO_CLEANUP_SCHEDULE ?= rate(48 hours)
1212
STAGE_CLEANUP_SCHEDULE ?= rate(24 hours)
13-
CDK_OPTIONS = $(if $(AWS_PROFILE),$(AWS_PROFILE_FLAG)) -c stage=$(STAGE) -c publish=$(PUBLISH) -c stackName=$(STACK) -c cognitoCleanupScheduleExp="$(strip $(COGNITO_CLEANUP_SCHEDULE))"
13+
CDK_OPTIONS = $(if $(AWS_PROFILE),$(AWS_PROFILE_FLAG)) -c stage=$(STAGE) -c publish=$(PUBLISH) -c stackName=$(STACK) -c cognitoCleanupScheduleExp="$(strip $(COGNITO_CLEANUP_SCHEDULE))" -c stageCleanupScheduleExp="$(strip $(STAGE_CLEANUP_SCHEDULE))"
1414
FE_DEPLOYMENT_STACK = UGC-Frontend-Deployment-$(STAGE)
1515
SEED_COUNT ?= 50
1616
OFFLINE_SESSION_COUNT ?= 1

‎cdk/api/stages/authRouter/createStage.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ const handler = async (request: FastifyRequest, reply: FastifyReply) => {
6464
],
6565
tags: {
6666
creationDate: stageCreationDate,
67-
stageOwnerChannelId: channelId
67+
stageOwnerChannelId: channelId,
68+
stack: process.env.STACK as string
6869
}
6970
};
7071

‎cdk/bin/cdk.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,8 @@ const app = new App();
1212
const stage = app.node.tryGetContext('stage');
1313
const stackName = app.node.tryGetContext('stackName');
1414
const shouldPublish = app.node.tryGetContext('publish') === 'true';
15-
const cognitoCleanupScheduleExp = app.node.tryGetContext(
16-
'cognitoCleanupScheduleExp'
17-
);
15+
const cognitoCleanupScheduleExp = app.node.tryGetContext('cognitoCleanupScheduleExp');
16+
const stageCleanupScheduleExp = app.node.tryGetContext('stageCleanupScheduleExp')
1817
// Get the config for the current stage
1918
const { resourceConfig }: { resourceConfig: UGCResourceWithChannelsConfig } =
2019
app.node.tryGetContext(stage);
@@ -26,7 +25,8 @@ new UGCStack(app, stackName, {
2625
tags: { stage, project: 'ugc' },
2726
resourceConfig,
2827
shouldPublish,
29-
cognitoCleanupScheduleExp
28+
cognitoCleanupScheduleExp,
29+
stageCleanupScheduleExp
3030
});
3131

3232
new UGCFrontendDeploymentStack(app, `UGC-Frontend-Deployment-${stage}`, {

‎cdk/lambdas/cleanupIdleStages.ts

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { ListStagesCommand } from '@aws-sdk/client-ivs-realtime';
2+
3+
import {
4+
ivsRealTimeClient,
5+
getIdleStageArns,
6+
deleteStagesWithRetry,
7+
updateMultipleChannelDynamoItems,
8+
getBatchChannelWriteUpdates,
9+
getIdleStages
10+
} from './helpers';
11+
12+
export const handler = async () => {
13+
try {
14+
const deleteIdleStages = async (nextToken = '') => {
15+
const listStagesCommand = new ListStagesCommand({
16+
maxResults: 100,
17+
nextToken
18+
});
19+
const response = await ivsRealTimeClient.send(listStagesCommand);
20+
21+
const stages = response?.stages || [];
22+
const _nextToken = response?.nextToken || '';
23+
24+
if (stages.length) {
25+
// Filter list of stages by stack name
26+
const projectStages = stages.filter(
27+
({ tags }) => !!tags?.stack && tags.stack === process.env.STACK_TAG
28+
);
29+
const idleStages = getIdleStages(projectStages);
30+
const idleStageArns = getIdleStageArns(idleStages);
31+
const batchChannelWriteUpdates =
32+
getBatchChannelWriteUpdates(idleStages);
33+
await Promise.all([
34+
deleteStagesWithRetry(idleStageArns),
35+
updateMultipleChannelDynamoItems(batchChannelWriteUpdates)
36+
]);
37+
}
38+
39+
if (_nextToken) await deleteIdleStages(_nextToken);
40+
};
41+
42+
await deleteIdleStages();
43+
} catch (error) {
44+
console.error(error);
45+
46+
throw new Error('Failed to remove idle stages due to unexpected error');
47+
}
48+
};
49+
50+
export default handler;

‎cdk/lambdas/helpers.ts

+9-5
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ export const batchDeleteItemsWithRetry = async (
8787
* Cleanup Idle Stages
8888
*/
8989

90-
export const getIdleStages = (stages: StageSummary[]) => {
90+
export const getIdleStages = (stages: StageSummary[] = []) => {
9191
const currentTimestamp = Date.now();
9292
const millisecondsPerHour = 60 * 60 * 1000;
9393
const hoursThreshold = 1;
@@ -105,7 +105,9 @@ export const getIdleStages = (stages: StageSummary[]) => {
105105
});
106106
};
107107

108-
export const getBatchChannelWriteUpdates = (idleStages: StageSummary[]) => {
108+
export const getBatchChannelWriteUpdates = (
109+
idleStages: StageSummary[] = []
110+
) => {
109111
const channelWriteUpdates: WriteRequest[] = [];
110112

111113
idleStages.forEach((idleAndOldStage) => {
@@ -132,7 +134,7 @@ export const getBatchChannelWriteUpdates = (idleStages: StageSummary[]) => {
132134
return channelWriteUpdates;
133135
};
134136

135-
export const getIdleStageArns = (idleStages: StageSummary[]) =>
137+
export const getIdleStageArns = (idleStages: StageSummary[] = []) =>
136138
idleStages
137139
.map((idleAndOldStage) => idleAndOldStage.arn)
138140
.filter((arn) => typeof arn === 'string') as string[];
@@ -209,7 +211,7 @@ const analyzeDeleteStageResponse = (
209211
};
210212
};
211213

212-
export const deleteStagesWithRetry = async (stageArns: string[]) => {
214+
export const deleteStagesWithRetry = async (stageArns: string[] = []) => {
213215
if (!stageArns.length) return;
214216

215217
const stagesToDelete = chunkIntoArrayBatches(stageArns, 5);
@@ -282,8 +284,10 @@ export const updateDynamoItemAttributes = ({
282284
};
283285

284286
export const updateMultipleChannelDynamoItems = (
285-
idleStagesChannelArns: WriteRequest[]
287+
idleStagesChannelArns: WriteRequest[] = []
286288
) => {
289+
if (!idleStagesChannelArns.length) return;
290+
287291
const batchWriteInput: BatchWriteItemInput = {
288292
RequestItems: {
289293
[process.env.CHANNELS_TABLE_NAME as string]: idleStagesChannelArns

‎cdk/lib/ChannelsStack/cdk-channels-stack.ts

+39-2
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ import {
77
aws_events as events,
88
aws_events_targets as targets,
99
aws_iam as iam,
10+
aws_lambda as lambda,
1011
aws_lambda_nodejs as nodejsLambda,
1112
aws_s3 as s3,
1213
aws_s3_notifications as s3n,
14+
aws_sqs as sqs,
1315
Duration,
1416
NestedStack,
1517
NestedStackProps,
@@ -41,6 +43,7 @@ interface ChannelsStackProps extends NestedStackProps {
4143
resourceConfig: ChannelsResourceConfig;
4244
tags: { [key: string]: string };
4345
cognitoCleanupScheduleExp: string;
46+
stageCleanupScheduleExp: string;
4447
}
4548

4649
export class ChannelsStack extends NestedStack {
@@ -67,7 +70,12 @@ export class ChannelsStack extends NestedStack {
6770
const region = Stack.of(this.nestedStackParent!).region;
6871
const nestedStackName = 'Channels';
6972
const stackNamePrefix = `${parentStackName}-${nestedStackName}`;
70-
const { resourceConfig, cognitoCleanupScheduleExp, tags } = props;
73+
const {
74+
resourceConfig,
75+
cognitoCleanupScheduleExp,
76+
stageCleanupScheduleExp,
77+
tags
78+
} = props;
7179

7280
// Configuration variables based on the stage (dev or prod)
7381
const {
@@ -471,7 +479,7 @@ export class ChannelsStack extends NestedStack {
471479

472480
// Cleanup idle stages users policies
473481
const deleteIdleStagesIvsPolicyStatement = new iam.PolicyStatement({
474-
actions: ['ivs:ListStages', 'ivs:DeleteStage'],
482+
actions: ['ivs:ListStages', 'ivs:DeleteStage', 'dynamodb:BatchWriteItem'],
475483
effect: iam.Effect.ALLOW,
476484
resources: ['*']
477485
});
@@ -488,6 +496,23 @@ export class ChannelsStack extends NestedStack {
488496
resources: [userPool.userPoolArn]
489497
});
490498

499+
// Cleanup idle stages lambda
500+
const cleanupIdleStagesHandler = new nodejsLambda.NodejsFunction(
501+
this,
502+
`${stackNamePrefix}-CleanupIdleStages-Handler`,
503+
{
504+
...defaultLambdaParams,
505+
logRetention: RetentionDays.ONE_WEEK,
506+
functionName: `${stackNamePrefix}-CleanupIdleStages`,
507+
entry: getLambdaEntryPath('cleanupIdleStages'),
508+
timeout: Duration.minutes(10),
509+
initialPolicy: [deleteIdleStagesIvsPolicyStatement],
510+
environment: {
511+
STACK_TAG: parentStackName
512+
}
513+
}
514+
);
515+
491516
// Cleanup unverified users lambda
492517
const cleanupUnverifiedUsersHandler = new nodejsLambda.NodejsFunction(
493518
this,
@@ -505,6 +530,18 @@ export class ChannelsStack extends NestedStack {
505530
}
506531
);
507532

533+
// Scheduled cleanup idle stages lambda function
534+
new events.Rule(this, 'Cleanup-Idle-Stages-Schedule-Rule', {
535+
schedule: events.Schedule.expression(stageCleanupScheduleExp),
536+
ruleName: `${stackNamePrefix}-CleanupIdleStages-Schedule`,
537+
targets: [
538+
new targets.LambdaFunction(cleanupIdleStagesHandler, {
539+
maxEventAge: Duration.minutes(2),
540+
retryAttempts: 2
541+
})
542+
]
543+
});
544+
508545
// Scheduled cleanup unverified users lambda function
509546
new events.Rule(this, 'Cleanup-Unverified-Users-Schedule-Rule', {
510547
schedule: events.Schedule.expression(cognitoCleanupScheduleExp),

‎cdk/lib/cdk-ugc-stack.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const DEFAULT_CLIENT_BASE_URLS = ['', 'http://localhost:3000'];
2828
interface UGCDashboardStackProps extends StackProps {
2929
resourceConfig: UGCResourceWithChannelsConfig;
3030
cognitoCleanupScheduleExp: string;
31+
stageCleanupScheduleExp: string;
3132
shouldPublish: boolean;
3233
}
3334

@@ -38,6 +39,7 @@ export class UGCStack extends Stack {
3839
const {
3940
resourceConfig,
4041
cognitoCleanupScheduleExp,
42+
stageCleanupScheduleExp,
4143
shouldPublish,
4244
tags = {}
4345
} = props;
@@ -141,6 +143,7 @@ export class UGCStack extends Stack {
141143
const channelsStack = new ChannelsStack(this, 'Channels', {
142144
resourceConfig,
143145
cognitoCleanupScheduleExp,
146+
stageCleanupScheduleExp,
144147
tags
145148
});
146149
const {
@@ -203,7 +206,8 @@ export class UGCStack extends Stack {
203206
PRODUCT_LINK_REGION_CODE: productLinkRegionCode,
204207
ENABLE_AMAZON_PRODUCT_STREAM_ACTION: `${enableAmazonProductStreamAction}`,
205208
PRODUCT_API_SECRET_NAME: productApiSecretName,
206-
APPSYNC_GRAPHQL_API_SECRET_NAME: appSyncGraphQlApi.secretName
209+
APPSYNC_GRAPHQL_API_SECRET_NAME: appSyncGraphQlApi.secretName,
210+
STACK: stackNamePrefix
207211
};
208212
const sharedContainerEnv = {
209213
...baseContainerEnv,

0 commit comments

Comments
 (0)