feat: add debug command

Author: pevisscher

package-lock.json

@@ -1,5 +1,6 @@
 {
   "name": "@skyleague/starchart",
+  "version": "1.2.0",
   "license": "MIT",
   "type": "module",
   "repository": {
@@ -29,16 +30,25 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "@skyleague/esbuild-lambda": "^5.2.0",
+    "@aws-sdk/client-iam": "^3.614.0",
+    "@aws-sdk/client-lambda": "^3.614.0",
+    "@skyleague/esbuild-lambda": "^5.3.2",
+    "@skyleague/event-horizon": "^9.0.0",
+    "aws-iot-device-sdk-v2": "^1.20.0",
     "camelcase": "^8.0.0",
     "find-root": "^1.1.0",
     "globby": "^14.0.2",
     "js-yaml": "^4.1.0",
+    "pino-pretty": "^11.2.1",
+    "tsx": "^4.16.2",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
-    "@skyleague/node-standards": "^7.2.0",
-    "@skyleague/therefore": "^5.9.1",
+    "@aws-sdk/client-iot": "^3.614.0",
+    "@aws-sdk/credential-providers": "^3.614.0",
+    "@skyleague/node-standards": "^7.3.1",
+    "@skyleague/therefore": "^5.9.2",
+    "@types/aws-lambda": "^8.10.141",
     "@types/find-root": "^1.1.4",
     "@types/js-yaml": "^4.0.9",
     "@types/yargs": "^17.0.32",

package.json

@@ -22,39 +22,68 @@ export function builder(yargs: Argv) {
             default: ['*'],
             string: true,
         })
+        .option('clean', {
+            type: 'boolean',
+            default: true,
+        })
+        .option('dot', {
+            type: 'boolean',
+            default: false,
+        })
+}
+
+export type BuildOptions = {
+    fnDirs?: string[]
+    stacks?: string[]
+    preBuild?: () => Promise<void> | void
+    postBuild?: () => Promise<void> | void
 }
 
-export async function handler(argv: ReturnType<typeof builder>['argv']): Promise<void> {
-    const { buildDir: _buildDir, artifactDir: _artifactDir, target } = await argv
+export async function handler(argv: ReturnType<typeof builder>['argv'], options: BuildOptions = {}): Promise<void> {
+    const { buildDir: _buildDir, artifactDir: _artifactDir, target, clean, dot } = await argv
+    const { fnDirs = ['src/**/functions'], preBuild, postBuild } = options
     const buildDir = path.join(rootDirectory, _buildDir)
     const artifactDir = path.join(rootDirectory, _artifactDir)
 
     const targets = target.flatMap((t) => t.split(','))
 
-    if (targets.includes('*')) {
-        await fs.promises.rm(artifactDir, { recursive: true }).catch(() => void {})
+    if (clean) {
+        if (targets.includes('*')) {
+            fs.rmSync(artifactDir, { recursive: true, force: true })
+        }
+        fs.rmSync(buildDir, { recursive: true, force: true })
     }
-    await fs.promises.mkdir(artifactDir).catch(() => void {})
+    fs.mkdirSync(artifactDir, { recursive: true })
+
+    await preBuild?.()
 
-    const stacks = await globby(['src/**/functions'], { cwd: rootDirectory, onlyDirectories: true })
-    const handlers = (await Promise.all(stacks.flatMap(async (fnDir) => listLambdaHandlers(path.join(rootDirectory, fnDir)))))
-        .flat()
-        .filter((handler) => targets.includes('*') || targets.some((t) => handler.includes(t)))
+    const stacks = await globby(fnDirs, { cwd: rootDirectory, onlyDirectories: true, dot })
+    const handlers = (
+        await Promise.all(
+            [
+                ...(options.stacks ?? []),
+                ...stacks.filter((handler) => targets.includes('*') || targets.some((t) => handler.includes(t))),
+            ].flatMap(async (fnDir) => listLambdaHandlers(path.join(rootDirectory, fnDir))),
+        )
+    ).flat()
 
-    const outbase = path.join(rootDirectory, 'src')
+    const outbase = rootDirectory
 
     await esbuildLambda(rootDirectory, {
         esbuild: {
             absWorkingDir: rootDirectory,
             tsconfig: path.join(rootDirectory, 'tsconfig.dist.json'),
-            outbase,
+            outbase: rootDirectory,
         },
         root: rootDirectory,
+        modulesRoot: rootDirectory,
         entryPoints: handlers.map((fnDir) => path.join(fnDir, 'index.ts')),
         outdir: () => buildDir,
-        forceBundle: () => true,
+        forceBundle: ({ packageName }) => packageName !== 'aws-crt',
     })
 
+    await postBuild?.()
+
     await zipHandlers(handlers, {
         outbase,
         buildDir,

src/commands/build/index.ts

@@ -2,10 +2,15 @@ import { spawnSync } from 'node:child_process'
 import path from 'node:path'
 import { globby } from 'globby'
 import type { Argv } from 'yargs'
-import { builder as buildBuilder, handler as buildHandler } from '../build/index.js'
+import { type BuildOptions, builder as buildBuilder, handler as buildHandler } from '../build/index.js'
 
 export function builder(yargs: Argv) {
     return buildBuilder(yargs)
+        .option('stack', {
+            type: 'array',
+            default: ['*'],
+            string: true,
+        })
         .option('dir', {
             describe: 'environment directory',
             type: 'string',
@@ -20,27 +25,34 @@ export function builder(yargs: Argv) {
         })
 }
 
-export async function handler(argv: ReturnType<typeof builder>['argv']): Promise<void> {
-    const { target, dir, refresh } = await argv
-    const targets = target.flatMap((t) => t.split(','))
+export interface DeployOptions extends BuildOptions {}
+
+export async function handler(argv: ReturnType<typeof builder>['argv'], options: DeployOptions = {}): Promise<void> {
+    const { dir, refresh, stack } = await argv
+
+    const stacks = stack.flatMap((t) => t.split(','))
     const cwd = path.join(process.cwd(), dir)
 
-    const [stacks] = await Promise.all([globby(['**/*.hcl'], { cwd: cwd, onlyDirectories: false }), buildHandler(argv)])
+    const [foundStacks] = await Promise.all([
+        globby(['**/*.hcl'], { cwd: cwd, onlyDirectories: false }),
+        buildHandler(argv, options),
+    ])
 
-    const groups = stacks
+    const groups = foundStacks
         .map((s) => path.join(cwd, path.dirname(s)))
-        .filter((stack) => targets.includes('*') || targets.some((t) => stack.includes(t)))
+        .filter((stack) => stacks.includes('*') || stacks.some((t) => stack.includes(t)))
 
     console.log('Deploying stacks')
 
     spawnSync(
         'terragrunt',
         [
             '--terragrunt-non-interactive',
-            ...(targets.includes('*')
+            ...(stacks.includes('*')
                 ? ['run-all', 'apply']
                 : ['run-all', 'apply', '--terragrunt-strict-include', ...groups.flatMap((t) => ['--terragrunt-include-dir', t])]),
             ...(refresh === false ? ['-refresh=false'] : []),
+            '--terragrunt-fetch-dependency-output-from-state',
         ],
         {
             stdio: 'inherit',

src/commands/deploy/index.ts

@@ -0,0 +1,45 @@
+import type { IAM } from '@aws-sdk/client-iam'
+import type { FunctionConfiguration, Lambda } from '@aws-sdk/client-lambda'
+
+export interface LambdaFunction {
+    configuration: FunctionConfiguration
+    tags: { [key: string]: string }
+}
+export async function patchDebugFunction({
+    fn,
+    lambda,
+    iam,
+    debugZip,
+    codeSha256,
+}: { fn: LambdaFunction; lambda: Lambda; iam: IAM; debugZip: Uint8Array; codeSha256: string }) {
+    if (codeSha256 !== fn.configuration.CodeSha256) {
+        await lambda.updateFunctionCode({
+            // biome-ignore lint/style/noNonNullAssertion: This is a debug function, so we can assume the configuration is present
+            FunctionName: fn.configuration.FunctionName!,
+            ZipFile: debugZip,
+        })
+    }
+
+    await iam.putRolePolicy({
+        // biome-ignore lint/style/noNonNullAssertion: <explanation>
+        RoleName: fn.configuration.Role!.split('/').pop()!,
+        PolicyName: '.localdebug',
+        PolicyDocument: JSON.stringify({
+            Version: '2012-10-17',
+            Statement: [
+                {
+                    Effect: 'Allow',
+                    Action: [
+                        'iot:DescribeEndpoint',
+                        'iot:Connect',
+                        'iot:Subscribe',
+                        'iot:Publish',
+                        'iot:RetainPublish',
+                        'iot:Receive',
+                    ],
+                    Resource: '*',
+                },
+            ],
+        }),
+    })
+}

src/commands/dev/function.ts

@@ -0,0 +1,22 @@
+import { tsImport } from 'tsx/esm/api'
+
+process.on('message', async ({ lambdaFn, event, context, rootDirectory }) => {
+    let handler: ((...args: unknown[]) => unknown) | undefined = undefined
+    try {
+        const loaded = await tsImport(`.${lambdaFn.tags.Path}.ts`, `${rootDirectory}/`)
+        if ('handler' in loaded && typeof loaded.handler === 'function') {
+            handler = loaded.handler
+        }
+    } catch (error) {
+        console.error(`Error loading handler for ${lambdaFn.tags.Path}`)
+        process.send?.({ left: (error as { message: string }).message })
+        return
+    }
+
+    if (handler !== undefined && context !== undefined) {
+        const response = await handler(event, { ...context, getRemainingTimeInMillis: () => 1000 })
+        process.send?.({ right: response })
+    } else {
+        process.send?.({ left: 'Handler function not found or context is undefined' })
+    }
+})

src/commands/dev/handler.ts

@@ -0,0 +1,140 @@
+import crypto from 'node:crypto'
+import fs from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { IAM } from '@aws-sdk/client-iam'
+import { type FunctionConfiguration, Lambda, paginateListFunctions } from '@aws-sdk/client-lambda'
+import {} from '@skyleague/esbuild-lambda'
+import type { Argv } from 'yargs'
+import { rootDirectory } from '../../lib/constants.js'
+import { handler as buildHandler } from '../build/index.js'
+import { builder as deployBuilder, handler as deployHandler } from '../deploy/index.js'
+import { type LambdaFunction, patchDebugFunction } from './function.js'
+import { local } from './local.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+function sha256Hash(data: Uint8Array) {
+    const hash = crypto.createHash('sha256')
+    hash.update(data)
+    return hash.digest('base64')
+}
+
+export function builder(yargs: Argv) {
+    return deployBuilder(yargs)
+        .option('stack', {
+            type: 'array',
+            default: ['*'],
+            string: true,
+        })
+        .option('dir', {
+            describe: 'environment directory',
+            type: 'string',
+            default: '.',
+            example: 'terraform/dev',
+            demandOption: true,
+        })
+        .option('refresh', {
+            describe: 'refresh the state before applying',
+            type: 'boolean',
+            default: true,
+        })
+        .option('deploy', {
+            describe: 'refresh the stacks before starting debug',
+            type: 'boolean',
+            default: false,
+        })
+}
+
+export async function handler(argv: ReturnType<typeof builder>['argv']): Promise<void> {
+    const options = await argv
+    const { deploy, buildDir: _buildDir, artifactDir: _artifactDir, stack } = options
+    const artifactDir = path.join(rootDirectory, _artifactDir)
+
+    const stacks = stack.flatMap((t) => t.split(','))
+
+    const debugPath = '.debug'
+    const debugDir = path.join(_buildDir, debugPath)
+    const debugArtifact = path.join(artifactDir, _buildDir, `${debugPath}.zip`)
+
+    const debugLambda = path.join(path.join(rootDirectory, debugDir), 'index.ts')
+
+    const hasDebugArtifact = fs.existsSync(debugArtifact) && false
+
+    const remoteLambda = fs.existsSync(`${__dirname}/lambda.ts`)
+        ? fs.readFileSync(`${__dirname}/lambda.ts`).toString()
+        : fs.existsSync(`${__dirname}/lambda.js`)
+          ? fs.readFileSync(`${__dirname}/lambda.js`).toString()
+          : undefined
+    const remoteLambdaContent = [remoteLambda, ' export const handler = proxyHandler()'].join('\n')
+
+    if (remoteLambda === undefined) {
+        throw new Error('No remote lambda contents found, please provide a lambda.ts or lambda.js file')
+    }
+
+    const buildRemoteLambdaArtifact =
+        !hasDebugArtifact || (fs.existsSync(debugLambda) && fs.readFileSync(debugLambda).toString() !== remoteLambdaContent)
+
+    const preBuild = () => {
+        fs.mkdirSync(path.join(rootDirectory, debugDir), { recursive: true })
+
+        fs.writeFileSync(debugLambda, remoteLambdaContent)
+    }
+
+    if (deploy) {
+        await deployHandler(
+            { ...options, clean: false },
+            {
+                stacks: buildRemoteLambdaArtifact ? [debugDir] : [],
+                preBuild,
+            },
+        )
+    } else {
+        await buildHandler(
+            { ...options, fnDir: [], clean: false },
+            {
+                fnDirs: [],
+                stacks: !hasDebugArtifact ? [debugDir] : [],
+                preBuild,
+            },
+        )
+    }
+    const lambda = new Lambda({
+        region: 'eu-west-1',
+    })
+
+    const iam = new IAM({
+        region: 'eu-west-1',
+    })
+
+    const paginator = paginateListFunctions({ client: lambda, pageSize: 10 }, {})
+
+    const configurations: FunctionConfiguration[] = []
+
+    for await (const page of paginator) {
+        // only keep funtions that start with the any of the given stacks
+        configurations.push(...(page.Functions?.filter((f) => stacks.some((s) => f.FunctionName?.startsWith(s))) ?? []))
+    }
+
+    const functions: LambdaFunction[] = (
+        await Promise.all(
+            configurations.map(async (f) => ({
+                configuration: f,
+                tags: (await lambda.listTags({ Resource: f.FunctionArn })).Tags ?? {},
+            })),
+        )
+    ).filter((f) => f.tags.Stack !== undefined && stack.includes(f.tags.Stack))
+
+    const zipFile = new Uint8Array(fs.readFileSync(debugArtifact).buffer)
+    const codeSha256 = sha256Hash(zipFile)
+    await Promise.all(functions.map((f) => patchDebugFunction({ fn: f, lambda, debugZip: zipFile, iam, codeSha256 })))
+
+    await local(functions)
+}
+
+export default {
+    command: 'dev',
+    describe: 'starts a dev deploy',
+    builder,
+    handler,
+}