Packaging optimization (#250)

This PR aims to improve how dependencies are bundled and benefit from
ESM builds of some packages.
It also improves how pkg is used after esbuild to prevent packaging some
dependencies twice.

Author: Corentin Mors

.github/workflows/manual-test-release.yml

@@ -35,7 +35,7 @@ jobs:
             - run: yarn workspaces focus --all --production
             # package final binaries
             - run: |
-                  yarn dlx @yao-pkg/pkg@5.11.1 . -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
+                  yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
 
             - name: Archive binary artifact
               uses: actions/upload-artifact@v4

.github/workflows/release.yml

@@ -38,7 +38,7 @@ jobs:
             - run: yarn workspaces focus --all --production
             # package final binaries
             - run: |
-                  yarn dlx @yao-pkg/pkg@5.11.1 . -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
+                  yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
 
             - name: Archive binary artifact
               uses: actions/upload-artifact@v4

package.json

@@ -9,12 +9,12 @@
     },
     "pkg": {
         "assets": [
-            "node_modules/better-sqlite3/build/Release/better_sqlite3.node",
-            "node_modules/@json2csv/plainjs/dist/**/*",
-            "node_modules/@json2csv/formatters/dist/**/*",
-            "node_modules/@json2csv/transforms/dist/**/*",
-            "node_modules/@streamparser/json/dist/**/*",
-            "node_modules/node-mac-auth/build/Release/auth.node"
+            "../node_modules/better-sqlite3/build/Release/better_sqlite3.node",
+            "../node_modules/@json2csv/plainjs/dist/**/*",
+            "../node_modules/@json2csv/formatters/dist/**/*",
+            "../node_modules/@json2csv/transforms/dist/**/*",
+            "../node_modules/@streamparser/json/dist/**/*",
+            "../node_modules/node-mac-auth/build/Release/auth.node"
         ]
     },
     "scripts": {
@@ -24,10 +24,10 @@
         "format": "prettier --write src && eslint --fix src",
         "start": "node dist/index.cjs",
         "start:dev": "node build/index.js",
-        "pkg:linux": "pkg . -t node18-linux-x64 -o bundle/dcli-linux -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
-        "pkg:macos": "pkg . -t node18-macos-x64 -o bundle/dcli-macos -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
-        "pkg:macos-arm": "pkg . -t node18-macos-arm64 -o bundle/dcli-macos-arm -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
-        "pkg:win": "pkg . -t node18-win-x64 -o bundle/dcli-win.exe -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
+        "pkg:linux": "pkg ./dist -t node18-linux-x64 -o bundle/dcli-linux -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
+        "pkg:macos": "pkg ./dist -t node18-macos-x64 -o bundle/dcli-macos -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
+        "pkg:macos-arm": "pkg ./dist -t node18-macos-arm64 -o bundle/dcli-macos-arm -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode --no-native-build",
+        "pkg:win": "pkg ./dist -t node18-win-x64 -o bundle/dcli-win.exe -C GZip --public --public-packages tslib,thirty-two,node-hkdf-sync,vows --no-bytecode",
         "pkg": "yarn run build && yarn run pkg:linux && yarn run pkg:macos && yarn run pkg:win",
         "version:bump": "yarn run build && node ./build/bumpVersion.js",
         "prepare": "husky",
@@ -56,14 +56,14 @@
         "@types/better-sqlite3": "^7.6.10",
         "@types/chai": "^4.3.16",
         "@types/inquirer": "^9.0.7",
-        "@types/libsodium-wrappers": "^0",
+        "@types/libsodium-wrappers": "^0.7.14",
         "@types/mocha": "^10.0.6",
         "@types/node": "^18.19.33",
         "@typescript-eslint/eslint-plugin": "^7.8.0",
         "@typescript-eslint/parser": "^7.8.0",
         "@yao-pkg/pkg": "^5.11.5",
         "chai": "^5.1.1",
-        "esbuild": "^0.21.2",
+        "esbuild": "^0.21.4",
         "eslint": "^8.57.0",
         "eslint-config-prettier": "^9.1.0",
         "eslint-plugin-import": "^2.29.1",
@@ -75,23 +75,22 @@
     },
     "dependencies": {
         "@dashlane/nsm-attestation": "^1.0.1",
+        "@inquirer/prompts": "^5.0.4",
         "@json2csv/plainjs": "^7.0.6",
         "@json2csv/transforms": "^7.0.6",
         "@napi-rs/clipboard": "^1.1.2",
         "@napi-rs/keyring": "^1.1.6",
         "@node-rs/argon2": "^1.8.3",
-        "ajv": "^8.13.0",
+        "ajv": "^8.14.0",
         "ajv-formats": "^3.0.1",
-        "better-sqlite3": "^10.0.0",
-        "commander": "^12.0.0",
-        "got": "^14.2.1",
-        "inquirer": "^9.2.21",
-        "inquirer-search-list": "^1.2.6",
+        "better-sqlite3": "^11.0.0",
+        "commander": "^12.1.0",
+        "got": "^14.3.0",
         "jsonpath-plus": "^9.0.0",
         "libsodium-wrappers": "^0.7.13",
         "node-mac-auth": "^1.0.0",
         "otplib": "^12.0.1",
-        "playwright-core": "^1.44.0",
+        "playwright-core": "^1.44.1",
         "winston": "^3.13.0",
         "xml-js": "^1.6.11",
         "zlib": "^1.0.5"

scripts/build.js

@@ -2,7 +2,7 @@
 
 import os from 'os';
 import fs from 'fs';
-import path, { format } from 'path';
+import path from 'path';
 import process from 'process';
 import childProcess from 'child_process';
 import esbuild from 'esbuild';
@@ -73,11 +73,36 @@ async function main(argv = process.argv) {
         // Minify and keep the original names
         minify: true,
         keepNames: true,
-        outfile: path.join(distPath, 'index.cjs')
+        outfile: path.join(distPath, 'index.cjs'),
+        metafile: true
     };
     console.error('Running esbuild:');
     console.error(esbuildOptions);
-    await esbuild.build(esbuildOptions);
+    const result = await esbuild.build(esbuildOptions);
+    fs.writeFileSync(path.join(distPath, 'index.meta.json'), JSON.stringify(result.metafile, null, 2));
+
+    // Copy package.json
+    const pkgJson = cleanPkgJson(packageJSON);
+    fs.writeFileSync(path.join(distPath, 'package.json'), JSON.stringify(pkgJson, null, 2));
 }
 
+const cleanPkgJson = (json) => {
+    delete json.devDependencies;
+    delete json.optionalDependencies;
+    const oldDependencies = json.dependencies;
+    delete json.dependencies;
+    json.dependencies = {};
+
+    for (const [name, version] of Object.entries(oldDependencies)) {
+        if (Object.keys(json.nativeDependencies).includes(name)) {
+            json.dependencies[name] = version;
+        }
+    }
+    delete json.nativeDependencies;
+    delete json.scripts;
+    json.bin.dcli = 'index.cjs';
+    json.main = 'index.cjs';
+    return json;
+};
+
 void main();

src/command-handlers/lock.ts

@@ -1,6 +1,6 @@
-import * as winston from 'winston';
 import { deleteLocalKey } from '../modules/crypto/keychainManager.js';
 import { connectAndPrepare } from '../modules/database/index.js';
+import { logger } from '../logger.js';
 
 export const runLock = async () => {
     const { db, localConfiguration } = await connectAndPrepare({
@@ -18,7 +18,7 @@ export const runLock = async () => {
         if (error instanceof Error) {
             errorMessage = error.message;
         }
-        winston.warn(`Unable to lock the vault: ${errorMessage}`);
+        logger.warn(`Unable to lock the vault: ${errorMessage}`);
     }
 
     db.prepare('UPDATE device SET masterPasswordEncrypted = ? WHERE login = ?')

src/modules/api-connect/signRequest.ts

@@ -1,4 +1,4 @@
-import * as crypto from 'crypto';
+import crypto from 'crypto';
 import { makeCanonicalRequest } from './makeCanonicalRequest.js';
 import { Authentication, SignRequestParams } from './types.js';
 import { logger } from '../../logger.js';

src/modules/crypto/decrypt.ts

@@ -1,5 +1,5 @@
-import * as argon2 from '@node-rs/argon2';
-import * as xmlJs from 'xml-js';
+import argon2 from '@node-rs/argon2';
+import xmlJs from 'xml-js';
 import crypto from 'crypto';
 import { promisify } from 'util';
 import zlib from 'zlib';

src/modules/crypto/hash.ts

@@ -1,4 +1,4 @@
-import * as crypto from 'crypto';
+import crypto from 'crypto';
 
 const toBuffer = (data: Buffer | string): Buffer => {
     return Buffer.isBuffer(data) ? data : Buffer.from(data);

src/modules/tunnel-api-connect/apiconnect.ts

@@ -1,4 +1,4 @@
-import * as sodium from 'libsodium-wrappers';
+import sodium from 'libsodium-wrappers';
 import { clientHello, terminateHello, SendSecureContentParams, sendSecureContent } from './steps/index.js';
 import { ApiConnectParams, ApiConnect, ApiData, ApiRequestsDefault } from './types.js';
 import { makeClientKeyPair, makeOrRefreshSession } from './utils/index.js';

src/modules/tunnel-api-connect/steps/types.ts

@@ -1,4 +1,4 @@
-import type * as sodium from 'libsodium-wrappers';
+import type sodium from 'libsodium-wrappers';
 import { ApiRequestsDefault } from '../types.js';
 
 export interface ApiEndpointResponse<T> {

src/modules/tunnel-api-connect/types.ts

@@ -1,5 +1,5 @@
 import type { EnclavePcr } from '@dashlane/nsm-attestation';
-import type * as sodium from 'libsodium-wrappers';
+import type sodium from 'libsodium-wrappers';
 import type { ClientHelloParsedResponse, SendSecureContentParams, TerminateHelloResponse } from './steps/index.js';
 
 export interface ApiRequestsDefault {

src/modules/typecheck/customize_ajv.ts

@@ -1,11 +1,5 @@
-import Ajv, { SchemaObjCxt } from 'ajv';
-import {
-    AnySchemaObject,
-    DataValidateFunction,
-    DataValidationCxt,
-    ErrorObject,
-    KeywordDefinition,
-} from 'ajv/dist/types';
+import Ajv, { SchemaObjCxt, AnySchemaObject, ErrorObject, KeywordDefinition } from 'ajv';
+import type { DataValidateFunction, DataValidationCxt } from 'ajv/dist/types';
 
 // https://github.com/ajv-validator/ajv-formats/blob/master/src/formats.ts#L237
 const BYTE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;

src/modules/typecheck/index.ts

@@ -1,6 +1,6 @@
-import * as Ajv from 'ajv';
-import addFormats from 'ajv-formats';
+import Ajv, { ErrorObject, ValidateFunction } from 'ajv';
 import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
 import { customizeAjv } from './customize_ajv.js';
 
 export class TypeCheckError extends Error {
@@ -22,7 +22,7 @@ export class JSONParsingError extends TypeCheckError {
 
 export class JSONValidationError extends TypeCheckError {
     constructor(
-        readonly reasonAjv: Ajv.ErrorObject,
+        readonly reasonAjv: ErrorObject,
         details?: string
     ) {
         super(
@@ -34,11 +34,11 @@ export class JSONValidationError extends TypeCheckError {
     }
 }
 
-export type DetailedAjvErrorObject = { errors: Ajv.ErrorObject[]; details: string };
+export type DetailedAjvErrorObject = { errors: ErrorObject[]; details: string };
 
-export const detailedErrorReportingOverride = (errors: Ajv.ErrorObject[]): DetailedAjvErrorObject => {
+export const detailedErrorReportingOverride = (errors: ErrorObject[]): DetailedAjvErrorObject => {
     const details = errors
-        .map((obj: Ajv.ErrorObject) => `[${obj.schemaPath}] [${JSON.stringify(obj.params)}] ${obj.message ?? ''}`)
+        .map((obj: ErrorObject) => `[${obj.schemaPath}] [${JSON.stringify(obj.params)}] ${obj.message ?? ''}`)
         .join('\n')
         .trim();
     return { errors, details };
@@ -47,11 +47,11 @@ export const detailedErrorReportingOverride = (errors: Ajv.ErrorObject[]): Detai
 type JSONSchema = Record<string, unknown>;
 
 export class TypeCheck<T> {
-    private validator: Ajv.ValidateFunction;
+    private validator: ValidateFunction;
 
     constructor(
         schema: JSONSchema,
-        readonly errorReportingOverride?: (errors: Ajv.ErrorObject[]) => DetailedAjvErrorObject,
+        readonly errorReportingOverride?: (errors: ErrorObject[]) => DetailedAjvErrorObject,
         openApiValidation?: boolean
     ) {
         if (openApiValidation) {
@@ -61,6 +61,7 @@ export class TypeCheck<T> {
                 strictTypes: false,
                 strictRequired: false,
                 allowMatchingProperties: true,
+                code: { esm: true },
             });
             ajv.addFormat('media-range', true); // used in OpenAPI 3.1
 
@@ -69,7 +70,7 @@ export class TypeCheck<T> {
 
             this.validator = ajv.compile(schema);
         } else {
-            const ajv = new Ajv.default({ allErrors: true });
+            const ajv = new Ajv({ allErrors: true, code: { esm: true } });
             addFormats(ajv);
             customizeAjv(ajv);
             this.validator = ajv.compile(schema);

src/requestApi.ts

@@ -1,12 +1,12 @@
 import { Response, HTTPError } from 'got';
-import * as apiConnect from './modules/api-connect/index.js';
+import { Authentication, postRequestAPI } from './modules/api-connect/index.js';
 import { CLI_VERSION, cliVersionToString } from './cliVersion.js';
 import { gotImplementation } from './utils/index.js';
 
 interface RequestApi {
     payload: Record<string, unknown>;
     path: string;
-    authentication: apiConnect.Authentication;
+    authentication: Authentication;
     isNitroEncryptionService?: boolean;
 }
 
@@ -46,7 +46,7 @@ const requestApi = async <T>(params: RequestApi): Promise<T> => {
 
     let response: Response<string>;
     try {
-        response = await apiConnect.postRequestAPI<Response<string>>({
+        response = await postRequestAPI<Response<string>>({
             requestFunction: gotImplementation,
             authentication,
             path: (isNitroEncryptionService ? 'v1-nitro-encryption-service/' : 'v1/') + path,