From 307ba119ca532073b07aea6add461885944f6fb9 Mon Sep 17 00:00:00 2001 From: Miguel Grinberg Date: Mon, 24 Feb 2025 19:45:45 +0000 Subject: [PATCH 1/3] reorganization to make it possible to run in the browser --- eslint.config.mjs | 2 +- jest.config.js | 5 ++++ package.json | 10 ++++---- scripts/compile-templates.mjs | 18 ++++++++++++++ src/convert.ts | 9 ++++++- src/exporters/javascript.ts | 20 +++++++++++++--- src/exporters/python.ts | 15 ++++++++---- src/parse.ts | 45 ++++++++++++++++++++++++----------- tests/parse.test.ts | 21 ++++++++++++++++ tsconfig.json | 1 + 10 files changed, 117 insertions(+), 29 deletions(-) create mode 100644 scripts/compile-templates.mjs diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a4b3cc..4be8e43 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -6,6 +6,6 @@ import tseslint from "typescript-eslint"; export default tseslint.config( eslint.configs.recommended, ...tseslint.configs.recommended, - {ignores: ["**/.venv/", "**/es-client/", "tests/wasm"]}, + {ignores: ["**/.venv/", "**/es-client/", "**/templates.js", "tests/wasm"]}, {languageOptions: {globals: {require: true}}}, ); diff --git a/jest.config.js b/jest.config.js index 1b60681..a6af151 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,5 +3,10 @@ module.exports = { testEnvironment: "node", setupFilesAfterEnv: [ "jest-expect-message" + ], + moduleFileExtensions: [ + "ts", + "js", + "tpl" ] }; diff --git a/package.json b/package.json index c11b1a1..0de8119 100644 --- a/package.json +++ b/package.json @@ -11,13 +11,14 @@ "scripts": { "update-schema": "node scripts/update-schema.mjs", "release": "./scripts/release.sh", - "build": "npm run clean && tsc && npm run copy-files", + "build": "npm run clean && tsc && npm run compile-templates && npm run copy-files", "clean": "rimraf dist/", - "copy-files": "copyfiles -u 1 src/**/*.js src/**/*.json src/**/*.tpl dist/", + "compile-templates": "node scripts/compile-templates.mjs", + "copy-files": "copyfiles -u 1 src/**/*.js src/**/*.json dist/", "docs": "typedoc src/index.ts", "lint": "eslint src tests --ignore-pattern tests/wasm/", "prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --list-different", - "test": "jest --test-path-ignore-patterns integration --coverage", + "test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage", "test:setup": "node scripts/download-examples.mjs && ./tests/integration/run-python.sh && ./tests/integration/run-javascript.sh", "test:setup-curl": "node scripts/download-examples.mjs", "test:setup-python": "node scripts/download-examples.mjs && ./tests/integration/run-python.sh", @@ -31,8 +32,7 @@ "fix:lint": "eslint src tests --fix --ignore-pattern tests/wasm/", "fix:prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --write", "fix:precommit": "./scripts/precommit.sh", - "watch:build": "tsc -w", - "watch:test": "jest --test-path-ignore-patterns integration --coverage --watch", + "watch:test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage --watch", "prepare": "husky install" }, "main": "dist/index.js", diff --git a/scripts/compile-templates.mjs b/scripts/compile-templates.mjs new file mode 100644 index 0000000..5a99454 --- /dev/null +++ b/scripts/compile-templates.mjs @@ -0,0 +1,18 @@ +import fs from 'fs'; +import { exec } from 'child_process'; + +const sourceDir = (process.platform !== "win32") + ? "src/exporters" + : "src\\exporters"; +process.chdir(sourceDir); +const cmd = "handlebars python.tpl javascript.tpl -f templates.js -c"; +exec(cmd, (error, stdout, stderr) => { + if (error) { + console.log(stdout); + console.log(stderr); + process.exit(1); + } + let templates = fs.readFileSync("templates.js", 'utf-8'); + templates = 'const Handlebars = require("handlebars");\n' + templates; + fs.writeFileSync("templates.js", templates); +}); diff --git a/src/convert.ts b/src/convert.ts index 1e25eac..f721501 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -6,7 +6,8 @@ import { CurlExporter } from "./exporters/curl"; import { JavaScriptExporter } from "./exporters/javascript"; import util from "util"; -const execAsync = util.promisify(childProcess.exec); +const isBrowser = typeof window !== "undefined"; +const execAsync = !isBrowser ? util.promisify(childProcess.exec) : undefined; export type ConvertOptions = { /** When `true`, the converter will only check if the conversion can be carried @@ -188,6 +189,9 @@ export class SubprocessExporter implements FormatExporter { const input = base64url.encode( JSON.stringify({ requests: getSlimRequests(requests) }), ); + if (execAsync === undefined) { + throw new Error("Cannot use exec()"); + } const { stdout, stderr } = await execAsync( `${this.baseCmd} check ${input}`, ); @@ -208,6 +212,9 @@ export class SubprocessExporter implements FormatExporter { const input = base64url.encode( JSON.stringify({ requests: getSlimRequests(requests), options }), ); + if (execAsync === undefined) { + throw new Error("Cannot use exec()"); + } const { stdout } = await execAsync(`${this.baseCmd} convert ${input}`); if (stdout) { const json = JSON.parse(base64url.decode(stdout)); diff --git a/src/exporters/javascript.ts b/src/exporters/javascript.ts index a93a24d..f2459af 100644 --- a/src/exporters/javascript.ts +++ b/src/exporters/javascript.ts @@ -2,8 +2,10 @@ import { readFileSync } from "fs"; import path from "path"; import Handlebars from "handlebars"; import prettier from "prettier"; +import prettierTypeScript from "prettier/parser-typescript"; import { FormatExporter, ConvertOptions } from "../convert"; import { ParsedRequest } from "../parse"; +import "./templates"; const UNSUPPORTED_APIS = new RegExp( "^query_rules.*$" + @@ -39,7 +41,10 @@ export class JavaScriptExporter implements FormatExporter { throw new Error("Cannot perform conversion"); } const output = this.template({ requests, ...options }); - return prettier.format(output, { parser: "typescript" }); + return prettier.format(output, { + parser: "typescript", + plugins: [prettierTypeScript], + }); } get template(): Handlebars.TemplateDelegate { @@ -115,8 +120,17 @@ export class JavaScriptExporter implements FormatExporter { Handlebars.registerHelper("camelCase", (text) => toCamelCase(text)); - const t = readFileSync(path.join(__dirname, "./javascript.tpl"), "utf-8"); - this._template = Handlebars.compile(t); + if (process.env.NODE_ENV !== "test") { + this._template = Handlebars.templates["javascript.tpl"]; + } else { + // when running tests we read the templates directly, in case the + // compiled file is not up to date + const t = readFileSync( + path.join(__dirname, "./javascript.tpl"), + "utf-8", + ); + this._template = Handlebars.compile(t); + } } return this._template; diff --git a/src/exporters/python.ts b/src/exporters/python.ts index c14cdb8..1ab67d2 100644 --- a/src/exporters/python.ts +++ b/src/exporters/python.ts @@ -1,8 +1,9 @@ -import { readFile } from "fs/promises"; +import { readFileSync } from "fs"; import path from "path"; import Handlebars from "handlebars"; import { FormatExporter, ConvertOptions } from "../convert"; import { ParsedRequest } from "../parse"; +import "./templates"; // this regex should match the list of APIs that do not have specific handlers // in the Python client. APIs in this list are rendered with a perform_request() @@ -159,10 +160,14 @@ export class PythonExporter implements FormatExporter { }, ); - const t = await readFile(path.join(__dirname, "./python.tpl"), { - encoding: "utf-8", - }); - this.template = Handlebars.compile(t); + if (process.env.NODE_ENV !== "test") { + this.template = Handlebars.templates["python.tpl"]; + } else { + // when running tests we read the templates directly, in case the + // compiled file is not up to date + const t = readFileSync(path.join(__dirname, "./python.tpl"), "utf-8"); + this.template = Handlebars.compile(t); + } } return this.template; } diff --git a/src/parse.ts b/src/parse.ts index f2af6e4..0dd425b 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -1,10 +1,11 @@ -import { METHODS } from "http"; import { URL } from "url"; import { readFile } from "fs/promises"; import path from "path"; import * as Router from "find-my-way-ts"; import { Model, Request } from "./metamodel"; +const isBrowser = typeof window !== "undefined"; +const httpMethods = ["HEAD", "GET", "POST", "PUT", "PATCH", "DELETE"]; export type JSONValue = string | number | boolean | JSONArray | JSONObject; interface JSONArray extends Array {} interface JSONObject { @@ -47,7 +48,7 @@ type ESRoute = { request: Request; }; -const router = Router.make({ +let router = Router.make({ ignoreTrailingSlash: true, maxParamLength: 1000, }); @@ -62,7 +63,7 @@ export function splitSource(source: string): string[] { let prev = 0; while (index < len) { // Beginning of a new command, we should find the method and proceede to the url. - for (const method of METHODS) { + for (const method of httpMethods) { if (source.slice(index, len).startsWith(method)) { index += method.length; break; @@ -80,7 +81,7 @@ export function splitSource(source: string): string[] { if (index == len) return; let brackets = 0; // If we found an http method, then we have found a new command. - for (const method of METHODS) { + for (const method of httpMethods) { if (source.slice(index, len).startsWith(method)) { return; } @@ -90,7 +91,7 @@ export function splitSource(source: string): string[] { // If we find an open curly bracket, we should also find the closing one // before to checking for the http method. if (source[index] == "{") { - for (;;) { + for (; index < len; ) { if (source[index] == "{") { brackets += 1; } else if (source[index] == "}") { @@ -131,7 +132,7 @@ function parseCommand(source: string, options: ParseOptions) { const len = source.length; let index = 0; // identify the method - for (const method of METHODS) { + for (const method of httpMethods) { if (source.slice(index, len).startsWith(method)) { data.method = method; index += method.length; @@ -264,16 +265,28 @@ function parseCommand(source: string, options: ParseOptions) { * function directly, but it can be used to load a different version of the * specification than the one bundled with this package. * - * @param filename The path to the schema.json file to load. + * @param filename_or_object The path to the schema.json file to load, or an + * object with a loaded schema. */ -export async function loadSchema(filename: string) { +export async function loadSchema(filename_or_object: string | object) { + let spec: Model; + + if (typeof filename_or_object === "string") { + spec = JSON.parse( + await readFile(filename_or_object, { encoding: "utf-8" }), + ) as Model; + } else { + spec = filename_or_object as Model; + } + if (router.find("GET", "/") != undefined) { - throw Error("A schema has already been loaded"); + // start from a clean router + router = Router.make({ + ignoreTrailingSlash: true, + maxParamLength: 1000, + }); } - const spec = JSON.parse( - await readFile(filename, { encoding: "utf-8" }), - ) as Model; for (const endpoint of spec.endpoints) { for (const url of endpoint.urls) { const { path, methods } = url; @@ -323,8 +336,12 @@ async function getAPI( endpointPath: string, ): Promise> { if (router.find("GET", "/") == undefined) { - // load the Elasticsearch spec - await loadSchema(path.join(__dirname, "./schema.json")); + if (!isBrowser) { + // load the Elasticsearch spec + await loadSchema(path.join(__dirname, "./schema.json")); + } else { + throw new Error("Specification is missing"); + } } const formattedPath = endpointPath.startsWith("/") diff --git a/tests/parse.test.ts b/tests/parse.test.ts index 4c6e434..1c7eb35 100644 --- a/tests/parse.test.ts +++ b/tests/parse.test.ts @@ -273,6 +273,27 @@ POST\n_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update\n }); }); + it("errors with incomplete bodies", async () => { + const script = `GET /my-index/_search +{`; + expect(async () => await parseRequests(script)).rejects.toThrowError( + "body cannot be parsed", + ); + const req = await parseRequest(script, { ignoreErrors: true }); + expect(req).toMatchObject({ + source: "GET /my-index/_search\n{", + service: "es", + api: "search", + body: "\n{", + method: "GET", + params: { + index: "my-index", + }, + path: "/my-index/_search", + rawPath: "/my-index/_search", + }); + }); + it("parses kibana requests", async () => { const req = await parseRequest( "GET kbn:/api/saved_objects/_find?type=dashboard", diff --git a/tsconfig.json b/tsconfig.json index 435e96e..315ade1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,7 @@ "esModuleInterop": true, "strictPropertyInitialization": false, "strict": true, + "allowJs": true, "rootDir": "./src", "outDir": "./dist" }, From 5149508eb2b14c1091c3a5addf6f1072490e92de Mon Sep 17 00:00:00 2001 From: Miguel Grinberg Date: Wed, 26 Feb 2025 15:49:28 +0000 Subject: [PATCH 2/3] Update scripts/compile-templates.mjs Co-authored-by: Josh Mock --- scripts/compile-templates.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/compile-templates.mjs b/scripts/compile-templates.mjs index 5a99454..70c173c 100644 --- a/scripts/compile-templates.mjs +++ b/scripts/compile-templates.mjs @@ -9,7 +9,7 @@ const cmd = "handlebars python.tpl javascript.tpl -f templates.js -c"; exec(cmd, (error, stdout, stderr) => { if (error) { console.log(stdout); - console.log(stderr); + console.error(stderr); process.exit(1); } let templates = fs.readFileSync("templates.js", 'utf-8'); From bc66471f7c3008d9462f0da6d1156d23041a993e Mon Sep 17 00:00:00 2001 From: Miguel Grinberg Date: Wed, 26 Feb 2025 15:49:47 +0000 Subject: [PATCH 3/3] Update src/parse.ts Co-authored-by: Josh Mock --- src/parse.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse.ts b/src/parse.ts index 0dd425b..910d9a6 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -91,7 +91,7 @@ export function splitSource(source: string): string[] { // If we find an open curly bracket, we should also find the closing one // before to checking for the http method. if (source[index] == "{") { - for (; index < len; ) { + while (index < len) { if (source[index] == "{") { brackets += 1; } else if (source[index] == "}") {