reorganization to make it possible to run in the browser

Author: miguelgrinberg

eslint.config.mjs

@@ -6,6 +6,6 @@ import tseslint from "typescript-eslint";
 export default tseslint.config(
   eslint.configs.recommended,
   ...tseslint.configs.recommended,
-  {ignores: ["**/.venv/", "**/es-client/", "tests/wasm"]},
+  {ignores: ["**/.venv/", "**/es-client/", "**/templates.js", "tests/wasm"]},
   {languageOptions: {globals: {require: true}}},
 );

jest.config.js

@@ -3,5 +3,10 @@ module.exports = {
   testEnvironment: "node",
   setupFilesAfterEnv: [
     "jest-expect-message"
+  ],
+  moduleFileExtensions: [
+    "ts",
+    "js",
+    "tpl"
   ]
 };

package.json

@@ -11,13 +11,14 @@
   "scripts": {
     "update-schema": "node scripts/update-schema.mjs",
     "release": "./scripts/release.sh",
-    "build": "npm run clean && tsc && npm run copy-files",
+    "build": "npm run clean && tsc && npm run compile-templates && npm run copy-files",
     "clean": "rimraf dist/",
-    "copy-files": "copyfiles -u 1 src/**/*.js src/**/*.json src/**/*.tpl dist/",
+    "compile-templates": "node scripts/compile-templates.mjs",
+    "copy-files": "copyfiles -u 1 src/**/*.js src/**/*.json dist/",
     "docs": "typedoc src/index.ts",
     "lint": "eslint src tests --ignore-pattern tests/wasm/",
     "prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --list-different",
-    "test": "jest --test-path-ignore-patterns integration --coverage",
+    "test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage",
     "test:setup": "node scripts/download-examples.mjs && ./tests/integration/run-python.sh && ./tests/integration/run-javascript.sh",
     "test:setup-curl": "node scripts/download-examples.mjs",
     "test:setup-python": "node scripts/download-examples.mjs && ./tests/integration/run-python.sh",
@@ -31,8 +32,7 @@
     "fix:lint": "eslint src tests --fix --ignore-pattern tests/wasm/",
     "fix:prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --write",
     "fix:precommit": "./scripts/precommit.sh",
-    "watch:build": "tsc -w",
-    "watch:test": "jest --test-path-ignore-patterns integration --coverage --watch",
+    "watch:test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage --watch",
     "prepare": "husky install"
   },
   "main": "dist/index.js",

scripts/compile-templates.mjs

@@ -0,0 +1,11 @@
+import fs from 'fs';
+import { exec }  from 'child_process';
+
+const cmd = (process.platform !== "win32")
+  ? "handlebars src/exporters/*.tpl -f src/exporters/templates.js -c"
+  : "handlebars src\\exporters\\*.tpl -f src\\exporters\\templates.js -c";
+exec(cmd, () => {
+  let templates = fs.readFileSync('src/exporters/templates.js', 'utf-8');
+  templates = 'const Handlebars = require("handlebars");\n' + templates;
+  fs.writeFileSync('src/exporters/templates.js', templates);
+});

src/convert.ts

@@ -6,7 +6,8 @@ import { CurlExporter } from "./exporters/curl";
 import { JavaScriptExporter } from "./exporters/javascript";
 import util from "util";
 
-const execAsync = util.promisify(childProcess.exec);
+const isBrowser = typeof window !== "undefined";
+const execAsync = !isBrowser ? util.promisify(childProcess.exec) : undefined;
 
 export type ConvertOptions = {
   /** When `true`, the converter will only check if the conversion can be carried
@@ -188,6 +189,9 @@ export class SubprocessExporter implements FormatExporter {
     const input = base64url.encode(
       JSON.stringify({ requests: getSlimRequests(requests) }),
     );
+    if (execAsync === undefined) {
+      throw new Error("Cannot use exec()");
+    }
     const { stdout, stderr } = await execAsync(
       `${this.baseCmd} check ${input}`,
     );
@@ -208,6 +212,9 @@ export class SubprocessExporter implements FormatExporter {
     const input = base64url.encode(
       JSON.stringify({ requests: getSlimRequests(requests), options }),
     );
+    if (execAsync === undefined) {
+      throw new Error("Cannot use exec()");
+    }
     const { stdout } = await execAsync(`${this.baseCmd} convert ${input}`);
     if (stdout) {
       const json = JSON.parse(base64url.decode(stdout));

src/exporters/javascript.ts

@@ -2,8 +2,10 @@ import { readFileSync } from "fs";
 import path from "path";
 import Handlebars from "handlebars";
 import prettier from "prettier";
+import prettierTypeScript from "prettier/parser-typescript";
 import { FormatExporter, ConvertOptions } from "../convert";
 import { ParsedRequest } from "../parse";
+import "./templates";
 
 const UNSUPPORTED_APIS = new RegExp(
   "^query_rules.*$" +
@@ -39,7 +41,10 @@ export class JavaScriptExporter implements FormatExporter {
       throw new Error("Cannot perform conversion");
     }
     const output = this.template({ requests, ...options });
-    return prettier.format(output, { parser: "typescript" });
+    return prettier.format(output, {
+      parser: "typescript",
+      plugins: [prettierTypeScript],
+    });
   }
 
   get template(): Handlebars.TemplateDelegate {
@@ -115,8 +120,17 @@ export class JavaScriptExporter implements FormatExporter {
 
       Handlebars.registerHelper("camelCase", (text) => toCamelCase(text));
 
-      const t = readFileSync(path.join(__dirname, "./javascript.tpl"), "utf-8");
-      this._template = Handlebars.compile(t);
+      if (process.env.NODE_ENV !== "test") {
+        this._template = Handlebars.templates["javascript.tpl"];
+      } else {
+        // when running tests we read the templates directly, in case the
+        // compiled file is not up to date
+        const t = readFileSync(
+          path.join(__dirname, "./javascript.tpl"),
+          "utf-8",
+        );
+        this._template = Handlebars.compile(t);
+      }
     }
 
     return this._template;

src/exporters/python.ts

@@ -1,8 +1,9 @@
-import { readFile } from "fs/promises";
+import { readFileSync } from "fs";
 import path from "path";
 import Handlebars from "handlebars";
 import { FormatExporter, ConvertOptions } from "../convert";
 import { ParsedRequest } from "../parse";
+import "./templates";
 
 // this regex should match the list of APIs that do not have specific handlers
 // in the Python client. APIs in this list are rendered with a perform_request()
@@ -159,10 +160,14 @@ export class PythonExporter implements FormatExporter {
         },
       );
 
-      const t = await readFile(path.join(__dirname, "./python.tpl"), {
-        encoding: "utf-8",
-      });
-      this.template = Handlebars.compile(t);
+      if (process.env.NODE_ENV !== "test") {
+        this.template = Handlebars.templates["python.tpl"];
+      } else {
+        // when running tests we read the templates directly, in case the
+        // compiled file is not up to date
+        const t = readFileSync(path.join(__dirname, "./python.tpl"), "utf-8");
+        this.template = Handlebars.compile(t);
+      }
     }
     return this.template;
   }

src/parse.ts

@@ -1,10 +1,11 @@
-import { METHODS } from "http";
 import { URL } from "url";
 import { readFile } from "fs/promises";
 import path from "path";
 import * as Router from "find-my-way-ts";
 import { Model, Request } from "./metamodel";
 
+const isBrowser = typeof window !== "undefined";
+const httpMethods = ["HEAD", "GET", "POST", "PUT", "PATCH", "DELETE"];
 export type JSONValue = string | number | boolean | JSONArray | JSONObject;
 interface JSONArray extends Array<JSONValue> {}
 interface JSONObject {
@@ -47,7 +48,7 @@ type ESRoute = {
   request: Request;
 };
 
-const router = Router.make<ESRoute>({
+let router = Router.make<ESRoute>({
   ignoreTrailingSlash: true,
   maxParamLength: 1000,
 });
@@ -62,7 +63,7 @@ export function splitSource(source: string): string[] {
   let prev = 0;
   while (index < len) {
     // Beginning of a new command, we should find the method and proceede to the url.
-    for (const method of METHODS) {
+    for (const method of httpMethods) {
       if (source.slice(index, len).startsWith(method)) {
         index += method.length;
         break;
@@ -80,7 +81,7 @@ export function splitSource(source: string): string[] {
     if (index == len) return;
     let brackets = 0;
     // If we found an http method, then we have found a new command.
-    for (const method of METHODS) {
+    for (const method of httpMethods) {
       if (source.slice(index, len).startsWith(method)) {
         return;
       }
@@ -90,7 +91,7 @@ export function splitSource(source: string): string[] {
     // If we find an open curly bracket, we should also find the closing one
     // before to checking for the http method.
     if (source[index] == "{") {
-      for (;;) {
+      for (; index < len; ) {
         if (source[index] == "{") {
           brackets += 1;
         } else if (source[index] == "}") {
@@ -131,7 +132,7 @@ function parseCommand(source: string, options: ParseOptions) {
   const len = source.length;
   let index = 0;
   // identify the method
-  for (const method of METHODS) {
+  for (const method of httpMethods) {
     if (source.slice(index, len).startsWith(method)) {
       data.method = method;
       index += method.length;
@@ -264,16 +265,28 @@ function parseCommand(source: string, options: ParseOptions) {
  * function directly, but it can be used to load a different version of the
  * specification than the one bundled with this package.
  *
- * @param filename The path to the schema.json file to load.
+ * @param filename_or_object The path to the schema.json file to load, or an
+ *   object with a loaded schema.
  */
-export async function loadSchema(filename: string) {
+export async function loadSchema(filename_or_object: string | object) {
+  let spec: Model;
+
+  if (typeof filename_or_object === "string") {
+    spec = JSON.parse(
+      await readFile(filename_or_object, { encoding: "utf-8" }),
+    ) as Model;
+  } else {
+    spec = filename_or_object as Model;
+  }
+
   if (router.find("GET", "/") != undefined) {
-    throw Error("A schema has already been loaded");
+    // start from a clean router
+    router = Router.make<ESRoute>({
+      ignoreTrailingSlash: true,
+      maxParamLength: 1000,
+    });
   }
 
-  const spec = JSON.parse(
-    await readFile(filename, { encoding: "utf-8" }),
-  ) as Model;
   for (const endpoint of spec.endpoints) {
     for (const url of endpoint.urls) {
       const { path, methods } = url;
@@ -323,8 +336,12 @@ async function getAPI(
   endpointPath: string,
 ): Promise<Router.FindResult<ESRoute>> {
   if (router.find("GET", "/") == undefined) {
-    // load the Elasticsearch spec
-    await loadSchema(path.join(__dirname, "./schema.json"));
+    if (!isBrowser) {
+      // load the Elasticsearch spec
+      await loadSchema(path.join(__dirname, "./schema.json"));
+    } else {
+      throw new Error("Specification is missing");
+    }
   }
 
   const formattedPath = endpointPath.startsWith("/")

tests/parse.test.ts

@@ -273,6 +273,27 @@ POST\n_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update\n
     });
   });
 
+  it("errors with incomplete bodies", async () => {
+    const script = `GET /my-index/_search
+{`;
+    expect(async () => await parseRequests(script)).rejects.toThrowError(
+      "body cannot be parsed",
+    );
+    const req = await parseRequest(script, { ignoreErrors: true });
+    expect(req).toMatchObject({
+      source: "GET /my-index/_search\n{",
+      service: "es",
+      api: "search",
+      body: "\n{",
+      method: "GET",
+      params: {
+        index: "my-index",
+      },
+      path: "/my-index/_search",
+      rawPath: "/my-index/_search",
+    });
+  });
+
   it("parses kibana requests", async () => {
     const req = await parseRequest(
       "GET kbn:/api/saved_objects/_find?type=dashboard",

tsconfig.json

@@ -7,6 +7,7 @@
     "esModuleInterop": true,
     "strictPropertyInitialization": false,
     "strict": true,
+    "allowJs": true,
     "rootDir": "./src",
     "outDir": "./dist"
   },