From 94eb5a622e990d35973dc4bd77bb3d8ed8bd8aac Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Wed, 11 Dec 2024 15:54:51 +0000 Subject: [PATCH 1/7] Add Ruby converter --- src/convert.ts | 2 + src/exporters/ruby.tpl | 44 ++++++++++++++ src/exporters/ruby.ts | 129 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 src/exporters/ruby.tpl create mode 100644 src/exporters/ruby.ts diff --git a/src/convert.ts b/src/convert.ts index f721501..1f76d57 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -4,6 +4,7 @@ import { parseRequests, ParsedRequest } from "./parse"; import { PythonExporter } from "./exporters/python"; import { CurlExporter } from "./exporters/curl"; import { JavaScriptExporter } from "./exporters/javascript"; +import { RubyExporter } from "./exporters/ruby"; import util from "util"; const isBrowser = typeof window !== "undefined"; @@ -41,6 +42,7 @@ const EXPORTERS: Record = { python: new PythonExporter(), curl: new CurlExporter(), javascript: new JavaScriptExporter(), + ruby: new RubyExporter(), }; /** diff --git a/src/exporters/ruby.tpl b/src/exporters/ruby.tpl new file mode 100644 index 0000000..d16cf11 --- /dev/null +++ b/src/exporters/ruby.tpl @@ -0,0 +1,44 @@ +{{#if complete}} +require 'elasticsearch' + +client = Elasticsearch::Client.new( + host: {{#if elasticsearchUrl}}'{{elasticsearchUrl}}'{{else}}ENV['ELASTICSEARCH_URL']{{/if}}, + api_key: ENV['ELASTIC_API_KEY'] +) + +{{/if}} +{{#each requests}} +{{#supportedApi}} +{{#hasArgs}} +response{{#if @index}}{{@index}}{{/if}} = client.{{this.api}}( + {{#each this.params}} + {{alias @key ../this.request.path}}: '{{{this}}}', + {{/each}} + {{#each this.query}} + {{alias @key ../this.request.query}}: {{{rubyprint this}}}, + {{/each}} + {{#if this.body}} + body: {{{rubyprint this.body}}} + {{/if}} +) +{{else}} +response{{#if @index}}{{@index}}{{/if}} = client.{{this.api}} +{{/hasArgs}} +{{else}} +response{{#if @index}}{{@index}}{{/if}} = client.perform_request( + '{{this.method}}', + '{{this.path}}', + {{#if this.query}} + params: {{{rubyprint this.query}}}, + {{/if}} + {{#if this.body}} + headers: { 'Content-Type': 'application/json' }, + body: {{{rubyprint this.body}}}, + {{/if}} +) +{{/supportedApi}} +{{#if ../printResponse}} +print(resp{{#if @index}}{{@index}}{{/if}}) +{{/if}} + +{{/each}} diff --git a/src/exporters/ruby.ts b/src/exporters/ruby.ts new file mode 100644 index 0000000..bae518d --- /dev/null +++ b/src/exporters/ruby.ts @@ -0,0 +1,129 @@ +import { readFile } from "fs/promises"; +import path from "path"; +import Handlebars from "handlebars"; +import { FormatExporter, ConvertOptions } from "../convert"; +import { ParsedRequest } from "../parse"; + +// this regex should match the list of APIs that do not have specific handlers +// in the Ruby client. APIs in this list are rendered with a perform_request() +// call +const UNSUPPORTED_APIS = new RegExp("^_internal.*$"); + +const RUBYCONSTANTS: Record = { null: "nil" }; + +export class RubyExporter implements FormatExporter { + template: Handlebars.TemplateDelegate | undefined; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async check(requests: ParsedRequest[]): Promise { + // only return true if all requests are for Elasticsearch + return requests + .map((req) => req.service == "es") + .reduce((prev, curr) => prev && curr, true); + } + + async convert( + requests: ParsedRequest[], + options: ConvertOptions, + ): Promise { + if (!(await this.check(requests))) { + throw new Error("Cannot perform conversion"); + } + return (await this.getTemplate())({ requests, ...options }); + } + + async getTemplate(): Promise { + if (!this.template) { + // custom data renderer for Ruby + Handlebars.registerHelper("rubyprint", (context) => { + const lines = JSON.stringify(context ?? null, null, 2).split(/\r?\n/); + for (let i = 1; i < lines.length; i++) { + lines[i] = " " + lines[i]; + } + if (lines.length > 1) { + let result = lines.join("\n"); + for (const k of Object.keys(RUBYCONSTANTS)) { + result = result.replaceAll(`${k},\n`, `${RUBYCONSTANTS[k]},\n`); + result = result.replaceAll(`${k}\n`, `${RUBYCONSTANTS[k]}\n`); + } + return result; + } else if (RUBYCONSTANTS[lines[0]]) { + return RUBYCONSTANTS[lines[0]]; + } else if (lines[0].startsWith('"') && lines[0].endsWith('"')) { + // special case: handle strings such as "null" as + // their native types + const s = lines[0].substring(1, lines[0].length - 1); + if (RUBYCONSTANTS[s]) { + return RUBYCONSTANTS[s]; + } else { + return lines[0]; + } + } else { + return lines[0]; + } + }); + + // custom conditional for requests without any arguments + Handlebars.registerHelper( + "hasArgs", + function (this: ParsedRequest, options) { + if ( + Object.keys(this.params ?? {}).length + + Object.keys(this.query ?? {}).length + + Object.keys(this.body ?? {}).length > + 0 + ) { + return options.fn(this); + } else { + return options.inverse(this); + } + }, + ); + + // custom conditional to separate supported vs unsupported APIs + Handlebars.registerHelper( + "supportedApi", + function (this: ParsedRequest, options) { + if (!UNSUPPORTED_APIS.test(this.api as string) && this.request) { + return options.fn(this); + } else { + return options.inverse(this); + } + }, + ); + + // attribute name renderer that considers aliases and code-specific names + // arguments: + // name: the name of the attribute + // props: the list of schema properties this attribute belongs to + // TODO: I don't think any of these are necessary in Ruby, but check for others + Handlebars.registerHelper("alias", (name, props) => { + const aliases: Record = { + _meta: "meta", + _field_names: "field_names", + _routing: "routing", + _source: "source", + _source_excludes: "source_excludes", + _source_includes: "source_includes", + }; + if (aliases[name]) { + return aliases[name]; + } + if (props) { + for (const prop of props) { + if (prop.name == name && prop.codegenName != undefined) { + return prop.codegenName; + } + } + } + return name; + }); + + const t = await readFile(path.join(__dirname, "./ruby.tpl"), { + encoding: "utf-8", + }); + this.template = Handlebars.compile(t); + } + return this.template; + } +} From a585a474e2a55ea0e681cb5fef88c02a216c1472 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 7 Jan 2025 13:07:17 +0000 Subject: [PATCH 2/7] Update README to add Ruby --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 624fc7c..89bc184 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ const { convertRequests, listFormats } = require("@elastic/request-converter"); ## Available Formats -At this time the converter supports `curl` and `python`. Work is currently in -progress to add support for `javascript`, `ruby` and `php`. +At this time the converter supports `curl`, `python`, `javascript` and `ruby`. Work is currently in +progress to add support for `php`. ### curl @@ -107,6 +107,18 @@ Supported options: | `complete` | `boolean` | no | If `true`, generate a complete script. If `false`, only generate the request code. The default is `false`. | | `elasticsearchUrl` | `string` | no | The Elasticsearch endpoint to use. The default is `http://localhost:9200`. | +### ruby + +The Ruby exporter generates code for the Elasticsearch Ruby client. + +Supported options: + +| Option name | Type | Required | Description | +| ----------- | ---- | -------- | ----------- | +| `printResponse` | `boolean` | no | If `true`, add code to print the response. The default is `false`. | +| `complete` | `boolean` | no | If `true`, generate a complete script. If `false`, only generate the request code. The default is `false`. | +| `elasticsearchUrl` | `string` | no | The Elasticsearch endpoint to use. The default is `http://localhost:9200`. | + ## Command-Line Interface For convenience, a CLI that wraps the `convertRequests` function is also available. From 7ad3e73b4f0d4f61ed4cf391fb903fd9df9713c9 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 7 Jan 2025 13:44:11 +0000 Subject: [PATCH 3/7] Set up integration testing for Ruby --- .github/workflows/daily.yml | 23 +++++++++++++++++++++++ .github/workflows/weekly.yml | 23 +++++++++++++++++++++++ package.json | 2 ++ tests/integration/convert.test.ts | 1 + tests/integration/run-ruby.sh | 26 ++++++++++++++++++++++++++ 5 files changed, 75 insertions(+) create mode 100755 tests/integration/run-ruby.sh diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 2102c39..52fca1b 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -124,3 +124,26 @@ jobs: - name: Integration tests run: | npm run test:integration-javascript + integration-tests-ruby: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: 8.17 + - name: Setup node.js + uses: actions/setup-node@v4 + - name: Install + run: | + npm install + - name: Build + run: | + npm run build + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.4 + - name: Prepare for testing + run: | + npm run test:setup-ruby + - name: Integration tests + run: | + npm run test:integration-ruby diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index e7bf64e..7b6ebcd 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -64,3 +64,26 @@ jobs: - name: Integration tests run: | npm run test:integration-javascript + integration-tests-ruby: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: main + - name: Setup node.js + uses: actions/setup-node@v4 + - name: Install + run: | + npm install + - name: Build + run: | + npm run build + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.4 + - name: Prepare for testing + run: | + npm run test:setup-ruby + - name: Integration tests + run: | + npm run test:integration-ruby diff --git a/package.json b/package.json index 0de8119..08825d2 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,12 @@ "test:setup-curl": "node scripts/download-examples.mjs", "test:setup-python": "node scripts/download-examples.mjs && ./tests/integration/run-python.sh", "test:setup-javascript": "node scripts/download-examples.mjs && ./tests/integration/run-javascript.sh", + "test:setup-ruby": "node scripts/download-examples.mjs && ./tests/integration/run-ruby.sh", "test:integration": "jest tests/integration", "test:integration-curl": "./scripts/test-format.sh curl", "test:integration-python": "./scripts/test-format.sh python", "test:integration-javascript": "./scripts/test-format.sh javascript", + "test:integration-ruby": "./scripts/test-format.sh ruby", "test:example": "./scripts/test-example.sh", "fix": "npm run fix:lint && npm run fix:prettier", "fix:lint": "eslint src tests --fix --ignore-pattern tests/wasm/", diff --git a/tests/integration/convert.test.ts b/tests/integration/convert.test.ts index e6cadac..db2de99 100644 --- a/tests/integration/convert.test.ts +++ b/tests/integration/convert.test.ts @@ -14,6 +14,7 @@ const TEST_FORMATS: Record = { python: "py", javascript: "js", curl: "sh", + ruby: "rb", }; interface Example { diff --git a/tests/integration/run-ruby.sh b/tests/integration/run-ruby.sh new file mode 100755 index 0000000..b638567 --- /dev/null +++ b/tests/integration/run-ruby.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eo pipefail +set -x + +CLIENT_DIR=/tmp/es-client +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BRANCH=$(jq -r .version package.json | grep -Eo "^[0-9]+\.[0-9]+") + +if [ ! -d "$CLIENT_DIR" ]; then + echo "Installing from branch $BRANCH" + git clone -b "$BRANCH" --depth=1 "https://github.com/elastic/elasticsearch-ruby.git" "$CLIENT_DIR" || + (echo "Branch $BRANCH not found. Cloning main branch." && + git clone -b "main" --depth=1 "https://github.com/elastic/elasticsearch-ruby.git" "$CLIENT_DIR") + pushd "$CLIENT_DIR" + bundle install + bundle exec rake bundle:install + bundle exec rake automation:build_gems + gem install build/elasticsearch-api* + gem install build/elasticsearch* + popd +fi + +# if [[ "$1" != "" ]]; then +# $SCRIPT_DIR/.venv/bin/python $1 +# fi From 219df8dd36217d29207421e24662eb7166de5557 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 7 Jan 2025 16:28:01 +0000 Subject: [PATCH 4/7] Fix run-ruby to actually run ruby --- tests/integration/run-ruby.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/run-ruby.sh b/tests/integration/run-ruby.sh index b638567..db7dcbd 100755 --- a/tests/integration/run-ruby.sh +++ b/tests/integration/run-ruby.sh @@ -3,8 +3,7 @@ set -eo pipefail set -x -CLIENT_DIR=/tmp/es-client -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CLIENT_DIR=/tmp/ruby-client BRANCH=$(jq -r .version package.json | grep -Eo "^[0-9]+\.[0-9]+") if [ ! -d "$CLIENT_DIR" ]; then @@ -21,6 +20,6 @@ if [ ! -d "$CLIENT_DIR" ]; then popd fi -# if [[ "$1" != "" ]]; then -# $SCRIPT_DIR/.venv/bin/python $1 -# fi +if [[ "$1" != "" ]]; then + ruby $1 +fi From 668f3a113c2cae121b363dc08f60a6e3b704532b Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Thu, 27 Feb 2025 16:04:39 +0000 Subject: [PATCH 5/7] Updates ruby exporter and compiles template --- scripts/compile-templates.mjs | 2 +- src/exporters/ruby.ts | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/compile-templates.mjs b/scripts/compile-templates.mjs index 70c173c..9232151 100644 --- a/scripts/compile-templates.mjs +++ b/scripts/compile-templates.mjs @@ -5,7 +5,7 @@ const sourceDir = (process.platform !== "win32") ? "src/exporters" : "src\\exporters"; process.chdir(sourceDir); -const cmd = "handlebars python.tpl javascript.tpl -f templates.js -c"; +const cmd = "handlebars python.tpl javascript.tpl ruby.tpl -f templates.js -c"; exec(cmd, (error, stdout, stderr) => { if (error) { console.log(stdout); diff --git a/src/exporters/ruby.ts b/src/exporters/ruby.ts index bae518d..edbe868 100644 --- a/src/exporters/ruby.ts +++ b/src/exporters/ruby.ts @@ -1,8 +1,9 @@ -import { readFile } from "fs/promises"; +import { readFileSync } from "fs"; import path from "path"; import Handlebars from "handlebars"; import { FormatExporter, ConvertOptions } from "../convert"; import { ParsedRequest } from "../parse"; +import "./templates"; // this regex should match the list of APIs that do not have specific handlers // in the Ruby client. APIs in this list are rendered with a perform_request() @@ -119,10 +120,14 @@ export class RubyExporter implements FormatExporter { return name; }); - const t = await readFile(path.join(__dirname, "./ruby.tpl"), { - encoding: "utf-8", - }); - this.template = Handlebars.compile(t); + if (process.env.NODE_ENV !== "test") { + this.template = Handlebars.templates["ruby.tpl"]; + } else { + // when running tests we read the templates directly, in case the + // compiled file is not up to date + const t = readFileSync(path.join(__dirname, "./ruby.tpl"), "utf-8"); + this.template = Handlebars.compile(t); + } } return this.template; } From 0ed3c1dddb8c649840cc74e23a609844bd0fe2d8 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Thu, 27 Feb 2025 16:08:28 +0000 Subject: [PATCH 6/7] Update to use double quotes in Ruby for consistency --- src/exporters/ruby.tpl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/exporters/ruby.tpl b/src/exporters/ruby.tpl index d16cf11..fdd95ee 100644 --- a/src/exporters/ruby.tpl +++ b/src/exporters/ruby.tpl @@ -1,9 +1,9 @@ {{#if complete}} -require 'elasticsearch' +require "elasticsearch" client = Elasticsearch::Client.new( - host: {{#if elasticsearchUrl}}'{{elasticsearchUrl}}'{{else}}ENV['ELASTICSEARCH_URL']{{/if}}, - api_key: ENV['ELASTIC_API_KEY'] + host: {{#if elasticsearchUrl}}"{{elasticsearchUrl}}"{{else}}ENV["ELASTICSEARCH_URL"]{{/if}}, + api_key: ENV["ELASTIC_API_KEY"] ) {{/if}} @@ -12,7 +12,7 @@ client = Elasticsearch::Client.new( {{#hasArgs}} response{{#if @index}}{{@index}}{{/if}} = client.{{this.api}}( {{#each this.params}} - {{alias @key ../this.request.path}}: '{{{this}}}', + {{alias @key ../this.request.path}}: "{{{this}}}", {{/each}} {{#each this.query}} {{alias @key ../this.request.query}}: {{{rubyprint this}}}, @@ -26,13 +26,13 @@ response{{#if @index}}{{@index}}{{/if}} = client.{{this.api}} {{/hasArgs}} {{else}} response{{#if @index}}{{@index}}{{/if}} = client.perform_request( - '{{this.method}}', - '{{this.path}}', + "{{this.method}}", + "{{this.path}}", {{#if this.query}} params: {{{rubyprint this.query}}}, {{/if}} {{#if this.body}} - headers: { 'Content-Type': 'application/json' }, + headers: { "Content-Type": "application/json" }, body: {{{rubyprint this.body}}}, {{/if}} ) From 3884d3210606cee4702b88c0881262bfc81c7924 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Thu, 27 Feb 2025 16:47:38 +0000 Subject: [PATCH 7/7] Updates Ruby template for commas usage at end of line Makes it so there are no commas for the last parameter in a function call, for more idiomatic Ruby. --- src/exporters/ruby.tpl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/exporters/ruby.tpl b/src/exporters/ruby.tpl index fdd95ee..49930d7 100644 --- a/src/exporters/ruby.tpl +++ b/src/exporters/ruby.tpl @@ -11,15 +11,28 @@ client = Elasticsearch::Client.new( {{#supportedApi}} {{#hasArgs}} response{{#if @index}}{{@index}}{{/if}} = client.{{this.api}}( +{{#if this.body}} {{#each this.params}} {{alias @key ../this.request.path}}: "{{{this}}}", {{/each}} {{#each this.query}} {{alias @key ../this.request.query}}: {{{rubyprint this}}}, {{/each}} - {{#if this.body}} body: {{{rubyprint this.body}}} +{{else}} + {{#if this.query}} + {{#each this.params}} + {{alias @key ../this.request.path}}: "{{{this}}}", + {{/each}} + {{#each this.query}} + {{alias @key ../this.request.query}}: {{{rubyprint this}}}{{#unless @last}},{{/unless}} + {{/each}} + {{else}} + {{#each this.params}} + {{alias @key ../this.request.path}}: "{{{this}}}"{{#unless @last}},{{/unless}} + {{/each}} {{/if}} +{{/if}} ) {{else}} response{{#if @index}}{{@index}}{{/if}} = client.{{this.api}}