From df6c6f07dde0d7b5cdf0669c8480996d80bc8b08 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Mon, 12 Feb 2024 16:13:38 +0000 Subject: [PATCH 1/7] Adds Ruby script to generate a report of tested/untested endpoints --- .gitignore | 1 + report/.gitignore | 1 + report/Gemfile | 5 +++ report/Rakefile | 15 +++++++++ report/download_artifacts.rb | 65 ++++++++++++++++++++++++++++++++++++ report/reporter.rb | 38 +++++++++++++++++++++ report/template.erb | 17 ++++++++++ 7 files changed, 142 insertions(+) create mode 100644 .gitignore create mode 100644 report/.gitignore create mode 100644 report/Gemfile create mode 100644 report/Rakefile create mode 100644 report/download_artifacts.rb create mode 100644 report/reporter.rb create mode 100644 report/template.erb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c66eeb8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +report/tmp \ No newline at end of file diff --git a/report/.gitignore b/report/.gitignore new file mode 100644 index 0000000..68feb7d --- /dev/null +++ b/report/.gitignore @@ -0,0 +1 @@ +Gemfile.lock \ No newline at end of file diff --git a/report/Gemfile b/report/Gemfile new file mode 100644 index 0000000..027a254 --- /dev/null +++ b/report/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +group :development, :test do + gem 'debug' +end diff --git a/report/Rakefile b/report/Rakefile new file mode 100644 index 0000000..732cb11 --- /dev/null +++ b/report/Rakefile @@ -0,0 +1,15 @@ +require 'erb' +require './download_artifacts' +require './reporter' + +desc "Generate report" +task :report do + @reporter = Elastic::Reporter.new + template = ERB.new(File.read('./template.erb'), trim_mode = true) + File.write('../apis_report.md', template.result(binding)) +end + +desc "Download artifacts" +task :download do + Elastic::download_artifacts(ENV['STACK_VERSION'] || '8.13.0-SNAPSHOT') +end diff --git a/report/download_artifacts.rb b/report/download_artifacts.rb new file mode 100644 index 0000000..c363792 --- /dev/null +++ b/report/download_artifacts.rb @@ -0,0 +1,65 @@ +require 'pathname' +require 'open-uri' +require 'json' + +module Elastic + CURRENT_PATH = Pathname(File.expand_path(__dir__)) + class << self + def download_artifacts(version) + json_filename = CURRENT_PATH.join('tmp/artifacts.json') + + # Create ./tmp if it doesn't exist + Dir.mkdir(CURRENT_PATH.join('tmp'), 0700) unless File.directory?(CURRENT_PATH.join('tmp')) + + # Download json file with package information for version: + json_url = "https://artifacts-api.elastic.co/v1/versions/#{version}" + download_file!(json_url, json_filename) + + # Parse the downloaded JSON + begin + artifacts = JSON.parse(File.read(json_filename)) + rescue StandardError => e + STDERR.puts "[!] Couldn't read JSON file #{json_filename}" + exit 1 + end + + # Either find the artifacts for the exact same build hash from the current running cluster or + # use the first one from the list of builds: + build_hash_artifact = artifacts['version']['builds'].first + zip_url = build_hash_artifact.dig('projects', 'elasticsearch', 'packages').select { |k, _| k =~ /rest-resources-zip/ }.map { |_, v| v['url'] }.first + + # Dig into the elasticsearch packages, search for the rest-resources-zip package and return the URL: + build_hash_artifact.dig('projects', 'elasticsearch', 'packages').select { |k, _| k =~ /rest-resources-zip/ }.map { |_, v| v['url'] }.first + + # Download the zip file + filename = CURRENT_PATH.join("tmp/#{zip_url.split('/').last}") + download_file!(zip_url, filename) + + spec = CURRENT_PATH.join('tmp/rest-api-spec') + FileUtils.remove_dir(spec) if File.directory?(spec) + + puts "Unzipping file #{filename}" + `unzip -o #{filename} -d tmp/` + `rm #{filename}` + puts "Artifacts downloaded in ./tmp, build hash #{@build_hash}" + File.write(CURRENT_PATH.join('tmp/rest-api-spec/build_hash'), @build_hash) + end + + def download_file!(url, filename) + puts "Downloading #{filename} from #{url}" + File.open(filename, "w") do |downloaded_file| + URI.open(url, "rb") do |artifact_file| + downloaded_file.write(artifact_file.read) + end + end + puts "Successfully downloaded #{filename}" + + unless File.exist?(filename) + warn "[!] Couldn't download #{filename}" + exit 1 + end + rescue StandardError => e + abort e.message + end + end +end diff --git a/report/reporter.rb b/report/reporter.rb new file mode 100644 index 0000000..0ce084b --- /dev/null +++ b/report/reporter.rb @@ -0,0 +1,38 @@ +module Elastic + class Reporter + JSON_FILES = "#{File.expand_path('./tmp/rest-api-spec/api')}/*.json".freeze + TESTS_PATH = File.expand_path('../tests/**/*.yml') + + attr_reader :apis, :tested, :untested + + def initialize + @apis = gather_apis + @tested = [] + @untested = [] + report! + end + + def gather_apis + endpoints = Dir[JSON_FILES].map do |path| + path.split('/').last.gsub('.json','') + end.reject { |a| a.split('/').last.gsub('.json','').start_with?('_') } + end + + def report! + @apis.each do |api| + if find_in_tests(api) + @tested << api + else + @untested << api + end + end + end + + def find_in_tests(endpoint) + Dir[TESTS_PATH].map do |path| + return true if File.readlines(path).grep(/#{endpoint}/).any? + end + false + end + end +end diff --git a/report/template.erb b/report/template.erb new file mode 100644 index 0000000..b4fa524 --- /dev/null +++ b/report/template.erb @@ -0,0 +1,17 @@ +# Tests report + +Endpoints: <%= @reporter.apis.count %> +[Tested](#tested): <%= @reporter.tested.count %> +[Untested](#untested): <%= @reporter.untested.count %> + +## Tested + +<% @reporter.tested.each do |test| %> +- [x] <%= test %> +<% end %> + +## Untested + +<% @reporter.untested.each do |test| %> +- [ ] <%= test %> +<% end %> From 15e0d1504fb7e5eb032942a705463adb284ad5c8 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Mon, 12 Feb 2024 16:16:20 +0000 Subject: [PATCH 2/7] Adds report file --- apis_report.md | 1004 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1004 insertions(+) create mode 100644 apis_report.md diff --git a/apis_report.md b/apis_report.md new file mode 100644 index 0000000..4c43d84 --- /dev/null +++ b/apis_report.md @@ -0,0 +1,1004 @@ +# Tests report + +* Endpoints in JSON spec: 494 +* Endpoints in elasticsearch-specification: 456 +* Endpoints in Serverless: 208 +* [Tested](#tested): 198 +* [Untested](#untested): 296 +* Coverage: ![](https://geps.dev/progress/40) + +## Tested + + +- [x] async_search.delete + +- [x] async_search.get + +- [x] async_search.status + +- [x] async_search.submit + +- [x] bulk + +- [x] cat.aliases + +- [x] cat.component_templates + +- [x] cat.count + +- [x] cat.help + +- [x] cat.indices + +- [x] cat.ml_data_frame_analytics + +- [x] cat.ml_datafeeds + +- [x] cat.ml_jobs + +- [x] cat.ml_trained_models + +- [x] cat.transforms + +- [x] clear_scroll + +- [x] close_point_in_time + +- [x] cluster.delete_component_template + +- [x] cluster.exists_component_template + +- [x] cluster.get_component_template + +- [x] cluster.info + +- [x] cluster.put_component_template + +- [x] count + +- [x] create + +- [x] delete + +- [x] delete_by_query + +- [x] delete_script + +- [x] enrich.delete_policy + +- [x] enrich.execute_policy + +- [x] enrich.get_policy + +- [x] enrich.put_policy + +- [x] enrich.stats + +- [x] eql.delete + +- [x] eql.get + +- [x] eql.get_status + +- [x] eql.search + +- [x] exists + +- [x] exists_source + +- [x] explain + +- [x] field_caps + +- [x] get + +- [x] get_script + +- [x] get_source + +- [x] graph.explore + +- [x] index + +- [x] indices.add_block + +- [x] indices.analyze + +- [x] indices.create + +- [x] indices.create_data_stream + +- [x] indices.data_streams_stats + +- [x] indices.delete + +- [x] indices.delete_alias + +- [x] indices.delete_data_lifecycle + +- [x] indices.delete_data_stream + +- [x] indices.delete_index_template + +- [x] indices.exists + +- [x] indices.exists_alias + +- [x] indices.exists_index_template + +- [x] indices.explain_data_lifecycle + +- [x] indices.get + +- [x] indices.get_alias + +- [x] indices.get_data_lifecycle + +- [x] indices.get_data_stream + +- [x] indices.get_index_template + +- [x] indices.get_mapping + +- [x] indices.get_settings + +- [x] indices.migrate_to_data_stream + +- [x] indices.modify_data_stream + +- [x] indices.put_alias + +- [x] indices.put_data_lifecycle + +- [x] indices.put_index_template + +- [x] indices.put_mapping + +- [x] indices.put_settings + +- [x] indices.refresh + +- [x] indices.resolve_index + +- [x] indices.rollover + +- [x] indices.simulate_index_template + +- [x] indices.simulate_template + +- [x] indices.update_aliases + +- [x] indices.validate_query + +- [x] info + +- [x] ingest.delete_pipeline + +- [x] ingest.get_pipeline + +- [x] ingest.processor_grok + +- [x] ingest.put_pipeline + +- [x] ingest.simulate + +- [x] license.get + +- [x] logstash.delete_pipeline + +- [x] logstash.get_pipeline + +- [x] logstash.put_pipeline + +- [x] mget + +- [x] ml.close_job + +- [x] ml.delete_calendar + +- [x] ml.delete_calendar_event + +- [x] ml.delete_calendar_job + +- [x] ml.delete_data_frame_analytics + +- [x] ml.delete_datafeed + +- [x] ml.delete_filter + +- [x] ml.delete_job + +- [x] ml.delete_trained_model + +- [x] ml.delete_trained_model_alias + +- [x] ml.estimate_model_memory + +- [x] ml.evaluate_data_frame + +- [x] ml.flush_job + +- [x] ml.get_calendar_events + +- [x] ml.get_calendars + +- [x] ml.get_data_frame_analytics + +- [x] ml.get_data_frame_analytics_stats + +- [x] ml.get_datafeed_stats + +- [x] ml.get_datafeeds + +- [x] ml.get_filters + +- [x] ml.get_job_stats + +- [x] ml.get_jobs + +- [x] ml.get_overall_buckets + +- [x] ml.get_trained_models + +- [x] ml.get_trained_models_stats + +- [x] ml.open_job + +- [x] ml.post_calendar_events + +- [x] ml.preview_data_frame_analytics + +- [x] ml.preview_datafeed + +- [x] ml.put_calendar + +- [x] ml.put_calendar_job + +- [x] ml.put_data_frame_analytics + +- [x] ml.put_datafeed + +- [x] ml.put_filter + +- [x] ml.put_job + +- [x] ml.put_trained_model + +- [x] ml.put_trained_model_alias + +- [x] ml.put_trained_model_definition_part + +- [x] ml.put_trained_model_vocabulary + +- [x] ml.reset_job + +- [x] ml.start_data_frame_analytics + +- [x] ml.start_datafeed + +- [x] ml.start_trained_model_deployment + +- [x] ml.stop_data_frame_analytics + +- [x] ml.stop_datafeed + +- [x] ml.stop_trained_model_deployment + +- [x] ml.update_data_frame_analytics + +- [x] ml.update_datafeed + +- [x] ml.update_filter + +- [x] ml.update_job + +- [x] msearch + +- [x] msearch_template + +- [x] mtermvectors + +- [x] open_point_in_time + +- [x] ping + +- [x] put_script + +- [x] query_ruleset.delete + +- [x] query_ruleset.get + +- [x] query_ruleset.list + +- [x] query_ruleset.put + +- [x] rank_eval + +- [x] reindex + +- [x] render_search_template + +- [x] scripts_painless_execute + +- [x] scroll + +- [x] search + +- [x] search_application.delete + +- [x] search_application.delete_behavioral_analytics + +- [x] search_application.get + +- [x] search_application.get_behavioral_analytics + +- [x] search_application.list + +- [x] search_application.put + +- [x] search_application.put_behavioral_analytics + +- [x] search_mvt + +- [x] search_template + +- [x] security.authenticate + +- [x] security.create_api_key + +- [x] security.get_api_key + +- [x] security.has_privileges + +- [x] security.invalidate_api_key + +- [x] security.query_api_keys + +- [x] sql.clear_cursor + +- [x] sql.delete_async + +- [x] sql.get_async + +- [x] sql.get_async_status + +- [x] sql.query + +- [x] sql.translate + +- [x] synonyms.delete_synonym + +- [x] synonyms.delete_synonym_rule + +- [x] synonyms.get_synonym + +- [x] synonyms.get_synonym_rule + +- [x] synonyms.put_synonym + +- [x] synonyms.put_synonym_rule + +- [x] terms_enum + +- [x] termvectors + +- [x] transform.delete_transform + +- [x] transform.get_transform + +- [x] transform.get_transform_stats + +- [x] transform.preview_transform + +- [x] transform.put_transform + +- [x] transform.reset_transform + +- [x] transform.schedule_now_transform + +- [x] transform.start_transform + +- [x] transform.stop_transform + +- [x] transform.update_transform + +- [x] update + +- [x] update_by_query + + +## Untested + + +- [ ] autoscaling.delete_autoscaling_policy + +- [ ] autoscaling.get_autoscaling_capacity + +- [ ] autoscaling.get_autoscaling_policy + +- [ ] autoscaling.put_autoscaling_policy + +- [ ] cat.allocation + +- [ ] cat.fielddata + +- [ ] cat.health + +- [ ] cat.master + +- [ ] cat.nodeattrs + +- [ ] cat.nodes + +- [ ] cat.pending_tasks + +- [ ] cat.plugins + +- [ ] cat.recovery + +- [ ] cat.repositories + +- [ ] cat.segments + +- [ ] cat.shards + +- [ ] cat.snapshots + +- [ ] cat.tasks + +- [ ] cat.templates + +- [ ] cat.thread_pool + +- [ ] ccr.delete_auto_follow_pattern + +- [ ] ccr.follow + +- [ ] ccr.follow_info + +- [ ] ccr.follow_stats + +- [ ] ccr.forget_follower + +- [ ] ccr.get_auto_follow_pattern + +- [ ] ccr.pause_auto_follow_pattern + +- [ ] ccr.pause_follow + +- [ ] ccr.put_auto_follow_pattern + +- [ ] ccr.resume_auto_follow_pattern + +- [ ] ccr.resume_follow + +- [ ] ccr.stats + +- [ ] ccr.unfollow + +- [ ] cluster.allocation_explain + +- [ ] cluster.delete_voting_config_exclusions + +- [ ] cluster.get_settings + +- [ ] cluster.health + +- [ ] cluster.pending_tasks + +- [ ] cluster.post_voting_config_exclusions + +- [ ] cluster.put_settings + +- [ ] cluster.remote_info + +- [ ] cluster.reroute + +- [ ] cluster.state + +- [ ] cluster.stats + +- [ ] connector.check_in + +- [ ] connector.delete + +- [ ] connector.get + +- [ ] connector.last_sync + +- [ ] connector.list + +- [ ] connector.post + +- [ ] connector.put + +- [ ] connector.update_api_key_id + +- [ ] connector.update_configuration + +- [ ] connector.update_error + +- [ ] connector.update_filtering + +- [ ] connector.update_index_name + +- [ ] connector.update_name + +- [ ] connector.update_native + +- [ ] connector.update_pipeline + +- [ ] connector.update_scheduling + +- [ ] connector.update_service_type + +- [ ] connector.update_status + +- [ ] connector_secret.delete + +- [ ] connector_secret.get + +- [ ] connector_secret.post + +- [ ] connector_secret.put + +- [ ] connector_sync_job.cancel + +- [ ] connector_sync_job.check_in + +- [ ] connector_sync_job.delete + +- [ ] connector_sync_job.error + +- [ ] connector_sync_job.get + +- [ ] connector_sync_job.list + +- [ ] connector_sync_job.post + +- [ ] connector_sync_job.update_stats + +- [ ] dangling_indices.delete_dangling_index + +- [ ] dangling_indices.import_dangling_index + +- [ ] dangling_indices.list_dangling_indices + +- [ ] delete_by_query_rethrottle + +- [ ] esql.async_query + +- [ ] esql.async_query_get + +- [ ] esql.query + +- [ ] features.get_features + +- [ ] features.reset_features + +- [ ] fleet.delete_secret + +- [ ] fleet.get_secret + +- [ ] fleet.global_checkpoints + +- [ ] fleet.msearch + +- [ ] fleet.post_secret + +- [ ] fleet.search + +- [ ] get_script_context + +- [ ] get_script_languages + +- [ ] health_report + +- [ ] ilm.delete_lifecycle + +- [ ] ilm.explain_lifecycle + +- [ ] ilm.get_lifecycle + +- [ ] ilm.get_status + +- [ ] ilm.migrate_to_data_tiers + +- [ ] ilm.move_to_step + +- [ ] ilm.put_lifecycle + +- [ ] ilm.remove_policy + +- [ ] ilm.retry + +- [ ] ilm.start + +- [ ] ilm.stop + +- [ ] indices.clear_cache + +- [ ] indices.clone + +- [ ] indices.close + +- [ ] indices.delete_template + +- [ ] indices.disk_usage + +- [ ] indices.downsample + +- [ ] indices.exists_template + +- [ ] indices.field_usage_stats + +- [ ] indices.flush + +- [ ] indices.forcemerge + +- [ ] indices.get_field_mapping + +- [ ] indices.get_template + +- [ ] indices.open + +- [ ] indices.promote_data_stream + +- [ ] indices.put_template + +- [ ] indices.recovery + +- [ ] indices.reload_search_analyzers + +- [ ] indices.resolve_cluster + +- [ ] indices.segments + +- [ ] indices.shard_stores + +- [ ] indices.shrink + +- [ ] indices.split + +- [ ] indices.stats + +- [ ] indices.unfreeze + +- [ ] inference.delete_model + +- [ ] inference.get_model + +- [ ] inference.inference + +- [ ] inference.put_model + +- [ ] ingest.geo_ip_stats + +- [ ] knn_search + +- [ ] license.delete + +- [ ] license.get_basic_status + +- [ ] license.get_trial_status + +- [ ] license.post + +- [ ] license.post_start_basic + +- [ ] license.post_start_trial + +- [ ] migration.deprecations + +- [ ] migration.get_feature_upgrade_status + +- [ ] migration.post_feature_upgrade + +- [ ] ml.clear_trained_model_deployment_cache + +- [ ] ml.delete_expired_data + +- [ ] ml.delete_forecast + +- [ ] ml.delete_model_snapshot + +- [ ] ml.explain_data_frame_analytics + +- [ ] ml.forecast + +- [ ] ml.get_buckets + +- [ ] ml.get_categories + +- [ ] ml.get_influencers + +- [ ] ml.get_memory_stats + +- [ ] ml.get_model_snapshot_upgrade_stats + +- [ ] ml.get_model_snapshots + +- [ ] ml.get_records + +- [ ] ml.infer_trained_model + +- [ ] ml.info + +- [ ] ml.post_data + +- [ ] ml.revert_model_snapshot + +- [ ] ml.set_upgrade_mode + +- [ ] ml.update_model_snapshot + +- [ ] ml.update_trained_model_deployment + +- [ ] ml.upgrade_job_snapshot + +- [ ] ml.validate + +- [ ] ml.validate_detector + +- [ ] monitoring.bulk + +- [ ] nodes.clear_repositories_metering_archive + +- [ ] nodes.get_repositories_metering_info + +- [ ] nodes.hot_threads + +- [ ] nodes.info + +- [ ] nodes.reload_secure_settings + +- [ ] nodes.stats + +- [ ] nodes.usage + +- [ ] profiling.flamegraph + +- [ ] profiling.stacktraces + +- [ ] profiling.status + +- [ ] reindex_rethrottle + +- [ ] rollup.delete_job + +- [ ] rollup.get_jobs + +- [ ] rollup.get_rollup_caps + +- [ ] rollup.get_rollup_index_caps + +- [ ] rollup.put_job + +- [ ] rollup.rollup_search + +- [ ] rollup.start_job + +- [ ] rollup.stop_job + +- [ ] search_application.post_behavioral_analytics_event + +- [ ] search_application.render_query + +- [ ] search_application.search + +- [ ] search_shards + +- [ ] searchable_snapshots.cache_stats + +- [ ] searchable_snapshots.clear_cache + +- [ ] searchable_snapshots.mount + +- [ ] searchable_snapshots.stats + +- [ ] security.activate_user_profile + +- [ ] security.bulk_update_api_keys + +- [ ] security.change_password + +- [ ] security.clear_api_key_cache + +- [ ] security.clear_cached_privileges + +- [ ] security.clear_cached_realms + +- [ ] security.clear_cached_roles + +- [ ] security.clear_cached_service_tokens + +- [ ] security.create_cross_cluster_api_key + +- [ ] security.create_service_token + +- [ ] security.delete_privileges + +- [ ] security.delete_role + +- [ ] security.delete_role_mapping + +- [ ] security.delete_service_token + +- [ ] security.delete_user + +- [ ] security.disable_user + +- [ ] security.disable_user_profile + +- [ ] security.enable_user + +- [ ] security.enable_user_profile + +- [ ] security.enroll_kibana + +- [ ] security.enroll_node + +- [ ] security.get_builtin_privileges + +- [ ] security.get_privileges + +- [ ] security.get_role + +- [ ] security.get_role_mapping + +- [ ] security.get_service_accounts + +- [ ] security.get_service_credentials + +- [ ] security.get_settings + +- [ ] security.get_token + +- [ ] security.get_user + +- [ ] security.get_user_privileges + +- [ ] security.get_user_profile + +- [ ] security.grant_api_key + +- [ ] security.has_privileges_user_profile + +- [ ] security.invalidate_token + +- [ ] security.oidc_authenticate + +- [ ] security.oidc_logout + +- [ ] security.oidc_prepare_authentication + +- [ ] security.put_privileges + +- [ ] security.put_role + +- [ ] security.put_role_mapping + +- [ ] security.put_user + +- [ ] security.query_user + +- [ ] security.saml_authenticate + +- [ ] security.saml_complete_logout + +- [ ] security.saml_invalidate + +- [ ] security.saml_logout + +- [ ] security.saml_prepare_authentication + +- [ ] security.saml_service_provider_metadata + +- [ ] security.suggest_user_profiles + +- [ ] security.update_api_key + +- [ ] security.update_cross_cluster_api_key + +- [ ] security.update_settings + +- [ ] security.update_user_profile_data + +- [ ] shutdown.delete_node + +- [ ] shutdown.get_node + +- [ ] shutdown.put_node + +- [ ] simulate.ingest + +- [ ] slm.delete_lifecycle + +- [ ] slm.execute_lifecycle + +- [ ] slm.execute_retention + +- [ ] slm.get_lifecycle + +- [ ] slm.get_stats + +- [ ] slm.get_status + +- [ ] slm.put_lifecycle + +- [ ] slm.start + +- [ ] slm.stop + +- [ ] snapshot.cleanup_repository + +- [ ] snapshot.clone + +- [ ] snapshot.create + +- [ ] snapshot.create_repository + +- [ ] snapshot.delete + +- [ ] snapshot.delete_repository + +- [ ] snapshot.get + +- [ ] snapshot.get_repository + +- [ ] snapshot.repository_analyze + +- [ ] snapshot.restore + +- [ ] snapshot.status + +- [ ] snapshot.verify_repository + +- [ ] ssl.certificates + +- [ ] synonyms.get_synonyms_sets + +- [ ] tasks.cancel + +- [ ] tasks.get + +- [ ] tasks.list + +- [ ] text_structure.find_structure + +- [ ] text_structure.test_grok_pattern + +- [ ] transform.upgrade_transforms + +- [ ] update_by_query_rethrottle + +- [ ] watcher.ack_watch + +- [ ] watcher.activate_watch + +- [ ] watcher.deactivate_watch + +- [ ] watcher.delete_watch + +- [ ] watcher.execute_watch + +- [ ] watcher.get_settings + +- [ ] watcher.get_watch + +- [ ] watcher.put_watch + +- [ ] watcher.query_watches + +- [ ] watcher.start + +- [ ] watcher.stats + +- [ ] watcher.stop + +- [ ] watcher.update_settings + +- [ ] xpack.info + +- [ ] xpack.usage + From 634b0c6f820f39be73dddca1179cf1d981b8b466 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Mon, 12 Feb 2024 16:38:51 +0000 Subject: [PATCH 3/7] Adds GitHub Action to generate report --- .github/workflows/report.yml | 33 +++++++++++++++++++++++++++++++++ report/Gemfile | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 .github/workflows/report.yml diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml new file mode 100644 index 0000000..ff7cf82 --- /dev/null +++ b/.github/workflows/report.yml @@ -0,0 +1,33 @@ +name: report +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + generate_report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.3 + env: + STACK_VERSION: 8.13.0-SNAPSHOT + - name: Build + run: | + cd report && bundle install + - name: Download Artifacts + run: cd report && bundle exec rake download_all + - name: Generate report + run: cd report && bundle exec rake report + - uses: gr2m/create-or-update-pull-request-action@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + commit-message: "Updates report `date '+%Y-%m-%d|%H:%M:%S'`" diff --git a/report/Gemfile b/report/Gemfile index 027a254..0a7555f 100644 --- a/report/Gemfile +++ b/report/Gemfile @@ -1,5 +1,7 @@ source 'https://rubygems.org' +gem 'rake' + group :development, :test do gem 'debug' end From 30816921dff110477f17a69e3a13e2e302f3a5ee Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 13 Feb 2024 09:41:11 +0000 Subject: [PATCH 4/7] Adds link to the test file in tested report --- apis_report.md | 396 ++++++++++++++++++++++---------------------- report/reporter.rb | 9 +- report/template.erb | 2 +- 3 files changed, 204 insertions(+), 203 deletions(-) diff --git a/apis_report.md b/apis_report.md index 4c43d84..3a81965 100644 --- a/apis_report.md +++ b/apis_report.md @@ -10,401 +10,401 @@ ## Tested -- [x] async_search.delete +- [x] [async_search.delete](./tests/async_search/10_basic.yml) -- [x] async_search.get +- [x] [async_search.get](./tests/async_search/10_basic.yml) -- [x] async_search.status +- [x] [async_search.status](./tests/async_search/10_basic.yml) -- [x] async_search.submit +- [x] [async_search.submit](./tests/async_search/10_basic.yml) -- [x] bulk +- [x] [bulk](./tests/bulk/10_basic.yml) -- [x] cat.aliases +- [x] [cat.aliases](./tests/cat/aliases.yml) -- [x] cat.component_templates +- [x] [cat.component_templates](./tests/cat/component_templates.yml) -- [x] cat.count +- [x] [cat.count](./tests/cat/count.yml) -- [x] cat.help +- [x] [cat.help](./tests/cat/help.yml) -- [x] cat.indices +- [x] [cat.indices](./tests/cat/indices.yml) -- [x] cat.ml_data_frame_analytics +- [x] [cat.ml_data_frame_analytics](./tests/cat/ml.yml) -- [x] cat.ml_datafeeds +- [x] [cat.ml_datafeeds](./tests/cat/ml.yml) -- [x] cat.ml_jobs +- [x] [cat.ml_jobs](./tests/cat/ml.yml) -- [x] cat.ml_trained_models +- [x] [cat.ml_trained_models](./tests/cat/ml.yml) -- [x] cat.transforms +- [x] [cat.transforms](./tests/cat/transform.yml) -- [x] clear_scroll +- [x] [clear_scroll](./tests/scroll/10_basic.yml) -- [x] close_point_in_time +- [x] [close_point_in_time](./tests/point_in_time/10_basic.yml) -- [x] cluster.delete_component_template +- [x] [cluster.delete_component_template](./tests/cluster/component_templates.yml) -- [x] cluster.exists_component_template +- [x] [cluster.exists_component_template](./tests/cluster/component_templates.yml) -- [x] cluster.get_component_template +- [x] [cluster.get_component_template](./tests/cluster/component_templates.yml) -- [x] cluster.info +- [x] [cluster.info](./tests/cluster/cluster_info.yml) -- [x] cluster.put_component_template +- [x] [cluster.put_component_template](./tests/cluster/component_templates.yml) -- [x] count +- [x] [count](./tests/async_search/10_basic.yml) -- [x] create +- [x] [create](./tests/cat/aliases.yml) -- [x] delete +- [x] [delete](./tests/async_search/10_basic.yml) -- [x] delete_by_query +- [x] [delete_by_query](./tests/delete_by_query/10_basic.yml) -- [x] delete_script +- [x] [delete_script](./tests/script/10_basic.yml) -- [x] enrich.delete_policy +- [x] [enrich.delete_policy](./tests/enrich/10_basic.yml) -- [x] enrich.execute_policy +- [x] [enrich.execute_policy](./tests/enrich/10_basic.yml) -- [x] enrich.get_policy +- [x] [enrich.get_policy](./tests/enrich/10_basic.yml) -- [x] enrich.put_policy +- [x] [enrich.put_policy](./tests/enrich/10_basic.yml) -- [x] enrich.stats +- [x] [enrich.stats](./tests/enrich/10_basic.yml) -- [x] eql.delete +- [x] [eql.delete](./tests/eql/10_basic.yml) -- [x] eql.get +- [x] [eql.get](./tests/eql/10_basic.yml) -- [x] eql.get_status +- [x] [eql.get_status](./tests/eql/10_basic.yml) -- [x] eql.search +- [x] [eql.search](./tests/eql/10_basic.yml) -- [x] exists +- [x] [exists](./tests/cluster/component_templates.yml) -- [x] exists_source +- [x] [exists_source](./tests/exists_source/10_basic.yml) -- [x] explain +- [x] [explain](./tests/explain/10_basic.yml) -- [x] field_caps +- [x] [field_caps](./tests/field_caps/10_basic.yml) -- [x] get +- [x] [get](./tests/async_search/10_basic.yml) -- [x] get_script +- [x] [get_script](./tests/script/10_basic.yml) -- [x] get_source +- [x] [get_source](./tests/get_source/10_basic.yml) -- [x] graph.explore +- [x] [graph.explore](./tests/graph/explore.yml) -- [x] index +- [x] [index](./tests/async_search/10_basic.yml) -- [x] indices.add_block +- [x] [indices.add_block](./tests/indices/block.yml) -- [x] indices.analyze +- [x] [indices.analyze](./tests/indices/analyze.yml) -- [x] indices.create +- [x] [indices.create](./tests/cat/aliases.yml) -- [x] indices.create_data_stream +- [x] [indices.create_data_stream](./tests/indices/data_streams.yml) -- [x] indices.data_streams_stats +- [x] [indices.data_streams_stats](./tests/indices/data_streams.yml) -- [x] indices.delete +- [x] [indices.delete](./tests/async_search/10_basic.yml) -- [x] indices.delete_alias +- [x] [indices.delete_alias](./tests/indices/alias.yml) -- [x] indices.delete_data_lifecycle +- [x] [indices.delete_data_lifecycle](./tests/indices/data_lifecycle.yml) -- [x] indices.delete_data_stream +- [x] [indices.delete_data_stream](./tests/indices/data_streams.yml) -- [x] indices.delete_index_template +- [x] [indices.delete_index_template](./tests/indices/data_streams.yml) -- [x] indices.exists +- [x] [indices.exists](./tests/indices/alias.yml) -- [x] indices.exists_alias +- [x] [indices.exists_alias](./tests/indices/alias.yml) -- [x] indices.exists_index_template +- [x] [indices.exists_index_template](./tests/indices/index_template.yml) -- [x] indices.explain_data_lifecycle +- [x] [indices.explain_data_lifecycle](./tests/indices/data_lifecycle.yml) -- [x] indices.get +- [x] [indices.get](./tests/indices/alias.yml) -- [x] indices.get_alias +- [x] [indices.get_alias](./tests/indices/alias.yml) -- [x] indices.get_data_lifecycle +- [x] [indices.get_data_lifecycle](./tests/indices/data_lifecycle.yml) -- [x] indices.get_data_stream +- [x] [indices.get_data_stream](./tests/indices/data_streams.yml) -- [x] indices.get_index_template +- [x] [indices.get_index_template](./tests/indices/index_template.yml) -- [x] indices.get_mapping +- [x] [indices.get_mapping](./tests/indices/mapping.yml) -- [x] indices.get_settings +- [x] [indices.get_settings](./tests/indices/settings.yml) -- [x] indices.migrate_to_data_stream +- [x] [indices.migrate_to_data_stream](./tests/indices/migrate_modify_data_stream.yml) -- [x] indices.modify_data_stream +- [x] [indices.modify_data_stream](./tests/indices/migrate_modify_data_stream.yml) -- [x] indices.put_alias +- [x] [indices.put_alias](./tests/cat/aliases.yml) -- [x] indices.put_data_lifecycle +- [x] [indices.put_data_lifecycle](./tests/indices/data_lifecycle.yml) -- [x] indices.put_index_template +- [x] [indices.put_index_template](./tests/indices/data_streams.yml) -- [x] indices.put_mapping +- [x] [indices.put_mapping](./tests/indices/mapping.yml) -- [x] indices.put_settings +- [x] [indices.put_settings](./tests/indices/settings.yml) -- [x] indices.refresh +- [x] [indices.refresh](./tests/graph/explore.yml) -- [x] indices.resolve_index +- [x] [indices.resolve_index](./tests/indices/resolve.yml) -- [x] indices.rollover +- [x] [indices.rollover](./tests/indices/rollover.yml) -- [x] indices.simulate_index_template +- [x] [indices.simulate_index_template](./tests/indices/simulate_template.yml) -- [x] indices.simulate_template +- [x] [indices.simulate_template](./tests/indices/simulate_index_template.yml) -- [x] indices.update_aliases +- [x] [indices.update_aliases](./tests/indices/alias.yml) -- [x] indices.validate_query +- [x] [indices.validate_query](./tests/validate_query/10_basic.yml) -- [x] info +- [x] [info](./tests/cluster/cluster_info.yml) -- [x] ingest.delete_pipeline +- [x] [ingest.delete_pipeline](./tests/ingest/10_basic.yml) -- [x] ingest.get_pipeline +- [x] [ingest.get_pipeline](./tests/ingest/10_basic.yml) -- [x] ingest.processor_grok +- [x] [ingest.processor_grok](./tests/ingest/10_basic.yml) -- [x] ingest.put_pipeline +- [x] [ingest.put_pipeline](./tests/ingest/10_basic.yml) -- [x] ingest.simulate +- [x] [ingest.simulate](./tests/ingest/10_basic.yml) -- [x] license.get +- [x] [license.get](./tests/license.yml) -- [x] logstash.delete_pipeline +- [x] [logstash.delete_pipeline](./tests/logstash/10_basic.yml) -- [x] logstash.get_pipeline +- [x] [logstash.get_pipeline](./tests/logstash/10_basic.yml) -- [x] logstash.put_pipeline +- [x] [logstash.put_pipeline](./tests/logstash/10_basic.yml) -- [x] mget +- [x] [mget](./tests/mget.yml) -- [x] ml.close_job +- [x] [ml.close_job](./tests/machine_learning/jobs_crud.yml) -- [x] ml.delete_calendar +- [x] [ml.delete_calendar](./tests/machine_learning/calendar_crud.yml) -- [x] ml.delete_calendar_event +- [x] [ml.delete_calendar_event](./tests/machine_learning/calendar_events_crud.yml) -- [x] ml.delete_calendar_job +- [x] [ml.delete_calendar_job](./tests/machine_learning/calendar_job.yml) -- [x] ml.delete_data_frame_analytics +- [x] [ml.delete_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.delete_datafeed +- [x] [ml.delete_datafeed](./tests/machine_learning/datafeed_crud.yml) -- [x] ml.delete_filter +- [x] [ml.delete_filter](./tests/machine_learning/filter_crud.yml) -- [x] ml.delete_job +- [x] [ml.delete_job](./tests/machine_learning/calendar_crud.yml) -- [x] ml.delete_trained_model +- [x] [ml.delete_trained_model](./tests/machine_learning/trained_model.yml) -- [x] ml.delete_trained_model_alias +- [x] [ml.delete_trained_model_alias](./tests/machine_learning/trained_model_aliases.yml) -- [x] ml.estimate_model_memory +- [x] [ml.estimate_model_memory](./tests/machine_learning/estimate_model_memory.yml) -- [x] ml.evaluate_data_frame +- [x] [ml.evaluate_data_frame](./tests/machine_learning/data_frame_evaluate.yml) -- [x] ml.flush_job +- [x] [ml.flush_job](./tests/machine_learning/jobs_crud.yml) -- [x] ml.get_calendar_events +- [x] [ml.get_calendar_events](./tests/machine_learning/calendar_events_crud.yml) -- [x] ml.get_calendars +- [x] [ml.get_calendars](./tests/machine_learning/calendar_crud.yml) -- [x] ml.get_data_frame_analytics +- [x] [ml.get_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.get_data_frame_analytics_stats +- [x] [ml.get_data_frame_analytics_stats](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.get_datafeed_stats +- [x] [ml.get_datafeed_stats](./tests/machine_learning/datafeed_crud.yml) -- [x] ml.get_datafeeds +- [x] [ml.get_datafeeds](./tests/machine_learning/datafeed_crud.yml) -- [x] ml.get_filters +- [x] [ml.get_filters](./tests/machine_learning/filter_crud.yml) -- [x] ml.get_job_stats +- [x] [ml.get_job_stats](./tests/machine_learning/jobs_crud.yml) -- [x] ml.get_jobs +- [x] [ml.get_jobs](./tests/machine_learning/jobs_crud.yml) -- [x] ml.get_overall_buckets +- [x] [ml.get_overall_buckets](./tests/machine_learning/get_overall_buckets.yml) -- [x] ml.get_trained_models +- [x] [ml.get_trained_models](./tests/machine_learning/trained_model.yml) -- [x] ml.get_trained_models_stats +- [x] [ml.get_trained_models_stats](./tests/machine_learning/trained_model_more.yml) -- [x] ml.open_job +- [x] [ml.open_job](./tests/machine_learning/jobs_crud.yml) -- [x] ml.post_calendar_events +- [x] [ml.post_calendar_events](./tests/machine_learning/calendar_events_crud.yml) -- [x] ml.preview_data_frame_analytics +- [x] [ml.preview_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.preview_datafeed +- [x] [ml.preview_datafeed](./tests/machine_learning/preview_datafeed.yml) -- [x] ml.put_calendar +- [x] [ml.put_calendar](./tests/machine_learning/calendar_crud.yml) -- [x] ml.put_calendar_job +- [x] [ml.put_calendar_job](./tests/machine_learning/calendar_job.yml) -- [x] ml.put_data_frame_analytics +- [x] [ml.put_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.put_datafeed +- [x] [ml.put_datafeed](./tests/machine_learning/datafeed_crud.yml) -- [x] ml.put_filter +- [x] [ml.put_filter](./tests/machine_learning/filter_crud.yml) -- [x] ml.put_job +- [x] [ml.put_job](./tests/machine_learning/calendar_crud.yml) -- [x] ml.put_trained_model +- [x] [ml.put_trained_model](./tests/machine_learning/trained_model.yml) -- [x] ml.put_trained_model_alias +- [x] [ml.put_trained_model_alias](./tests/machine_learning/trained_model_aliases.yml) -- [x] ml.put_trained_model_definition_part +- [x] [ml.put_trained_model_definition_part](./tests/machine_learning/trained_model_more.yml) -- [x] ml.put_trained_model_vocabulary +- [x] [ml.put_trained_model_vocabulary](./tests/machine_learning/trained_model_more.yml) -- [x] ml.reset_job +- [x] [ml.reset_job](./tests/machine_learning/jobs_reset.yml) -- [x] ml.start_data_frame_analytics +- [x] [ml.start_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.start_datafeed +- [x] [ml.start_datafeed](./tests/machine_learning/start_stop_datafeed.yml) -- [x] ml.start_trained_model_deployment +- [x] [ml.start_trained_model_deployment](./tests/machine_learning/trained_model_more.yml) -- [x] ml.stop_data_frame_analytics +- [x] [ml.stop_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.stop_datafeed +- [x] [ml.stop_datafeed](./tests/machine_learning/start_stop_datafeed.yml) -- [x] ml.stop_trained_model_deployment +- [x] [ml.stop_trained_model_deployment](./tests/machine_learning/trained_model_more.yml) -- [x] ml.update_data_frame_analytics +- [x] [ml.update_data_frame_analytics](./tests/machine_learning/data_frame_analytics.yml) -- [x] ml.update_datafeed +- [x] [ml.update_datafeed](./tests/machine_learning/datafeed_crud.yml) -- [x] ml.update_filter +- [x] [ml.update_filter](./tests/machine_learning/filter_crud.yml) -- [x] ml.update_job +- [x] [ml.update_job](./tests/machine_learning/jobs_crud.yml) -- [x] msearch +- [x] [msearch](./tests/msearch.yml) -- [x] msearch_template +- [x] [msearch_template](./tests/msearch_template.yml) -- [x] mtermvectors +- [x] [mtermvectors](./tests/mtermvectors/10_basic.yml) -- [x] open_point_in_time +- [x] [open_point_in_time](./tests/point_in_time/10_basic.yml) -- [x] ping +- [x] [ping](./tests/cluster/component_templates.yml) -- [x] put_script +- [x] [put_script](./tests/msearch_template.yml) -- [x] query_ruleset.delete +- [x] [query_ruleset.delete](./tests/query_ruleset/10_basic.yml) -- [x] query_ruleset.get +- [x] [query_ruleset.get](./tests/query_ruleset/10_basic.yml) -- [x] query_ruleset.list +- [x] [query_ruleset.list](./tests/query_ruleset/10_basic.yml) -- [x] query_ruleset.put +- [x] [query_ruleset.put](./tests/query_ruleset/10_basic.yml) -- [x] rank_eval +- [x] [rank_eval](./tests/rank_eval.yml) -- [x] reindex +- [x] [reindex](./tests/reindex/10_basic.yml) -- [x] render_search_template +- [x] [render_search_template](./tests/search_template/10_basic.yml) -- [x] scripts_painless_execute +- [x] [scripts_painless_execute](./tests/script/10_basic.yml) -- [x] scroll +- [x] [scroll](./tests/machine_learning/datafeed_crud.yml) -- [x] search +- [x] [search](./tests/async_search/10_basic.yml) -- [x] search_application.delete +- [x] [search_application.delete](./tests/search_application/10_basic.yml) -- [x] search_application.delete_behavioral_analytics +- [x] [search_application.delete_behavioral_analytics](./tests/search_application/20_behavioral_analytics.yml) -- [x] search_application.get +- [x] [search_application.get](./tests/search_application/10_basic.yml) -- [x] search_application.get_behavioral_analytics +- [x] [search_application.get_behavioral_analytics](./tests/search_application/20_behavioral_analytics.yml) -- [x] search_application.list +- [x] [search_application.list](./tests/search_application/10_basic.yml) -- [x] search_application.put +- [x] [search_application.put](./tests/search_application/10_basic.yml) -- [x] search_application.put_behavioral_analytics +- [x] [search_application.put_behavioral_analytics](./tests/search_application/20_behavioral_analytics.yml) -- [x] search_mvt +- [x] [search_mvt](./tests/search_mvt/10_basic.yml) -- [x] search_template +- [x] [search_template](./tests/msearch_template.yml) -- [x] security.authenticate +- [x] [security.authenticate](./tests/security/20_authenticate.yml) -- [x] security.create_api_key +- [x] [security.create_api_key](./tests/security/10_api_key_basic.yml) -- [x] security.get_api_key +- [x] [security.get_api_key](./tests/security/10_api_key_basic.yml) -- [x] security.has_privileges +- [x] [security.has_privileges](./tests/security/30_has_privileges.yml) -- [x] security.invalidate_api_key +- [x] [security.invalidate_api_key](./tests/security/10_api_key_basic.yml) -- [x] security.query_api_keys +- [x] [security.query_api_keys](./tests/security/10_api_key_basic.yml) -- [x] sql.clear_cursor +- [x] [sql.clear_cursor](./tests/sql/10_basic.yml) -- [x] sql.delete_async +- [x] [sql.delete_async](./tests/sql/10_basic.yml) -- [x] sql.get_async +- [x] [sql.get_async](./tests/sql/10_basic.yml) -- [x] sql.get_async_status +- [x] [sql.get_async_status](./tests/sql/10_basic.yml) -- [x] sql.query +- [x] [sql.query](./tests/sql/10_basic.yml) -- [x] sql.translate +- [x] [sql.translate](./tests/sql/10_basic.yml) -- [x] synonyms.delete_synonym +- [x] [synonyms.delete_synonym](./tests/synonyms/10_basic.yml) -- [x] synonyms.delete_synonym_rule +- [x] [synonyms.delete_synonym_rule](./tests/synonyms/10_basic.yml) -- [x] synonyms.get_synonym +- [x] [synonyms.get_synonym](./tests/synonyms/10_basic.yml) -- [x] synonyms.get_synonym_rule +- [x] [synonyms.get_synonym_rule](./tests/synonyms/10_basic.yml) -- [x] synonyms.put_synonym +- [x] [synonyms.put_synonym](./tests/synonyms/10_basic.yml) -- [x] synonyms.put_synonym_rule +- [x] [synonyms.put_synonym_rule](./tests/synonyms/10_basic.yml) -- [x] terms_enum +- [x] [terms_enum](./tests/terms_enum/10_basic.yml) -- [x] termvectors +- [x] [termvectors](./tests/mtermvectors/10_basic.yml) -- [x] transform.delete_transform +- [x] [transform.delete_transform](./tests/cat/transform.yml) -- [x] transform.get_transform +- [x] [transform.get_transform](./tests/transform/10_basic.yml) -- [x] transform.get_transform_stats +- [x] [transform.get_transform_stats](./tests/transform/10_basic.yml) -- [x] transform.preview_transform +- [x] [transform.preview_transform](./tests/transform/10_basic.yml) -- [x] transform.put_transform +- [x] [transform.put_transform](./tests/cat/transform.yml) -- [x] transform.reset_transform +- [x] [transform.reset_transform](./tests/transform/10_basic.yml) -- [x] transform.schedule_now_transform +- [x] [transform.schedule_now_transform](./tests/transform/10_basic.yml) -- [x] transform.start_transform +- [x] [transform.start_transform](./tests/transform/10_basic.yml) -- [x] transform.stop_transform +- [x] [transform.stop_transform](./tests/transform/10_basic.yml) -- [x] transform.update_transform +- [x] [transform.update_transform](./tests/transform/10_basic.yml) -- [x] update +- [x] [update](./tests/indices/alias.yml) -- [x] update_by_query +- [x] [update_by_query](./tests/update_by_query/10_basic.yml) ## Untested diff --git a/report/reporter.rb b/report/reporter.rb index 0ce084b..be10a41 100644 --- a/report/reporter.rb +++ b/report/reporter.rb @@ -20,17 +20,18 @@ def gather_apis def report! @apis.each do |api| - if find_in_tests(api) - @tested << api + if (test = find_test(api)) + @tested << test else @untested << api end end end - def find_in_tests(endpoint) + def find_test(endpoint) Dir[TESTS_PATH].map do |path| - return true if File.readlines(path).grep(/#{endpoint}/).any? + relative_path = path[path.index('/tests')..-1] + return { endpoint: endpoint, file: ".#{relative_path}" } if File.readlines(path).grep(/#{endpoint}/).any? end false end diff --git a/report/template.erb b/report/template.erb index b4fa524..23a9697 100644 --- a/report/template.erb +++ b/report/template.erb @@ -7,7 +7,7 @@ Endpoints: <%= @reporter.apis.count %> ## Tested <% @reporter.tested.each do |test| %> -- [x] <%= test %> +- [x] [<%= test[:endpoint] %>](<%= test[:file] %>) <% end %> ## Untested From db97fa756440a257024feb288beeb41b14692982 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Mon, 19 Feb 2024 17:06:07 +0000 Subject: [PATCH 5/7] Adds elasticsearch-specification to downloads and report --- report/Rakefile | 17 ++++++++-- report/download_artifacts.rb | 8 ++++- report/reporter.rb | 61 +++++++++++++++++++++++++++++++----- report/template.erb | 5 ++- 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/report/Rakefile b/report/Rakefile index 732cb11..b649fdc 100644 --- a/report/Rakefile +++ b/report/Rakefile @@ -9,7 +9,18 @@ task :report do File.write('../apis_report.md', template.result(binding)) end -desc "Download artifacts" -task :download do - Elastic::download_artifacts(ENV['STACK_VERSION'] || '8.13.0-SNAPSHOT') +desc "Download Elasticsearch Stack artifacts" +task :download_json do + Elastic::download_json_spec(ENV['STACK_VERSION'] || '8.13.0-SNAPSHOT') +end + +desc "Download Elasticsearch Serverless artifacts" +task :download_spec do + Elastic::download_es_specification +end + +desc "Download all artifacts" +task :download_all do + Rake::Task['download_json'].invoke + Rake::Task['download_spec'].invoke end diff --git a/report/download_artifacts.rb b/report/download_artifacts.rb index c363792..6d3ab8f 100644 --- a/report/download_artifacts.rb +++ b/report/download_artifacts.rb @@ -5,7 +5,7 @@ module Elastic CURRENT_PATH = Pathname(File.expand_path(__dir__)) class << self - def download_artifacts(version) + def download_json_spec(version) json_filename = CURRENT_PATH.join('tmp/artifacts.json') # Create ./tmp if it doesn't exist @@ -45,6 +45,12 @@ def download_artifacts(version) File.write(CURRENT_PATH.join('tmp/rest-api-spec/build_hash'), @build_hash) end + def download_es_specification(branch = 'main') + filename = CURRENT_PATH.join('tmp/schema.json') + url = "https://github.com/elastic/elasticsearch-specification/raw/#{branch}/output/schema/schema.json" + download_file!(url, filename) + end + def download_file!(url, filename) puts "Downloading #{filename} from #{url}" File.open(filename, "w") do |downloaded_file| diff --git a/report/reporter.rb b/report/reporter.rb index be10a41..42892b3 100644 --- a/report/reporter.rb +++ b/report/reporter.rb @@ -1,25 +1,50 @@ module Elastic class Reporter - JSON_FILES = "#{File.expand_path('./tmp/rest-api-spec/api')}/*.json".freeze + STACK_FILES = "#{File.expand_path('./tmp/rest-api-spec/api')}/*.json".freeze TESTS_PATH = File.expand_path('../tests/**/*.yml') attr_reader :apis, :tested, :untested def initialize - @apis = gather_apis + @apis = {} + @apis[:specification] = specification_apis + @apis[:json] = json_apis @tested = [] @untested = [] report! end - def gather_apis - endpoints = Dir[JSON_FILES].map do |path| - path.split('/').last.gsub('.json','') - end.reject { |a| a.split('/').last.gsub('.json','').start_with?('_') } + # Stack APIs are obtained from the Elasticsearch Rest JSON specification. + # Use `rake download_stack` to download the spec files to ../tmp. + # + def json_apis + apis = Dir[STACK_FILES].map { |path| path.split('/').last.gsub('.json','') } + reject_internal(apis) + end + + # Serverless APIs are obtained from elastic/elasticsearch-specification. + # Use `rake download_serverless` to download the files to ../tmp. + def specification_apis + apis = JSON.parse(File.read('./tmp/schema.json'))['endpoints'].map do |s| + { 'name' => s['name'], 'availability' => s['availability'] } + end + reject_internal(apis) + end + + def serverless_apis + # The absence of an 'availability' field on a property implies that the property is + # available in all flavors. + @apis[:specification].select do |api| + api.dig('availability').nil? || + ( + !!api.dig('availability', 'serverless') && + api.dig('availability', 'serverless', 'visibility') == 'public' + ) + end end def report! - @apis.each do |api| + @apis[:json].each do |api| if (test = find_test(api)) @tested << test else @@ -28,12 +53,32 @@ def report! end end + def coverage + percentage = @tested.count * 100 / @apis[:json].count + "![](https://geps.dev/progress/#{percentage})" + end + + private + def find_test(endpoint) Dir[TESTS_PATH].map do |path| relative_path = path[path.index('/tests')..-1] - return { endpoint: endpoint, file: ".#{relative_path}" } if File.readlines(path).grep(/#{endpoint}/).any? + + if File.readlines(path).grep(/#{endpoint}/).any? + return { endpoint: endpoint, file: ".#{relative_path}" } + end end false end + + def reject_internal(apis) + apis.reject! do |api| + if api.is_a?(Hash) + api.dig('name').start_with?('_') + elsif api.is_a?(String) + api.start_with?('_') + end + end + end end end diff --git a/report/template.erb b/report/template.erb index 23a9697..e2d4cf0 100644 --- a/report/template.erb +++ b/report/template.erb @@ -1,8 +1,11 @@ # Tests report -Endpoints: <%= @reporter.apis.count %> +Endpoints in JSON spec: <%= @reporter.apis[:json].count %> +Endpoints in elasticsearch-specification: <%= @reporter.apis[:specification].count %> +Endpoints in Serverless: <%= @reporter.serverless_apis.count %> [Tested](#tested): <%= @reporter.tested.count %> [Untested](#untested): <%= @reporter.untested.count %> +<%= @reporter.coverage %> ## Tested From 4bdbc588b463bb9469305ad4fd11ddc55d274c1c Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Mon, 19 Feb 2024 17:07:54 +0000 Subject: [PATCH 6/7] Reformats template --- report/template.erb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/report/template.erb b/report/template.erb index e2d4cf0..bcaea26 100644 --- a/report/template.erb +++ b/report/template.erb @@ -1,11 +1,11 @@ # Tests report -Endpoints in JSON spec: <%= @reporter.apis[:json].count %> -Endpoints in elasticsearch-specification: <%= @reporter.apis[:specification].count %> -Endpoints in Serverless: <%= @reporter.serverless_apis.count %> -[Tested](#tested): <%= @reporter.tested.count %> -[Untested](#untested): <%= @reporter.untested.count %> -<%= @reporter.coverage %> +* Endpoints in JSON spec: <%= @reporter.apis[:json].count %> +* Endpoints in elasticsearch-specification: <%= @reporter.apis[:specification].count %> +* Endpoints in Serverless: <%= @reporter.serverless_apis.count %> +* [Tested](#tested): <%= @reporter.tested.count %> +* [Untested](#untested): <%= @reporter.untested.count %> +* Coverage: <%= @reporter.coverage %> ## Tested From ba83b228ff0ff2ddf07a928f3a3be1a68946fe7d Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Mon, 12 Feb 2024 16:14:43 +0000 Subject: [PATCH 7/7] Adds README for report generator --- report/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 report/README.md diff --git a/report/README.md b/report/README.md new file mode 100644 index 0000000..d0ef5f1 --- /dev/null +++ b/report/README.md @@ -0,0 +1,15 @@ +# Test report + +This is a Ruby script that generates the [`../apis_report.md`](../apis_report.md) file. + +## Usage: + +### Download artifacts + +Download the necessary files with `rake download_all`. This will download and unzip the Elasticsearch JSON API spec and the [elasticsearch-specification](https://github.com/elastic/elasticsearch-specification/) spec. The files will be downloaded to `./tmp/rest-api-spec` and `./tmp/schema.json` respectively. The `tmp` directory is added to `.gitignore`. + +### Generate report + +Run `rake report` to check the available endpoints and which ones have been tested. At the time of writing this, the script checks the `../tests` folder for ocurrences of each endpoint name. The script will generate a Markdown file in `../apis_report.md`. + +There are three main files: `reporter.rb`, `Rakefile` and `template.erb`. The first one has the code for the `Elastic::Reporter` class. When it is initialized, it gathers all the API names from the spec (excluding names starting with `_` like `_common` and `_internal`). Then it checks for each API endpoint name, if it's being used in any of the yml files in `./tests`. The reporter is initialized in the `report` task in the Rakefile and the result is passed to the ERB template to generate the markdown file and write it to `./apis_report.md`.