From 603838335628644778dbbd6c32651b7b42feae87 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 1 Feb 2023 16:11:57 +0000 Subject: [PATCH 1/4] use resolved `template_api` when injecting ILM --- lib/logstash/outputs/elasticsearch/template_manager.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 6cf76199..b39a745d 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -44,7 +44,7 @@ def self.install(client, template_endpoint, template_name, template, template_ov def self.add_ilm_settings_to_template(plugin, template) # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern # definition - remove any existing definition of 'template' - template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8 + template.delete('template') if template_endpoint(plugin) == LEGACY_TEMPLATE_ENDPOINT template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" settings = template_settings(plugin, template) if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias']) @@ -54,7 +54,12 @@ def self.add_ilm_settings_to_template(plugin, template) end def self.template_settings(plugin, template) - plugin.maximum_seen_major_version < 8 ? template['settings']: template['template']['settings'] + if template_endpoint(plugin) == LEGACY_TEMPLATE_ENDPOINT + return template['settings'] ||= {} + end + + template['template'] ||= {} + template['template']['settings'] ||= {} end # Template name - if template_name set, use it From 1bb2161bc255510620296f3f952305660e3e47f6 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 1 Feb 2023 17:23:32 +0000 Subject: [PATCH 2/4] make `template` required when `template_api` is provided --- docs/index.asciidoc | 19 ++++++-------- lib/logstash/outputs/elasticsearch.rb | 2 ++ .../outputs/elasticsearch/template_manager.rb | 25 +++++++++---------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index a0ce80fd..6f9b6d69 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -1077,6 +1077,7 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1. * Value type is <> * There is no default value for this setting. + * When a `template` is provided, you may also configure <> as a hint about the format of the provided template. You can set the path to your own template here, if you so desire. If not set, the included template will be used. @@ -1084,18 +1085,14 @@ If not set, the included template will be used. [id="plugins-{type}s-{plugin}-template_api"] ===== `template_api` - * Value can be any of: `auto`, `legacy`, `composable` - * Default value is `auto` - -The default setting of `auto` will use -{ref}/index-templates.html[index template API] to create index template, if the -Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, -and use {ref}/indices-templates-v1.html[legacy template API] otherwise. - -Setting this flag to `legacy` will use legacy template API to create index template. -Setting this flag to `composable` will use index template API to create index template. + * Value can be any of: + ** `composable`: maps to composable {ref}/index-templates.html[index template API] + ** `legacy`: maps to the {ref}/indices-templates-v1.html[legacy template API] + * The default value is determined based on which version of Elasticsearch we are connected to: + ** for {es} 7.x or older, the default is `legacy` + ** for {es} 8.0+, the default is `composable` -NOTE: The format of template provided to <> needs to match the template API being used. +This is a hint for the format of the provided <>. [id="plugins-{type}s-{plugin}-template_name"] ===== `template_name` diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 4af248ba..72a837de 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -279,6 +279,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base def initialize(*params) super setup_ecs_compatibility_related_defaults + + raise LogStash::ConfigurationError("`template_api` is not allowed unless a `template` is also provided") if original_params.include?('template_api') && !original_params.include?('template') end def register diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index b39a745d..f9d53e29 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -7,15 +7,15 @@ class TemplateManager def self.install_template(plugin) return unless plugin.manage_template - if plugin.maximum_seen_major_version < 8 && plugin.template_api == 'auto' - plugin.logger.warn("`template_api => auto` resolved to `legacy` since we are connected to " + "Elasticsearch #{plugin.maximum_seen_major_version}, " + - "but will resolve to `composable` the first time it connects to Elasticsearch 8+. " + - "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " + - "or migrating your template to the composable style and setting `template_api => composable`. " + - "The legacy template API is slated for removal in Elasticsearch 9.") - end - if plugin.template + if plugin.maximum_seen_major_version < 8 && plugin.template_api == 'auto' + plugin.logger.warn("`template_api => auto` resolved to `legacy` since we are connected to " + "Elasticsearch #{plugin.maximum_seen_major_version}, " + + "but will resolve to `composable` the first time it connects to Elasticsearch 8+. " + + "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " + + "or migrating your template to the composable style and setting `template_api => composable`. " + + "The legacy template API is slated for removal in Elasticsearch 9.") + end + plugin.logger.info("Using mapping template from", :path => plugin.template) template = read_template_file(plugin.template) else @@ -82,12 +82,11 @@ def self.read_template_file(template_path) end def self.template_endpoint(plugin) - if plugin.template_api == 'auto' - plugin.maximum_seen_major_version < 8 ? LEGACY_TEMPLATE_ENDPOINT : INDEX_TEMPLATE_ENDPOINT - elsif plugin.template_api.to_s == 'legacy' - LEGACY_TEMPLATE_ENDPOINT + case plugin.template_api.to_s + when 'composable' then INDEX_TEMPLATE_ENDPOINT + when 'legacy' then LEGACY_TEMPLATE_ENDPOINT else - INDEX_TEMPLATE_ENDPOINT + plugin.maximum_seen_major_version < 8 ? LEGACY_TEMPLATE_ENDPOINT : INDEX_TEMPLATE_ENDPOINT end end From 37ab2afe863e32ab2fd8bc153b123ef5259f7ff6 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 1 Feb 2023 18:46:06 +0000 Subject: [PATCH 3/4] deprecate combining ILM injection with user-provided template --- docs/index.asciidoc | 3 +++ lib/logstash/outputs/elasticsearch/template_manager.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 6f9b6d69..e546a3fb 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -1082,6 +1082,9 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1. You can set the path to your own template here, if you so desire. If not set, the included template will be used. +WARNING: Injecting ILM configuration into a custom `template` is deprecated. + If you are providing your own template, your ILM configuration should be a part of the template you provide. + [id="plugins-{type}s-{plugin}-template_api"] ===== `template_api` diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index f9d53e29..50c2b8b8 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -42,6 +42,9 @@ def self.install(client, template_endpoint, template_name, template, template_ov end def self.add_ilm_settings_to_template(plugin, template) + if plugin.template + plugin.deprecation_logger.deprecated("Injecting Index Lifecycle Management configuration into a provided `template` is deprecated, and support will be removed in a future version. Please add the configuration directly to your template.") + end # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern # definition - remove any existing definition of 'template' template.delete('template') if template_endpoint(plugin) == LEGACY_TEMPLATE_ENDPOINT From 0db40159bb68229f4e3bc7d6d41f23342f3e0511 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 1 Feb 2023 18:50:20 +0000 Subject: [PATCH 4/4] raise meaningful exception when ILM injection fails --- lib/logstash/outputs/elasticsearch/template_manager.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 50c2b8b8..2c64579a 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -54,6 +54,8 @@ def self.add_ilm_settings_to_template(plugin, template) plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled") end settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) + rescue Exception => e + fail("Failed to apply ILM settings to template: #{e.message}") end def self.template_settings(plugin, template)