From a2779368b1203b9c65270773f77c810c0cbe480c Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 5 Nov 2021 11:24:19 +0100 Subject: [PATCH 1/4] Log Prometheus Query (cherry picked from commit 6c61ea713e7b33c431ce1b87fd5d48a10a6df4c4) (cherry picked from commit 884a43a7c72c8372d07e3c63a515dae6a8623fbc) (cherry picked from commit 572ad448dba7ad78339688c3a03ff9215bf76c28) (cherry picked from commit fa02093432f5a3d5d56b495399cd2f906714f7a7) (cherry picked from commit 011085ebcb52d7818bdb02624970e805b4a6a39d) --- cloudkitty/collector/prometheus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudkitty/collector/prometheus.py b/cloudkitty/collector/prometheus.py index fe94d47a..e2586f16 100644 --- a/cloudkitty/collector/prometheus.py +++ b/cloudkitty/collector/prometheus.py @@ -212,6 +212,8 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): if query_suffix: query = "{0} {1}".format(query, query_suffix) + LOG.debug("Calling Prometheus with query: %s", query) + try: res = self._conn.get_instant( query, From cbe645314aeb03587164aca704cf05d5429bb44a Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 8 Feb 2023 11:55:04 +0100 Subject: [PATCH 2/4] Install OpenSearch 2.5.0 using packages Change-Id: I42aa30754e2d26a3953059b27b1f612ba5ae4150 (cherry picked from commit bbe93d4a4e82a22aeb3035d2e26f3838dde77d55) --- cloudkitty/storage/v2/elasticsearch/client.py | 7 ++----- cloudkitty/tests/storage/v2/elasticsearch/test_client.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cloudkitty/storage/v2/elasticsearch/client.py b/cloudkitty/storage/v2/elasticsearch/client.py index 7716f039..98418e3f 100644 --- a/cloudkitty/storage/v2/elasticsearch/client.py +++ b/cloudkitty/storage/v2/elasticsearch/client.py @@ -158,12 +158,9 @@ def put_mapping(self, mapping): :rtype: requests.models.Response """ url = '/'.join( - (self._url, self._index_name, '_mapping', self._mapping_name)) - # NOTE(peschk_l): This is done for compatibility with - # Elasticsearch 6 and 7. - param = {"include_type_name": "true"} + (self._url, self._index_name, self._mapping_name)) return self._req( - self._sess.put, url, json.dumps(mapping), param, deserialize=False) + self._sess.post, url, json.dumps(mapping), {}, deserialize=False) def get_index(self): """Does a GET request against ES's index API. diff --git a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py index 5d3158b4..89853e67 100644 --- a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py +++ b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py @@ -186,9 +186,9 @@ def test_put_mapping(self): with mock.patch.object(self.client, '_req') as rmock: self.client.put_mapping(mapping) rmock.assert_called_once_with( - self.client._sess.put, - 'http://elasticsearch:9200/index_name/_mapping/test_mapping', - '{"a": "b"}', {'include_type_name': 'true'}, deserialize=False) + self.client._sess.post, + 'http://elasticsearch:9200/index_name/test_mapping', + '{"a": "b"}', {}, deserialize=False) def test_get_index(self): with mock.patch.object(self.client, '_req') as rmock: From 01a0057e00d1d0daa9077a4bc7f6a758052f2a47 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 7 Nov 2023 14:20:27 +0000 Subject: [PATCH 3/4] Remove mapping name from bulk API URL (cherry picked from commit e77911550281bcde19976df6501ad126dd12b411) --- cloudkitty/storage/v2/elasticsearch/client.py | 4 ++-- cloudkitty/tests/storage/v2/elasticsearch/test_client.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudkitty/storage/v2/elasticsearch/client.py b/cloudkitty/storage/v2/elasticsearch/client.py index 98418e3f..8f681b70 100644 --- a/cloudkitty/storage/v2/elasticsearch/client.py +++ b/cloudkitty/storage/v2/elasticsearch/client.py @@ -225,7 +225,7 @@ def bulk_with_instruction(self, instruction, terms): """Does a POST request against ES's bulk API The POST request will be done against - `///_bulk` + `//_bulk` The instruction will be appended before each term. For example, bulk_with_instruction('instr', ['one', 'two']) will produce:: @@ -246,7 +246,7 @@ def bulk_with_instruction(self, instruction, terms): *[(instruction, json.dumps(term)) for term in terms] )) + '\n' url = '/'.join( - (self._url, self._index_name, self._mapping_name, '_bulk')) + (self._url, self._index_name, '_bulk')) return self._req(self._sess.post, url, data, None, deserialize=False) def bulk_index(self, terms): diff --git a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py index 89853e67..460fa5e3 100644 --- a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py +++ b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py @@ -259,7 +259,7 @@ def test_bulk_with_instruction(self): self.client.bulk_with_instruction(instruction, terms) rmock.assert_called_once_with( self.client._sess.post, - 'http://elasticsearch:9200/index_name/test_mapping/_bulk', + 'http://elasticsearch:9200/index_name/_bulk', expected_data, None, deserialize=False) def test_bulk_index(self): From 1853c37c910312639ff720e478e27850bd91251a Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 26 Apr 2024 08:58:00 +0000 Subject: [PATCH 4/4] Patch the elasticsearch backend too Required as this patch is still up for review: https://review.opendev.org/c/openstack/kolla-ansible/+/898555 (cherry picked from commit 5888807e409cf94603d41f9b91a9164d887732f1) --- cloudkitty/storage/v2/elasticsearch/client.py | 5 +++-- cloudkitty/tests/storage/v2/elasticsearch/test_client.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudkitty/storage/v2/elasticsearch/client.py b/cloudkitty/storage/v2/elasticsearch/client.py index 8f681b70..aec09402 100644 --- a/cloudkitty/storage/v2/elasticsearch/client.py +++ b/cloudkitty/storage/v2/elasticsearch/client.py @@ -98,14 +98,15 @@ def _build_composite(self, groupby): sources = [] for elem in groupby: if elem == 'type': - sources.append({'type': {'terms': {'field': 'type'}}}) + sources.append({'type': {'terms': {'field': 'type.keyword'}}}) elif elem == 'time': # Not doing a date_histogram aggregation because we don't know # the period sources.append({'begin': {'terms': {'field': 'start'}}}) sources.append({'end': {'terms': {'field': 'end'}}}) else: - sources.append({elem: {'terms': {'field': 'groupby.' + elem}}}) + field = 'groupby.' + elem + '.keyword' + sources.append({elem: {'terms': {'field': field}}}) return {"sources": sources} diff --git a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py index 460fa5e3..3dae994f 100644 --- a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py +++ b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py @@ -86,9 +86,9 @@ def test_build_composite(self): self.assertEqual( self.client._build_composite(['one', 'type', 'two']), {'sources': [ - {'one': {'terms': {'field': 'groupby.one'}}}, - {'type': {'terms': {'field': 'type'}}}, - {'two': {'terms': {'field': 'groupby.two'}}}, + {'one': {'terms': {'field': 'groupby.one.keyword'}}}, + {'type': {'terms': {'field': 'type.keyword'}}}, + {'two': {'terms': {'field': 'groupby.two.keyword'}}}, ]}, )