Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Apply 2024.1 backports from 2023.1 #100

Merged
merged 4 commits into from
Jul 2, 2024
Merged

Apply 2024.1 backports from 2023.1 #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a277936
Log Prometheus Query
priteau Nov 5, 2021
cbe6453
Install OpenSearch 2.5.0 using packages
priteau Feb 8, 2023
01a0057
Remove mapping name from bulk API URL
MoteHue Nov 7, 2023
1853c37
Patch the elasticsearch backend too
MoteHue Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cloudkitty/collector/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,8 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None):
if query_suffix:
query = "{0} {1}".format(query, query_suffix)

LOG.debug("Calling Prometheus with query: %s", query)

try:
res = self._conn.get_instant(
query,
Expand Down
16 changes: 7 additions & 9 deletions cloudkitty/storage/v2/elasticsearch/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,15 @@ def _build_composite(self, groupby):
sources = []
for elem in groupby:
if elem == 'type':
sources.append({'type': {'terms': {'field': 'type'}}})
sources.append({'type': {'terms': {'field': 'type.keyword'}}})
elif elem == 'time':
# Not doing a date_histogram aggregation because we don't know
# the period
sources.append({'begin': {'terms': {'field': 'start'}}})
sources.append({'end': {'terms': {'field': 'end'}}})
else:
sources.append({elem: {'terms': {'field': 'groupby.' + elem}}})
field = 'groupby.' + elem + '.keyword'
sources.append({elem: {'terms': {'field': field}}})

return {"sources": sources}

Expand Down Expand Up @@ -158,12 +159,9 @@ def put_mapping(self, mapping):
:rtype: requests.models.Response
"""
url = '/'.join(
(self._url, self._index_name, '_mapping', self._mapping_name))
# NOTE(peschk_l): This is done for compatibility with
# Elasticsearch 6 and 7.
param = {"include_type_name": "true"}
(self._url, self._index_name, self._mapping_name))
return self._req(
self._sess.put, url, json.dumps(mapping), param, deserialize=False)
self._sess.post, url, json.dumps(mapping), {}, deserialize=False)

def get_index(self):
"""Does a GET request against ES's index API.
Expand Down Expand Up @@ -228,7 +226,7 @@ def bulk_with_instruction(self, instruction, terms):
"""Does a POST request against ES's bulk API

The POST request will be done against
`/<index_name>/<mapping_name>/_bulk`
`/<index_name>/_bulk`

The instruction will be appended before each term. For example,
bulk_with_instruction('instr', ['one', 'two']) will produce::
Expand All @@ -249,7 +247,7 @@ def bulk_with_instruction(self, instruction, terms):
*[(instruction, json.dumps(term)) for term in terms]
)) + '\n'
url = '/'.join(
(self._url, self._index_name, self._mapping_name, '_bulk'))
(self._url, self._index_name, '_bulk'))
return self._req(self._sess.post, url, data, None, deserialize=False)

def bulk_index(self, terms):
Expand Down
14 changes: 7 additions & 7 deletions cloudkitty/tests/storage/v2/elasticsearch/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ def test_build_composite(self):
self.assertEqual(
self.client._build_composite(['one', 'type', 'two']),
{'sources': [
{'one': {'terms': {'field': 'groupby.one'}}},
{'type': {'terms': {'field': 'type'}}},
{'two': {'terms': {'field': 'groupby.two'}}},
{'one': {'terms': {'field': 'groupby.one.keyword'}}},
{'type': {'terms': {'field': 'type.keyword'}}},
{'two': {'terms': {'field': 'groupby.two.keyword'}}},
]},
)

Expand Down Expand Up @@ -186,9 +186,9 @@ def test_put_mapping(self):
with mock.patch.object(self.client, '_req') as rmock:
self.client.put_mapping(mapping)
rmock.assert_called_once_with(
self.client._sess.put,
'http://elasticsearch:9200/index_name/_mapping/test_mapping',
'{"a": "b"}', {'include_type_name': 'true'}, deserialize=False)
self.client._sess.post,
'http://elasticsearch:9200/index_name/test_mapping',
'{"a": "b"}', {}, deserialize=False)

def test_get_index(self):
with mock.patch.object(self.client, '_req') as rmock:
Expand Down Expand Up @@ -259,7 +259,7 @@ def test_bulk_with_instruction(self):
self.client.bulk_with_instruction(instruction, terms)
rmock.assert_called_once_with(
self.client._sess.post,
'http://elasticsearch:9200/index_name/test_mapping/_bulk',
'http://elasticsearch:9200/index_name/_bulk',
expected_data, None, deserialize=False)

def test_bulk_index(self):
Expand Down