Skip to content

Commit

Permalink
Re-enable pylint too-many-branches check
Browse files Browse the repository at this point in the history
  • Loading branch information
@dosaboy
dosaboy committed Aug 5, 2024
1 parent f535fa9 commit 8d40394
Show file tree
Hide file tree
Showing 7 changed files with 315 additions and 241 deletions.
58 changes: 31 additions & 27 deletions hotsos/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,25 @@ class and produce two distinct dict's where the first
return (cleaned_kwargs, remainder)


def run_client(arguments, plugins_to_run, data_root_name, logmanager):
""" Run the hotsos client inside a progress spinner. """
with progress_spinner(
not arguments.quiet and not arguments.debug, data_root_name):
client = HotSOSClient(plugins_to_run)
try:
client.run()
except Exception as exc:
log.exception("An exception occurred while running "
"plugins")
print('\nException running plugins:', exc)
print('See temp log file for possible details:',
logmanager.temp_log_path)
logmanager.delete_temp_file = False
raise

return client.summary


# pylint: disable=R0915
def main():
@click.command(name='hotsos')
Expand Down Expand Up @@ -350,6 +369,16 @@ def cli(**kwargs):
# class and what not.
(cli_kwargs, excess_kwargs) = CLIArgs.filter_kwargs(**kwargs)
arguments = CLIArgs(**cli_kwargs)
# Each plugin has a bool cli opt that defaults to False which if set to
# True indicates that plugin should be run. One or more plugin can be
# be set to True.
plugins_to_run = {k for k, v in excess_kwargs.items()
if v is True}
if plugins_to_run:
# always run these
plugins_to_run.add('hotsos')
if 'system' not in plugins_to_run:
plugins_to_run.add('system')

_version = get_version()
if arguments.version:
Expand Down Expand Up @@ -394,33 +423,8 @@ def cli(**kwargs):
sys.stdout.write('\n')
return

with progress_spinner(
not arguments.quiet and not arguments.debug, drm.name):
plugins = []
for k, v in excess_kwargs.items():
if v is True:
plugins.append(k)

if plugins:
# always run these
plugins.append('hotsos')
if 'system' not in plugins:
plugins.append('system')

client = HotSOSClient(plugins)
try:
client.run()
except Exception as exc:
log.exception("An exception occurred while running "
"plugins")
print('\nException running plugins:', exc)
print('See temp log file for possible details:',
logmanager.temp_log_path)
logmanager.delete_temp_file = False
raise

summary = client.summary

summary = run_client(arguments, list(plugins_to_run), drm.name,
logmanager)
if arguments.save:
path = summary.save(
drm.basename,
Expand Down
81 changes: 42 additions & 39 deletions hotsos/core/host_helpers/systemd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,44 @@ def _unit_files_expr(self, svc_name_expr):
self._cached_unit_files_exprs[svc_name_expr] = re.compile(unit_expr)
return self._cached_unit_files_exprs[svc_name_expr]

def _get_service_obj(self, svc_name_expr, line, services_found):
# check each matched unit file for instances
ret = self._unit_files_expr(svc_name_expr).match(line)
if not ret:
return

unit = ret.group(1)
state = ret.group(2)
has_instances = False
units_expr = r"\*?\s+({})\.service\s+(\S+)\s+(\S+)\s+(\S+)"
if unit.endswith('@'):
# indirect or "template" units can have "instantiated"
# units where only the latter represents whether the
# unit is in use. If an indirect unit has instantiated
# units we use them to represent the state of the
# service.
units_expr = units_expr.format(unit + r'\S+')
unit = unit.partition('@')[0]
else:
units_expr = units_expr.format(unit)

if state != 'disabled':
if self._has_unit_instances(units_expr):
has_instances = True
elif unit in services_found:
# don't override enabled with disabled
return

if state == 'indirect' and not has_instances:
return

if unit in services_found:
if (services_found[unit].state == 'indirect' and
services_found[unit].has_instances):
return

services_found[unit] = SystemdService(unit, state, has_instances)

@cached_property
def services(self):
"""
Expand All @@ -245,53 +283,18 @@ def services(self):
instances. Enabled units are aggregated but masked units are not so
that they can be identified and reported.
"""
_services = {}
services_found = {}
for line in CLIHelper().systemctl_list_unit_files():
for svc_name_expr in self._service_exprs:
# check each matched unit file for instances
ret = self._unit_files_expr(svc_name_expr).match(line)
if not ret:
continue

unit = ret.group(1)
state = ret.group(2)
has_instances = False
units_expr = r"\*?\s+({})\.service\s+(\S+)\s+(\S+)\s+(\S+)"
if unit.endswith('@'):
# indirect or "template" units can have "instantiated"
# units where only the latter represents whether the
# unit is in use. If an indirect unit has instantiated
# units we use them to represent the state of the
# service.
units_expr = units_expr.format(unit + r'\S+')
unit = unit.partition('@')[0]
else:
units_expr = units_expr.format(unit)

if state != 'disabled':
if self._has_unit_instances(units_expr):
has_instances = True
elif unit in _services:
# don't override enabled with disabled
continue

if state == 'indirect' and not has_instances:
continue

if unit in _services:
if (_services[unit].state == 'indirect' and
_services[unit].has_instances):
continue

_services[unit] = SystemdService(unit, state, has_instances)
self._get_service_obj(svc_name_expr, line, services_found)

# NOTE: assumes that indirect instances always supersede direct ones
# i.e. you can't have both.
for info in _services.values():
for info in services_found.values():
if info.state == 'indirect':
info.state = 'enabled'

return _services
return services_found

@property
def masked_services(self):
Expand Down
87 changes: 49 additions & 38 deletions hotsos/core/ycheck/events.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,15 +131,20 @@ def _get_event_results(cls, event):
"fields",
_fail_count, event.name)

@staticmethod
def _get_tally(result, info, options: EventProcessingOptions):
@classmethod
def _get_tally_keys(cls, options, result):
"""
By default we tally by 'key' unless options.key_by_date is True in
which case we tally by value i.e. the date.
"""
if options.key_by_date:
key = result['date']
value = result['key']
else:
key = result['key']
value = result['date']
return (result['date'], result['key'])

return (result['key'], result['date'])

@classmethod
def _get_tally(cls, result, info, options: EventProcessingOptions):
key, value = cls._get_tally_keys(options, result)
if key not in info:
info[key] = {}

Expand All @@ -148,21 +153,22 @@ def _get_tally(result, info, options: EventProcessingOptions):
info[key] = 1
else:
info[key] += 1
else:
ts_time = result.get('time')
if value not in info[key]:
if ts_time is not None and options.include_time:
info[key][value] = {}
else:
info[key][value] = 0
return

ts_time = result.get('time')
if value not in info[key]:
if ts_time is not None and options.include_time:
if ts_time not in info[key][value]:
info[key][value][ts_time] = 1
else:
info[key][value][ts_time] += 1
info[key][value] = {}
else:
info[key][value] += 1
info[key][value] = 0

if ts_time is not None and options.include_time:
if ts_time not in info[key][value]:
info[key][value][ts_time] = 1
else:
info[key][value][ts_time] += 1
else:
info[key][value] += 1

@staticmethod
def _sort_results(categorised_results, options: EventProcessingOptions):
Expand Down Expand Up @@ -458,6 +464,28 @@ def _get_event_search_results(global_results, search_tag,

return {}

def _exec_callback(self, event, section_name, search_results,
event_search):
# We want this to throw an exception if the callback is not
# defined.
callback_name = f'{self.event_group}.{event}'
if callback_name not in CALLBACKS:
msg = f"no callback found for event {callback_name}"
raise EventCallbackNotFound(msg)

callback = CALLBACKS[callback_name]
event_result = EventCheckResult(
name=event,
section_name=section_name,
results=search_results,
search_tag=event_search['search_tag'],
searcher=self.global_searcher.searcher,
sequence_def=event_search['sequence_search']
)
log.debug("executing event %s.%s callback '%s'",
event_result.section_name, event, callback_name)
return callback()(event_result)

def run(self):
"""
Process each event and call respective callback functions when results
Expand Down Expand Up @@ -489,25 +517,8 @@ def run(self):
event, event_search['search_tag'])
continue

# We want this to throw an exception if the callback is not
# defined.
callback_name = f'{self.event_group}.{event}'
if callback_name not in CALLBACKS:
msg = f"no callback found for event {callback_name}"
raise EventCallbackNotFound(msg)

callback = CALLBACKS[callback_name]
event_result = EventCheckResult(
name=event,
section_name=section_name,
results=search_results,
search_tag=search_tag,
searcher=self.global_searcher.searcher,
sequence_def=seq_def
)
log.debug("executing event %s.%s callback '%s'",
event_result.section_name, event, callback_name)
ret = callback()(event_result)
ret = self._exec_callback(event, section_name, search_results,
event_search)
if not ret:
continue

Expand Down
50 changes: 27 additions & 23 deletions hotsos/plugin_extensions/juju/summary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,31 @@ class UnitLogInfo():
Create a tally of log errors and warnings for each unit.
"""

@staticmethod
def error_and_warnings():
@classmethod
def _tally_result(cls, app_name, events, result, tally_key):
if app_name not in events:
events[app_name] = {}

tag = result.tag.lower()
if tag not in events[app_name]:
events[app_name][tag] = {}

origin = result.get(3)
origin_child = origin.rpartition('.')[2]
if origin_child not in events[app_name][tag]:
events[app_name][tag][origin_child] = {}

mod = result.get(4)
if mod not in events[app_name][tag][origin_child]:
events[app_name][tag][origin_child][mod] = {}

if tally_key not in events[app_name][tag][origin_child][mod]:
events[app_name][tag][origin_child][mod][tally_key] = 1
else:
events[app_name][tag][origin_child][mod][tally_key] += 1

@classmethod
def error_and_warnings(cls):
log.debug("searching unit logs for errors and warnings")
c = SearchConstraintSearchSince(ts_matcher_cls=CommonTimestampMatcher)
searchobj = FileSearcher(constraint=c)
Expand Down Expand Up @@ -68,27 +91,8 @@ def error_and_warnings():
continue

path = searchobj.resolve_source_id(result.source_id)
name = re.search(r".+/unit-(\S+).log.*", path).group(1)
if name not in events:
events[name] = {}

tag = tag.lower()
if tag not in events[name]:
events[name][tag] = {}

origin = result.get(3)
origin_child = origin.rpartition('.')[2]
if origin_child not in events[name][tag]:
events[name][tag][origin_child] = {}

mod = result.get(4)
if mod not in events[name][tag][origin_child]:
events[name][tag][origin_child][mod] = {}

if key not in events[name][tag][origin_child][mod]:
events[name][tag][origin_child][mod][key] = 1
else:
events[name][tag][origin_child][mod][key] += 1
app_name = re.search(r".+/unit-(\S+).log.*", path).group(1)
cls._tally_result(app_name, events, result, key)

# ensure consistent ordering of results
for tag, units in events.items():
Expand Down
Loading

0 comments on commit 8d40394

Please sign in to comment.