Skip to content

Commit

Permalink
reverting to sequential node norm calls
Browse files Browse the repository at this point in the history
  • Loading branch information
EvanDietzMorris committed Jul 25, 2024
1 parent 1613f62 commit 350f7b6
Showing 1 changed file with 17 additions and 8 deletions.
25 changes: 17 additions & 8 deletions Common/normalization.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,18 +165,27 @@ def normalize_node_data(self, node_list: list, batch_size: int = 1000) -> list:
else:
break

# we should be able to do the following, but it's causing RemoteDisconnected errors with node norm
#
# hit the node norm api with the chunks of curies in parallel
# we could try to optimize the number of max_workers for ThreadPoolExecutor more specifically,
# by default python attempts to find a reasonable # based on os.cpu_count()
# with ThreadPoolExecutor() as executor:
# executor_results = executor.map(self.hit_node_norm_service, chunks_of_ids)
#
# normalization_results = list(executor_results)
# for normalization_json, ids in zip(normalization_results, chunks_of_ids):
# if not normalization_json:
# raise NormalizationFailedError(f'!!! Normalization json results missing for ids: {ids}')
# else:
# merge the normalization results into one dictionary
# node_normalization_results.update(**normalization_json)

# until we can get threading working, hit node norm sequentially
node_normalization_results: dict = {}
with ThreadPoolExecutor() as executor:
normalization_results = list(executor.map(self.hit_node_norm_service, chunks_of_ids))
for normalization_json, ids in zip(normalization_results, chunks_of_ids):
if not normalization_json:
self.logger.error(f'!!! Normalization json results missing for ids: {ids}')
else:
# merge the normalization results into one dictionary
node_normalization_results.update(**normalization_json)
for chunk in chunks_of_ids:
results = self.hit_node_norm_service(chunk)
node_normalization_results.update(**results)

# reset the node index
node_idx = 0
Expand Down

0 comments on commit 350f7b6

Please sign in to comment.