Fix/79 gridded hazard dups

kororaa_graphql_api/schema/toshi_hazard/gridded_hazard.py

@@ -16,7 +16,7 @@
 
 from kororaa_graphql_api.cloudwatch import ServerlessMetricWriter
 
-from .gridded_hazard_helpers import CustomPolygon, clip_tiles, nz_simplified_polgons
+from .gridded_hazard_helpers import CustomPolygon, clip_tiles, nz_simplified_polygons
 from .hazard_schema import GriddedLocation
 
 log = logging.getLogger(__name__)
@@ -88,7 +88,7 @@ def get_colour_values(
 
 
 @lru_cache
-def get_tile_polygons(grid_id: str, poes: Tuple[Any]) -> Tuple[CustomPolygon, ...]:
+def get_tile_polygons(grid_id: str) -> Tuple[CustomPolygon, ...]:
     # build the hazard_map
     t0 = dt.utcnow()
     region_grid = RegionGrid[grid_id]
@@ -142,24 +142,25 @@ def cacheable_hazard_map(
         % (vs30, imt, poe, agg, hazard_model, grid_id)
     )
 
-    nz_parts = nz_simplified_polgons()  # cached
-    log.debug('nz_simplified_polgons cache_info: %s' % str(nz_simplified_polgons.cache_info()))
+    nz_parts = nz_simplified_polygons()  # cached
+    log.debug('nz_simplified_polygons cache_info: %s' % str(nz_simplified_polygons.cache_info()))
 
-    polygons = get_tile_polygons(grid_id, values)
-    # print('get_tile_polygon cache_info')
+    polygons = get_tile_polygons(grid_id)
     log.debug('get_tile_polygon cache_info: %s' % str(get_tile_polygons.cache_info()))
 
     new_geometry = clip_tiles(nz_parts, polygons)
     log.debug('clip_tiles cache_info: %s' % str(clip_tiles.cache_info()))
 
+    log.debug('len(values) %s; len(polygons) %s; len(new_geometry) %s' % (len(values), len(polygons), len(new_geometry)))
+
     t1 = dt.utcnow()
-    log.debug('time to build geom took %s' % (t1 - t0))
+    log.debug('time to build geometry of %s polygons took %s' % (len(new_geometry), (t1 - t0)))
 
     values = values_for_clipped_tiles(new_geometry, polygons, values)
     assert len(new_geometry) == len(values)
     t2 = dt.utcnow()
     log.debug('values_for_clipped_tiles cache_info: %s' % str(values_for_clipped_tiles.cache_info()))
-    log.debug('time to build values %s' % (t2 - t1))
+    log.debug('time to build %s values %s' % (len(values), (t2 - t1)))
 
     color_scale_vmax = (
         color_scale_vmax

kororaa_graphql_api/schema/toshi_hazard/gridded_hazard_helpers.py

@@ -105,7 +105,7 @@ def load_polygon_file(file_name: str) -> gpd.GeoDataFrame:
 
 
 @lru_cache
-def nz_simplified_polgons() -> Tuple[CustomPolygon, ...]:
+def nz_simplified_polygons() -> Tuple[CustomPolygon, ...]:
 
     small_nz = Path(__file__).parent.parent.parent / 'resources' / 'small-nz.wkt.csv.zip'
     nzdf = load_polygon_file(str(small_nz))
@@ -122,17 +122,17 @@ def nz_simplified_polgons() -> Tuple[CustomPolygon, ...]:
 @lru_cache
 def clip_tiles(clipping_parts: Tuple[CustomPolygon], tiles: Tuple[CustomPolygon]):
     t0 = dt.utcnow()
-    covered_tiles: List[CustomPolygon] = list(inner_tiles(clipping_parts, tiles))
+    covered_tiles = set(inner_tiles(clipping_parts, tiles))
     db_metrics.put_duration(__name__, 'filter_inner_tiles', dt.utcnow() - t0)
 
-    outer_tiles: List[CustomPolygon] = list(set(tiles).difference(set(covered_tiles)))
+    outer_tiles = set(tiles).difference(covered_tiles)
 
     t0 = dt.utcnow()
-    clipped_tiles: List[CustomPolygon] = list(edge_tiles(clipping_parts, outer_tiles))
+    clipped_tiles = set(edge_tiles(clipping_parts, outer_tiles))
     db_metrics.put_duration(__name__, 'clip_outer_tiles', dt.utcnow() - t0)
 
     log.info('filtered %s tiles to %s inner in %s' % (len(tiles), len(covered_tiles), dt.utcnow() - t0))
     log.info('clipped %s edge tiles to %s in %s' % (len(outer_tiles), len(clipped_tiles), dt.utcnow() - t0))
 
-    new_geometry = covered_tiles + clipped_tiles
+    new_geometry = covered_tiles.union(clipped_tiles)
     return tuple(new_geometry)

pyproject.toml

@@ -38,15 +38,18 @@ pyproj = "3.3.0"
 nzshm-model = "^0.2.0"
 toshi-hazard-store = "^0.7.2"
 
+# see https://github.com/orgs/python-poetry/discussions/7937
+urllib3 = "<2" 
+numpy = "<1.25"
+
+
 [tool.poetry.group.dev.dependencies]
 tox = "^4.2.8"
 black  = { version = "^22.3"}
 isort  = { version = "^5.8.0"}
 flake8  = { version = "^3.9.2"}
 flake8-docstrings = { version = "^1.6.0", optional = true }
 mypy = {version = "^0.900"}
-pytest = { version = "^6.2.4"}
-pytest-cov  = { version = "^2.12.0"}
 virtualenv  = { version = "^20.2.2", optional = true}
 pip  = { version = "^20.3.1", optional = true}
 
@@ -70,7 +73,9 @@ jinja2 = {version = "3.0.3"}
 #moto = "^3.1.10"
 types-python-dateutil = "^2.8.16"
 types-PyYAML = "^6.0.7"
-moto = "^4.0.2"
+moto = "^4.1.12"
+pytest = "^7.4.0"
+pytest-cov = "^4.1.0"
 
 
 [tool.poetry.extras]

setup.cfg

@@ -42,13 +42,12 @@ exclude_lines =
 
 [tox:tox]
 isolated_build = true
-envlist = py37, py38, py39, format, lint, build
+envlist = py39, py310, format, lint, build
 
 [gh-actions]
 python =
-    3.9: py39
-    3.8: py38, format, lint, build
-    3.7: py37
+    3.10: py310
+    3.9: py39, format, lint, build
 
 [testenv]
 allowlist_externals = pytest
@@ -60,6 +59,7 @@ setenv =
     PYTHONWARNINGS = ignore
     TESTING=1
 commands =
+    pytest --version
     pytest --cov=kororaa_graphql_api --cov-branch --cov-report=xml --cov-report=term-missing tests
 
 [testenv:format]

tests/conftest.py

@@ -0,0 +1,28 @@
+# import boto3
+import os
+import pytest
+
+# from moto import mock_s3  # , mock_cloudwatch
+
+
+@pytest.fixture(scope="module")
+def aws_credentials():
+    """Mocked AWS Credentials for moto."""
+    os.unsetenv("AWS_PROFILE")
+    os.unsetenv("PROFILE")
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+
+
+# @pytest.fixture(scope="module")
+# def s3(aws_credentials):
+#     with mock_s3():
+#         yield boto3.client("s3", region_name="us-east-1")
+
+# @pytest.fixture(scope="module")
+# def cloudwatch(aws_credentials):
+#     with mock_mock_cloudwatch():
+#         yield boto3.client("cloudwatch", region_name="us-east-1")

tests/test_clipping.py

@@ -4,13 +4,17 @@
 from datetime import datetime as dt
 from nzshm_common.grids import RegionGrid
 from nzshm_common.geometry.geometry import create_square_tile
-from kororaa_graphql_api.schema.toshi_hazard.gridded_hazard_helpers import (
-    CustomPolygon,
-    clip_tiles,
-    nz_simplified_polgons,
-)
+
+from moto import mock_cloudwatch
 from shapely.geometry import Polygon
 
+with mock_cloudwatch():
+    from kororaa_graphql_api.schema.toshi_hazard.gridded_hazard_helpers import (
+        CustomPolygon,
+        clip_tiles,
+        nz_simplified_polygons,
+    )
+
 GRID = 'WLG_0_01_nb_1_1'
 
 
@@ -44,7 +48,7 @@ def test_get_clipped_grid(self):
             geometry.append(tile)
         print('built %s tiles in %s' % (len(geometry), dt.utcnow() - t0))
 
-        nz_parts = nz_simplified_polgons()
+        nz_parts = nz_simplified_polygons()
 
         new_geometry = clip_tiles(nz_parts, tuple(geometry))
         self.assertEqual(len(new_geometry), 763)

tests/test_disaggregations.py

@@ -1,13 +1,15 @@
 import json
 import unittest
-import boto3
 import io
 from graphene.test import Client
-from moto import mock_s3
+import boto3
+
+from moto import mock_s3, mock_cloudwatch
 
-# from kororaa_graphql_api.datastore import *
-from kororaa_graphql_api.schema import schema_root
-from kororaa_graphql_api.config import S3_BUCKET_NAME, DISAGGS_KEY
+
+with mock_cloudwatch():
+    from kororaa_graphql_api.schema import schema_root
+    from kororaa_graphql_api.config import S3_BUCKET_NAME, DISAGGS_KEY
 
 DISAGGS = [
     {
@@ -42,6 +44,15 @@
     },
 ]
 
+# def test_create_bucket(s3):
+#     # s3 is a fixture defined above that yields a boto3 s3 client.
+#     # Feel free to instantiate another boto3 S3 client -- Keep note of the region though.
+#     s3.create_bucket(Bucket="somebucket")
+
+#     result = s3.list_buckets()
+#     assert len(result["Buckets"]) == 1
+#     assert result["Buckets"][0]["Name"] == "somebucket"
+
 
 def setup_disaggs():
     s3 = boto3.resource('s3')

tests/test_gridded_hazard.py

@@ -7,7 +7,10 @@
 from unittest import mock
 
 from graphene.test import Client
-from kororaa_graphql_api.schema import schema_root
+from moto import mock_cloudwatch
+
+with mock_cloudwatch():
+    from kororaa_graphql_api.schema import schema_root
 
 from toshi_hazard_store import model
 from nzshm_common.grids import RegionGrid

tests/test_gridded_hazard_fix_issue_79.py

@@ -0,0 +1,145 @@
+"""Tests for toshi_hazard_rev module."""
+
+import unittest
+import itertools
+import random
+import json
+from unittest import mock
+
+from graphene.test import Client
+from moto import mock_cloudwatch
+import pytest
+
+with mock_cloudwatch():
+    from kororaa_graphql_api.schema import schema_root
+    from kororaa_graphql_api.schema.toshi_hazard.gridded_hazard_helpers import (
+        CustomPolygon,
+        clip_tiles,
+        nz_simplified_polygons,
+    )
+
+from toshi_hazard_store import model
+from nzshm_common.grids import RegionGrid
+
+# log = logging.getLogger()
+# logging.basicConfig(level=logging.DEBUG)
+# logging.getLogger('botocore').setLevel(logging.INFO)
+# logging.getLogger('pynamodb').setLevel(logging.DEBUG)
+# logging.getLogger('toshi_hazard_store').setLevel(logging.DEBUG)
+
+HAZARD_MODEL_ID = 'GRIDDED_THE_THIRD'
+vs30s = [250, 400]
+imts = ['PGA', 'SA(0.5)']
+aggs = ['mean', '0.10']
+
+
+def build_hazard_aggregation_models(*args, **kwargs):
+    print('args', args)
+    grid_id = args[1]['location_grid_id']
+    grid = RegionGrid[grid_id].load()
+
+    grid_poes = [random.randint(0, int(4.7e6)) / int(1e6) for x in range(len(grid))]
+    grid_poes[0] = 0.1
+
+    for (imt, vs30, agg) in itertools.product(imts, vs30s, aggs):
+
+        obj = model.GriddedHazard.new_model(
+            hazard_model_id=args[1]['hazard_model_id'],
+            location_grid_id=grid_id,
+            vs30=vs30,
+            imt=imt,
+            agg=agg,
+            poe=0.02,
+            grid_poes=grid_poes,
+        )
+        # print('OBJ', obj)
+        yield obj
+
+
+def mock_query_response(*args, **kwargs):
+    return list(build_hazard_aggregation_models(args, kwargs))
+
+
+# query.get_one_gridded_hazard
+@mock.patch(
+    'kororaa_graphql_api.schema.toshi_hazard.gridded_hazard.cacheable_gridded_hazard_query',
+    side_effect=mock_query_response,
+)
+class TestGriddedHazard(unittest.TestCase):
+    def setUp(self):
+        self.client = Client(schema_root)
+
+    @pytest.mark.skip('WHAT IZZZ')
+    def test_get_gridded_hazard_uniqueness(self, mocked_qry):
+
+        QUERY = """
+        query {
+            gridded_hazard (
+                grid_id: NZ_0_1_NB_1_1
+                hazard_model_id: "%s"
+                imt: "PGA"
+                agg: "mean"
+                vs30: 400
+                poe: 0.02
+                )
+            {
+                ok
+                gridded_hazard {
+                    hazard_model
+                    values
+                    grid_id
+                    hazard_map {
+                        geojson
+
+                    }
+                }
+            }
+        }
+        """ % (
+            HAZARD_MODEL_ID
+        )  # , json.dumps(locs))
+
+        executed = self.client.execute(QUERY)
+        #print(executed)
+        res = executed['data']['gridded_hazard']
+        self.assertEqual(res['ok'], True)
+        self.assertEqual(mocked_qry.call_count, 1)
+
+        mocked_qry.assert_called_with(
+            hazard_model_id='GRIDDED_THE_THIRD',
+            location_grid_id='NZ_0_1_NB_1_1',
+            vs30=400.0,
+            imt='PGA',
+            agg='mean',
+            poe=0.02,
+        )
+        grid = RegionGrid['NZ_0_1_NB_1_1'].load()
+
+        self.assertEqual(res['gridded_hazard'][0]['grid_id'], 'NZ_0_1_NB_1_1')
+        self.assertEqual(len(res['gridded_hazard'][0]['values']), len(grid))
+
+        df_json = json.loads(res['gridded_hazard'][0]['hazard_map']['geojson'])
+
+        features = df_json.get('features')
+        print(features[0])
+
+        # should be 3402 unique locations
+        nz_parts = nz_simplified_polygons()
+        new_geometry = clip_tiles(nz_parts, (None,0))
+        assert len(new_geometry) == 3402
+        print(new_geometry[0])
+
+        assert 0
+
+        self.assertEqual(len(features), len(new_geometry))
+
+        locs = [tuple(f["properties"]["loc"]) for f in features]
+        assert len(locs) == len(set(locs))
+        print(features[0])
+        self.assertEqual(len(features), len(grid))  # one tile dropped
+
+        self.assertTrue(max(res['gridded_hazard'][0]['values']) < 4.7)
+        self.assertTrue(max(res['gridded_hazard'][0]['values']) > 4.5)
+
+
+