clean-up of code prior to PR. Add is_copilot switch for instantiating s3 client

Author: PaulWheatcroft

.copilot/image_build_run.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-# Exit early if something goes wrong
 set -e
 
 # Add commands below to run inside the container after all the other buildpacks have been applied

.copilot/phases/build.sh

@@ -1,9 +1,3 @@
 #!/usr/bin/env bash
 
-set -ex
-
-#python -m pip install --upgrade pip
-#pip install -r requirements.txt
-#
-#npm install
-#npm run build
+set -e

.copilot/phases/install.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-# Exit early if something goes wrong
 set -e
 
 # Add commands below to run as part of the install phase

.copilot/phases/post_build.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-# Exit early if something goes wrong
 set -e
 
 # Add commands below to run as part of the post_build phase

.copilot/phases/pre_build.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-# Exit early if something goes wrong
 set -e
 
 # Add commands below to run as part of the pre_build phase

common/views.py

@@ -14,6 +14,7 @@
 import kombu.exceptions
 from botocore.exceptions import ClientError
 from botocore.exceptions import EndpointConnectionError
+from dbt_copilot_python.utility import is_copilot
 from django.conf import settings
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.auth.mixins import PermissionRequiredMixin
@@ -302,12 +303,23 @@ def check_celery_broker(self) -> Tuple[str, int]:
 
     def check_s3(self) -> Tuple[str, int]:
         try:
-            client = boto3.client(
-                "s3",
-                endpoint_url=settings.S3_ENDPOINT_URL,
-                region_name=settings.HMRC_PACKAGING_S3_REGION_NAME,
+            if is_copilot():
+                client = boto3.client(
+                    "s3",
+                    endpoint_url=settings.S3_ENDPOINT_URL,
+                    region_name=settings.HMRC_PACKAGING_S3_REGION_NAME,
+                )
+            else:
+                client = boto3.client(
+                    "s3",
+                    aws_access_key_id=settings.HMRC_PACKAGING_S3_ACCESS_KEY_ID,
+                    aws_secret_access_key=settings.HMRC_PACKAGING_S3_SECRET_ACCESS_KEY,
+                    endpoint_url=settings.S3_ENDPOINT_URL,
+                    region_name=settings.HMRC_PACKAGING_S3_REGION_NAME,
+                )
+            client.head_bucket(
+                Bucket=settings.HMRC_PACKAGING_STORAGE_BUCKET_NAME,
             )
-            client.head_bucket(Bucket=settings.HMRC_PACKAGING_STORAGE_BUCKET_NAME)
             return "OK", 200
         except (ClientError, EndpointConnectionError):
             return "S3 health check failed", 503
@@ -429,10 +441,14 @@ def get_context_data(self, **kwargs):
             data["APP_UPDATED_TIME"] = AppInfoView.timestamp_to_datetime_string(
                 os.path.getmtime(__file__),
             )
-            last_transaction = Transaction.objects.order_by("updated_at").last()
+            last_transaction = Transaction.objects.order_by(
+                "updated_at",
+            ).last()
             data["LAST_TRANSACTION_TIME"] = (
                 format(
-                    last_transaction.updated_at.strftime(AppInfoView.DATETIME_FORMAT),
+                    last_transaction.updated_at.strftime(
+                        AppInfoView.DATETIME_FORMAT,
+                    ),
                 )
                 if last_transaction
                 else "No transactions"
@@ -519,7 +535,9 @@ def get_object(self, queryset: Optional[QuerySet] = None) -> Model:
         try:
             obj = queryset.get()
         except queryset.model.DoesNotExist:
-            raise Http404(f"No {self.model.__name__} matching the query {self.kwargs}")
+            raise Http404(
+                f"No {self.model.__name__} matching the query {self.kwargs}",
+            )
 
         return obj
 
@@ -652,10 +670,15 @@ def get_ordering(self):
             self,
             "sort_by_fields",
         ), "SortingMixin requires class attribute sort_by_fields to be set"
-        assert isinstance(self.sort_by_fields, list), "sort_by_fields must be a list"
+        assert isinstance(
+            self.sort_by_fields,
+            list,
+        ), "sort_by_fields must be a list"
 
         if sort_by and sort_by in self.sort_by_fields:
-            if hasattr(self, "custom_sorting") and self.custom_sorting.get(sort_by):
+            if hasattr(self, "custom_sorting") and self.custom_sorting.get(
+                sort_by,
+            ):
                 sort_by = self.custom_sorting.get(sort_by)
 
             if ordered == "desc":
@@ -668,11 +691,19 @@ def get_ordering(self):
 
 
 def handler403(request, *args, **kwargs):
-    return TemplateResponse(request=request, template="common/403.jinja", status=403)
+    return TemplateResponse(
+        request=request,
+        template="common/403.jinja",
+        status=403,
+    )
 
 
 def handler500(request, *args, **kwargs):
-    return TemplateResponse(request=request, template="common/500.jinja", status=500)
+    return TemplateResponse(
+        request=request,
+        template="common/500.jinja",
+        status=500,
+    )
 
 
 class MaintenanceView(TemplateView):

scripts/web-worker-entrypoint.sh

@@ -1,8 +1,11 @@
 #!/bin/sh -e
 
-# Django Migrations
 WORKER_COMMAND="gunicorn wsgi --bind 0.0.0.0:${PORT} --timeout 1000 --worker-class=gevent --worker-connections=1000 --workers 9"
 
+if [ -n "${COPILOT_ENVIRONMENT_NAME}" ]; then
+    WORKER_COMMAND="opentelemetry-instrument ${WORKER_COMMAND}"
+fi
+
 echo "Starting worker using the command: ${WORKER_COMMAND}"
 
 eval ${WORKER_COMMAND}

settings/common.py

@@ -176,9 +176,7 @@
     INSTALLED_APPS.remove("django.contrib.admin")
 
     MIDDLEWARE.remove("django.contrib.sessions.middleware.SessionMiddleware")
-    MIDDLEWARE.remove(
-        "django.contrib.auth.middleware.AuthenticationMiddleware",
-    )
+    MIDDLEWARE.remove("django.contrib.auth.middleware.AuthenticationMiddleware")
     MIDDLEWARE.remove("django.contrib.messages.middleware.MessageMiddleware")
     MIDDLEWARE.remove("common.models.utils.TransactionMiddleware")
     MIDDLEWARE.remove("common.models.utils.ValidateUserWorkBasketMiddleware")
@@ -284,8 +282,7 @@
     ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS)
 # Govuk PaaS
 elif "VCAP_APPLICATION" in os.environ:
-    # Under PaaS, if ALLOW_PAAS_URIS is set
-    # fetch trusted domains from VCAP_APPLICATION env var
+    # Under PaaS, if ALLOW_PAAS_URIS is set fetch trusted domains from VCAP_APPLICATION env var
     paas_hosts = json.loads(os.environ["VCAP_APPLICATION"])["uris"]
     ALLOWED_HOSTS.extend(paas_hosts)
 
@@ -387,9 +384,7 @@
 # Settings about retrying uploads if the bucket or endpoint cannot be contacted.
 # Names correspond to celery settings for retrying tasks:
 #   https://docs.celeryproject.org/en/master/userguide/tasks.html#automatic-retry-for-known-exceptions
-EXPORTER_UPLOAD_MAX_RETRIES = int(
-    os.environ.get("EXPORTER_UPLOAD_MAX_RETRIES", "3"),
-)
+EXPORTER_UPLOAD_MAX_RETRIES = int(os.environ.get("EXPORTER_UPLOAD_MAX_RETRIES", "3"))
 EXPORTER_UPLOAD_RETRY_BACKOFF_MAX = int(
     os.environ.get("EXPORTER_UPLOAD_RETRY_BACKOFF_MAX", "600"),
 )
@@ -432,10 +427,7 @@
 
 # HMRC AWS settings (override the defaults) - DEPRECATED.
 HMRC_STORAGE_BUCKET_NAME = os.environ.get("HMRC_STORAGE_BUCKET_NAME", "hmrc")
-HMRC_STORAGE_DIRECTORY = os.environ.get(
-    "HMRC_STORAGE_DIRECTORY",
-    "tohmrc/staging/",
-)
+HMRC_STORAGE_DIRECTORY = os.environ.get("HMRC_STORAGE_DIRECTORY", "tohmrc/staging/")
 
 
 # S3 settings for packaging automation.
@@ -545,18 +537,12 @@
         "test_sqlite_key",
     )
 
-SQLITE_STORAGE_BUCKET_NAME = os.environ.get(
-    "SQLITE_STORAGE_BUCKET_NAME",
-    "sqlite",
-)
+SQLITE_STORAGE_BUCKET_NAME = os.environ.get("SQLITE_STORAGE_BUCKET_NAME", "sqlite")
 SQLITE_S3_ENDPOINT_URL = os.environ.get(
     "SQLITE_S3_ENDPOINT_URL",
     "https://test-sqlite-url.local/",
 )
-SQLITE_STORAGE_DIRECTORY = os.environ.get(
-    "SQLITE_STORAGE_DIRECTORY",
-    "sqlite/",
-)
+SQLITE_STORAGE_DIRECTORY = os.environ.get("SQLITE_STORAGE_DIRECTORY", "sqlite/")
 
 # Default AWS settings.
 if is_copilot():
@@ -586,14 +572,8 @@
     "CROWN_DEPENDENCIES_API_URL_PATH",
     "api/v1/taricfiles/",
 )
-CROWN_DEPENDENCIES_GET_API_KEY = os.environ.get(
-    "CROWN_DEPENDENCIES_GET_API_KEY",
-    "",
-)
-CROWN_DEPENDENCIES_POST_API_KEY = os.environ.get(
-    "CROWN_DEPENDENCIES_POST_API_KEY",
-    "",
-)
+CROWN_DEPENDENCIES_GET_API_KEY = os.environ.get("CROWN_DEPENDENCIES_GET_API_KEY", "")
+CROWN_DEPENDENCIES_POST_API_KEY = os.environ.get("CROWN_DEPENDENCIES_POST_API_KEY", "")
 
 
 if is_copilot():
@@ -614,10 +594,7 @@
         CACHES["default"]["LOCATION"],
     )
 
-CELERY_RESULT_BACKEND = os.environ.get(
-    "CELERY_RESULT_BACKEND",
-    CELERY_BROKER_URL,
-)
+CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", CELERY_BROKER_URL)
 CELERY_TRACK_STARTED = True
 CELERY_TASK_TRACK_STARTED = True
 CELERY_RESULT_PERSISTENT = True
@@ -689,19 +666,14 @@
 
 # -- Google Tag Manager
 GOOGLE_ANALYTICS_ID = os.environ.get("GOOGLE_ANALYTICS_ID")
-GOOGLE_ANALYTICS_APP_ID = os.environ.get(
-    "GOOGLE_ANALYTICS_APP_ID",
-    GOOGLE_ANALYTICS_ID,
-)
+GOOGLE_ANALYTICS_APP_ID = os.environ.get("GOOGLE_ANALYTICS_APP_ID", GOOGLE_ANALYTICS_ID)
 
 # -- Logging
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "default": {
-            "format": "%(asctime)s %(name)s %(levelname)s %(message)s",
-        },
+        "default": {"format": "%(asctime)s %(name)s %(levelname)s %(message)s"},
         "asim_formatter": {
             "()": ASIMFormatter,
         },
@@ -825,9 +797,7 @@
 
 # -- REST Framework
 REST_FRAMEWORK = {
-    "DEFAULT_FILTER_BACKENDS": [
-        "django_filters.rest_framework.DjangoFilterBackend",
-    ],
+    "DEFAULT_FILTER_BACKENDS": ["django_filters.rest_framework.DjangoFilterBackend"],
     # Use Django's standard `django.contrib.auth` permissions,
     # or allow read-only access for unauthenticated users.
     "DEFAULT_PERMISSION_CLASSES": [
@@ -854,10 +824,7 @@
     PATH_COMMODITIES_ASSETS,
     "commodities_envelope.xsd",
 )
-PATH_XSD_COMMODITIES_TARIC = Path(
-    PATH_COMMODITIES_ASSETS,
-    "commodities_taric3.xsd",
-)
+PATH_XSD_COMMODITIES_TARIC = Path(PATH_COMMODITIES_ASSETS, "commodities_taric3.xsd")
 
 
 # Default username for envelope data imports
@@ -875,16 +842,12 @@
     "client_id": os.environ.get("HMRC_API_CLIENT_ID"),
     "client_secret": os.environ.get("HMRC_API_CLIENT_SECRET"),
     "token_url": os.environ.get("HMRC_API_TOKEN_URL", "/oauth/token"),
-    "service_reference_number": os.environ.get(
-        "HMRC_API_SERVICE_REFERENCE_NUMBER",
-    ),
+    "service_reference_number": os.environ.get("HMRC_API_SERVICE_REFERENCE_NUMBER"),
     "device_id": str(uuid.uuid4()),
 }
 
 SKIP_VALIDATION = is_truthy(os.getenv("SKIP_VALIDATION", False))
-SKIP_WORKBASKET_VALIDATION = is_truthy(
-    os.getenv("SKIP_WORKBASKET_VALIDATION", False),
-)
+SKIP_WORKBASKET_VALIDATION = is_truthy(os.getenv("SKIP_WORKBASKET_VALIDATION", False))
 USE_IMPORTER_CACHE = is_truthy(os.getenv("USE_IMPORTER_CACHE", True))
 
 CRISPY_ALLOWED_TEMPLATE_PACKS = ["gds"]
@@ -898,10 +861,8 @@
     },
 }
 
-TRANSACTION_SCHEMA = os.getenv(
-    "TRANSACTION_SCHEMA",
-    "workbaskets.models.SEED_FIRST",
-)
+
+TRANSACTION_SCHEMA = os.getenv("TRANSACTION_SCHEMA", "workbaskets.models.SEED_FIRST")
 
 # Default max number of objects that will be accurately counted by LimitedPaginator.
 LIMITED_PAGINATOR_MAX_COUNT = 200
@@ -924,12 +885,8 @@
 READY_FOR_CDS_TEMPLATE_ID = os.environ.get("READY_FOR_CDS_TEMPLATE_ID")
 CDS_ACCEPTED_TEMPLATE_ID = os.environ.get("CDS_ACCEPTED_TEMPLATE_ID")
 CDS_REJECTED_TEMPLATE_ID = os.environ.get("CDS_REJECTED_TEMPLATE_ID")
-API_PUBLISH_SUCCESS_TEMPLATE_ID = os.environ.get(
-    "API_PUBLISH_SUCCESS_TEMPLATE_ID",
-)
-API_PUBLISH_FAILED_TEMPLATE_ID = os.environ.get(
-    "API_PUBLISH_FAILED_TEMPLATE_ID",
-)
+API_PUBLISH_SUCCESS_TEMPLATE_ID = os.environ.get("API_PUBLISH_SUCCESS_TEMPLATE_ID")
+API_PUBLISH_FAILED_TEMPLATE_ID = os.environ.get("API_PUBLISH_FAILED_TEMPLATE_ID")
 GOODS_REPORT_TEMPLATE_ID = os.environ.get("GOODS_REPORT_TEMPLATE_ID")
 
 # Base service URL - required when constructing an absolute TAP URL to a page
@@ -951,12 +908,8 @@
     "django.core.files.uploadhandler.MemoryFileUploadHandler",  # defaults
     "django.core.files.uploadhandler.TemporaryFileUploadHandler",  # defaults
 )  # Order is important
-DATA_MIGRATION_BATCH_SIZE = int(
-    os.environ.get("DATA_MIGRATION_BATCH_SIZE", "10000"),
-)
+DATA_MIGRATION_BATCH_SIZE = int(os.environ.get("DATA_MIGRATION_BATCH_SIZE", "10000"))
 
 
 # Asynchronous / background (bulk) object creation and editing config.
-MEASURES_ASYNC_CREATION = is_truthy(
-    os.environ.get("MEASURES_ASYNC_CREATION", "true"),
-)
+MEASURES_ASYNC_CREATION = is_truthy(os.environ.get("MEASURES_ASYNC_CREATION", "true"))