From 7c5be30a27fdab1af06a072393ed5c5f3d145014 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Wed, 12 Feb 2025 17:09:49 +0000 Subject: [PATCH 01/16] Open data --- open_data/__init__.py | 0 open_data/admin.py | 1 + open_data/apps.py | 7 + open_data/commodities.py | 57 + open_data/geo_areas.py | 29 + open_data/management/commands/__init__.py | 0 .../management/commands/refresh_open_data.py | 21 + open_data/measures.py | 93 ++ open_data/migrations/0001_create_schema.py | 20 + open_data/migrations/0002_initial.py | 1484 +++++++++++++++++ .../migrations/0003_drop_fk_constrains.py | 38 + .../migrations/0004_materialised_view.py | 30 + .../migrations/0005_measureasdefinedreport.py | 112 ++ open_data/migrations/__init__.py | 0 open_data/models/__init__.py | 86 + open_data/models/additional_codes_models.py | 45 + open_data/models/certificates_models.py | 45 + open_data/models/commodities_model.py | 110 ++ open_data/models/footnotes_models.py | 44 + open_data/models/geo_areas_models.py | 53 + open_data/models/measures_models.py | 469 ++++++ open_data/models/quotas_models.py | 174 ++ open_data/models/regulations_models.py | 132 ++ open_data/models/tests/__init__.py | 0 .../models/tests/test_populate_models.py | 97 ++ open_data/models/utils.py | 171 ++ open_data/tasks.py | 174 ++ open_data/views.py | 1 + settings/common.py | 3 + 29 files changed, 3496 insertions(+) create mode 100644 open_data/__init__.py create mode 100644 open_data/admin.py create mode 100644 open_data/apps.py create mode 100644 open_data/commodities.py create mode 100644 open_data/geo_areas.py create mode 100644 open_data/management/commands/__init__.py create mode 100644 open_data/management/commands/refresh_open_data.py create mode 100644 open_data/measures.py create mode 100644 open_data/migrations/0001_create_schema.py create mode 100644 open_data/migrations/0002_initial.py create mode 100644 open_data/migrations/0003_drop_fk_constrains.py create mode 100644 open_data/migrations/0004_materialised_view.py create mode 100644 open_data/migrations/0005_measureasdefinedreport.py create mode 100644 open_data/migrations/__init__.py create mode 100644 open_data/models/__init__.py create mode 100644 open_data/models/additional_codes_models.py create mode 100644 open_data/models/certificates_models.py create mode 100644 open_data/models/commodities_model.py create mode 100644 open_data/models/footnotes_models.py create mode 100644 open_data/models/geo_areas_models.py create mode 100644 open_data/models/measures_models.py create mode 100644 open_data/models/quotas_models.py create mode 100644 open_data/models/regulations_models.py create mode 100644 open_data/models/tests/__init__.py create mode 100644 open_data/models/tests/test_populate_models.py create mode 100644 open_data/models/utils.py create mode 100644 open_data/tasks.py create mode 100644 open_data/views.py diff --git a/open_data/__init__.py b/open_data/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/open_data/admin.py b/open_data/admin.py new file mode 100644 index 000000000..846f6b406 --- /dev/null +++ b/open_data/admin.py @@ -0,0 +1 @@ +# Register your models here. diff --git a/open_data/apps.py b/open_data/apps.py new file mode 100644 index 000000000..da1b83729 --- /dev/null +++ b/open_data/apps.py @@ -0,0 +1,7 @@ +from common.app_config import CommonConfig + +APP_LABEL = "open_data" + + +class OpenDataConfig(CommonConfig): + name = APP_LABEL diff --git a/open_data/commodities.py b/open_data/commodities.py new file mode 100644 index 000000000..fe023c3ee --- /dev/null +++ b/open_data/commodities.py @@ -0,0 +1,57 @@ +import time +from datetime import date + +from commodities.models.dc import CommodityCollectionLoader +from commodities.models.dc import CommodityTreeSnapshot +from commodities.models.dc import SnapshotMoment +from open_data.models import ReportGoodsNomenclature + + +def tree_edge_to_db(tree_edges): + for comm in tree_edges: + parent = tree_edges[comm] + if parent: + parent_obj_pk = parent.obj.pk + else: + parent_obj_pk = None + try: + commodity = ReportGoodsNomenclature.objects.get( + trackedmodel_ptr=comm.obj.pk, + ) + commodity.indent = comm.indent + commodity.parent_trackedmodel_ptr_id = parent_obj_pk + commodity.description = comm.description + commodity.save() + except ReportGoodsNomenclature.DoesNotExist: + pass + + +def save_commodities_parent(verbose=False): + # Brute force approach to find the commodity parents. + # CommodityTreeSnapshot creates the list of commodities and parent, + # given a two number prefix. + # Provide 99 prefix to be sure to cover all the possible + # combination. + # Once the tree is created, the parents are saved to + # ReportGoodsNomenclature + # In this way, Tomato code finds the correct information, without the need to + # replicate it in sql + + moment = SnapshotMoment(transaction=None, date=date.today()) + start = time.time() + for i in range(0, 100): + prefix = f"{i:02d}" + if verbose: + print(f"Starting prefix {prefix}") + commodities_collection = CommodityCollectionLoader(prefix=prefix).load( + current_only=True, + effective_only=True, + ) + snapshot = CommodityTreeSnapshot( + commodities=commodities_collection.commodities, + moment=moment, + ) + # snapshot = commodities_collection.get_snapshot(None, date.today()) + tree_edge_to_db(snapshot.edges) + if verbose: + print(f"Elapsed time {time.time() - start}") diff --git a/open_data/geo_areas.py b/open_data/geo_areas.py new file mode 100644 index 000000000..936e5c3a6 --- /dev/null +++ b/open_data/geo_areas.py @@ -0,0 +1,29 @@ +import time + +from geo_areas.models import GeographicalArea +from open_data.models import ReportGeographicalArea + + +def save_geo_areas(verbose): + report_geo_areas = ReportGeographicalArea.objects.select_related( + "trackedmodel_ptr", + ).all() + start = time.time() + for report_geo_area in report_geo_areas: + geo_area = report_geo_area.trackedmodel_ptr + report_geo_area.is_single_region_or_country = ( + geo_area.is_single_region_or_country() + ) + report_geo_area.is_all_countries = geo_area.is_all_countries() + report_geo_area.is_group = geo_area.is_group() + report_geo_area.is_all_countries = geo_area.is_all_countries() + + description = ( + GeographicalArea.objects.get(pk=report_geo_area.trackedmodel_ptr_id) + .get_description() + .description + ) + report_geo_area.description = description + report_geo_area.save() + if verbose: + print(f"Elapsed time {time.time() - start}") diff --git a/open_data/management/commands/__init__.py b/open_data/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/open_data/management/commands/refresh_open_data.py b/open_data/management/commands/refresh_open_data.py new file mode 100644 index 000000000..45d5a3dc6 --- /dev/null +++ b/open_data/management/commands/refresh_open_data.py @@ -0,0 +1,21 @@ +import logging + +from django.core.management.base import BaseCommand + +from open_data.tasks import populate_open_data + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = ( + "It deletes all the data in the reporting tables, and copy a fresh set of data" + "from the tracked tables in the database." + ) + + def handle(self, *args, **options): + logger.info(f"Starting the update of all the tables in the database") + populate_open_data(True) + self.stdout.write( + self.style.SUCCESS("Successfully updated the reporting tables."), + ) diff --git a/open_data/measures.py b/open_data/measures.py new file mode 100644 index 000000000..bf66e8d46 --- /dev/null +++ b/open_data/measures.py @@ -0,0 +1,93 @@ +import time + +from django.db import connection + +from common.models.transactions import Transaction +from common.models.utils import override_current_transaction +from open_data.models import ReportMeasure +from open_data.models import ReportMeasureCondition + + +def update_measure_components(verbose): + # Unless there is a current transaction, reading the latest description will fail in a misterious way + # Because this is called in a command, there is no transaction set""" + counter = 0 + tx = Transaction.objects.last() + start = time.time() + if verbose: + print("Updating measure components") + with override_current_transaction(tx): + measures_qs = ( + ReportMeasure.objects.filter(sid__gte=20000000) + .only("trackedmodel_ptr") + .select_related("trackedmodel_ptr") + ) + component_list = [] + for measure in measures_qs: + counter += 1 + if counter % 1000 == 0: + print(f"Measure count {counter}") + # comp_counter = 0 + for ( + component + ) in ( + measure.trackedmodel_ptr.conditions.latest_approved().with_reference_price_string() + ): + # comp_counter += 1 + # print(f" Condition count {comp_counter}") + component_list.append( + ReportMeasureCondition( + trackedmodel_ptr_id=component.trackedmodel_ptr_id, + sid=component.sid, + component_sequence_number=component.component_sequence_number, + duty_amount=component.duty_amount, + action_id=component.action_id, + condition_code_id=component.condition_code_id, + condition_measurement_id=component.condition_measurement_id, + dependent_measure_id=measure.trackedmodel_ptr_id, + monetary_unit_id=component.monetary_unit_id, + required_certificate_id=component.required_certificate_id, + reference_price=component.reference_price_string, + ), + ) + + ReportMeasureCondition.objects.bulk_create(component_list) + print("Completed Measure condition creation") + # The required_certificate_id is not updated when the certificate is updated + # In the UI it works because the certificate is selected using the SID and + # 'approved to last Transaction'. In data workspace works because when a + # certificate is updated, only the validity is changed so even if the data is not read from the latest, + # the SID is correct. I am not sure what is the best way to fix this!!! + # I'll try patching the required_certificate_id and hope for the best + fk_query_list = ReportMeasureCondition.update_fk_queries() + if fk_query_list: + with connection.cursor() as cursor: + for query in fk_query_list: + cursor.execute(query) + + if verbose: + print(f"Elapsed time {time.time() - start}") + + +def update_measure(verbose): + # Unless there is a current transaction, reading the latest description will fail in a misterious way + # Because this is called in a command, there is no transaction set""" + tx = Transaction.objects.last() + start = time.time() + if verbose: + print("Updating measure") + + with override_current_transaction(tx): + measures_qs = ( + ReportMeasure.objects.filter(sid__gte=20000000) + .only("trackedmodel_ptr", "duty_sentence") + .select_related("trackedmodel_ptr") + ) + for measure in measures_qs: + duty_sentence = measure.trackedmodel_ptr.duty_sentence + if duty_sentence: + measure.duty_sentence = duty_sentence + measure.save() + + if verbose: + print(f"Elapsed time {time.time() - start}") diff --git a/open_data/migrations/0001_create_schema.py b/open_data/migrations/0001_create_schema.py new file mode 100644 index 000000000..de4c4c318 --- /dev/null +++ b/open_data/migrations/0001_create_schema.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.15 on 2024-10-30 11:07 + +from django.db import migrations + +from open_data.models.utils import migrate_to_postgres + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + if migrate_to_postgres(): + operations = [ + migrations.RunSQL( + sql=[("CREATE SCHEMA reporting;")], + reverse_sql=[("DROP SCHEMA reporting;")], + ), + ] + else: + operations = [] diff --git a/open_data/migrations/0002_initial.py b/open_data/migrations/0002_initial.py new file mode 100644 index 000000000..cbc0151df --- /dev/null +++ b/open_data/migrations/0002_initial.py @@ -0,0 +1,1484 @@ +# Generated by Django 4.2.16 on 2025-02-03 16:56 + +import django.db.models.deletion +from django.db import migrations +from django.db import models + +import common.fields +from open_data.models.utils import migrate_to_postgres + + +class Migration(migrations.Migration): + # schema is not supported by sqlite. + if migrate_to_postgres(): + schema_name = 'reporting"."' + else: + schema_name = "" + + dependencies = [ + ("open_data", "0001_create_schema"), + ] + + operations = [ + migrations.CreateModel( + name="ReportAdditionalCode", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="additional_codes.additionalcode", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("code", models.CharField(max_length=3)), + ("description", models.TextField(blank=True, null=True)), + ], + options={ + "db_table": f"{schema_name}open_data_reportadditionalcode", + }, + ), + migrations.CreateModel( + name="ReportAdditionalCodeType", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="additional_codes.additionalcodetype", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.CharField(max_length=1)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ("application_code", models.SmallIntegerField()), + ], + options={ + "db_table": f"{schema_name}open_data_reportadditionalcodetype", + }, + ), + migrations.CreateModel( + name="ReportCertificate", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="certificates.certificate", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.CharField(max_length=3)), + ("description", models.TextField(blank=True, null=True)), + ], + options={ + "db_table": f"{schema_name}open_data_reportcertificate", + }, + ), + migrations.CreateModel( + name="ReportCertificateType", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="certificates.certificatetype", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.CharField(max_length=1)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportcertificatetype", + }, + ), + migrations.CreateModel( + name="ReportDutyExpression", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.dutyexpression", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("prefix", models.CharField(max_length=14)), + ("duty_amount_applicability_code", models.SmallIntegerField()), + ("measurement_unit_applicability_code", models.SmallIntegerField()), + ("monetary_unit_applicability_code", models.SmallIntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportdutyexpression", + }, + ), + migrations.CreateModel( + name="ReportFootnote", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="footnotes.footnote", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("footnote_id", models.CharField(max_length=5)), + ("description", models.TextField(blank=True, null=True)), + ], + options={ + "db_table": f"{schema_name}open_data_reportfootnote", + }, + ), + migrations.CreateModel( + name="ReportFootnoteType", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="footnotes.footnotetype", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("footnote_type_id", models.CharField(max_length=3)), + ("application_code", models.IntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportfootnotetype", + }, + ), + migrations.CreateModel( + name="ReportGeographicalArea", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="geo_areas.geographicalarea", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("area_id", models.CharField(max_length=4)), + ("area_code", models.SmallIntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "is_single_region_or_country", + models.BooleanField(blank=True, null=True), + ), + ("is_group", models.BooleanField(blank=True, null=True)), + ("is_all_countries", models.BooleanField(blank=True, null=True)), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgeographicalarea", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportgeographicalarea", + }, + ), + migrations.CreateModel( + name="ReportGoodsNomenclature", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="commodities.goodsnomenclature", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("item_id", models.CharField(max_length=10)), + ("suffix", models.CharField(max_length=2)), + ("statistical", models.BooleanField()), + ("indent", models.IntegerField(null=True)), + ("description", models.TextField(blank=True, null=True)), + ( + "parent_trackedmodel_ptr", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgoodsnomenclature", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportgoodsnomenclature", + }, + ), + migrations.CreateModel( + name="ReportGroup", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="regulations.group", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("group_id", models.CharField(max_length=3)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportgroup", + }, + ), + migrations.CreateModel( + name="ReportMeasure", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measure", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ( + "dead_additional_code", + models.CharField(blank=True, max_length=16, null=True), + ), + ( + "dead_order_number", + models.CharField(blank=True, max_length=6, null=True), + ), + ("reduction", models.SmallIntegerField(blank=True, null=True)), + ("stopped", models.BooleanField()), + ( + "export_refund_nomenclature_sid", + models.IntegerField(blank=True, null=True), + ), + ("duty_sentence", models.TextField(blank=True, null=True)), + ( + "additional_code", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportadditionalcode", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasure", + }, + ), + migrations.CreateModel( + name="ReportMeasureAction", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measureaction", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("code", models.CharField(max_length=3)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ("requires_duty", models.BooleanField()), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasureaction", + }, + ), + migrations.CreateModel( + name="ReportMeasureCondition", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measurecondition", + ), + ), + ("sid", models.IntegerField()), + ("component_sequence_number", models.SmallIntegerField()), + ( + "duty_amount", + models.DecimalField( + blank=True, + decimal_places=3, + max_digits=10, + null=True, + ), + ), + ("reference_price", models.TextField(blank=True, null=True)), + ( + "action", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasureaction", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasurecondition", + }, + ), + migrations.CreateModel( + name="ReportMeasureConditionCode", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measureconditioncode", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("code", models.CharField(max_length=2)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ("accepts_certificate", models.BooleanField()), + ("accepts_price", models.BooleanField()), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasureconditioncode", + }, + ), + migrations.CreateModel( + name="ReportMeasurementUnit", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measurementunit", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("code", models.CharField(max_length=3)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ("abbreviation", models.CharField(max_length=32)), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasurementunit", + }, + ), + migrations.CreateModel( + name="ReportMeasurementUnitQualifier", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measurementunitqualifier", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("code", models.CharField(max_length=1)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ("abbreviation", models.CharField(max_length=32)), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasurementunitqualifier", + }, + ), + migrations.CreateModel( + name="ReportMeasureTypeSeries", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measuretypeseries", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.CharField(max_length=2)), + ("measure_type_combination", models.SmallIntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasuretypeseries", + }, + ), + migrations.CreateModel( + name="ReportMonetaryUnit", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.monetaryunit", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("code", models.CharField(max_length=3)), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmonetaryunit", + }, + ), + migrations.CreateModel( + name="ReportQuotaDefinition", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotadefinition", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("volume", models.DecimalField(decimal_places=3, max_digits=14)), + ( + "initial_volume", + models.DecimalField(decimal_places=3, max_digits=14), + ), + ("maximum_precision", models.SmallIntegerField()), + ("quota_critical", models.BooleanField()), + ("quota_critical_threshold", models.SmallIntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measurement_unit", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurementunit", + ), + ), + ( + "measurement_unit_qualifier", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurementunitqualifier", + ), + ), + ( + "monetary_unit", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmonetaryunit", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotadefinition", + }, + ), + migrations.CreateModel( + name="ReportQuotaOrderNumber", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotaordernumber", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("order_number", models.CharField(max_length=6)), + ("mechanism", models.SmallIntegerField()), + ("category", models.SmallIntegerField()), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotaordernumber", + }, + ), + migrations.CreateModel( + name="ReportQuotaOrderNumberOrigin", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotaordernumberorigin", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ( + "geographical_area", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgeographicalarea", + ), + ), + ( + "order_number", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotaordernumber", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotaordernumberorigin", + }, + ), + migrations.CreateModel( + name="ReportRegulation", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="regulations.regulation", + ), + ), + ("role_type", models.IntegerField()), + ("regulation_id", models.CharField(max_length=8)), + ( + "official_journal_number", + models.CharField(blank=True, max_length=5, null=True), + ), + ( + "official_journal_page", + models.SmallIntegerField(blank=True, null=True), + ), + ("published_at", models.DateField(blank=True, null=True)), + ( + "information_text", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "public_identifier", + models.CharField(blank=True, max_length=50, null=True), + ), + ("url", models.CharField(blank=True, max_length=200, null=True)), + ("approved", models.BooleanField()), + ("replacement_indicator", models.IntegerField()), + ("valid_between", models.TextField(blank=True, null=True)), + ("effective_end_date", models.DateField(blank=True, null=True)), + ("stopped", models.BooleanField()), + ("community_code", models.IntegerField(blank=True, null=True)), + ( + "regulation_group", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgroup", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportregulation", + }, + ), + migrations.CreateModel( + name="ReportSuspension", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="regulations.suspension", + ), + ), + ("effective_end_date", models.DateField(blank=True, null=True)), + ( + "enacting_regulation", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportregulation", + ), + ), + ( + "target_regulation", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="regulationssuspension_target_regulation_set", + to="open_data.reportregulation", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportsuspension", + }, + ), + migrations.CreateModel( + name="ReportReplacement", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="regulations.replacement", + ), + ), + ( + "measure_type_id", + models.CharField(blank=True, max_length=6, null=True), + ), + ( + "geographical_area_id", + models.CharField(blank=True, max_length=4, null=True), + ), + ( + "chapter_heading", + models.CharField(blank=True, max_length=2, null=True), + ), + ( + "enacting_regulation", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportregulation", + ), + ), + ( + "target_regulation", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="regulationsreplacement_target_regulation_set", + to="open_data.reportregulation", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportreplacement", + }, + ), + migrations.CreateModel( + name="ReportQuotaSuspension", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotasuspension", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "quota_definition", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotadefinition", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotasuspension", + }, + ), + migrations.CreateModel( + name="ReportQuotaOrderNumberOriginExclusion", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotaordernumberoriginexclusion", + ), + ), + ( + "excluded_geographical_area", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgeographicalarea", + ), + ), + ( + "origin", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotaordernumberorigin", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotaordernumberoriginexclusion", + }, + ), + migrations.AddField( + model_name="reportquotadefinition", + name="order_number", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotaordernumber", + ), + ), + migrations.CreateModel( + name="ReportQuotaBlocking", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotablocking", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.IntegerField()), + ("blocking_period_type", models.SmallIntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "quota_definition", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotadefinition", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotablocking", + }, + ), + migrations.CreateModel( + name="ReportQuotaAssociation", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="quotas.quotaassociation", + ), + ), + ("sub_quota_relation_type", models.CharField(max_length=2)), + ("coefficient", models.DecimalField(decimal_places=5, max_digits=16)), + ( + "main_quota", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotadefinition", + ), + ), + ( + "sub_quota", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="quotasquotaassociation_sub_quota_set", + to="open_data.reportquotadefinition", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportquotaassociation", + }, + ), + migrations.CreateModel( + name="ReportMeasureType", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measuretype", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ("sid", models.CharField(max_length=6)), + ("trade_movement_code", models.SmallIntegerField()), + ("priority_code", models.SmallIntegerField()), + ("measure_component_applicability_code", models.SmallIntegerField()), + ("origin_destination_code", models.SmallIntegerField()), + ("order_number_capture_code", models.SmallIntegerField()), + ("measure_explosion_level", models.SmallIntegerField()), + ( + "description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_type_series", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasuretypeseries", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasuretype", + }, + ), + migrations.CreateModel( + name="ReportMeasurement", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measurement", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ( + "measurement_unit", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurementunit", + ), + ), + ( + "measurement_unit_qualifier", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurementunitqualifier", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasurement", + }, + ), + migrations.CreateModel( + name="ReportMeasureExcludedGeographicalArea", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measureexcludedgeographicalarea", + ), + ), + ( + "excluded_geographical_area", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgeographicalarea", + ), + ), + ( + "modified_measure", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasure", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasureexcludedgeographicalarea", + }, + ), + migrations.CreateModel( + name="ReportMeasureConditionComponent", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measureconditioncomponent", + ), + ), + ( + "duty_amount", + models.DecimalField( + blank=True, + decimal_places=3, + max_digits=10, + null=True, + ), + ), + ( + "component_measurement", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurement", + ), + ), + ( + "condition", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurecondition", + ), + ), + ( + "duty_expression", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportdutyexpression", + ), + ), + ( + "monetary_unit", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmonetaryunit", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasureconditioncomponent", + }, + ), + migrations.AddField( + model_name="reportmeasurecondition", + name="condition_code", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasureconditioncode", + ), + ), + migrations.AddField( + model_name="reportmeasurecondition", + name="condition_measurement", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurement", + ), + ), + migrations.AddField( + model_name="reportmeasurecondition", + name="dependent_measure", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasure", + ), + ), + migrations.AddField( + model_name="reportmeasurecondition", + name="monetary_unit", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmonetaryunit", + ), + ), + migrations.AddField( + model_name="reportmeasurecondition", + name="required_certificate", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportcertificate", + ), + ), + migrations.CreateModel( + name="ReportMeasureComponent", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.measurecomponent", + ), + ), + ( + "duty_amount", + models.DecimalField( + blank=True, + decimal_places=3, + max_digits=10, + null=True, + ), + ), + ( + "component_measure", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasure", + ), + ), + ( + "component_measurement", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasurement", + ), + ), + ( + "duty_expression", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportdutyexpression", + ), + ), + ( + "monetary_unit", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmonetaryunit", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasurecomponent", + }, + ), + migrations.AddField( + model_name="reportmeasure", + name="generating_regulation", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportregulation", + ), + ), + migrations.AddField( + model_name="reportmeasure", + name="geographical_area", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgeographicalarea", + ), + ), + migrations.AddField( + model_name="reportmeasure", + name="goods_nomenclature", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgoodsnomenclature", + ), + ), + migrations.AddField( + model_name="reportmeasure", + name="measure_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasuretype", + ), + ), + migrations.AddField( + model_name="reportmeasure", + name="order_number", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportquotaordernumber", + ), + ), + migrations.AddField( + model_name="reportmeasure", + name="terminating_regulation", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="measuresmeasure_terminating_regulation_set", + to="open_data.reportregulation", + ), + ), + migrations.CreateModel( + name="ReportGoodsNomenclatureSuccessor", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="commodities.goodsnomenclaturesuccessor", + ), + ), + ( + "absorbed_into_goods_nomenclature", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgoodsnomenclature", + ), + ), + ( + "replaced_goods_nomenclature", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="commoditiesgoodsnomenclaturesuccessor_replaced_goods_nomenclature_set", + to="open_data.reportgoodsnomenclature", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportgoodsnomenclaturesuccessor", + }, + ), + migrations.CreateModel( + name="ReportGoodsNomenclatureOrigin", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="commodities.goodsnomenclatureorigin", + ), + ), + ( + "derived_from_goods_nomenclature", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgoodsnomenclature", + ), + ), + ( + "new_goods_nomenclature", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="commoditiesgoodsnomenclatureorigin_new_goods_nomenclature_set", + to="open_data.reportgoodsnomenclature", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportgoodsnomenclatureorigin", + }, + ), + migrations.CreateModel( + name="ReportGeographicalMembership", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="geo_areas.geographicalmembership", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ( + "geo_group", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgeographicalarea", + ), + ), + ( + "member", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="geoareasgeographicalmembership_member_set", + to="open_data.reportgeographicalarea", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportgeographicalmembership", + }, + ), + migrations.CreateModel( + name="ReportFootnoteAssociationMeasure", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.footnoteassociationmeasure", + ), + ), + ( + "associated_footnote", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportfootnote", + ), + ), + ( + "footnoted_measure", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasure", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportfootnoteassociationmeasure", + }, + ), + migrations.CreateModel( + name="ReportFootnoteAssociationGoodsNomenclature", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="commodities.footnoteassociationgoodsnomenclature", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ( + "associated_footnote", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportfootnote", + ), + ), + ( + "goods_nomenclature", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportgoodsnomenclature", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportfootnoteassociationgoodsnomenclature", + }, + ), + migrations.AddField( + model_name="reportfootnote", + name="footnote_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportfootnotetype", + ), + ), + migrations.AddField( + model_name="reportcertificate", + name="certificate_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportcertificatetype", + ), + ), + migrations.CreateModel( + name="ReportAmendment", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="regulations.amendment", + ), + ), + ( + "enacting_regulation", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportregulation", + ), + ), + ( + "target_regulation", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="regulationsamendment_target_regulation_set", + to="open_data.reportregulation", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportamendment", + }, + ), + migrations.CreateModel( + name="ReportAdditionalCodeTypeMeasureType", + fields=[ + ( + "trackedmodel_ptr", + models.OneToOneField( + db_column="trackedmodel_ptr_id", + on_delete=django.db.models.deletion.DO_NOTHING, + primary_key=True, + serialize=False, + to="measures.additionalcodetypemeasuretype", + ), + ), + ("valid_between", common.fields.TaricDateRangeField(db_index=True)), + ( + "additional_code_type", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportadditionalcodetype", + ), + ), + ( + "measure_type", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportmeasuretype", + ), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportadditionalcodetypemeasuretype", + }, + ), + migrations.AddField( + model_name="reportadditionalcode", + name="type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="open_data.reportadditionalcodetype", + ), + ), + ] diff --git a/open_data/migrations/0003_drop_fk_constrains.py b/open_data/migrations/0003_drop_fk_constrains.py new file mode 100644 index 000000000..f26762e7f --- /dev/null +++ b/open_data/migrations/0003_drop_fk_constrains.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-10-30 11:17 + +from django.db import migrations + +from open_data.models.utils import migrate_to_postgres + +# It will be impossible to update the tables in the open data area with the +# foreign keys constrain in place. But it is useful to declare them in the Django +# models, so Django will create the correct queryset: the following query t +# dropped them in the database while they are still the model definition. +# The 'magic' query has been copied from somewhere in Stackoverflow! + + +drop_fk_sql = """ +DO $$DECLARE r record; + BEGIN + FOR r IN SELECT table_schema, table_name, constraint_name + FROM information_schema.table_constraints AS tc + WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema='reporting' + LOOP + EXECUTE 'ALTER TABLE '|| quote_ident(r.table_schema) || '.' || quote_ident(r.table_name)|| ' DROP CONSTRAINT '|| quote_ident(r.constraint_name) || ';'; + END LOOP; + END$$; +""" + + +class Migration(migrations.Migration): + + dependencies = [ + ("open_data", "0002_initial"), + ] + + if migrate_to_postgres(): + operations = [ + migrations.RunSQL(drop_fk_sql), + ] + else: + operations = [] diff --git a/open_data/migrations/0004_materialised_view.py b/open_data/migrations/0004_materialised_view.py new file mode 100644 index 000000000..c63d15dca --- /dev/null +++ b/open_data/migrations/0004_materialised_view.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.15 on 2024-10-30 11:33 + +from django.db import migrations + +from open_data.models.utils import migrate_to_postgres + +create_view_sql = """ +CREATE MATERIALIZED VIEW reporting.foreign_key_lookup AS + SELECT common_trackedmodel.ID as old_id, current_version_id + FROM public.common_trackedmodel + INNER JOIN common_versiongroup + ON (common_trackedmodel.version_group_id = common_versiongroup.id) +WHERE (current_version_id IS NOT NULL AND NOT (common_trackedmodel.update_type = 2)); + +CREATE UNIQUE INDEX old_id_idx ON reporting.foreign_key_lookup (old_id); +CREATE INDEX current_version_id_idx ON reporting.foreign_key_lookup (current_version_id); +""" + + +class Migration(migrations.Migration): + + dependencies = [ + ("open_data", "0003_drop_fk_constrains"), + ] + if migrate_to_postgres(): + operations = [ + migrations.RunSQL(create_view_sql), + ] + else: + operations = [] diff --git a/open_data/migrations/0005_measureasdefinedreport.py b/open_data/migrations/0005_measureasdefinedreport.py new file mode 100644 index 000000000..006c24652 --- /dev/null +++ b/open_data/migrations/0005_measureasdefinedreport.py @@ -0,0 +1,112 @@ +# Generated by Django 4.2.16 on 2025-02-07 12:23 + +from django.db import migrations +from django.db import models + +from open_data.models.utils import migrate_to_postgres + + +class Migration(migrations.Migration): + + # schema is not supported by sqlite. + if migrate_to_postgres(): + schema_name = 'reporting"."' + else: + schema_name = "" + + dependencies = [ + ("open_data", "0004_materialised_view"), + ] + + operations = [ + migrations.CreateModel( + name="MeasureAsDefinedReport", + fields=[ + ("id_counter", models.IntegerField()), + ( + "trackedmodel_ptr_id", + models.IntegerField(primary_key=True, serialize=False), + ), + ( + "commodity_sid", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "commodity_code", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "commodity_indent", + models.CharField(blank=True, max_length=500, null=True), + ), + ("commodity_description", models.TextField(blank=True, null=True)), + ("measure_sid", models.IntegerField()), + ( + "measure_type_id", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_type_description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_additional_code_code", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_additional_code_description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_duty_expression", + models.CharField(blank=True, max_length=500, null=True), + ), + ("measure_effective_start_date", models.DateField()), + ("measure_effective_end_date", models.DateField(blank=True, null=True)), + ( + "measure_reduction_indicator", + models.CharField(blank=True, max_length=500, null=True), + ), + ("measure_footnotes", models.TextField(blank=True, null=True)), + ( + "measure_conditions", + models.TextField(blank=True, null=True), + ), + ( + "measure_geographical_area_sid", + models.IntegerField(blank=True, null=True), + ), + ( + "measure_geographical_area_id", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_geographical_area_description", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_excluded_geographical_areas_ids", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_excluded_geographical_areas_descriptions", + models.TextField(blank=True, null=True), + ), + ( + "measure_quota_order_number", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_regulation_id", + models.CharField(blank=True, max_length=500, null=True), + ), + ( + "measure_regulation_url", + models.CharField(blank=True, max_length=500, null=True), + ), + ], + options={ + "db_table": f"{schema_name}open_data_reportmeasure_as_defined_report", + }, + ), + ] diff --git a/open_data/migrations/__init__.py b/open_data/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/open_data/models/__init__.py b/open_data/models/__init__.py new file mode 100644 index 000000000..ea1023f9e --- /dev/null +++ b/open_data/models/__init__.py @@ -0,0 +1,86 @@ +from open_data.models.additional_codes_models import ReportAdditionalCode +from open_data.models.additional_codes_models import ReportAdditionalCodeType +from open_data.models.certificates_models import ReportCertificate +from open_data.models.certificates_models import ReportCertificateType +from open_data.models.commodities_model import ( + ReportFootnoteAssociationGoodsNomenclature, +) +from open_data.models.commodities_model import ReportGoodsNomenclature +from open_data.models.commodities_model import ReportGoodsNomenclatureOrigin +from open_data.models.commodities_model import ReportGoodsNomenclatureSuccessor +from open_data.models.footnotes_models import ReportFootnote +from open_data.models.footnotes_models import ReportFootnoteType +from open_data.models.geo_areas_models import ReportGeographicalArea +from open_data.models.geo_areas_models import ReportGeographicalMembership +from open_data.models.measures_models import ReportAdditionalCodeTypeMeasureType +from open_data.models.measures_models import ReportDutyExpression +from open_data.models.measures_models import ReportFootnoteAssociationMeasure +from open_data.models.measures_models import ReportMeasure +from open_data.models.measures_models import ReportMeasureAction +from open_data.models.measures_models import ReportMeasureComponent +from open_data.models.measures_models import ReportMeasureCondition +from open_data.models.measures_models import ReportMeasureConditionCode +from open_data.models.measures_models import ReportMeasureConditionComponent +from open_data.models.measures_models import ReportMeasureExcludedGeographicalArea +from open_data.models.measures_models import ReportMeasurement +from open_data.models.measures_models import ReportMeasurementUnit +from open_data.models.measures_models import ReportMeasurementUnitQualifier +from open_data.models.measures_models import ReportMeasureType +from open_data.models.measures_models import ReportMeasureTypeSeries +from open_data.models.measures_models import ReportMonetaryUnit +from open_data.models.quotas_models import ReportQuotaAssociation +from open_data.models.quotas_models import ReportQuotaBlocking +from open_data.models.quotas_models import ReportQuotaDefinition +from open_data.models.quotas_models import ReportQuotaOrderNumber +from open_data.models.quotas_models import ReportQuotaOrderNumberOrigin +from open_data.models.quotas_models import ReportQuotaOrderNumberOriginExclusion +from open_data.models.quotas_models import ReportQuotaSuspension +from open_data.models.regulations_models import ReportAmendment +from open_data.models.regulations_models import ReportGroup +from open_data.models.regulations_models import ReportRegulation +from open_data.models.regulations_models import ReportReplacement +from open_data.models.regulations_models import ReportSuspension + +__all__ = [ + "ReportAdditionalCode", + "ReportAdditionalCodeType", + "ReportAdditionalCode", + "ReportCertificateType", + "ReportCertificate", + "ReportGoodsNomenclature", + "ReportGoodsNomenclatureSuccessor", + "ReportGoodsNomenclatureOrigin", + "ReportFootnoteAssociationGoodsNomenclature", + "ReportFootnoteType", + "ReportFootnote", + "ReportGeographicalArea", + "ReportGeographicalMembership", + "ReportAdditionalCodeTypeMeasureType", + "ReportDutyExpression", + "ReportFootnoteAssociationMeasure", + "ReportMeasure", + "ReportMeasureAction", + "ReportMeasureConditionComponent", + "ReportMeasureCondition", + "ReportMeasureConditionCode", + "ReportMeasurementUnit", + "ReportMeasurementUnitQualifier", + "ReportMeasureTypeSeries", + "ReportMonetaryUnit", + "ReportMeasureType", + "ReportMeasurement", + "ReportMeasureExcludedGeographicalArea", + "ReportMeasureComponent", + "ReportQuotaAssociation", + "ReportQuotaDefinition", + "ReportQuotaOrderNumber", + "ReportQuotaOrderNumberOrigin", + "ReportQuotaSuspension", + "ReportQuotaOrderNumberOriginExclusion", + "ReportQuotaBlocking", + "ReportAmendment", + "ReportGroup", + "ReportRegulation", + "ReportSuspension", + "ReportReplacement", +] diff --git a/open_data/models/additional_codes_models.py b/open_data/models/additional_codes_models.py new file mode 100644 index 000000000..a5dc8c02f --- /dev/null +++ b/open_data/models/additional_codes_models.py @@ -0,0 +1,45 @@ +from django.db import models + +from additional_codes.models import AdditionalCode +from additional_codes.models import AdditionalCodeType +from common.fields import TaricDateRangeField +from open_data.models.utils import ReportModel + + +class ReportAdditionalCodeType(ReportModel): + shadowed_model = AdditionalCodeType + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.CharField(max_length=1) + description = models.CharField(max_length=500, blank=True, null=True) + application_code = models.SmallIntegerField() + + class Meta: + db_table = ReportModel.create_table_name(AdditionalCodeType) + + +class ReportAdditionalCode(ReportModel): + shadowed_model = AdditionalCode + update_description = True + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + code = models.CharField(max_length=3) + type = models.ForeignKey(ReportAdditionalCodeType, models.DO_NOTHING) + # Field completed using orm functions + description = models.TextField(blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(AdditionalCode) diff --git a/open_data/models/certificates_models.py b/open_data/models/certificates_models.py new file mode 100644 index 000000000..6bd6758bb --- /dev/null +++ b/open_data/models/certificates_models.py @@ -0,0 +1,45 @@ +from django.db import models + +from certificates.models import Certificate +from certificates.models import CertificateType +from common.fields import TaricDateRangeField +from open_data.models.utils import ReportModel + + +class ReportCertificateType(ReportModel): + shadowed_model = CertificateType + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.CharField(max_length=1) + description = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(CertificateType) + + +class ReportCertificate(ReportModel): + shadowed_model = Certificate + update_description = True + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + valid_between = TaricDateRangeField(db_index=True) + sid = models.CharField(max_length=3) + certificate_type = models.ForeignKey( + "ReportCertificateType", + models.DO_NOTHING, + ) + # Field completed using orm functions + description = models.TextField(blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(Certificate) diff --git a/open_data/models/commodities_model.py b/open_data/models/commodities_model.py new file mode 100644 index 000000000..bf186a79e --- /dev/null +++ b/open_data/models/commodities_model.py @@ -0,0 +1,110 @@ +from django.db import models + +from commodities.models import FootnoteAssociationGoodsNomenclature +from commodities.models import GoodsNomenclature +from commodities.models import GoodsNomenclatureOrigin +from commodities.models import GoodsNomenclatureSuccessor +from commodities.models.code import CommodityCode +from common.fields import TaricDateRangeField +from open_data.models.utils import ReportModel + + +class ReportGoodsNomenclature(ReportModel): + shadowed_model = GoodsNomenclature + extra_where = " AND valid_between @> CURRENT_DATE" + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + item_id = models.CharField(max_length=10) + suffix = models.CharField(max_length=2) + statistical = models.BooleanField() + + indent = models.IntegerField(null=True) + description = models.TextField(blank=True, null=True) + parent_trackedmodel_ptr = models.ForeignKey( + "self", + models.DO_NOTHING, + null=True, + ) + + @property + def code(self) -> CommodityCode: + """Returns a CommodityCode instance for the good.""" + return CommodityCode(code=self.item_id) + + class Meta: + db_table = ReportModel.create_table_name(GoodsNomenclature) + + +class ReportGoodsNomenclatureSuccessor(ReportModel): + shadowed_model = GoodsNomenclatureSuccessor + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + absorbed_into_goods_nomenclature = models.ForeignKey( + ReportGoodsNomenclature, + models.DO_NOTHING, + ) + replaced_goods_nomenclature = models.ForeignKey( + ReportGoodsNomenclature, + models.DO_NOTHING, + related_name="commoditiesgoodsnomenclaturesuccessor_replaced_goods_nomenclature_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(GoodsNomenclatureSuccessor) + + +class ReportGoodsNomenclatureOrigin(ReportModel): + shadowed_model = GoodsNomenclatureOrigin + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + derived_from_goods_nomenclature = models.ForeignKey( + ReportGoodsNomenclature, + models.DO_NOTHING, + ) + new_goods_nomenclature = models.ForeignKey( + ReportGoodsNomenclature, + models.DO_NOTHING, + related_name="commoditiesgoodsnomenclatureorigin_new_goods_nomenclature_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(GoodsNomenclatureOrigin) + + +class ReportFootnoteAssociationGoodsNomenclature(ReportModel): + shadowed_model = FootnoteAssociationGoodsNomenclature + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + associated_footnote = models.ForeignKey("ReportFootnote", models.DO_NOTHING) + goods_nomenclature = models.ForeignKey( + ReportGoodsNomenclature, + models.DO_NOTHING, + ) + + class Meta: + db_table = ReportModel.create_table_name(FootnoteAssociationGoodsNomenclature) diff --git a/open_data/models/footnotes_models.py b/open_data/models/footnotes_models.py new file mode 100644 index 000000000..9462545ab --- /dev/null +++ b/open_data/models/footnotes_models.py @@ -0,0 +1,44 @@ +from django.db import models + +from common.fields import TaricDateRangeField +from footnotes.models import Footnote +from footnotes.models import FootnoteType +from open_data.models.utils import ReportModel + + +class ReportFootnoteType(ReportModel): + shadowed_model = FootnoteType + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + footnote_type_id = models.CharField(max_length=3) + application_code = models.IntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(FootnoteType) + + +class ReportFootnote(ReportModel): + shadowed_model = Footnote + update_description = True + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + valid_between = TaricDateRangeField(db_index=True) + footnote_id = models.CharField(max_length=5) + footnote_type = models.ForeignKey("ReportFootnoteType", models.DO_NOTHING) + # Field completed using orm functions + description = models.TextField(blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(Footnote) diff --git a/open_data/models/geo_areas_models.py b/open_data/models/geo_areas_models.py new file mode 100644 index 000000000..ba6209b5e --- /dev/null +++ b/open_data/models/geo_areas_models.py @@ -0,0 +1,53 @@ +from django.db import models + +from common.fields import TaricDateRangeField +from geo_areas.models import GeographicalArea +from geo_areas.models import GeographicalMembership +from open_data.models.utils import ReportModel + + +class ReportGeographicalArea(ReportModel): + shadowed_model = GeographicalArea + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + area_id = models.CharField(max_length=4) + area_code = models.SmallIntegerField() + parent = models.ForeignKey("self", models.DO_NOTHING, blank=True, null=True) + + # Field completed using orm functions + description = models.CharField(max_length=500, blank=True, null=True) + is_single_region_or_country = models.BooleanField(blank=True, null=True) + is_all_countries = models.BooleanField(blank=True, null=True) + is_group = models.BooleanField(blank=True, null=True) + is_all_countries = models.BooleanField(blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(GeographicalArea) + + +class ReportGeographicalMembership(ReportModel): + shadowed_model = GeographicalMembership + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + geo_group = models.ForeignKey(ReportGeographicalArea, models.DO_NOTHING) + member = models.ForeignKey( + ReportGeographicalArea, + models.DO_NOTHING, + related_name="geoareasgeographicalmembership_member_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(GeographicalMembership) diff --git a/open_data/models/measures_models.py b/open_data/models/measures_models.py new file mode 100644 index 000000000..27a77709e --- /dev/null +++ b/open_data/models/measures_models.py @@ -0,0 +1,469 @@ +from django.db import models + +from common.fields import TaricDateRangeField +from measures.models.tracked_models import AdditionalCodeTypeMeasureType +from measures.models.tracked_models import DutyExpression +from measures.models.tracked_models import FootnoteAssociationMeasure +from measures.models.tracked_models import Measure +from measures.models.tracked_models import MeasureAction +from measures.models.tracked_models import MeasureComponent +from measures.models.tracked_models import MeasureCondition +from measures.models.tracked_models import MeasureConditionCode +from measures.models.tracked_models import MeasureConditionComponent +from measures.models.tracked_models import MeasureExcludedGeographicalArea +from measures.models.tracked_models import Measurement +from measures.models.tracked_models import MeasurementUnit +from measures.models.tracked_models import MeasurementUnitQualifier +from measures.models.tracked_models import MeasureType +from measures.models.tracked_models import MeasureTypeSeries +from measures.models.tracked_models import MonetaryUnit +from open_data.models.utils import ReportModel + + +class ReportAdditionalCodeTypeMeasureType(ReportModel): + shadowed_model = AdditionalCodeTypeMeasureType + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + additional_code_type = models.ForeignKey( + "ReportAdditionalCodeType", + models.DO_NOTHING, + ) + measure_type = models.ForeignKey("ReportMeasureType", models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(AdditionalCodeTypeMeasureType) + + +class ReportDutyExpression(ReportModel): + shadowed_model = DutyExpression + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + prefix = models.CharField(max_length=14) + duty_amount_applicability_code = models.SmallIntegerField() + measurement_unit_applicability_code = models.SmallIntegerField() + monetary_unit_applicability_code = models.SmallIntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(DutyExpression) + + +class ReportFootnoteAssociationMeasure(ReportModel): + shadowed_model = FootnoteAssociationMeasure + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + associated_footnote = models.ForeignKey("ReportFootnote", models.DO_NOTHING) + footnoted_measure = models.ForeignKey("ReportMeasure", models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(FootnoteAssociationMeasure) + + @classmethod + def get_ignore_fk_list(cls): + return ["footnoted_measure"] + + +class ReportMeasure(ReportModel): + shadowed_model = Measure + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + extra_where = ( + " AND (upper(valid_between) > CURRENT_DATE or upper(valid_between) is NULL) " + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + dead_additional_code = models.CharField(max_length=16, blank=True, null=True) + dead_order_number = models.CharField(max_length=6, blank=True, null=True) + reduction = models.SmallIntegerField(blank=True, null=True) + stopped = models.BooleanField() + export_refund_nomenclature_sid = models.IntegerField(blank=True, null=True) + additional_code = models.ForeignKey( + "ReportAdditionalCode", + models.DO_NOTHING, + blank=True, + null=True, + ) + generating_regulation = models.ForeignKey( + "ReportRegulation", + models.DO_NOTHING, + ) + geographical_area = models.ForeignKey("ReportGeographicalArea", models.DO_NOTHING) + goods_nomenclature = models.ForeignKey( + "ReportGoodsNomenclature", + models.DO_NOTHING, + blank=True, + null=True, + ) + measure_type = models.ForeignKey("ReportMeasureType", models.DO_NOTHING) + order_number = models.ForeignKey( + "ReportQuotaOrderNumber", + models.DO_NOTHING, + blank=True, + null=True, + ) + terminating_regulation = models.ForeignKey( + "ReportRegulation", + models.DO_NOTHING, + related_name="measuresmeasure_terminating_regulation_set", + blank=True, + null=True, + ) + + # ORM derived + duty_sentence = models.TextField(null=True, blank=True) + + class Meta: + db_table = ReportModel.create_table_name(Measure) + + +class ReportMeasureAction(ReportModel): + shadowed_model = MeasureAction + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + code = models.CharField(max_length=3) + description = models.CharField(max_length=500, blank=True, null=True) + requires_duty = models.BooleanField() + + class Meta: + db_table = ReportModel.create_table_name(MeasureAction) + + +class ReportMeasureConditionComponent(ReportModel): + shadowed_model = MeasureConditionComponent + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + duty_amount = models.DecimalField( + max_digits=10, + decimal_places=3, + blank=True, + null=True, + ) + condition = models.ForeignKey("ReportMeasureCondition", models.DO_NOTHING) + component_measurement = models.ForeignKey( + "ReportMeasurement", + models.DO_NOTHING, + blank=True, + null=True, + ) + duty_expression = models.ForeignKey("ReportDutyExpression", models.DO_NOTHING) + monetary_unit = models.ForeignKey( + "ReportMonetaryUnit", + models.DO_NOTHING, + blank=True, + null=True, + ) + + class Meta: + db_table = ReportModel.create_table_name(MeasureConditionComponent) + + +class ReportMeasureCondition(ReportModel): + shadowed_model = MeasureCondition + update_table = False + # This table is populated using the ORM + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + sid = models.IntegerField() + component_sequence_number = models.SmallIntegerField() + duty_amount = models.DecimalField( + max_digits=10, + decimal_places=3, + blank=True, + null=True, + ) + action = models.ForeignKey( + ReportMeasureAction, + models.DO_NOTHING, + blank=True, + null=True, + ) + condition_code = models.ForeignKey( + "ReportMeasureConditionCode", + models.DO_NOTHING, + ) + condition_measurement = models.ForeignKey( + "ReportMeasurement", + models.DO_NOTHING, + blank=True, + null=True, + ) + dependent_measure = models.ForeignKey(ReportMeasure, models.DO_NOTHING) + monetary_unit = models.ForeignKey( + "ReportMonetaryUnit", + models.DO_NOTHING, + blank=True, + null=True, + ) + required_certificate = models.ForeignKey( + "ReportCertificate", + models.DO_NOTHING, + blank=True, + null=True, + ) + # Calculated by the ORM + reference_price = models.TextField(blank=True, null=True) + + @classmethod + def get_ignore_fk_list(cls): + """The ORM used to populate this table does not return the latest + required_certificate To fix it, I am using the function update_fk, + excluding all the other FK.""" + return [ + "dependent_measure", + "action", + "condition_code", + "condition_measurement", + "monetary_unit", + ] + + class Meta: + db_table = ReportModel.create_table_name(MeasureCondition) + + +class ReportMeasureConditionCode(ReportModel): + shadowed_model = MeasureConditionCode + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + code = models.CharField(max_length=2) + description = models.CharField(max_length=500, blank=True, null=True) + accepts_certificate = models.BooleanField() + accepts_price = models.BooleanField() + + class Meta: + db_table = ReportModel.create_table_name(MeasureConditionCode) + + +class ReportMeasurementUnit(ReportModel): + shadowed_model = MeasurementUnit + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + code = models.CharField(max_length=3) + description = models.CharField(max_length=500, blank=True, null=True) + abbreviation = models.CharField(max_length=32) + + class Meta: + db_table = ReportModel.create_table_name(MeasurementUnit) + + +class ReportMeasurementUnitQualifier(ReportModel): + shadowed_model = MeasurementUnitQualifier + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + code = models.CharField(max_length=1) + description = models.CharField(max_length=500, blank=True, null=True) + abbreviation = models.CharField(max_length=32) + + class Meta: + db_table = ReportModel.create_table_name(MeasurementUnitQualifier) + + +class ReportMeasureTypeSeries(ReportModel): + shadowed_model = MeasureTypeSeries + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.CharField(max_length=2) + measure_type_combination = models.SmallIntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(MeasureTypeSeries) + + +class ReportMonetaryUnit(ReportModel): + shadowed_model = MonetaryUnit + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + code = models.CharField(max_length=3) + description = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(MonetaryUnit) + + +class ReportMeasureType(ReportModel): + shadowed_model = MeasureType + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.CharField(max_length=6) + trade_movement_code = models.SmallIntegerField() + priority_code = models.SmallIntegerField() + measure_component_applicability_code = models.SmallIntegerField() + origin_destination_code = models.SmallIntegerField() + order_number_capture_code = models.SmallIntegerField() + measure_explosion_level = models.SmallIntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + measure_type_series = models.ForeignKey( + ReportMeasureTypeSeries, + models.DO_NOTHING, + ) + + class Meta: + db_table = ReportModel.create_table_name(MeasureType) + + +class ReportMeasurement(ReportModel): + shadowed_model = Measurement + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + measurement_unit = models.ForeignKey(ReportMeasurementUnit, models.DO_NOTHING) + measurement_unit_qualifier = models.ForeignKey( + ReportMeasurementUnitQualifier, + models.DO_NOTHING, + blank=True, + null=True, + ) + + class Meta: + db_table = ReportModel.create_table_name(Measurement) + + +class ReportMeasureExcludedGeographicalArea(ReportModel): + shadowed_model = MeasureExcludedGeographicalArea + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + excluded_geographical_area = models.ForeignKey( + "ReportGeographicalArea", + models.DO_NOTHING, + ) + modified_measure = models.ForeignKey(ReportMeasure, models.DO_NOTHING) + + # @classmethod + # def ignore_fk_list(cls): + # return ["modified_measure"] + + class Meta: + db_table = ReportModel.create_table_name(MeasureExcludedGeographicalArea) + + +class ReportMeasureComponent(ReportModel): + shadowed_model = MeasureComponent + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + duty_amount = models.DecimalField( + max_digits=10, + decimal_places=3, + blank=True, + null=True, + ) + component_measure = models.ForeignKey(ReportMeasure, models.DO_NOTHING) + component_measurement = models.ForeignKey( + ReportMeasurement, + models.DO_NOTHING, + blank=True, + null=True, + ) + duty_expression = models.ForeignKey(ReportDutyExpression, models.DO_NOTHING) + monetary_unit = models.ForeignKey( + ReportMonetaryUnit, + models.DO_NOTHING, + blank=True, + null=True, + ) + + class Meta: + db_table = ReportModel.create_table_name(MeasureComponent) diff --git a/open_data/models/quotas_models.py b/open_data/models/quotas_models.py new file mode 100644 index 000000000..537ba2aae --- /dev/null +++ b/open_data/models/quotas_models.py @@ -0,0 +1,174 @@ +from django.db import models + +from common.fields import TaricDateRangeField +from open_data.models.utils import ReportModel +from quotas.models import QuotaAssociation +from quotas.models import QuotaBlocking +from quotas.models import QuotaDefinition +from quotas.models import QuotaOrderNumber +from quotas.models import QuotaOrderNumberOrigin +from quotas.models import QuotaOrderNumberOriginExclusion +from quotas.models import QuotaSuspension + + +class ReportQuotaAssociation(ReportModel): + shadowed_model = QuotaAssociation + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + sub_quota_relation_type = models.CharField(max_length=2) + coefficient = models.DecimalField(max_digits=16, decimal_places=5) + main_quota = models.ForeignKey("ReportQuotaDefinition", models.DO_NOTHING) + sub_quota = models.ForeignKey( + "ReportQuotaDefinition", + models.DO_NOTHING, + related_name="quotasquotaassociation_sub_quota_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(QuotaAssociation) + + +class ReportQuotaDefinition(ReportModel): + shadowed_model = QuotaDefinition + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + volume = models.DecimalField(max_digits=14, decimal_places=3) + initial_volume = models.DecimalField(max_digits=14, decimal_places=3) + maximum_precision = models.SmallIntegerField() + quota_critical = models.BooleanField() + quota_critical_threshold = models.SmallIntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + measurement_unit = models.ForeignKey( + "ReportMeasurementUnit", + models.DO_NOTHING, + blank=True, + null=True, + ) + measurement_unit_qualifier = models.ForeignKey( + "ReportMeasurementUnitQualifier", + models.DO_NOTHING, + blank=True, + null=True, + ) + monetary_unit = models.ForeignKey( + "ReportMonetaryunit", + models.DO_NOTHING, + blank=True, + null=True, + ) + order_number = models.ForeignKey("ReportQuotaOrderNumber", models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(QuotaDefinition) + + +class ReportQuotaOrderNumber(ReportModel): + shadowed_model = QuotaOrderNumber + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + order_number = models.CharField(max_length=6) + mechanism = models.SmallIntegerField() + category = models.SmallIntegerField() + + class Meta: + db_table = ReportModel.create_table_name(QuotaOrderNumber) + + +class ReportQuotaOrderNumberOrigin(ReportModel): + shadowed_model = QuotaOrderNumberOrigin + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + geographical_area = models.ForeignKey("ReportGeographicalArea", models.DO_NOTHING) + order_number = models.ForeignKey(ReportQuotaOrderNumber, models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(QuotaOrderNumberOrigin) + + +class ReportQuotaSuspension(ReportModel): + shadowed_model = QuotaSuspension + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + quota_definition = models.ForeignKey(ReportQuotaDefinition, models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(QuotaSuspension) + + +class ReportQuotaOrderNumberOriginExclusion(ReportModel): + shadowed_model = QuotaOrderNumberOriginExclusion + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + excluded_geographical_area = models.ForeignKey( + "ReportGeographicalArea", + models.DO_NOTHING, + ) + origin = models.ForeignKey(ReportQuotaOrderNumberOrigin, models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(QuotaOrderNumberOriginExclusion) + + +class ReportQuotaBlocking(ReportModel): + shadowed_model = QuotaBlocking + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + sid = models.IntegerField() + blocking_period_type = models.SmallIntegerField() + description = models.CharField(max_length=500, blank=True, null=True) + quota_definition = models.ForeignKey(ReportQuotaDefinition, models.DO_NOTHING) + + class Meta: + db_table = ReportModel.create_table_name(QuotaBlocking) diff --git a/open_data/models/regulations_models.py b/open_data/models/regulations_models.py new file mode 100644 index 000000000..8052f70d6 --- /dev/null +++ b/open_data/models/regulations_models.py @@ -0,0 +1,132 @@ +from django.db import models + +from common.fields import TaricDateRangeField +from open_data.models.utils import ReportModel +from regulations.models import Amendment +from regulations.models import Group +from regulations.models import Regulation +from regulations.models import Replacement +from regulations.models import Suspension + + +class ReportAmendment(ReportModel): + shadowed_model = Amendment + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + enacting_regulation = models.ForeignKey("ReportRegulation", models.DO_NOTHING) + target_regulation = models.ForeignKey( + "ReportRegulation", + models.DO_NOTHING, + related_name="regulationsamendment_target_regulation_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(Amendment) + + +class ReportGroup(ReportModel): + shadowed_model = Group + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + valid_between = TaricDateRangeField(db_index=True) + group_id = models.CharField(max_length=3) + description = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = ReportModel.create_table_name(Group) + + +class ReportRegulation(ReportModel): + shadowed_model = Regulation + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + role_type = models.IntegerField() + regulation_id = models.CharField(max_length=8) + official_journal_number = models.CharField(max_length=5, blank=True, null=True) + official_journal_page = models.SmallIntegerField(blank=True, null=True) + published_at = models.DateField(blank=True, null=True) + information_text = models.CharField(max_length=500, blank=True, null=True) + public_identifier = models.CharField(max_length=50, blank=True, null=True) + url = models.CharField(max_length=200, blank=True, null=True) + approved = models.BooleanField() + replacement_indicator = models.IntegerField() + valid_between = models.TextField( + blank=True, + null=True, + ) # This field type is a guess. + effective_end_date = models.DateField(blank=True, null=True) + stopped = models.BooleanField() + community_code = models.IntegerField(blank=True, null=True) + regulation_group = models.ForeignKey( + ReportGroup, + models.DO_NOTHING, + blank=True, + null=True, + ) + + class Meta: + db_table = ReportModel.create_table_name(Regulation) + + +class ReportSuspension(ReportModel): + shadowed_model = Suspension + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + effective_end_date = models.DateField(blank=True, null=True) + enacting_regulation = models.ForeignKey(ReportRegulation, models.DO_NOTHING) + target_regulation = models.ForeignKey( + ReportRegulation, + models.DO_NOTHING, + related_name="regulationssuspension_target_regulation_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(Suspension) + + +class ReportReplacement(ReportModel): + shadowed_model = Replacement + + trackedmodel_ptr = models.OneToOneField( + shadowed_model, + models.DO_NOTHING, + primary_key=True, + db_column="trackedmodel_ptr_id", + ) + + measure_type_id = models.CharField(max_length=6, blank=True, null=True) + geographical_area_id = models.CharField(max_length=4, blank=True, null=True) + chapter_heading = models.CharField(max_length=2, blank=True, null=True) + enacting_regulation = models.ForeignKey(ReportRegulation, models.DO_NOTHING) + target_regulation = models.ForeignKey( + ReportRegulation, + models.DO_NOTHING, + related_name="regulationsreplacement_target_regulation_set", + ) + + class Meta: + db_table = ReportModel.create_table_name(Replacement) diff --git a/open_data/models/tests/__init__.py b/open_data/models/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/open_data/models/tests/test_populate_models.py b/open_data/models/tests/test_populate_models.py new file mode 100644 index 000000000..38802e028 --- /dev/null +++ b/open_data/models/tests/test_populate_models.py @@ -0,0 +1,97 @@ +import pytest + +from common.models import TrackedModel +from common.tests import factories +from footnotes.models import Footnote +from footnotes.models import FootnoteDescription +from measures.models import Measure +from open_data.models import ReportFootnote +from open_data.models import ReportMeasure +from open_data.models.utils import ReportModel +from open_data.tasks import populate_open_data + +pytestmark = pytest.mark.django_db + +excluded_models = [ + "QuotaEvent", # excluded because not available in sqlite exported to data.gov + "GeographicalAreaDescription", # Description are merged in the described table + "GoodsNomenclatureIndent", + "GoodsNomenclatureDescription", + "AdditionalCodeDescription", + "FootnoteAssociationAdditionalCode", + "CertificateDescription", + "FootnoteDescription", + "Extension", # excluded because not available in sqlite exported to data.gov + "Termination", # excluded because not available in sqlite exported to data.gov +] + +# The following tracked models are created only for testing +test_models = ["TestModel1", "TestModel2", "TestModel3", "TestModelDescription1"] + + +def test_models_are_included(): + # This test will fail when a new tracked model has been created, + # without creating an equivalent model in the open data app + # If the new model is not relevant to open data, its name should be added + # to excluded_models + tracked_models = [cls.__name__ for cls in TrackedModel.__subclasses__()] + report_models = ( + [cls.shadowed_model.__name__ for cls in ReportModel.__subclasses__()] + + excluded_models + + test_models + ) + missing_models = [el for el in tracked_models if el not in report_models] + if len(missing_models): + print(missing_models) + assert len(missing_models) == 0 + + +def test_measures_unpublished_and_unapproved(): + factories.MeasureFactory.create( + transaction=factories.UnapprovedTransactionFactory.create(), + ) + factories.MeasureFactory.create( + transaction=factories.ApprovedTransactionFactory.create(), + ) + factories.MeasureFactory.create( + transaction=factories.PublishedTransactionFactory.create(), + ) + + assert Measure.objects.count() == 3 + assert ReportMeasure.objects.count() == 0 + assert Measure.objects.published().count() == 1 + populate_open_data() + assert Measure.objects.count() == 3 + assert ReportMeasure.objects.count() == 1 + assert Measure.objects.published().count() == 1 + + +def test_footnotes(): + published_transaction = factories.PublishedTransactionFactory.create() + factories.ApprovedTransactionFactory.create() + test_description = "Test description" + assert Footnote.objects.count() == 0 + test_footnote_id = "001" + factories.FootnoteFactory( + footnote_id=test_footnote_id, + transaction=published_transaction, + description__description=test_description, + ) + assert Footnote.objects.count() == 1 + assert FootnoteDescription.objects.count() == 1 + # Open data is empty + assert ReportFootnote.objects.count() == 0 + assert Footnote.objects.published().count() == 1 + populate_open_data() + + # Nothing created or deleted in the footnotes + assert Footnote.objects.count() == 1 + assert Footnote.objects.published().count() == 1 + assert FootnoteDescription.objects.count() == 1 + # Open data has a record + assert ReportFootnote.objects.count() == 1 + # and the description is correct + assert ( + ReportFootnote.objects.get(footnote_id=test_footnote_id).description + == test_description + ) diff --git a/open_data/models/utils.py b/open_data/models/utils.py new file mode 100644 index 000000000..52c5bf6b2 --- /dev/null +++ b/open_data/models/utils.py @@ -0,0 +1,171 @@ +import os + +from django.db import models + +from open_data.apps import APP_LABEL + +SCHEMA_NAME = "reporting" +LOOK_UP_VIEW = f"{SCHEMA_NAME}.foreign_key_lookup" + + +def migrate_to_postgres(): + data_base_url = os.environ.get("DATABASE_URL", "Postgres").lower() + # schema is not supported by sqlite. + # During sqlite dump creation, DATABASE_URL is set to sqlitexxxxx + # so we can establish if we are handlind sqlite, and ignore the schema name + if "sqlite" in data_base_url: + return False + return True + + +def create_name_with_schema(name): + # NOTE: the " around the stop are really important. + # without them, the table will not be created in the correct schema + # But is we are exporting to sqlite, we must omit the schma name + if migrate_to_postgres(): + return f'{SCHEMA_NAME}"."{APP_LABEL}_report{name}' + else: + return f"{APP_LABEL}_report{name}" + + +class ReportModel(models.Model): + # Individual classes can define extra restrictions on the copy to the report data. + # For instance, they may remove rows that have validity in the past + # The 'AND' clause must be defined in the extra_where, otherwise it will give + # an error. I could parse it and check if AND is in the clause, but it is too + # much extra effort, as the error will be identified immediately. + extra_where = "" + # Some tables contain several version of the same object, but with different + # validity. To make report creation easier, the old version will be removed + # when remove_obsolete is set to True + # The structure of the tables is identical, they always have a validity start + # The field to use for aggregation has to be specified + remove_obsolete = False + # The foreign keys in tracked models sometimes don't link to the latest object. + # Set patch_fk to run the query updating the foreign keys to the latest one. + patch_fk = True + # objects with a description in a separate table to allow several versions + # must set update_description to True. The description will be updated + # using the orm 'get_description' + update_description = False + # If false, the data will not be copied using the generated SQL. + # Required for complex data that is reconstructed using ORM calls. + update_table = True + + @staticmethod + def create_table_name(shadowed_model): + # to create a table in a different schema, we need to specify the schema + # in the db_table. The new name is constructed by removing + # the app name from the shadowed table, and prefixing the name + # with 'report', the app_label and the schema + shadowed_tb_name = shadowed_model._meta.db_table + new_name = shadowed_tb_name.split(shadowed_model._meta.app_label + "_")[1] + return create_name_with_schema(new_name) + + @classmethod + def extra_queries(cls): + # Individual classes can define a list of sql command to be executed + # after the table has been updated + return [] + + @classmethod + def copy_data_query(cls): + # used the generated queries for 'latest update' to create the query + # updating the report table with the latest values + db_from_table = cls.shadowed_model._meta.db_table + insert_fields = f'INSERT INTO "{cls._meta.db_table}" (' + read_field = ") SELECT " + # Find the SQL query that return the latest_approved on the + # model we are shadowing + if hasattr(cls.shadowed_model.objects.all(), "published"): + published_query = str( + cls.shadowed_model.objects.published().latest_approved().query, + ) + latest_query = published_query.replace("PUBLISHED", "'PUBLISHED'") + else: + latest_query = str(cls.shadowed_model.objects.latest_approved().query) + split_query = latest_query.split(" FROM") + # The query is split at the FROM keyword, and the second part contains + # the correct table we want to copy from and the correct WHERE clause + query_from_and_where = f" FROM {split_query[1]}" + # The first part of the query contains all the fields retrieved + # by latest_approved. + field_list = split_query[0].split() + # The Django generated latest_approved query contains fields from the + # track models. We only want to copy the fields from the table being shadowed + # The next loop eliminates fields that don't belong to the table. + for field in field_list: + if db_from_table in field: + read_field += field + # Remove the table name from the field definition + insert_fields += field.split(".")[1] + # return a correct SQL query for copying the fields from the tracked table" + return ( + f"{insert_fields} {read_field} {query_from_and_where} {cls.extra_where} ;" + ) + + @classmethod + def get_ignore_fk_list(cls): + return [] + + @classmethod + def update_fk_queries(cls): + # The foreign keys in a table don't always point to the latest version of the object. + # It is an unfortunate fact, with several causes. It will not be fixed, at least + # for now. So there is a materialised view, mapping every single primary key to + # the equivalent latest version. + # The following code generate the queries required to update the foreign keys + # in the tables. + # There is no attempt to make the process efficient! + query_list = [] + if cls.patch_fk: + ignore_list = cls.get_ignore_fk_list() + for f in cls._meta.get_fields(): + if ( + type(f) is models.fields.related.ForeignKey + and not f.name in ignore_list + and not f.primary_key + ): + query_list.append( + f'UPDATE "{cls._meta.db_table}" ' + f"SET {f.column}=current_version_id " + f" FROM {LOOK_UP_VIEW} " + f" WHERE {f.column} = old_id;", + ) + return query_list + + @classmethod + def remove_obsolete_row_query(cls): + query = "" + if cls.remove_obsolete: + # Don't ask about the next horrible lines. It works + partition_field_db = cls._meta.get_field( + cls.partition_field, + ).get_attname_column()[1] + query = f""" + DELETE + FROM \"{cls._meta.db_table}\" + WHERE trackedmodel_ptr_id in + (SELECT trackedmodel_ptr_id + FROM + (SELECT trackedmodel_ptr_id, + ROW_NUMBER() OVER ( + PARTITION BY {partition_field_db} + ORDER BY validity_start DESC + ) as DupRank + FROM \"{cls._meta.db_table}\") AS T + WHERE t.DupRank > 1 ) + """ + return query + + @classmethod + def referenced_models(cls): + """Returns a dictionary of the FK fields and the model they piont to.""" + referenced = {} + for f in cls._meta.get_fields(): + if type(f) is models.fields.related.ForeignKey and not f.primary_key: + referenced[f.column] = f.related_model + return referenced + + class Meta: + abstract = True diff --git a/open_data/tasks.py b/open_data/tasks.py new file mode 100644 index 000000000..7ea38f286 --- /dev/null +++ b/open_data/tasks.py @@ -0,0 +1,174 @@ +import time + +import django.apps +from django.db import connection +from django.db.models import Subquery + +from common.models.mixins.description import DescribedMixin +from common.models.transactions import Transaction +from common.models.utils import override_current_transaction +from open_data.apps import APP_LABEL +from open_data.commodities import save_commodities_parent +from open_data.geo_areas import save_geo_areas +from open_data.measures import update_measure +from open_data.measures import update_measure_components +from open_data.models.utils import LOOK_UP_VIEW +from open_data.models.utils import ReportModel + + +def add_description(model, verbose=True): + # The open data description is in the main table, not in a different table, + # because we only need the current version. + # The field is populated using the orm get_description() on the tracked table + # Not efficient, but correct + if not issubclass(model, ReportModel): + return + if model.update_description: + if issubclass(model.shadowed_model, DescribedMixin): + queryset = model.objects.select_related( + "trackedmodel_ptr", + ).all() + start = time.time() + for row in queryset: + description = model.shadowed_model.objects.get( + pk=row.trackedmodel_ptr_id, + ).get_description() + if description: + row.description = description.description + row.save() + if verbose: + print(f"Elapsed time {model._meta.db_table} {time.time() - start}") + + +def update_model(model, cursor, verbose=True): + if verbose: + print(f"Delete data from {model._meta.db_table}") + + cursor.execute(f'TRUNCATE TABLE "{model._meta.db_table}"') + if model.update_table: + cursor.execute(model.copy_data_query()) + + fk_query_list = model.update_fk_queries() + + if fk_query_list: + for query in fk_query_list: + cursor.execute(query) + + if model.remove_obsolete: + cursor.execute(model.remove_obsolete_row_query()) + + extra_queries = model.extra_queries() + + if extra_queries: + for sql_query in extra_queries: + cursor.execute(sql_query) + if verbose: + print(f"{model._meta.db_table} updated") + + +def populate_open_data(verbose=False): + + config = django.apps.apps.get_app_config(APP_LABEL) + + with connection.cursor() as cursor: + cursor.execute(f"REFRESH MATERIALIZED VIEW {LOOK_UP_VIEW};") + for model in config.get_models(): + if issubclass(model, ReportModel): + if verbose: + print(f'Starting update of "{model._meta.db_table}"') + start_time = time.time() + update_model(model, cursor, verbose) + if verbose: + elapsed_time = time.time() - start_time + print( + f'Completed update of "{model._meta.db_table}" in {elapsed_time} seconds', + ) + # The following are changes specific to different tables. + # They update fields using Django routines, created specifically for the task. + # Unless there is a current transaction, + # reading the latest description will fail in a misterious way + # Because this is called in a command, there is no transaction set""" + tx = Transaction.objects.last() + with override_current_transaction(tx): + for model in config.get_models(): + if verbose: + print(f'Starting update of "{model._meta.db_table}"') + start_time = time.time() + add_description(model) + if verbose: + elapsed_time = time.time() - start_time + print( + f'Completed update of "{model._meta.db_table}" in {elapsed_time} seconds', + ) + save_commodities_parent(verbose) + save_geo_areas(verbose) + update_measure(verbose) + update_measure_components(verbose) + + +def update_model_and_description(model): + print(f'Starting update of "{model._meta.db_table}"') + start_time = time.time() + with connection.cursor() as cursor: + update_model(model, cursor) + elapsed_time = time.time() - start_time + print( + f'Completed update of "{model._meta.db_table}" in {elapsed_time} seconds', + ) + tx = Transaction.objects.last() + with override_current_transaction(tx): + add_description(model) + + +def orphan_fk_queryset(model, fk_list): + queryset = model.objects.all() + for fk in fk_list: + filter = f"{fk[1]}__isnull" + subquery = fk[0].objects.exclude(**{filter: True}).values(fk[1]) + queryset = queryset.exclude(trackedmodel_ptr__in=Subquery(subquery)) + return queryset + + +def find_relations(): + """ + Creates a dictionary of tables and fk pointing to it. + + It uses the dictionary to find orphaned records, ie records that are in the + table, but are not used by any fk. The queries generated are very, very + slow, and I am not sure we need to delete the orphaned records. I am leaving + the code in, just in case. + """ + config = django.apps.apps.get_app_config(APP_LABEL) + relations = {} + inverse_relations = {} + for model in config.get_models(): + if issubclass(model, ReportModel): + references = model.referenced_models() + if references: + relations[model] = references + for field, referenced_model in references.items(): + fk = (model, field) + if referenced_model in inverse_relations: + inverse_relations[referenced_model].append(fk) + else: + inverse_relations[referenced_model] = [fk] + + # print("=====Relations===") + # rel = relations + # for x in rel: + # print(x) + # print(rel[x]) + # + # print("=====inverse_relations===") + # rel = inverse_relations + # for x in rel: + # print(f"{x} {len(rel[x])}") + # for n in rel[x]: + # print(f"\t{n}") + + for referenced_model, fk_list in inverse_relations.items(): + qs = orphan_fk_queryset(referenced_model, fk_list) + print(f"{referenced_model}, {fk_list}") + print(qs.count()) + print(qs.query) + # return relations, inverse_relations diff --git a/open_data/views.py b/open_data/views.py new file mode 100644 index 000000000..60f00ef0e --- /dev/null +++ b/open_data/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/settings/common.py b/settings/common.py index 20bba4846..bbeba63bf 100644 --- a/settings/common.py +++ b/settings/common.py @@ -128,6 +128,8 @@ "taric_parsers.apps.TaricParsersConfig", ] +OPEN_DATA_APPS = ["open_data.apps.OpenDataConfig"] + TAMATO_APPS = [ "hmrc_sdes", "importer", @@ -150,6 +152,7 @@ *DJANGO_CORE_APPS, *THIRD_PARTY_APPS, *TAMATO_APPS, + *OPEN_DATA_APPS, *DOMAIN_APPS, *APPS_THAT_MUST_COME_LAST, ] From 81a4dc558f6c6a2bae5019ad154663ed1b88a6f3 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Wed, 12 Feb 2025 17:29:07 +0000 Subject: [PATCH 02/16] Removed useless migration. --- .../migrations/0005_measureasdefinedreport.py | 112 ------------------ 1 file changed, 112 deletions(-) delete mode 100644 open_data/migrations/0005_measureasdefinedreport.py diff --git a/open_data/migrations/0005_measureasdefinedreport.py b/open_data/migrations/0005_measureasdefinedreport.py deleted file mode 100644 index 006c24652..000000000 --- a/open_data/migrations/0005_measureasdefinedreport.py +++ /dev/null @@ -1,112 +0,0 @@ -# Generated by Django 4.2.16 on 2025-02-07 12:23 - -from django.db import migrations -from django.db import models - -from open_data.models.utils import migrate_to_postgres - - -class Migration(migrations.Migration): - - # schema is not supported by sqlite. - if migrate_to_postgres(): - schema_name = 'reporting"."' - else: - schema_name = "" - - dependencies = [ - ("open_data", "0004_materialised_view"), - ] - - operations = [ - migrations.CreateModel( - name="MeasureAsDefinedReport", - fields=[ - ("id_counter", models.IntegerField()), - ( - "trackedmodel_ptr_id", - models.IntegerField(primary_key=True, serialize=False), - ), - ( - "commodity_sid", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "commodity_code", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "commodity_indent", - models.CharField(blank=True, max_length=500, null=True), - ), - ("commodity_description", models.TextField(blank=True, null=True)), - ("measure_sid", models.IntegerField()), - ( - "measure_type_id", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_type_description", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_additional_code_code", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_additional_code_description", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_duty_expression", - models.CharField(blank=True, max_length=500, null=True), - ), - ("measure_effective_start_date", models.DateField()), - ("measure_effective_end_date", models.DateField(blank=True, null=True)), - ( - "measure_reduction_indicator", - models.CharField(blank=True, max_length=500, null=True), - ), - ("measure_footnotes", models.TextField(blank=True, null=True)), - ( - "measure_conditions", - models.TextField(blank=True, null=True), - ), - ( - "measure_geographical_area_sid", - models.IntegerField(blank=True, null=True), - ), - ( - "measure_geographical_area_id", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_geographical_area_description", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_excluded_geographical_areas_ids", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_excluded_geographical_areas_descriptions", - models.TextField(blank=True, null=True), - ), - ( - "measure_quota_order_number", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_regulation_id", - models.CharField(blank=True, max_length=500, null=True), - ), - ( - "measure_regulation_url", - models.CharField(blank=True, max_length=500, null=True), - ), - ], - options={ - "db_table": f"{schema_name}open_data_reportmeasure_as_defined_report", - }, - ), - ] From c2074e6acf755d99e5376777fe8e8e7755ab363d Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Wed, 19 Feb 2025 16:10:27 +0000 Subject: [PATCH 03/16] Deliberate error in CREATE SCHEMA to check if it gets executed or not --- open_data/migrations/0001_create_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_data/migrations/0001_create_schema.py b/open_data/migrations/0001_create_schema.py index de4c4c318..234186391 100644 --- a/open_data/migrations/0001_create_schema.py +++ b/open_data/migrations/0001_create_schema.py @@ -12,7 +12,7 @@ class Migration(migrations.Migration): if migrate_to_postgres(): operations = [ migrations.RunSQL( - sql=[("CREATE SCHEMA reporting;")], + sql=[("CREATE ASCHEMA reporting;")], reverse_sql=[("DROP SCHEMA reporting;")], ), ] From bd373193620f874968bed77bfbeedc9834c6db1d Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Wed, 19 Feb 2025 16:35:50 +0000 Subject: [PATCH 04/16] Rename migrations hoping to fix the error in git testing --- .../migrations/{0001_create_schema.py => 0001_initial.py} | 4 +++- .../migrations/{0002_initial.py => 0002_create_tables.py} | 2 +- open_data/migrations/0003_drop_fk_constrains.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) rename open_data/migrations/{0001_create_schema.py => 0001_initial.py} (79%) rename open_data/migrations/{0002_initial.py => 0002_create_tables.py} (99%) diff --git a/open_data/migrations/0001_create_schema.py b/open_data/migrations/0001_initial.py similarity index 79% rename from open_data/migrations/0001_create_schema.py rename to open_data/migrations/0001_initial.py index 234186391..fc9a6b64e 100644 --- a/open_data/migrations/0001_create_schema.py +++ b/open_data/migrations/0001_initial.py @@ -4,6 +4,8 @@ from open_data.models.utils import migrate_to_postgres +# common 0013_versiongroup_common_vers_current_04c358_idx + class Migration(migrations.Migration): initial = True @@ -12,7 +14,7 @@ class Migration(migrations.Migration): if migrate_to_postgres(): operations = [ migrations.RunSQL( - sql=[("CREATE ASCHEMA reporting;")], + sql=[("CREATE SCHEMA reporting;")], reverse_sql=[("DROP SCHEMA reporting;")], ), ] diff --git a/open_data/migrations/0002_initial.py b/open_data/migrations/0002_create_tables.py similarity index 99% rename from open_data/migrations/0002_initial.py rename to open_data/migrations/0002_create_tables.py index cbc0151df..f9c222b87 100644 --- a/open_data/migrations/0002_initial.py +++ b/open_data/migrations/0002_create_tables.py @@ -16,7 +16,7 @@ class Migration(migrations.Migration): schema_name = "" dependencies = [ - ("open_data", "0001_create_schema"), + ("open_data", "0001_initial"), ] operations = [ diff --git a/open_data/migrations/0003_drop_fk_constrains.py b/open_data/migrations/0003_drop_fk_constrains.py index f26762e7f..2a6d35caf 100644 --- a/open_data/migrations/0003_drop_fk_constrains.py +++ b/open_data/migrations/0003_drop_fk_constrains.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): dependencies = [ - ("open_data", "0002_initial"), + ("open_data", "0002_create_tables"), ] if migrate_to_postgres(): From 9d05e81a8203629277140227a207c5ce746328e9 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Wed, 19 Feb 2025 16:54:39 +0000 Subject: [PATCH 05/16] Add migration dependenciencies, hoping to fix the git test error --- open_data/migrations/0001_initial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/open_data/migrations/0001_initial.py b/open_data/migrations/0001_initial.py index fc9a6b64e..a3904ce4d 100644 --- a/open_data/migrations/0001_initial.py +++ b/open_data/migrations/0001_initial.py @@ -4,13 +4,13 @@ from open_data.models.utils import migrate_to_postgres -# common 0013_versiongroup_common_vers_current_04c358_idx - class Migration(migrations.Migration): initial = True - dependencies = [] + dependencies = [ + ("common", "0013_versiongroup_common_vers_current_04c358_idx"), + ] if migrate_to_postgres(): operations = [ migrations.RunSQL( From 50f39fa7673b966d489ee472519d0e1b35c99fc8 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Thu, 20 Feb 2025 11:44:46 +0000 Subject: [PATCH 06/16] Moved test directory to proper place --- open_data/{models => }/tests/__init__.py | 0 open_data/{models => }/tests/test_populate_models.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename open_data/{models => }/tests/__init__.py (100%) rename open_data/{models => }/tests/test_populate_models.py (100%) diff --git a/open_data/models/tests/__init__.py b/open_data/tests/__init__.py similarity index 100% rename from open_data/models/tests/__init__.py rename to open_data/tests/__init__.py diff --git a/open_data/models/tests/test_populate_models.py b/open_data/tests/test_populate_models.py similarity index 100% rename from open_data/models/tests/test_populate_models.py rename to open_data/tests/test_populate_models.py From 412664e84170722976800bbec547cb71bd021529 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Fri, 21 Feb 2025 15:32:36 +0000 Subject: [PATCH 07/16] Better comment --- open_data/tests/test_populate_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_data/tests/test_populate_models.py b/open_data/tests/test_populate_models.py index 38802e028..6861682d7 100644 --- a/open_data/tests/test_populate_models.py +++ b/open_data/tests/test_populate_models.py @@ -29,7 +29,7 @@ test_models = ["TestModel1", "TestModel2", "TestModel3", "TestModelDescription1"] -def test_models_are_included(): +def test_models_are_included(create_schema): # This test will fail when a new tracked model has been created, # without creating an equivalent model in the open data app # If the new model is not relevant to open data, its name should be added From 0ecaea56b3e95914f396d13d16e714bcb828dc4f Mon Sep 17 00:00:00 2001 From: Paul Pepper Date: Fri, 21 Feb 2025 15:54:42 +0000 Subject: [PATCH 08/16] Implement in_test and use to generate correct schema names to satisfy pytest --- common/util.py | 7 +++++++ open_data/models/utils.py | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/common/util.py b/common/util.py index bbdecf610..8c73f6b46 100644 --- a/common/util.py +++ b/common/util.py @@ -783,3 +783,10 @@ def make_real_edit( cls.objects.filter(pk=obj.pk).update(update_type=UpdateType.DELETE) return None + + +def in_test() -> bool: + """Returns True if we're currently in a test situation (e.g. running via + pytest or python manage.py test), False otherwise.""" + + return os.environ.get("DJANGO_SETTINGS_MODULE") == "settings.test" diff --git a/open_data/models/utils.py b/open_data/models/utils.py index 52c5bf6b2..67d6851ff 100644 --- a/open_data/models/utils.py +++ b/open_data/models/utils.py @@ -2,6 +2,7 @@ from django.db import models +from common.util import in_test from open_data.apps import APP_LABEL SCHEMA_NAME = "reporting" @@ -22,7 +23,10 @@ def create_name_with_schema(name): # NOTE: the " around the stop are really important. # without them, the table will not be created in the correct schema # But is we are exporting to sqlite, we must omit the schma name - if migrate_to_postgres(): + + if in_test(): + return name + elif migrate_to_postgres(): return f'{SCHEMA_NAME}"."{APP_LABEL}_report{name}' else: return f"{APP_LABEL}_report{name}" From ce1c7615ef286c6a2e9958fb7787d487136ad549 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Fri, 21 Feb 2025 16:31:28 +0000 Subject: [PATCH 09/16] Changed one-to-one relationships to foreign key, to avoid errors when testing importers --- open_data/migrations/0002_create_tables.py | 181 ++++++++++---------- open_data/models/additional_codes_models.py | 4 +- open_data/models/certificates_models.py | 4 +- open_data/models/commodities_model.py | 8 +- open_data/models/footnotes_models.py | 4 +- open_data/models/geo_areas_models.py | 4 +- open_data/models/measures_models.py | 32 ++-- open_data/models/quotas_models.py | 14 +- open_data/models/regulations_models.py | 10 +- 9 files changed, 134 insertions(+), 127 deletions(-) diff --git a/open_data/migrations/0002_create_tables.py b/open_data/migrations/0002_create_tables.py index f9c222b87..8fb1cf93a 100644 --- a/open_data/migrations/0002_create_tables.py +++ b/open_data/migrations/0002_create_tables.py @@ -1,21 +1,28 @@ -# Generated by Django 4.2.16 on 2025-02-03 16:56 +# Generated by Django 4.2.16 on 2025-02-21 12:28 import django.db.models.deletion from django.db import migrations from django.db import models import common.fields -from open_data.models.utils import migrate_to_postgres class Migration(migrations.Migration): - # schema is not supported by sqlite. - if migrate_to_postgres(): - schema_name = 'reporting"."' - else: - schema_name = "" + + initial = True dependencies = [ + ("footnotes", "0006_allow_blank_descriptions"), + ("commodities", "0013_alter_goodsnomenclature_origins_and_more"), + ("quotas", "0009_alter_quotadefinition_sub_quotas_and_more"), + ("certificates", "0004_validity_start"), + ("geo_areas", "0006_alter_geographicalarea_memberships"), + ( + "regulations", + "0010_alter_regulation_amends_alter_regulation_extends_and_more", + ), + ("measures", "0017_measuresbulkeditor"), + ("additional_codes", "0007_allow_blank_descriptions"), ("open_data", "0001_initial"), ] @@ -25,7 +32,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -39,7 +46,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ], options={ - "db_table": f"{schema_name}open_data_reportadditionalcode", + "db_table": 'reporting"."open_data_reportadditionalcode', }, ), migrations.CreateModel( @@ -47,7 +54,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -64,7 +71,7 @@ class Migration(migrations.Migration): ("application_code", models.SmallIntegerField()), ], options={ - "db_table": f"{schema_name}open_data_reportadditionalcodetype", + "db_table": 'reporting"."open_data_reportadditionalcodetype', }, ), migrations.CreateModel( @@ -72,7 +79,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -85,7 +92,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ], options={ - "db_table": f"{schema_name}open_data_reportcertificate", + "db_table": 'reporting"."open_data_reportcertificate', }, ), migrations.CreateModel( @@ -93,7 +100,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -109,7 +116,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportcertificatetype", + "db_table": 'reporting"."open_data_reportcertificatetype', }, ), migrations.CreateModel( @@ -117,7 +124,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -137,7 +144,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportdutyexpression", + "db_table": 'reporting"."open_data_reportdutyexpression', }, ), migrations.CreateModel( @@ -145,7 +152,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -158,7 +165,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ], options={ - "db_table": f"{schema_name}open_data_reportfootnote", + "db_table": 'reporting"."open_data_reportfootnote', }, ), migrations.CreateModel( @@ -166,7 +173,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -183,7 +190,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportfootnotetype", + "db_table": 'reporting"."open_data_reportfootnotetype', }, ), migrations.CreateModel( @@ -191,7 +198,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -224,7 +231,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportgeographicalarea", + "db_table": 'reporting"."open_data_reportgeographicalarea', }, ), migrations.CreateModel( @@ -232,7 +239,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -257,7 +264,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportgoodsnomenclature", + "db_table": 'reporting"."open_data_reportgoodsnomenclature', }, ), migrations.CreateModel( @@ -265,7 +272,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -281,7 +288,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportgroup", + "db_table": 'reporting"."open_data_reportgroup', }, ), migrations.CreateModel( @@ -289,7 +296,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -325,7 +332,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasure", + "db_table": 'reporting"."open_data_reportmeasure', }, ), migrations.CreateModel( @@ -333,7 +340,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -350,7 +357,7 @@ class Migration(migrations.Migration): ("requires_duty", models.BooleanField()), ], options={ - "db_table": f"{schema_name}open_data_reportmeasureaction", + "db_table": 'reporting"."open_data_reportmeasureaction', }, ), migrations.CreateModel( @@ -358,7 +365,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -389,7 +396,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasurecondition", + "db_table": 'reporting"."open_data_reportmeasurecondition', }, ), migrations.CreateModel( @@ -397,7 +404,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -415,7 +422,7 @@ class Migration(migrations.Migration): ("accepts_price", models.BooleanField()), ], options={ - "db_table": f"{schema_name}open_data_reportmeasureconditioncode", + "db_table": 'reporting"."open_data_reportmeasureconditioncode', }, ), migrations.CreateModel( @@ -423,7 +430,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -440,7 +447,7 @@ class Migration(migrations.Migration): ("abbreviation", models.CharField(max_length=32)), ], options={ - "db_table": f"{schema_name}open_data_reportmeasurementunit", + "db_table": 'reporting"."open_data_reportmeasurementunit', }, ), migrations.CreateModel( @@ -448,7 +455,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -465,7 +472,7 @@ class Migration(migrations.Migration): ("abbreviation", models.CharField(max_length=32)), ], options={ - "db_table": f"{schema_name}open_data_reportmeasurementunitqualifier", + "db_table": 'reporting"."open_data_reportmeasurementunitqualifier', }, ), migrations.CreateModel( @@ -473,7 +480,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -490,7 +497,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasuretypeseries", + "db_table": 'reporting"."open_data_reportmeasuretypeseries', }, ), migrations.CreateModel( @@ -498,7 +505,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -514,7 +521,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmonetaryunit", + "db_table": 'reporting"."open_data_reportmonetaryunit', }, ), migrations.CreateModel( @@ -522,7 +529,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -573,7 +580,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportquotadefinition", + "db_table": 'reporting"."open_data_reportquotadefinition', }, ), migrations.CreateModel( @@ -581,7 +588,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -596,7 +603,7 @@ class Migration(migrations.Migration): ("category", models.SmallIntegerField()), ], options={ - "db_table": f"{schema_name}open_data_reportquotaordernumber", + "db_table": 'reporting"."open_data_reportquotaordernumber', }, ), migrations.CreateModel( @@ -604,7 +611,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -630,7 +637,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportquotaordernumberorigin", + "db_table": 'reporting"."open_data_reportquotaordernumberorigin', }, ), migrations.CreateModel( @@ -638,7 +645,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -683,7 +690,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportregulation", + "db_table": 'reporting"."open_data_reportregulation', }, ), migrations.CreateModel( @@ -691,7 +698,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -717,7 +724,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportsuspension", + "db_table": 'reporting"."open_data_reportsuspension', }, ), migrations.CreateModel( @@ -725,7 +732,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -762,7 +769,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportreplacement", + "db_table": 'reporting"."open_data_reportreplacement', }, ), migrations.CreateModel( @@ -770,7 +777,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -793,7 +800,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportquotasuspension", + "db_table": 'reporting"."open_data_reportquotasuspension', }, ), migrations.CreateModel( @@ -801,7 +808,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -825,7 +832,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportquotaordernumberoriginexclusion", + "db_table": 'reporting"."open_data_reportquotaordernumberoriginexclusion', }, ), migrations.AddField( @@ -841,7 +848,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -865,7 +872,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportquotablocking", + "db_table": 'reporting"."open_data_reportquotablocking', }, ), migrations.CreateModel( @@ -873,7 +880,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -900,7 +907,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportquotaassociation", + "db_table": 'reporting"."open_data_reportquotaassociation', }, ), migrations.CreateModel( @@ -908,7 +915,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -937,7 +944,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasuretype", + "db_table": 'reporting"."open_data_reportmeasuretype', }, ), migrations.CreateModel( @@ -945,7 +952,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -972,7 +979,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasurement", + "db_table": 'reporting"."open_data_reportmeasurement', }, ), migrations.CreateModel( @@ -980,7 +987,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1004,7 +1011,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasureexcludedgeographicalarea", + "db_table": 'reporting"."open_data_reportmeasureexcludedgeographicalarea', }, ), migrations.CreateModel( @@ -1012,7 +1019,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1063,7 +1070,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasureconditioncomponent", + "db_table": 'reporting"."open_data_reportmeasureconditioncomponent', }, ), migrations.AddField( @@ -1117,7 +1124,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1168,7 +1175,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportmeasurecomponent", + "db_table": 'reporting"."open_data_reportmeasurecomponent', }, ), migrations.AddField( @@ -1231,7 +1238,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1256,7 +1263,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportgoodsnomenclaturesuccessor", + "db_table": 'reporting"."open_data_reportgoodsnomenclaturesuccessor', }, ), migrations.CreateModel( @@ -1264,7 +1271,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1289,7 +1296,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportgoodsnomenclatureorigin", + "db_table": 'reporting"."open_data_reportgoodsnomenclatureorigin', }, ), migrations.CreateModel( @@ -1297,7 +1304,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1323,7 +1330,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportgeographicalmembership", + "db_table": 'reporting"."open_data_reportgeographicalmembership', }, ), migrations.CreateModel( @@ -1331,7 +1338,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1355,7 +1362,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportfootnoteassociationmeasure", + "db_table": 'reporting"."open_data_reportfootnoteassociationmeasure', }, ), migrations.CreateModel( @@ -1363,7 +1370,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1388,7 +1395,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportfootnoteassociationgoodsnomenclature", + "db_table": 'reporting"."open_data_reportfootnoteassociationgoodsnomenclature', }, ), migrations.AddField( @@ -1412,7 +1419,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1437,7 +1444,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportamendment", + "db_table": 'reporting"."open_data_reportamendment', }, ), migrations.CreateModel( @@ -1445,7 +1452,7 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.OneToOneField( + models.ForeignKey( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, @@ -1470,7 +1477,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": f"{schema_name}open_data_reportadditionalcodetypemeasuretype", + "db_table": 'reporting"."open_data_reportadditionalcodetypemeasuretype', }, ), migrations.AddField( diff --git a/open_data/models/additional_codes_models.py b/open_data/models/additional_codes_models.py index a5dc8c02f..c0b2ec308 100644 --- a/open_data/models/additional_codes_models.py +++ b/open_data/models/additional_codes_models.py @@ -8,7 +8,7 @@ class ReportAdditionalCodeType(ReportModel): shadowed_model = AdditionalCodeType - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -27,7 +27,7 @@ class Meta: class ReportAdditionalCode(ReportModel): shadowed_model = AdditionalCode update_description = True - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/certificates_models.py b/open_data/models/certificates_models.py index 6bd6758bb..ddf15d5c2 100644 --- a/open_data/models/certificates_models.py +++ b/open_data/models/certificates_models.py @@ -8,7 +8,7 @@ class ReportCertificateType(ReportModel): shadowed_model = CertificateType - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -26,7 +26,7 @@ class Meta: class ReportCertificate(ReportModel): shadowed_model = Certificate update_description = True - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/commodities_model.py b/open_data/models/commodities_model.py index bf186a79e..7fa7f3fa0 100644 --- a/open_data/models/commodities_model.py +++ b/open_data/models/commodities_model.py @@ -13,7 +13,7 @@ class ReportGoodsNomenclature(ReportModel): shadowed_model = GoodsNomenclature extra_where = " AND valid_between @> CURRENT_DATE" - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -44,7 +44,7 @@ class Meta: class ReportGoodsNomenclatureSuccessor(ReportModel): shadowed_model = GoodsNomenclatureSuccessor - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -68,7 +68,7 @@ class Meta: class ReportGoodsNomenclatureOrigin(ReportModel): shadowed_model = GoodsNomenclatureOrigin - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -92,7 +92,7 @@ class Meta: class ReportFootnoteAssociationGoodsNomenclature(ReportModel): shadowed_model = FootnoteAssociationGoodsNomenclature - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/footnotes_models.py b/open_data/models/footnotes_models.py index 9462545ab..a388e5a2b 100644 --- a/open_data/models/footnotes_models.py +++ b/open_data/models/footnotes_models.py @@ -9,7 +9,7 @@ class ReportFootnoteType(ReportModel): shadowed_model = FootnoteType - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -28,7 +28,7 @@ class Meta: class ReportFootnote(ReportModel): shadowed_model = Footnote update_description = True - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/geo_areas_models.py b/open_data/models/geo_areas_models.py index ba6209b5e..953623ec9 100644 --- a/open_data/models/geo_areas_models.py +++ b/open_data/models/geo_areas_models.py @@ -8,7 +8,7 @@ class ReportGeographicalArea(ReportModel): shadowed_model = GeographicalArea - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -34,7 +34,7 @@ class Meta: class ReportGeographicalMembership(ReportModel): shadowed_model = GeographicalMembership - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/measures_models.py b/open_data/models/measures_models.py index 27a77709e..267f8a60e 100644 --- a/open_data/models/measures_models.py +++ b/open_data/models/measures_models.py @@ -22,7 +22,7 @@ class ReportAdditionalCodeTypeMeasureType(ReportModel): shadowed_model = AdditionalCodeTypeMeasureType - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -43,7 +43,7 @@ class Meta: class ReportDutyExpression(ReportModel): shadowed_model = DutyExpression - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -65,7 +65,7 @@ class Meta: class ReportFootnoteAssociationMeasure(ReportModel): shadowed_model = FootnoteAssociationMeasure - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -85,7 +85,7 @@ def get_ignore_fk_list(cls): class ReportMeasure(ReportModel): shadowed_model = Measure - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -145,7 +145,7 @@ class Meta: class ReportMeasureAction(ReportModel): shadowed_model = MeasureAction - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -164,7 +164,7 @@ class Meta: class ReportMeasureConditionComponent(ReportModel): shadowed_model = MeasureConditionComponent - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -200,7 +200,7 @@ class ReportMeasureCondition(ReportModel): shadowed_model = MeasureCondition update_table = False # This table is populated using the ORM - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -267,7 +267,7 @@ class Meta: class ReportMeasureConditionCode(ReportModel): shadowed_model = MeasureConditionCode - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -287,7 +287,7 @@ class Meta: class ReportMeasurementUnit(ReportModel): shadowed_model = MeasurementUnit - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -306,7 +306,7 @@ class Meta: class ReportMeasurementUnitQualifier(ReportModel): shadowed_model = MeasurementUnitQualifier - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -325,7 +325,7 @@ class Meta: class ReportMeasureTypeSeries(ReportModel): shadowed_model = MeasureTypeSeries - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -344,7 +344,7 @@ class Meta: class ReportMonetaryUnit(ReportModel): shadowed_model = MonetaryUnit - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -362,7 +362,7 @@ class Meta: class ReportMeasureType(ReportModel): shadowed_model = MeasureType - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -390,7 +390,7 @@ class Meta: class ReportMeasurement(ReportModel): shadowed_model = Measurement - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -413,7 +413,7 @@ class Meta: class ReportMeasureExcludedGeographicalArea(ReportModel): shadowed_model = MeasureExcludedGeographicalArea - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -437,7 +437,7 @@ class Meta: class ReportMeasureComponent(ReportModel): shadowed_model = MeasureComponent - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/quotas_models.py b/open_data/models/quotas_models.py index 537ba2aae..d0ba599e6 100644 --- a/open_data/models/quotas_models.py +++ b/open_data/models/quotas_models.py @@ -14,7 +14,7 @@ class ReportQuotaAssociation(ReportModel): shadowed_model = QuotaAssociation - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -37,7 +37,7 @@ class Meta: class ReportQuotaDefinition(ReportModel): shadowed_model = QuotaDefinition - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -79,7 +79,7 @@ class Meta: class ReportQuotaOrderNumber(ReportModel): shadowed_model = QuotaOrderNumber - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -99,7 +99,7 @@ class Meta: class ReportQuotaOrderNumberOrigin(ReportModel): shadowed_model = QuotaOrderNumberOrigin - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -118,7 +118,7 @@ class Meta: class ReportQuotaSuspension(ReportModel): shadowed_model = QuotaSuspension - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -137,7 +137,7 @@ class Meta: class ReportQuotaOrderNumberOriginExclusion(ReportModel): shadowed_model = QuotaOrderNumberOriginExclusion - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -157,7 +157,7 @@ class Meta: class ReportQuotaBlocking(ReportModel): shadowed_model = QuotaBlocking - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, diff --git a/open_data/models/regulations_models.py b/open_data/models/regulations_models.py index 8052f70d6..f47dd8fb1 100644 --- a/open_data/models/regulations_models.py +++ b/open_data/models/regulations_models.py @@ -12,7 +12,7 @@ class ReportAmendment(ReportModel): shadowed_model = Amendment - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -33,7 +33,7 @@ class Meta: class ReportGroup(ReportModel): shadowed_model = Group - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -51,7 +51,7 @@ class Meta: class ReportRegulation(ReportModel): shadowed_model = Regulation - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -89,7 +89,7 @@ class Meta: class ReportSuspension(ReportModel): shadowed_model = Suspension - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, @@ -111,7 +111,7 @@ class Meta: class ReportReplacement(ReportModel): shadowed_model = Replacement - trackedmodel_ptr = models.OneToOneField( + trackedmodel_ptr = models.ForeignKey( shadowed_model, models.DO_NOTHING, primary_key=True, From afb0b776c67aa2c0c7205b0e6c7e1fef59eb2c73 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Fri, 21 Feb 2025 16:56:24 +0000 Subject: [PATCH 10/16] Better function names --- open_data/migrations/0001_initial.py | 4 ++-- open_data/migrations/0003_drop_fk_constrains.py | 4 ++-- open_data/migrations/0004_materialised_view.py | 4 ++-- open_data/models/utils.py | 15 ++++++--------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/open_data/migrations/0001_initial.py b/open_data/migrations/0001_initial.py index a3904ce4d..7be9d55df 100644 --- a/open_data/migrations/0001_initial.py +++ b/open_data/migrations/0001_initial.py @@ -2,7 +2,7 @@ from django.db import migrations -from open_data.models.utils import migrate_to_postgres +from open_data.models.utils import schema_required class Migration(migrations.Migration): @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ ("common", "0013_versiongroup_common_vers_current_04c358_idx"), ] - if migrate_to_postgres(): + if schema_required(): operations = [ migrations.RunSQL( sql=[("CREATE SCHEMA reporting;")], diff --git a/open_data/migrations/0003_drop_fk_constrains.py b/open_data/migrations/0003_drop_fk_constrains.py index 2a6d35caf..a47402805 100644 --- a/open_data/migrations/0003_drop_fk_constrains.py +++ b/open_data/migrations/0003_drop_fk_constrains.py @@ -2,7 +2,7 @@ from django.db import migrations -from open_data.models.utils import migrate_to_postgres +from open_data.models.utils import schema_required # It will be impossible to update the tables in the open data area with the # foreign keys constrain in place. But it is useful to declare them in the Django @@ -30,7 +30,7 @@ class Migration(migrations.Migration): ("open_data", "0002_create_tables"), ] - if migrate_to_postgres(): + if schema_required(): operations = [ migrations.RunSQL(drop_fk_sql), ] diff --git a/open_data/migrations/0004_materialised_view.py b/open_data/migrations/0004_materialised_view.py index c63d15dca..3919342db 100644 --- a/open_data/migrations/0004_materialised_view.py +++ b/open_data/migrations/0004_materialised_view.py @@ -2,7 +2,7 @@ from django.db import migrations -from open_data.models.utils import migrate_to_postgres +from open_data.models.utils import schema_required create_view_sql = """ CREATE MATERIALIZED VIEW reporting.foreign_key_lookup AS @@ -22,7 +22,7 @@ class Migration(migrations.Migration): dependencies = [ ("open_data", "0003_drop_fk_constrains"), ] - if migrate_to_postgres(): + if schema_required(): operations = [ migrations.RunSQL(create_view_sql), ] diff --git a/open_data/models/utils.py b/open_data/models/utils.py index 67d6851ff..9d32fc4d0 100644 --- a/open_data/models/utils.py +++ b/open_data/models/utils.py @@ -9,24 +9,21 @@ LOOK_UP_VIEW = f"{SCHEMA_NAME}.foreign_key_lookup" -def migrate_to_postgres(): +def schema_required(): data_base_url = os.environ.get("DATABASE_URL", "Postgres").lower() # schema is not supported by sqlite. # During sqlite dump creation, DATABASE_URL is set to sqlitexxxxx # so we can establish if we are handlind sqlite, and ignore the schema name - if "sqlite" in data_base_url: + # And we can't use the schema during testing + if "sqlite" in data_base_url or in_test(): return False return True -def create_name_with_schema(name): +def create_open_data_name(name): # NOTE: the " around the stop are really important. # without them, the table will not be created in the correct schema - # But is we are exporting to sqlite, we must omit the schma name - - if in_test(): - return name - elif migrate_to_postgres(): + if schema_required(): return f'{SCHEMA_NAME}"."{APP_LABEL}_report{name}' else: return f"{APP_LABEL}_report{name}" @@ -64,7 +61,7 @@ def create_table_name(shadowed_model): # with 'report', the app_label and the schema shadowed_tb_name = shadowed_model._meta.db_table new_name = shadowed_tb_name.split(shadowed_model._meta.app_label + "_")[1] - return create_name_with_schema(new_name) + return create_open_data_name(new_name) @classmethod def extra_queries(cls): From 80d67a9b2d434d22cd08953a3757bb0e27a8214b Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Mon, 24 Feb 2025 09:34:24 +0000 Subject: [PATCH 11/16] Safety commit. --- open_data/migrations/0002_create_tables.py | 87 +++++++++++---------- open_data/models/additional_codes_models.py | 6 +- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/open_data/migrations/0002_create_tables.py b/open_data/migrations/0002_create_tables.py index 8fb1cf93a..e29268c2c 100644 --- a/open_data/migrations/0002_create_tables.py +++ b/open_data/migrations/0002_create_tables.py @@ -5,12 +5,11 @@ from django.db import models import common.fields +from open_data.models.utils import schema_required class Migration(migrations.Migration): - initial = True - dependencies = [ ("footnotes", "0006_allow_blank_descriptions"), ("commodities", "0013_alter_goodsnomenclature_origins_and_more"), @@ -26,6 +25,10 @@ class Migration(migrations.Migration): ("open_data", "0001_initial"), ] + if schema_required(): + schema_name = 'reporting"."' + else: + schema_name = "" operations = [ migrations.CreateModel( name="ReportAdditionalCode", @@ -46,7 +49,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ], options={ - "db_table": 'reporting"."open_data_reportadditionalcode', + "db_table": f"{schema_name}open_data_reportadditionalcode", }, ), migrations.CreateModel( @@ -71,7 +74,7 @@ class Migration(migrations.Migration): ("application_code", models.SmallIntegerField()), ], options={ - "db_table": 'reporting"."open_data_reportadditionalcodetype', + "db_table": f"{schema_name}open_data_reportadditionalcodetype", }, ), migrations.CreateModel( @@ -92,7 +95,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ], options={ - "db_table": 'reporting"."open_data_reportcertificate', + "db_table": f"{schema_name}open_data_reportcertificate", }, ), migrations.CreateModel( @@ -116,7 +119,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportcertificatetype', + "db_table": f"{schema_name}open_data_reportcertificatetype", }, ), migrations.CreateModel( @@ -144,7 +147,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportdutyexpression', + "db_table": f"{schema_name}open_data_reportdutyexpression", }, ), migrations.CreateModel( @@ -165,7 +168,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ], options={ - "db_table": 'reporting"."open_data_reportfootnote', + "db_table": f"{schema_name}open_data_reportfootnote", }, ), migrations.CreateModel( @@ -190,7 +193,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportfootnotetype', + "db_table": f"{schema_name}open_data_reportfootnotetype", }, ), migrations.CreateModel( @@ -231,7 +234,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportgeographicalarea', + "db_table": f"{schema_name}open_data_reportgeographicalarea", }, ), migrations.CreateModel( @@ -264,7 +267,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportgoodsnomenclature', + "db_table": f"{schema_name}open_data_reportgoodsnomenclature", }, ), migrations.CreateModel( @@ -288,7 +291,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportgroup', + "db_table": f"{schema_name}open_data_reportgroup", }, ), migrations.CreateModel( @@ -332,7 +335,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasure', + "db_table": f"{schema_name}open_data_reportmeasure", }, ), migrations.CreateModel( @@ -357,7 +360,7 @@ class Migration(migrations.Migration): ("requires_duty", models.BooleanField()), ], options={ - "db_table": 'reporting"."open_data_reportmeasureaction', + "db_table": f"{schema_name}open_data_reportmeasureaction", }, ), migrations.CreateModel( @@ -396,7 +399,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasurecondition', + "db_table": f"{schema_name}open_data_reportmeasurecondition", }, ), migrations.CreateModel( @@ -422,7 +425,7 @@ class Migration(migrations.Migration): ("accepts_price", models.BooleanField()), ], options={ - "db_table": 'reporting"."open_data_reportmeasureconditioncode', + "db_table": f"{schema_name}open_data_reportmeasureconditioncode", }, ), migrations.CreateModel( @@ -447,7 +450,7 @@ class Migration(migrations.Migration): ("abbreviation", models.CharField(max_length=32)), ], options={ - "db_table": 'reporting"."open_data_reportmeasurementunit', + "db_table": f"{schema_name}open_data_reportmeasurementunit", }, ), migrations.CreateModel( @@ -472,7 +475,7 @@ class Migration(migrations.Migration): ("abbreviation", models.CharField(max_length=32)), ], options={ - "db_table": 'reporting"."open_data_reportmeasurementunitqualifier', + "db_table": f"{schema_name}open_data_reportmeasurementunitqualifier", }, ), migrations.CreateModel( @@ -497,7 +500,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasuretypeseries', + "db_table": f"{schema_name}open_data_reportmeasuretypeseries", }, ), migrations.CreateModel( @@ -521,7 +524,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmonetaryunit', + "db_table": f"{schema_name}open_data_reportmonetaryunit", }, ), migrations.CreateModel( @@ -580,7 +583,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportquotadefinition', + "db_table": f"{schema_name}open_data_reportquotadefinition", }, ), migrations.CreateModel( @@ -603,7 +606,7 @@ class Migration(migrations.Migration): ("category", models.SmallIntegerField()), ], options={ - "db_table": 'reporting"."open_data_reportquotaordernumber', + "db_table": f"{schema_name}open_data_reportquotaordernumber", }, ), migrations.CreateModel( @@ -637,7 +640,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportquotaordernumberorigin', + "db_table": f"{schema_name}open_data_reportquotaordernumberorigin", }, ), migrations.CreateModel( @@ -690,7 +693,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportregulation', + "db_table": f"{schema_name}open_data_reportregulation", }, ), migrations.CreateModel( @@ -724,7 +727,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportsuspension', + "db_table": f"{schema_name}open_data_reportsuspension", }, ), migrations.CreateModel( @@ -769,7 +772,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportreplacement', + "db_table": f"{schema_name}open_data_reportreplacement", }, ), migrations.CreateModel( @@ -800,7 +803,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportquotasuspension', + "db_table": f"{schema_name}open_data_reportquotasuspension", }, ), migrations.CreateModel( @@ -832,7 +835,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportquotaordernumberoriginexclusion', + "db_table": f"{schema_name}open_data_reportquotaordernumberoriginexclusion", }, ), migrations.AddField( @@ -872,7 +875,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportquotablocking', + "db_table": f"{schema_name}open_data_reportquotablocking", }, ), migrations.CreateModel( @@ -907,7 +910,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportquotaassociation', + "db_table": f"{schema_name}open_data_reportquotaassociation", }, ), migrations.CreateModel( @@ -944,7 +947,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasuretype', + "db_table": f"{schema_name}open_data_reportmeasuretype", }, ), migrations.CreateModel( @@ -979,7 +982,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasurement', + "db_table": f"{schema_name}open_data_reportmeasurement", }, ), migrations.CreateModel( @@ -1011,7 +1014,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasureexcludedgeographicalarea', + "db_table": f"{schema_name}open_data_reportmeasureexcludedgeographicalarea", }, ), migrations.CreateModel( @@ -1070,7 +1073,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasureconditioncomponent', + "db_table": f"{schema_name}open_data_reportmeasureconditioncomponent", }, ), migrations.AddField( @@ -1175,7 +1178,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportmeasurecomponent', + "db_table": f"{schema_name}open_data_reportmeasurecomponent", }, ), migrations.AddField( @@ -1263,7 +1266,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportgoodsnomenclaturesuccessor', + "db_table": f"{schema_name}open_data_reportgoodsnomenclaturesuccessor", }, ), migrations.CreateModel( @@ -1296,7 +1299,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportgoodsnomenclatureorigin', + "db_table": f"{schema_name}open_data_reportgoodsnomenclatureorigin", }, ), migrations.CreateModel( @@ -1330,7 +1333,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportgeographicalmembership', + "db_table": f"{schema_name}open_data_reportgeographicalmembership", }, ), migrations.CreateModel( @@ -1362,7 +1365,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportfootnoteassociationmeasure', + "db_table": f"{schema_name}open_data_reportfootnoteassociationmeasure", }, ), migrations.CreateModel( @@ -1395,7 +1398,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportfootnoteassociationgoodsnomenclature', + "db_table": f"{schema_name}open_data_reportfootnoteassociationgoodsnomenclature", }, ), migrations.AddField( @@ -1444,7 +1447,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportamendment', + "db_table": f"{schema_name}open_data_reportamendment", }, ), migrations.CreateModel( @@ -1477,7 +1480,7 @@ class Migration(migrations.Migration): ), ], options={ - "db_table": 'reporting"."open_data_reportadditionalcodetypemeasuretype', + "db_table": f"{schema_name}open_data_reportadditionalcodetypemeasuretype", }, ), migrations.AddField( diff --git a/open_data/models/additional_codes_models.py b/open_data/models/additional_codes_models.py index c0b2ec308..12a5a28d2 100644 --- a/open_data/models/additional_codes_models.py +++ b/open_data/models/additional_codes_models.py @@ -5,10 +5,14 @@ from common.fields import TaricDateRangeField from open_data.models.utils import ReportModel +# class OneToOneNWField(models.OneToOneField): +# def __init__(self, *args, **kwargs): +# super().__init__(*args, **kwargs) + class ReportAdditionalCodeType(ReportModel): shadowed_model = AdditionalCodeType - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, From 799a510f62f46f86279820b788623de6b48b0c06 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Mon, 24 Feb 2025 09:57:01 +0000 Subject: [PATCH 12/16] Better comments --- open_data/models/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/open_data/models/utils.py b/open_data/models/utils.py index 9d32fc4d0..1e05a5c44 100644 --- a/open_data/models/utils.py +++ b/open_data/models/utils.py @@ -14,7 +14,9 @@ def schema_required(): # schema is not supported by sqlite. # During sqlite dump creation, DATABASE_URL is set to sqlitexxxxx # so we can establish if we are handlind sqlite, and ignore the schema name - # And we can't use the schema during testing + # And we can't use the schema during testing, because pytest does not run the + # migrations, but generates the sql from the table definition. So the migration + # creating the partition is not run, and the table creation fails. if "sqlite" in data_base_url or in_test(): return False return True From b13888ed3289a76750b390c15ff136b31c42b285 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Mon, 24 Feb 2025 10:38:36 +0000 Subject: [PATCH 13/16] Safety commit. --- open_data/migrations/0001_initial.py | 5 +- open_data/migrations/0002_create_tables.py | 138 ++++++++++++-------- open_data/models/additional_codes_models.py | 4 +- open_data/models/certificates_models.py | 6 +- open_data/models/commodities_model.py | 14 +- open_data/models/footnotes_models.py | 6 +- open_data/models/geo_areas_models.py | 6 +- open_data/models/measures_models.py | 48 ++++--- open_data/models/quotas_models.py | 21 ++- open_data/models/regulations_models.py | 15 ++- 10 files changed, 167 insertions(+), 96 deletions(-) diff --git a/open_data/migrations/0001_initial.py b/open_data/migrations/0001_initial.py index 7be9d55df..1ea2d2a1e 100644 --- a/open_data/migrations/0001_initial.py +++ b/open_data/migrations/0001_initial.py @@ -8,9 +8,8 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ("common", "0013_versiongroup_common_vers_current_04c358_idx"), - ] + dependencies = [] + if schema_required(): operations = [ migrations.RunSQL( diff --git a/open_data/migrations/0002_create_tables.py b/open_data/migrations/0002_create_tables.py index e29268c2c..d37d074e1 100644 --- a/open_data/migrations/0002_create_tables.py +++ b/open_data/migrations/0002_create_tables.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2025-02-21 12:28 +# Generated by Django 4.2.16 on 2025-02-24 10:32 import django.db.models.deletion from django.db import migrations @@ -10,18 +10,9 @@ class Migration(migrations.Migration): + initial = True + dependencies = [ - ("footnotes", "0006_allow_blank_descriptions"), - ("commodities", "0013_alter_goodsnomenclature_origins_and_more"), - ("quotas", "0009_alter_quotadefinition_sub_quotas_and_more"), - ("certificates", "0004_validity_start"), - ("geo_areas", "0006_alter_geographicalarea_memberships"), - ( - "regulations", - "0010_alter_regulation_amends_alter_regulation_extends_and_more", - ), - ("measures", "0017_measuresbulkeditor"), - ("additional_codes", "0007_allow_blank_descriptions"), ("open_data", "0001_initial"), ] @@ -29,16 +20,18 @@ class Migration(migrations.Migration): schema_name = 'reporting"."' else: schema_name = "" + operations = [ migrations.CreateModel( name="ReportAdditionalCode", fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="additional_codes.additionalcode", ), @@ -57,10 +50,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="additional_codes.additionalcodetype", ), @@ -82,10 +76,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="certificates.certificate", ), @@ -103,10 +98,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="certificates.certificatetype", ), @@ -127,10 +123,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.dutyexpression", ), @@ -155,10 +152,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="footnotes.footnote", ), @@ -176,10 +174,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="footnotes.footnotetype", ), @@ -201,10 +200,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="geo_areas.geographicalarea", ), @@ -242,10 +242,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="commodities.goodsnomenclature", ), @@ -259,7 +260,7 @@ class Migration(migrations.Migration): ("description", models.TextField(blank=True, null=True)), ( "parent_trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( null=True, on_delete=django.db.models.deletion.DO_NOTHING, to="open_data.reportgoodsnomenclature", @@ -275,10 +276,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="regulations.group", ), @@ -299,10 +301,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measure", ), @@ -343,10 +346,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measureaction", ), @@ -368,10 +372,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measurecondition", ), @@ -407,10 +412,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measureconditioncode", ), @@ -433,10 +439,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measurementunit", ), @@ -458,10 +465,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measurementunitqualifier", ), @@ -483,10 +491,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measuretypeseries", ), @@ -508,10 +517,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.monetaryunit", ), @@ -532,10 +542,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotadefinition", ), @@ -591,10 +602,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotaordernumber", ), @@ -614,10 +626,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotaordernumberorigin", ), @@ -648,10 +661,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="regulations.regulation", ), @@ -701,10 +715,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="regulations.suspension", ), @@ -735,10 +750,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="regulations.replacement", ), @@ -780,10 +796,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotasuspension", ), @@ -811,10 +828,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotaordernumberoriginexclusion", ), @@ -851,10 +869,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotablocking", ), @@ -883,10 +902,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="quotas.quotaassociation", ), @@ -918,10 +938,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measuretype", ), @@ -955,10 +976,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measurement", ), @@ -990,10 +1012,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measureexcludedgeographicalarea", ), @@ -1022,10 +1045,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measureconditioncomponent", ), @@ -1127,10 +1151,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.measurecomponent", ), @@ -1241,10 +1266,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="commodities.goodsnomenclaturesuccessor", ), @@ -1274,10 +1300,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="commodities.goodsnomenclatureorigin", ), @@ -1307,10 +1334,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="geo_areas.geographicalmembership", ), @@ -1341,10 +1369,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.footnoteassociationmeasure", ), @@ -1373,10 +1402,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="commodities.footnoteassociationgoodsnomenclature", ), @@ -1422,10 +1452,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="regulations.amendment", ), @@ -1455,10 +1486,11 @@ class Migration(migrations.Migration): fields=[ ( "trackedmodel_ptr", - models.ForeignKey( + models.OneToOneField( db_column="trackedmodel_ptr_id", on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, + related_name="+", serialize=False, to="measures.additionalcodetypemeasuretype", ), diff --git a/open_data/models/additional_codes_models.py b/open_data/models/additional_codes_models.py index 12a5a28d2..99147314f 100644 --- a/open_data/models/additional_codes_models.py +++ b/open_data/models/additional_codes_models.py @@ -17,6 +17,7 @@ class ReportAdditionalCodeType(ReportModel): models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -31,11 +32,12 @@ class Meta: class ReportAdditionalCode(ReportModel): shadowed_model = AdditionalCode update_description = True - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) diff --git a/open_data/models/certificates_models.py b/open_data/models/certificates_models.py index ddf15d5c2..af6260410 100644 --- a/open_data/models/certificates_models.py +++ b/open_data/models/certificates_models.py @@ -8,11 +8,12 @@ class ReportCertificateType(ReportModel): shadowed_model = CertificateType - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -26,11 +27,12 @@ class Meta: class ReportCertificate(ReportModel): shadowed_model = Certificate update_description = True - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) sid = models.CharField(max_length=3) diff --git a/open_data/models/commodities_model.py b/open_data/models/commodities_model.py index 7fa7f3fa0..6f0b171dd 100644 --- a/open_data/models/commodities_model.py +++ b/open_data/models/commodities_model.py @@ -13,11 +13,12 @@ class ReportGoodsNomenclature(ReportModel): shadowed_model = GoodsNomenclature extra_where = " AND valid_between @> CURRENT_DATE" - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) sid = models.IntegerField() @@ -27,7 +28,7 @@ class ReportGoodsNomenclature(ReportModel): indent = models.IntegerField(null=True) description = models.TextField(blank=True, null=True) - parent_trackedmodel_ptr = models.ForeignKey( + parent_trackedmodel_ptr = models.OneToOneField( "self", models.DO_NOTHING, null=True, @@ -44,11 +45,12 @@ class Meta: class ReportGoodsNomenclatureSuccessor(ReportModel): shadowed_model = GoodsNomenclatureSuccessor - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) absorbed_into_goods_nomenclature = models.ForeignKey( @@ -68,11 +70,12 @@ class Meta: class ReportGoodsNomenclatureOrigin(ReportModel): shadowed_model = GoodsNomenclatureOrigin - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) derived_from_goods_nomenclature = models.ForeignKey( @@ -92,11 +95,12 @@ class Meta: class ReportFootnoteAssociationGoodsNomenclature(ReportModel): shadowed_model = FootnoteAssociationGoodsNomenclature - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) diff --git a/open_data/models/footnotes_models.py b/open_data/models/footnotes_models.py index a388e5a2b..ba5f8c1cb 100644 --- a/open_data/models/footnotes_models.py +++ b/open_data/models/footnotes_models.py @@ -9,11 +9,12 @@ class ReportFootnoteType(ReportModel): shadowed_model = FootnoteType - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -28,11 +29,12 @@ class Meta: class ReportFootnote(ReportModel): shadowed_model = Footnote update_description = True - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) footnote_id = models.CharField(max_length=5) diff --git a/open_data/models/geo_areas_models.py b/open_data/models/geo_areas_models.py index 953623ec9..75c93cde1 100644 --- a/open_data/models/geo_areas_models.py +++ b/open_data/models/geo_areas_models.py @@ -8,11 +8,12 @@ class ReportGeographicalArea(ReportModel): shadowed_model = GeographicalArea - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -34,11 +35,12 @@ class Meta: class ReportGeographicalMembership(ReportModel): shadowed_model = GeographicalMembership - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) diff --git a/open_data/models/measures_models.py b/open_data/models/measures_models.py index 267f8a60e..fc543a49c 100644 --- a/open_data/models/measures_models.py +++ b/open_data/models/measures_models.py @@ -22,11 +22,12 @@ class ReportAdditionalCodeTypeMeasureType(ReportModel): shadowed_model = AdditionalCodeTypeMeasureType - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -43,11 +44,12 @@ class Meta: class ReportDutyExpression(ReportModel): shadowed_model = DutyExpression - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -65,11 +67,12 @@ class Meta: class ReportFootnoteAssociationMeasure(ReportModel): shadowed_model = FootnoteAssociationMeasure - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) associated_footnote = models.ForeignKey("ReportFootnote", models.DO_NOTHING) @@ -85,11 +88,12 @@ def get_ignore_fk_list(cls): class ReportMeasure(ReportModel): shadowed_model = Measure - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) extra_where = ( @@ -145,11 +149,12 @@ class Meta: class ReportMeasureAction(ReportModel): shadowed_model = MeasureAction - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -164,11 +169,12 @@ class Meta: class ReportMeasureConditionComponent(ReportModel): shadowed_model = MeasureConditionComponent - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) duty_amount = models.DecimalField( @@ -200,11 +206,12 @@ class ReportMeasureCondition(ReportModel): shadowed_model = MeasureCondition update_table = False # This table is populated using the ORM - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) sid = models.IntegerField() @@ -267,11 +274,12 @@ class Meta: class ReportMeasureConditionCode(ReportModel): shadowed_model = MeasureConditionCode - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -287,11 +295,12 @@ class Meta: class ReportMeasurementUnit(ReportModel): shadowed_model = MeasurementUnit - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -306,11 +315,12 @@ class Meta: class ReportMeasurementUnitQualifier(ReportModel): shadowed_model = MeasurementUnitQualifier - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -325,11 +335,12 @@ class Meta: class ReportMeasureTypeSeries(ReportModel): shadowed_model = MeasureTypeSeries - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -344,11 +355,12 @@ class Meta: class ReportMonetaryUnit(ReportModel): shadowed_model = MonetaryUnit - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -362,11 +374,12 @@ class Meta: class ReportMeasureType(ReportModel): shadowed_model = MeasureType - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -390,11 +403,12 @@ class Meta: class ReportMeasurement(ReportModel): shadowed_model = Measurement - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -413,11 +427,12 @@ class Meta: class ReportMeasureExcludedGeographicalArea(ReportModel): shadowed_model = MeasureExcludedGeographicalArea - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) excluded_geographical_area = models.ForeignKey( @@ -437,11 +452,12 @@ class Meta: class ReportMeasureComponent(ReportModel): shadowed_model = MeasureComponent - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) duty_amount = models.DecimalField( diff --git a/open_data/models/quotas_models.py b/open_data/models/quotas_models.py index d0ba599e6..cab73ffc6 100644 --- a/open_data/models/quotas_models.py +++ b/open_data/models/quotas_models.py @@ -14,11 +14,12 @@ class ReportQuotaAssociation(ReportModel): shadowed_model = QuotaAssociation - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) sub_quota_relation_type = models.CharField(max_length=2) @@ -37,11 +38,12 @@ class Meta: class ReportQuotaDefinition(ReportModel): shadowed_model = QuotaDefinition - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -79,11 +81,12 @@ class Meta: class ReportQuotaOrderNumber(ReportModel): shadowed_model = QuotaOrderNumber - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -99,11 +102,12 @@ class Meta: class ReportQuotaOrderNumberOrigin(ReportModel): shadowed_model = QuotaOrderNumberOrigin - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -118,11 +122,12 @@ class Meta: class ReportQuotaSuspension(ReportModel): shadowed_model = QuotaSuspension - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -137,11 +142,12 @@ class Meta: class ReportQuotaOrderNumberOriginExclusion(ReportModel): shadowed_model = QuotaOrderNumberOriginExclusion - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) excluded_geographical_area = models.ForeignKey( @@ -157,11 +163,12 @@ class Meta: class ReportQuotaBlocking(ReportModel): shadowed_model = QuotaBlocking - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) diff --git a/open_data/models/regulations_models.py b/open_data/models/regulations_models.py index f47dd8fb1..99fa402e6 100644 --- a/open_data/models/regulations_models.py +++ b/open_data/models/regulations_models.py @@ -12,11 +12,12 @@ class ReportAmendment(ReportModel): shadowed_model = Amendment - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) enacting_regulation = models.ForeignKey("ReportRegulation", models.DO_NOTHING) @@ -33,11 +34,12 @@ class Meta: class ReportGroup(ReportModel): shadowed_model = Group - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) valid_between = TaricDateRangeField(db_index=True) @@ -51,11 +53,12 @@ class Meta: class ReportRegulation(ReportModel): shadowed_model = Regulation - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) role_type = models.IntegerField() @@ -89,11 +92,12 @@ class Meta: class ReportSuspension(ReportModel): shadowed_model = Suspension - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) effective_end_date = models.DateField(blank=True, null=True) @@ -111,11 +115,12 @@ class Meta: class ReportReplacement(ReportModel): shadowed_model = Replacement - trackedmodel_ptr = models.ForeignKey( + trackedmodel_ptr = models.OneToOneField( shadowed_model, models.DO_NOTHING, primary_key=True, db_column="trackedmodel_ptr_id", + related_name="+", ) measure_type_id = models.CharField(max_length=6, blank=True, null=True) From 314bbd58fc32750e484be87f9702c8057e1443b5 Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Mon, 24 Feb 2025 12:08:31 +0000 Subject: [PATCH 14/16] Test passing --- open_data/migrations/0003_drop_fk_constrains.py | 5 +++-- open_data/migrations/0004_materialised_view.py | 12 +++++++----- open_data/models/utils.py | 11 +++++++++-- open_data/tasks.py | 4 ++-- open_data/tests/test_populate_models.py | 2 +- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/open_data/migrations/0003_drop_fk_constrains.py b/open_data/migrations/0003_drop_fk_constrains.py index a47402805..59ab93ccd 100644 --- a/open_data/migrations/0003_drop_fk_constrains.py +++ b/open_data/migrations/0003_drop_fk_constrains.py @@ -2,6 +2,7 @@ from django.db import migrations +from common.util import in_test from open_data.models.utils import schema_required # It will be impossible to update the tables in the open data area with the @@ -16,7 +17,7 @@ BEGIN FOR r IN SELECT table_schema, table_name, constraint_name FROM information_schema.table_constraints AS tc - WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema='reporting' + WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name like 'open_data%' LOOP EXECUTE 'ALTER TABLE '|| quote_ident(r.table_schema) || '.' || quote_ident(r.table_name)|| ' DROP CONSTRAINT '|| quote_ident(r.constraint_name) || ';'; END LOOP; @@ -30,7 +31,7 @@ class Migration(migrations.Migration): ("open_data", "0002_create_tables"), ] - if schema_required(): + if schema_required() or in_test(): operations = [ migrations.RunSQL(drop_fk_sql), ] diff --git a/open_data/migrations/0004_materialised_view.py b/open_data/migrations/0004_materialised_view.py index 3919342db..97ecd4a0c 100644 --- a/open_data/migrations/0004_materialised_view.py +++ b/open_data/migrations/0004_materialised_view.py @@ -2,18 +2,20 @@ from django.db import migrations +from common.util import in_test +from open_data.models.utils import get_lookup_name from open_data.models.utils import schema_required -create_view_sql = """ -CREATE MATERIALIZED VIEW reporting.foreign_key_lookup AS +create_view_sql = f""" +CREATE MATERIALIZED VIEW {get_lookup_name()} AS SELECT common_trackedmodel.ID as old_id, current_version_id FROM public.common_trackedmodel INNER JOIN common_versiongroup ON (common_trackedmodel.version_group_id = common_versiongroup.id) WHERE (current_version_id IS NOT NULL AND NOT (common_trackedmodel.update_type = 2)); -CREATE UNIQUE INDEX old_id_idx ON reporting.foreign_key_lookup (old_id); -CREATE INDEX current_version_id_idx ON reporting.foreign_key_lookup (current_version_id); +CREATE UNIQUE INDEX old_id_idx ON {get_lookup_name()} (old_id); +CREATE INDEX current_version_id_idx ON {get_lookup_name()} (current_version_id); """ @@ -22,7 +24,7 @@ class Migration(migrations.Migration): dependencies = [ ("open_data", "0003_drop_fk_constrains"), ] - if schema_required(): + if schema_required() or in_test(): operations = [ migrations.RunSQL(create_view_sql), ] diff --git a/open_data/models/utils.py b/open_data/models/utils.py index 1e05a5c44..b6f9e3f12 100644 --- a/open_data/models/utils.py +++ b/open_data/models/utils.py @@ -6,7 +6,6 @@ from open_data.apps import APP_LABEL SCHEMA_NAME = "reporting" -LOOK_UP_VIEW = f"{SCHEMA_NAME}.foreign_key_lookup" def schema_required(): @@ -22,6 +21,14 @@ def schema_required(): return True +def get_lookup_name(): + if schema_required(): + prefix = f"{SCHEMA_NAME}." + else: + prefix = "" + return f"{prefix}foreign_key_lookup" + + def create_open_data_name(name): # NOTE: the " around the stop are really important. # without them, the table will not be created in the correct schema @@ -132,7 +139,7 @@ def update_fk_queries(cls): query_list.append( f'UPDATE "{cls._meta.db_table}" ' f"SET {f.column}=current_version_id " - f" FROM {LOOK_UP_VIEW} " + f" FROM {get_lookup_name()} " f" WHERE {f.column} = old_id;", ) return query_list diff --git a/open_data/tasks.py b/open_data/tasks.py index 7ea38f286..aa677e60a 100644 --- a/open_data/tasks.py +++ b/open_data/tasks.py @@ -12,8 +12,8 @@ from open_data.geo_areas import save_geo_areas from open_data.measures import update_measure from open_data.measures import update_measure_components -from open_data.models.utils import LOOK_UP_VIEW from open_data.models.utils import ReportModel +from open_data.models.utils import get_lookup_name def add_description(model, verbose=True): @@ -71,7 +71,7 @@ def populate_open_data(verbose=False): config = django.apps.apps.get_app_config(APP_LABEL) with connection.cursor() as cursor: - cursor.execute(f"REFRESH MATERIALIZED VIEW {LOOK_UP_VIEW};") + cursor.execute(f"REFRESH MATERIALIZED VIEW {get_lookup_name()};") for model in config.get_models(): if issubclass(model, ReportModel): if verbose: diff --git a/open_data/tests/test_populate_models.py b/open_data/tests/test_populate_models.py index 6861682d7..38802e028 100644 --- a/open_data/tests/test_populate_models.py +++ b/open_data/tests/test_populate_models.py @@ -29,7 +29,7 @@ test_models = ["TestModel1", "TestModel2", "TestModel3", "TestModelDescription1"] -def test_models_are_included(create_schema): +def test_models_are_included(): # This test will fail when a new tracked model has been created, # without creating an equivalent model in the open data app # If the new model is not relevant to open data, its name should be added From b9087d9014b644936eadf3bf77b92e7a75c8076a Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Mon, 24 Feb 2025 15:41:02 +0000 Subject: [PATCH 15/16] Safety commit --- open_data/migrations/0003_drop_fk_constrains.py | 5 +---- open_data/migrations/0004_materialised_view.py | 5 ++--- open_data/models/utils.py | 10 ++++++++-- open_data/tests/test_populate_models.py | 1 + 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/open_data/migrations/0003_drop_fk_constrains.py b/open_data/migrations/0003_drop_fk_constrains.py index 59ab93ccd..f0415532e 100644 --- a/open_data/migrations/0003_drop_fk_constrains.py +++ b/open_data/migrations/0003_drop_fk_constrains.py @@ -2,9 +2,6 @@ from django.db import migrations -from common.util import in_test -from open_data.models.utils import schema_required - # It will be impossible to update the tables in the open data area with the # foreign keys constrain in place. But it is useful to declare them in the Django # models, so Django will create the correct queryset: the following query t @@ -31,7 +28,7 @@ class Migration(migrations.Migration): ("open_data", "0002_create_tables"), ] - if schema_required() or in_test(): + if not is_sql(): operations = [ migrations.RunSQL(drop_fk_sql), ] diff --git a/open_data/migrations/0004_materialised_view.py b/open_data/migrations/0004_materialised_view.py index 97ecd4a0c..622722af8 100644 --- a/open_data/migrations/0004_materialised_view.py +++ b/open_data/migrations/0004_materialised_view.py @@ -2,9 +2,8 @@ from django.db import migrations -from common.util import in_test from open_data.models.utils import get_lookup_name -from open_data.models.utils import schema_required +from open_data.models.utils import in_sqlite create_view_sql = f""" CREATE MATERIALIZED VIEW {get_lookup_name()} AS @@ -24,7 +23,7 @@ class Migration(migrations.Migration): dependencies = [ ("open_data", "0003_drop_fk_constrains"), ] - if schema_required() or in_test(): + if not in_sqlite(): operations = [ migrations.RunSQL(create_view_sql), ] diff --git a/open_data/models/utils.py b/open_data/models/utils.py index b6f9e3f12..a771582d5 100644 --- a/open_data/models/utils.py +++ b/open_data/models/utils.py @@ -8,15 +8,21 @@ SCHEMA_NAME = "reporting" -def schema_required(): +def in_sqlite(): data_base_url = os.environ.get("DATABASE_URL", "Postgres").lower() + if "sqlite" in data_base_url: + return True + return False + + +def schema_required(): # schema is not supported by sqlite. # During sqlite dump creation, DATABASE_URL is set to sqlitexxxxx # so we can establish if we are handlind sqlite, and ignore the schema name # And we can't use the schema during testing, because pytest does not run the # migrations, but generates the sql from the table definition. So the migration # creating the partition is not run, and the table creation fails. - if "sqlite" in data_base_url or in_test(): + if in_sqlite() or in_test(): return False return True diff --git a/open_data/tests/test_populate_models.py b/open_data/tests/test_populate_models.py index 38802e028..6d2044cf9 100644 --- a/open_data/tests/test_populate_models.py +++ b/open_data/tests/test_populate_models.py @@ -12,6 +12,7 @@ pytestmark = pytest.mark.django_db + excluded_models = [ "QuotaEvent", # excluded because not available in sqlite exported to data.gov "GeographicalAreaDescription", # Description are merged in the described table From 3836a77b571087771bb03b4e43c3252335a31d4a Mon Sep 17 00:00:00 2001 From: Luisella Strona Date: Mon, 24 Feb 2025 17:03:29 +0000 Subject: [PATCH 16/16] More changes for tests --- open_data/direct_sql.py | 36 +++++++++++++++++++ .../migrations/0003_drop_fk_constrains.py | 24 +++---------- .../migrations/0004_materialised_view.py | 17 ++------- open_data/tests/test_populate_models.py | 11 ++++++ 4 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 open_data/direct_sql.py diff --git a/open_data/direct_sql.py b/open_data/direct_sql.py new file mode 100644 index 000000000..f9c4c454d --- /dev/null +++ b/open_data/direct_sql.py @@ -0,0 +1,36 @@ +from open_data.apps import APP_LABEL +from open_data.models.utils import get_lookup_name + + +def get_create_materialised_view_sql(): + return f""" + CREATE MATERIALIZED VIEW IF NOT EXISTS {get_lookup_name()} AS + SELECT common_trackedmodel.ID as old_id, current_version_id + FROM public.common_trackedmodel + INNER JOIN common_versiongroup + ON (common_trackedmodel.version_group_id = common_versiongroup.id) + WHERE (current_version_id IS NOT NULL AND NOT (common_trackedmodel.update_type = 2)); + + CREATE UNIQUE INDEX old_id_idx ON {get_lookup_name()} (old_id); + CREATE INDEX current_version_id_idx ON {get_lookup_name()} (current_version_id); + """ + + +def get_drop_fk_sql(): + # It will be impossible to update the tables in the open data area with the + # foreign keys constrain in place. But it is useful to declare them in the Django + # models, so Django will create the correct queryset: the following query t + # dropped them in the database while they are still the model definition. + # The 'magic' query has been copied from somewhere in Stackoverflow! + + return f""" + DO $$DECLARE r record; + BEGIN + FOR r IN SELECT table_schema, table_name, constraint_name + FROM information_schema.table_constraints AS tc + WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name like '{APP_LABEL}%' + LOOP + EXECUTE 'ALTER TABLE '|| quote_ident(r.table_schema) || '.' || quote_ident(r.table_name)|| ' DROP CONSTRAINT '|| quote_ident(r.constraint_name) || ';'; + END LOOP; + END$$; + """ diff --git a/open_data/migrations/0003_drop_fk_constrains.py b/open_data/migrations/0003_drop_fk_constrains.py index f0415532e..00c213398 100644 --- a/open_data/migrations/0003_drop_fk_constrains.py +++ b/open_data/migrations/0003_drop_fk_constrains.py @@ -2,24 +2,8 @@ from django.db import migrations -# It will be impossible to update the tables in the open data area with the -# foreign keys constrain in place. But it is useful to declare them in the Django -# models, so Django will create the correct queryset: the following query t -# dropped them in the database while they are still the model definition. -# The 'magic' query has been copied from somewhere in Stackoverflow! - - -drop_fk_sql = """ -DO $$DECLARE r record; - BEGIN - FOR r IN SELECT table_schema, table_name, constraint_name - FROM information_schema.table_constraints AS tc - WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name like 'open_data%' - LOOP - EXECUTE 'ALTER TABLE '|| quote_ident(r.table_schema) || '.' || quote_ident(r.table_name)|| ' DROP CONSTRAINT '|| quote_ident(r.constraint_name) || ';'; - END LOOP; - END$$; -""" +from open_data.direct_sql import get_drop_fk_sql +from open_data.models.utils import in_sqlite class Migration(migrations.Migration): @@ -28,9 +12,9 @@ class Migration(migrations.Migration): ("open_data", "0002_create_tables"), ] - if not is_sql(): + if not in_sqlite(): operations = [ - migrations.RunSQL(drop_fk_sql), + migrations.RunSQL(get_drop_fk_sql()), ] else: operations = [] diff --git a/open_data/migrations/0004_materialised_view.py b/open_data/migrations/0004_materialised_view.py index 622722af8..3079a0e1b 100644 --- a/open_data/migrations/0004_materialised_view.py +++ b/open_data/migrations/0004_materialised_view.py @@ -2,21 +2,9 @@ from django.db import migrations -from open_data.models.utils import get_lookup_name +from open_data.direct_sql import get_create_materialised_view_sql from open_data.models.utils import in_sqlite -create_view_sql = f""" -CREATE MATERIALIZED VIEW {get_lookup_name()} AS - SELECT common_trackedmodel.ID as old_id, current_version_id - FROM public.common_trackedmodel - INNER JOIN common_versiongroup - ON (common_trackedmodel.version_group_id = common_versiongroup.id) -WHERE (current_version_id IS NOT NULL AND NOT (common_trackedmodel.update_type = 2)); - -CREATE UNIQUE INDEX old_id_idx ON {get_lookup_name()} (old_id); -CREATE INDEX current_version_id_idx ON {get_lookup_name()} (current_version_id); -""" - class Migration(migrations.Migration): @@ -24,8 +12,9 @@ class Migration(migrations.Migration): ("open_data", "0003_drop_fk_constrains"), ] if not in_sqlite(): + operations = [ - migrations.RunSQL(create_view_sql), + migrations.RunSQL(get_create_materialised_view_sql()), ] else: operations = [] diff --git a/open_data/tests/test_populate_models.py b/open_data/tests/test_populate_models.py index 6d2044cf9..1a7ccd1fd 100644 --- a/open_data/tests/test_populate_models.py +++ b/open_data/tests/test_populate_models.py @@ -1,10 +1,13 @@ import pytest +from django.db import connection from common.models import TrackedModel from common.tests import factories from footnotes.models import Footnote from footnotes.models import FootnoteDescription from measures.models import Measure +from open_data.direct_sql import get_create_materialised_view_sql +from open_data.direct_sql import get_drop_fk_sql from open_data.models import ReportFootnote from open_data.models import ReportMeasure from open_data.models.utils import ReportModel @@ -47,7 +50,14 @@ def test_models_are_included(): assert len(missing_models) == 0 +def run_required_sql(): + with connection.cursor() as cursor: + cursor.execute(get_drop_fk_sql()) + cursor.execute(get_create_materialised_view_sql()) + + def test_measures_unpublished_and_unapproved(): + run_required_sql() factories.MeasureFactory.create( transaction=factories.UnapprovedTransactionFactory.create(), ) @@ -68,6 +78,7 @@ def test_measures_unpublished_and_unapproved(): def test_footnotes(): + run_required_sql() published_transaction = factories.PublishedTransactionFactory.create() factories.ApprovedTransactionFactory.create() test_description = "Test description"