diff --git a/docs/source/scripts/schema.py b/docs/source/scripts/schema.py index 08a36b576..4b9f3a3ff 100644 --- a/docs/source/scripts/schema.py +++ b/docs/source/scripts/schema.py @@ -1,61 +1,13 @@ import glob import json import os +import pathlib -import json_schema_for_humans.generate as Gen +from ravens.schema import RavensSchema, generate_schema_docs +from ravens.uml import UMLExclusions -from ravens.io import parse_uml_data -from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions -from ravens.cim_tools.graph import build_generalization_graph, build_attribute_graph -from ravens.cim_tools.template import CIMTemplate -from ravens.schema.build_definitions import build_definitions -from ravens.schema.build_map import add_attributes_to_template -from ravens.schema.build_schema import build_schema_from_map -from ravens.schema.add_copyright_notice import add_cim_copyright_notice_to_decomposed_schemas -from ravens.schema.decompose_schema import Schemas - - -def build_schema_docs(): - current_dir = os.path.dirname(os.path.abspath(__file__)) - template_path = os.path.join(current_dir, "../../../ravens/cim_tools/cim_conversion_template.json") - xmi_path = os.path.join(current_dir, "../../../cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi") - tmp_dir = os.path.join(current_dir, "../tmp") - static_schema_dir = os.path.join(current_dir, "../_static/schema") - schema_md_dir = os.path.join(current_dir, "../schema") - - uml_data = parse_uml_data(xmi_path) - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - schema = build_schema_from_map( - add_attributes_to_template( - CIMTemplate(template_path).template, - CIMTemplate(template_path).template, - uml_data, - build_generalization_graph(uml_data, exclude_packages, exclude_objects), - build_attribute_graph(uml_data, exclude_packages, exclude_objects), - ) - ) - - schema["$defs"] = build_definitions(uml_data) - - a = Schemas(schema, base_id_uri=f"file://{tmp_dir}") - - add_cim_copyright_notice_to_decomposed_schemas(a.schemas, uml_data) - - for k, v in a.schemas.items(): - filename = k.split("/")[-1].replace(".json", "") - with open(os.path.join(tmp_dir, f"{filename}.json"), "w") as f: - json.dump(v, f, indent=2) - - config = Gen.GenerationConfiguration(template_name="js") - Gen.generate_from_filename(tmp_dir, static_schema_dir, config=config) +def modify_schema_docs_resource_paths(static_schema_dir: pathlib.PosixPath): for file in glob.glob(os.path.join(static_schema_dir, "*.html")): with open(file, "r") as f: f_str = f.read() @@ -66,6 +18,8 @@ def build_schema_docs(): with open(file, "w") as f: f.write(f_str) + +def build_markdown_file(schema_md_dir: pathlib.PosixPath, static_schema_dir: pathlib.PosixPath): md_file = """# Schema Documentation ## Root Schema @@ -77,8 +31,25 @@ def build_schema_docs(): """ for file in sorted(glob.glob("*.html", root_dir=static_schema_dir)): - if file != "__main__.html": + if file != "Root.html": md_file = md_file + f"[{file.split(".html")[0]}](../_static/schema/{file})" + "{.external}" + "\n\n" with open(os.path.join(schema_md_dir, "index.md"), "w") as f: f.write(md_file) + + +def build_schema_docs(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + tmp_dir = os.path.join(current_dir, "../tmp") + static_schema_dir = os.path.join(current_dir, "../_static/schema") + schema_md_dir = os.path.join(current_dir, "../schema") + + a = RavensSchema(uml_exclusions=UMLExclusions().exclude_by_name_startswith(["Inf", "Mkt"])) + + a.export_schemas(tmp_dir) + + generate_schema_docs(tmp_dir, static_schema_dir) + + modify_schema_docs_resource_paths(static_schema_dir) + + build_markdown_file(schema_md_dir, static_schema_dir) diff --git a/docs/source/scripts/uml.py b/docs/source/scripts/uml.py index ed1b033b9..e80f4c8eb 100644 --- a/docs/source/scripts/uml.py +++ b/docs/source/scripts/uml.py @@ -1,20 +1,17 @@ import os -from ravens.io import parse_uml_data -from ravens.uml.d3 import save_uml_diagrams_from_package_name +from ravens.uml import UMLVisualizer def build_uml_docs(): - xmi_file = "../../cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - uml_data = parse_uml_data(xmi_file) - current_dir = os.path.dirname(os.path.abspath(__file__)) static_uml_path = os.path.join(current_dir, "../_static/uml") + uml_vis = UMLVisualizer() md_str = "# UML Diagrams for MG-RAVENS Schema\n" for package_name in ["EconomicDesign", "SimplifiedDiagrams", "EquipmentExtensions", "Software"]: - paths = save_uml_diagrams_from_package_name(uml_data, package_name, static_uml_path) + paths = uml_vis.save_uml_diagrams_from_package_name(package_name, static_uml_path) md_str = ( md_str + f"\n## {package_name}\n" diff --git a/examples/.gitignore b/examples/.gitignore index 3e52e1afe..3275bc961 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -4,6 +4,7 @@ !case3_balanced.dss !case3_balanced.xml +!case3_balanced.json !IEEE13_Assets.xml !IEEE13.xml !/schema diff --git a/examples/case3_balanced.json b/examples/case3_balanced.json new file mode 100644 index 000000000..2b37f3f4d --- /dev/null +++ b/examples/case3_balanced.json @@ -0,0 +1,724 @@ +{ + "Versions": { + "IEC61970CIMVersion": { + "Ravens.CimObjectType": "IEC61970CIMVersion", + "IEC61970CIMVersion.version": "IEC61970CIM100", + "IEC61970CIMVersion.date": "2019-04-01" + } + }, + "CoordinateSystem": { + "3bus_example_CrsUrn": { + "Ravens.CimObjectType": "CoordinateSystem", + "IdentifiedObject.mRID": "4B23ED1D-DE22-4C79-847D-8CEDB5ED3BB8", + "IdentifiedObject.name": "3bus_example_CrsUrn", + "CoordinateSystem.crsUrn": "OpenDSSLocalBusCoordinates" + } + }, + "Group": { + "GeographicalRegion": { + "3bus_example_Region": { + "Ravens.CimObjectType": "GeographicalRegion", + "IdentifiedObject.mRID": "06F63EA7-95CA-4C67-A8CD-75FC7134C783", + "IdentifiedObject.name": "3bus_example_Region" + } + }, + "SubGeographicalRegion": { + "3bus_example_SubRegion": { + "Ravens.CimObjectType": "SubGeographicalRegion", + "IdentifiedObject.mRID": "0A71D916-386B-437D-BA6D-829B0E8D56CC", + "IdentifiedObject.name": "3bus_example_SubRegion", + "SubGeographicalRegion.Region": "GeographicalRegion::'3bus_example_Region'" + } + }, + "EquipmentContainer": { + "Ravens.CimObjectType": "Substation", + "IdentifiedObject.mRID": "C2DA3682-25E9-4230-B4F5-F1CFE34BA032", + "IdentifiedObject.name": "3bus_example_Substation", + "PowerSystemResource.Location": "Location::'3bus_example_Location'", + "Substation.Region": "SubGeographicalRegion::'3bus_example_SubRegion'" + }, + "TopologicalIsland": { + "3bus_example_Island": { + "Ravens.CimObjectType": "TopologicalIsland", + "IdentifiedObject.mRID": "DC4C9999-F99D-4186-92E8-368FF2DB73A5", + "IdentifiedObject.name": "3bus_example_Island" + } + } + }, + "Location": { + "3bus_example_Location": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "5FD01F12-5088-43C6-9A2D-55ACA8D1B54B", + "IdentifiedObject.name": "3bus_example_Location", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'" + }, + "source_Loc": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "76B2F0D5-3220-4826-83D7-86F3A9377DB0", + "IdentifiedObject.name": "source_Loc", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'", + "Location.PositionPoints": [ + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 1, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + } + ] + }, + "ohline_Loc": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "73E41322-7292-4F70-984C-D8FDC5DEF3D6", + "IdentifiedObject.name": "ohline_Loc", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'", + "Location.PositionPoints": [ + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 1, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + }, + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 2, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + } + ] + }, + "quad_Loc": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "A9C5BFEE-FFFB-4CDC-94EC-813291D042A4", + "IdentifiedObject.name": "quad_Loc", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'", + "Location.PositionPoints": [ + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 1, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + }, + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 2, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + } + ] + }, + "l1_Loc": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "9AF605E6-C8C7-4C44-A1C7-7BA4A94EE055", + "IdentifiedObject.name": "l1_Loc", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'", + "Location.PositionPoints": [ + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 1, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + } + ] + }, + "l2_Loc": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "7F38296C-C2B4-4090-8D3D-4B9387DFF0F6", + "IdentifiedObject.name": "l2_Loc", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'", + "Location.PositionPoints": [ + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 1, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + } + ] + }, + "l3_Loc": { + "Ravens.CimObjectType": "Location", + "IdentifiedObject.mRID": "894CB6CD-6730-4B19-8CF3-3EA2A2DC499A", + "IdentifiedObject.name": "l3_Loc", + "Location.CoordinateSystem": "CoordinateSystem::'3bus_example_CrsUrn'", + "Location.PositionPoints": [ + { + "Ravens.CimObjectType": "PositionPoint", + "PositionPoint.sequenceNumber": 1, + "PositionPoint.xPosition": 0, + "PositionPoint.yPosition": 0 + } + ] + } + }, + "OperationalLimitSet": { + "OpLimI_400.0_600.0": { + "Ravens.CimObjectType": "OperationalLimitSet", + "IdentifiedObject.mRID": "33779B76-EAF3-464A-A2FD-F993F6771EE0", + "IdentifiedObject.name": "OpLimI_400.0_600.0", + "OperationalLimitSet.OperationalLimitValue": [ + { + "Ravens.CimObjectType": "CurrentLimit", + "IdentifiedObject.mRID": "757260ED-B962-4A95-AE16-D8DC2BC9D015", + "IdentifiedObject.name": "OpLimI_400.0_600.0_Norm", + "CurrentLimit.value": 400, + "OperationalLimit.OperationalLimitType": { + "Ravens.CimObjectType": "OperationalLimitType", + "IdentifiedObject.mRID": "FC11B84D-3580-4F3D-BC35-931EA7822FC1", + "IdentifiedObject.name": "3bus_example_NormAmpsType", + "OperationalLimitType.acceptableDuration": 5000000000.0, + "OperationalLimitType.direction": "OperationalLimitDirectionKind.absoluteValue" + } + }, + { + "Ravens.CimObjectType": "CurrentLimit", + "IdentifiedObject.mRID": "581B1409-BAAA-4413-A396-D673B8A1DB0A", + "IdentifiedObject.name": "OpLimI_400.0_600.0_Emerg", + "CurrentLimit.value": 600, + "OperationalLimit.OperationalLimitType": { + "Ravens.CimObjectType": "OperationalLimitType", + "IdentifiedObject.mRID": "B0F26B7C-A630-40A5-A740-714B60011595", + "IdentifiedObject.name": "3bus_example_EmergencyAmpsType", + "OperationalLimitType.acceptableDuration": 7200, + "OperationalLimitType.direction": "OperationalLimitDirectionKind.absoluteValue" + } + } + ] + }, + "OpLimV_0.4000": { + "Ravens.CimObjectType": "OperationalLimitSet", + "IdentifiedObject.mRID": "10562B54-D064-4798-AFA7-0EE310D8220D", + "IdentifiedObject.name": "OpLimV_0.4000", + "OperationalLimitSet.OperationalLimitValue": [ + { + "Ravens.CimObjectType": "VoltageLimit", + "IdentifiedObject.mRID": "1D9EA04D-38A0-4611-A93C-429246192389", + "IdentifiedObject.name": "OpLimV_0.4000_RangeAHi", + "VoltageLimit.value": 420, + "OperationalLimit.OperationalLimitType": { + "Ravens.CimObjectType": "OperationalLimitType", + "IdentifiedObject.mRID": "2E6F1693-801F-49A5-A99C-B96CB419E9EA", + "IdentifiedObject.name": "3bus_example_RangeAHiType", + "OperationalLimitType.acceptableDuration": 5000000000.0, + "OperationalLimitType.direction": "OperationalLimitDirectionKind.high" + } + }, + { + "Ravens.CimObjectType": "VoltageLimit", + "IdentifiedObject.mRID": "4E787E3C-1369-4672-A7E1-464772FA9BD9", + "IdentifiedObject.name": "OpLimV_0.4000_RangeALo", + "VoltageLimit.value": 380, + "OperationalLimit.OperationalLimitType": { + "Ravens.CimObjectType": "OperationalLimitType", + "IdentifiedObject.mRID": "4F67CAAE-4FB8-4C50-AA06-A87D2C2BDA14", + "IdentifiedObject.name": "3bus_example_RangeALoType", + "OperationalLimitType.acceptableDuration": 5000000000.0, + "OperationalLimitType.direction": "OperationalLimitDirectionKind.low" + } + }, + { + "Ravens.CimObjectType": "VoltageLimit", + "IdentifiedObject.mRID": "17E6DB10-E322-4707-B6DF-FD5FC6349CD2", + "IdentifiedObject.name": "OpLimV_0.4000_RangeBHi", + "VoltageLimit.value": 423.33332, + "OperationalLimit.OperationalLimitType": { + "Ravens.CimObjectType": "OperationalLimitType", + "IdentifiedObject.mRID": "1578AC7E-58BE-4219-8F17-14A85E6541B0", + "IdentifiedObject.name": "3bus_example_RangeBHiType", + "OperationalLimitType.acceptableDuration": 86400, + "OperationalLimitType.direction": "OperationalLimitDirectionKind.high" + } + }, + { + "Ravens.CimObjectType": "VoltageLimit", + "IdentifiedObject.mRID": "E5A76A0F-9788-4ED7-84B2-390AEF6F083B", + "IdentifiedObject.name": "OpLimV_0.4000_RangeBLo", + "VoltageLimit.value": 366.66668, + "OperationalLimit.OperationalLimitType": { + "Ravens.CimObjectType": "OperationalLimitType", + "IdentifiedObject.mRID": "29A347FA-302C-4644-BA80-DAC03E9E2EE2", + "IdentifiedObject.name": "3bus_example_RangeBLoType", + "OperationalLimitType.acceptableDuration": 86400, + "OperationalLimitType.direction": "OperationalLimitDirectionKind.low" + } + } + ] + } + }, + "BaseVoltage": { + "BaseV_0.4000": { + "Ravens.CimObjectType": "BaseVoltage", + "IdentifiedObject.mRID": "3B536889-9800-42EA-8CC7-6ECB47DF453A", + "IdentifiedObject.name": "BaseV_0.4000", + "BaseVoltage.nominalVoltage": 400 + } + }, + "TopologicalNode": { + "sourcebus": { + "Ravens.CimObjectType": "TopologicalNode", + "IdentifiedObject.mRID": "B1C9ACA6-3E8A-4A26-B9B0-A0B5E0D2C0DA", + "IdentifiedObject.name": "sourcebus", + "TopologicalNode.TopologicalIsland": "TopologicalIsland::'3bus_example_Island'" + }, + "primary": { + "Ravens.CimObjectType": "TopologicalNode", + "IdentifiedObject.mRID": "D7DE087B-5674-4A55-ACC5-4A7A25110B39", + "IdentifiedObject.name": "primary", + "TopologicalNode.TopologicalIsland": "TopologicalIsland::'3bus_example_Island'" + }, + "loadbus": { + "Ravens.CimObjectType": "TopologicalNode", + "IdentifiedObject.mRID": "CC48C523-717D-49D2-A7F3-0B6BA1A5FBC7", + "IdentifiedObject.name": "loadbus", + "TopologicalNode.TopologicalIsland": "TopologicalIsland::'3bus_example_Island'" + } + }, + "ConnectivityNode": { + "sourcebus": { + "Ravens.CimObjectType": "ConnectivityNode", + "IdentifiedObject.mRID": "43BB61C2-23BC-4B43-AC46-4455F666F253", + "IdentifiedObject.name": "sourcebus", + "ConnectivityNode.TopologicalNode": "TopologicalNode::'sourcebus'", + "ConnectivityNode.OperationalLimitSet": "OperationalLimitSet::'OpLimV_0.4000'" + }, + "primary": { + "Ravens.CimObjectType": "ConnectivityNode", + "IdentifiedObject.mRID": "91AB9A70-5830-4BD1-86BF-9C7B00E635AE", + "IdentifiedObject.name": "primary", + "ConnectivityNode.TopologicalNode": "TopologicalNode::'primary'", + "ConnectivityNode.OperationalLimitSet": "OperationalLimitSet::'OpLimV_0.4000'" + }, + "loadbus": { + "Ravens.CimObjectType": "ConnectivityNode", + "IdentifiedObject.mRID": "01B8247A-16DE-4A17-A96D-1915BA2AB0BB", + "IdentifiedObject.name": "loadbus", + "ConnectivityNode.TopologicalNode": "TopologicalNode::'loadbus'", + "ConnectivityNode.OperationalLimitSet": "OperationalLimitSet::'OpLimV_0.4000'" + } + }, + "PowerSystemResource": { + "Equipment": { + "ConductingEquipment": { + "EnergyConnection": { + "EnergySource": { + "source": { + "Ravens.CimObjectType": "EnergySource", + "IdentifiedObject.mRID": "5599BD15-FC8A-4D79-A255-3BF43D562255", + "IdentifiedObject.name": "source", + "ConductingEquipment.BaseVoltage": "BaseVoltage::'BaseV_0.4000'", + "EnergySource.nominalVoltage": 400, + "EnergySource.voltageMagnitude": 398.36, + "EnergySource.voltageAngle": 0, + "EnergySource.r": 3.88057e-08, + "EnergySource.x": 1.552228e-07, + "EnergySource.r0": 5.069596e-08, + "EnergySource.x0": 1.5208788e-07, + "PowerSystemResource.Location": "Location::'source_Loc'", + "ConductingEquipment.Terminals": [ + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "E1B3B1E0-1758-4040-A4F9-EB0025F6D7CF", + "IdentifiedObject.name": "source_T1", + "ACDCTerminal.sequenceNumber": 1, + "Terminal.ConnectivityNode": "ConnectivityNode::'sourcebus'" + } + ] + } + }, + "EnergyConsumer": { + "l1": { + "Ravens.CimObjectType": "EnergyConsumer", + "IdentifiedObject.mRID": "8609B5EE-B1D9-473A-B206-053AB1E9B53E", + "IdentifiedObject.name": "l1", + "ConductingEquipment.BaseVoltage": "BaseVoltage::'BaseV_0.4000'", + "EnergyConsumer.p": 6000, + "EnergyConsumer.q": 3000, + "EnergyConsumer.customerCount": 1, + "EnergyConsumer.phaseConnection": "PhaseShuntConnectionKind.Y", + "EnergyConsumer.grounded": "true", + "PowerSystemResource.Location": "Location::'l1_Loc'", + "ConductingEquipment.Terminals": [ + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "88DB59EE-A7AB-4E45-84F3-B029190C61F0", + "IdentifiedObject.name": "l1_T1", + "ACDCTerminal.sequenceNumber": 1, + "Terminal.ConnectivityNode": "ConnectivityNode::'loadbus'" + } + ], + "EnergyConsumer.LoadResponseCharacteristic": { + "Ravens.CimObjectType": "LoadResponseCharacteristic", + "IdentifiedObject.mRID": "0EBDA2D2-84CA-450B-B76D-6518BFFAB5B2", + "IdentifiedObject.name": "Constant kVA", + "LoadResponseCharacteristic.exponentModel": "false", + "LoadResponseCharacteristic.pConstantImpedance": 0, + "LoadResponseCharacteristic.pConstantCurrent": 0, + "LoadResponseCharacteristic.pConstantPower": 100, + "LoadResponseCharacteristic.qConstantImpedance": 0, + "LoadResponseCharacteristic.qConstantCurrent": 0, + "LoadResponseCharacteristic.qConstantPower": 100, + "LoadResponseCharacteristic.pVoltageExponent": 0, + "LoadResponseCharacteristic.qVoltageExponent": 0, + "LoadResponseCharacteristic.pFrequencyExponent": 0, + "LoadResponseCharacteristic.qFrequencyExponent": 0 + }, + "EnergyConsumer.EnergyConsumerPhase": [ + { + "Ravens.CimObjectType": "EnergyConsumerPhase", + "IdentifiedObject.mRID": "E91C8235-32EA-4EAF-8253-B232A3B7E286", + "IdentifiedObject.name": "l1_A", + "EnergyConsumerPhase.phase": "SinglePhaseKind.A", + "EnergyConsumerPhase.p": 6000, + "EnergyConsumerPhase.q": 3000, + "PowerSystemResource.Location": "Location::'l1_Loc'" + } + ] + }, + "l2": { + "Ravens.CimObjectType": "EnergyConsumer", + "IdentifiedObject.mRID": "5D2F5CD8-0201-40F7-B135-44FC6F90FD18", + "IdentifiedObject.name": "l2", + "ConductingEquipment.BaseVoltage": "BaseVoltage::'BaseV_0.4000'", + "EnergyConsumer.p": 6000, + "EnergyConsumer.q": 3000, + "EnergyConsumer.customerCount": 1, + "EnergyConsumer.phaseConnection": "PhaseShuntConnectionKind.Y", + "EnergyConsumer.grounded": "true", + "PowerSystemResource.Location": "Location::'l2_Loc'", + "ConductingEquipment.Terminals": [ + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "1AB70AF4-4ACA-4F20-AB08-AD98E453C970", + "IdentifiedObject.name": "l2_T1", + "ACDCTerminal.sequenceNumber": 1, + "Terminal.ConnectivityNode": "ConnectivityNode::'loadbus'" + } + ], + "PowerSystemResource": { + "Equipment": { + "ConductingEquipment": { + "EnergyConnection": { + "EnergyConsumer": { + "l1": { + "EnergyConsumer.LoadResponseCharacteristic": { + "Ravens.CimObjectType": "LoadResponseCharacteristic", + "IdentifiedObject.mRID": "0EBDA2D2-84CA-450B-B76D-6518BFFAB5B2", + "IdentifiedObject.name": "Constant kVA", + "LoadResponseCharacteristic.exponentModel": "false", + "LoadResponseCharacteristic.pConstantImpedance": 0, + "LoadResponseCharacteristic.pConstantCurrent": 0, + "LoadResponseCharacteristic.pConstantPower": 100, + "LoadResponseCharacteristic.qConstantImpedance": 0, + "LoadResponseCharacteristic.qConstantCurrent": 0, + "LoadResponseCharacteristic.qConstantPower": 100, + "LoadResponseCharacteristic.pVoltageExponent": 0, + "LoadResponseCharacteristic.qVoltageExponent": 0, + "LoadResponseCharacteristic.pFrequencyExponent": 0, + "LoadResponseCharacteristic.qFrequencyExponent": 0 + } + } + } + } + } + } + }, + "EnergyConsumer.EnergyConsumerPhase": [ + { + "Ravens.CimObjectType": "EnergyConsumerPhase", + "IdentifiedObject.mRID": "AAB10119-56FE-4DF2-892C-CCC6FC4C3E75", + "IdentifiedObject.name": "l2_B", + "EnergyConsumerPhase.phase": "SinglePhaseKind.B", + "EnergyConsumerPhase.p": 6000, + "EnergyConsumerPhase.q": 3000, + "PowerSystemResource.Location": "Location::'l2_Loc'" + } + ] + }, + "l3": { + "Ravens.CimObjectType": "EnergyConsumer", + "IdentifiedObject.mRID": "6B86B9A6-7373-4C71-9601-690490E2EF25", + "IdentifiedObject.name": "l3", + "ConductingEquipment.BaseVoltage": "BaseVoltage::'BaseV_0.4000'", + "EnergyConsumer.p": 6000, + "EnergyConsumer.q": 3000, + "EnergyConsumer.customerCount": 1, + "EnergyConsumer.phaseConnection": "PhaseShuntConnectionKind.Y", + "EnergyConsumer.grounded": "true", + "PowerSystemResource.Location": "Location::'l3_Loc'", + "ConductingEquipment.Terminals": [ + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "3B20F640-DA83-4A8E-8C85-66E2E8ACE996", + "IdentifiedObject.name": "l3_T1", + "ACDCTerminal.sequenceNumber": 1, + "Terminal.ConnectivityNode": "ConnectivityNode::'loadbus'" + } + ], + "PowerSystemResource": { + "Equipment": { + "ConductingEquipment": { + "EnergyConnection": { + "EnergyConsumer": { + "l1": { + "EnergyConsumer.LoadResponseCharacteristic": { + "Ravens.CimObjectType": "LoadResponseCharacteristic", + "IdentifiedObject.mRID": "0EBDA2D2-84CA-450B-B76D-6518BFFAB5B2", + "IdentifiedObject.name": "Constant kVA", + "LoadResponseCharacteristic.exponentModel": "false", + "LoadResponseCharacteristic.pConstantImpedance": 0, + "LoadResponseCharacteristic.pConstantCurrent": 0, + "LoadResponseCharacteristic.pConstantPower": 100, + "LoadResponseCharacteristic.qConstantImpedance": 0, + "LoadResponseCharacteristic.qConstantCurrent": 0, + "LoadResponseCharacteristic.qConstantPower": 100, + "LoadResponseCharacteristic.pVoltageExponent": 0, + "LoadResponseCharacteristic.qVoltageExponent": 0, + "LoadResponseCharacteristic.pFrequencyExponent": 0, + "LoadResponseCharacteristic.qFrequencyExponent": 0 + } + } + } + } + } + } + }, + "EnergyConsumer.EnergyConsumerPhase": [ + { + "Ravens.CimObjectType": "EnergyConsumerPhase", + "IdentifiedObject.mRID": "5C9A8BFE-3C2C-4B79-B111-56B37D39C802", + "IdentifiedObject.name": "l3_C", + "EnergyConsumerPhase.phase": "SinglePhaseKind.C", + "EnergyConsumerPhase.p": 6000, + "EnergyConsumerPhase.q": 3000, + "PowerSystemResource.Location": "Location::'l3_Loc'" + } + ] + } + } + }, + "Conductor": { + "ACLineSegment": { + "ohline": { + "Ravens.CimObjectType": "ACLineSegment", + "IdentifiedObject.mRID": "AE333ACB-2BA7-4C90-BA0E-3B52583DBC19", + "IdentifiedObject.name": "ohline", + "ConductingEquipment.BaseVoltage": "BaseVoltage::'BaseV_0.4000'", + "Conductor.length": 1, + "PowerSystemResource.Location": "Location::'ohline_Loc'", + "ACLineSegment.PerLengthImpedance": "PerLengthPhaseImpedance::'556mcm'", + "ConductingEquipment.Terminals": [ + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "B3A4C11A-6983-476A-B730-4DAB0E47A34D", + "IdentifiedObject.name": "ohline_T1", + "ACDCTerminal.sequenceNumber": 1, + "Terminal.ConnectivityNode": "ConnectivityNode::'sourcebus'" + }, + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "D7A2BCCF-EFB2-426D-B74D-9E228402BAC1", + "IdentifiedObject.name": "ohline_T2", + "ACDCTerminal.sequenceNumber": 2, + "Terminal.ConnectivityNode": "ConnectivityNode::'primary'" + } + ], + "ACLineSegment.ACLineSegmentPhase": [ + { + "Ravens.CimObjectType": "ACLineSegmentPhase", + "IdentifiedObject.mRID": "D17AD79B-4C9D-4145-96E5-3730A8B302B1", + "IdentifiedObject.name": "ohline_A", + "ACLineSegmentPhase.phase": "SinglePhaseKind.A", + "ACLineSegmentPhase.sequenceNumber": 1, + "PowerSystemResource.Location": "Location::'ohline_Loc'" + }, + { + "Ravens.CimObjectType": "ACLineSegmentPhase", + "IdentifiedObject.mRID": "BAF5D874-53AF-4BD7-843B-2AF5E5B51096", + "IdentifiedObject.name": "ohline_B", + "ACLineSegmentPhase.phase": "SinglePhaseKind.B", + "ACLineSegmentPhase.sequenceNumber": 2, + "PowerSystemResource.Location": "Location::'ohline_Loc'" + }, + { + "Ravens.CimObjectType": "ACLineSegmentPhase", + "IdentifiedObject.mRID": "64F4C2E5-4976-412A-9975-32561C4A58C4", + "IdentifiedObject.name": "ohline_C", + "ACLineSegmentPhase.phase": "SinglePhaseKind.C", + "ACLineSegmentPhase.sequenceNumber": 3, + "PowerSystemResource.Location": "Location::'ohline_Loc'" + } + ] + }, + "quad": { + "Ravens.CimObjectType": "ACLineSegment", + "IdentifiedObject.mRID": "8C41B256-2D52-4123-B104-4A8E339C754F", + "IdentifiedObject.name": "quad", + "ConductingEquipment.BaseVoltage": "BaseVoltage::'BaseV_0.4000'", + "Conductor.length": 1, + "PowerSystemResource.Location": "Location::'quad_Loc'", + "ACLineSegment.PerLengthImpedance": "PerLengthPhaseImpedance::'4/0quad'", + "ConductingEquipment.Terminals": [ + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "1D6BB695-87AD-462B-B466-D8E99EFE7BB5", + "IdentifiedObject.name": "quad_T1", + "ACDCTerminal.sequenceNumber": 1, + "Terminal.ConnectivityNode": "ConnectivityNode::'primary'" + }, + { + "Ravens.CimObjectType": "Terminal", + "IdentifiedObject.mRID": "B6F52D54-F107-4BAC-9817-ED8D7D19DC4C", + "IdentifiedObject.name": "quad_T2", + "ACDCTerminal.sequenceNumber": 2, + "Terminal.ConnectivityNode": "ConnectivityNode::'loadbus'" + } + ], + "ACLineSegment.ACLineSegmentPhase": [ + { + "Ravens.CimObjectType": "ACLineSegmentPhase", + "IdentifiedObject.mRID": "61916E4A-B45D-48CB-88E5-D101E54C6F0E", + "IdentifiedObject.name": "quad_A", + "ACLineSegmentPhase.phase": "SinglePhaseKind.A", + "ACLineSegmentPhase.sequenceNumber": 1, + "PowerSystemResource.Location": "Location::'quad_Loc'" + }, + { + "Ravens.CimObjectType": "ACLineSegmentPhase", + "IdentifiedObject.mRID": "461ADBEA-56C7-47FF-AA0F-9924CF296B4A", + "IdentifiedObject.name": "quad_B", + "ACLineSegmentPhase.phase": "SinglePhaseKind.B", + "ACLineSegmentPhase.sequenceNumber": 2, + "PowerSystemResource.Location": "Location::'quad_Loc'" + }, + { + "Ravens.CimObjectType": "ACLineSegmentPhase", + "IdentifiedObject.mRID": "FC99A019-D6AE-4A1D-85DA-613397524D13", + "IdentifiedObject.name": "quad_C", + "ACLineSegmentPhase.phase": "SinglePhaseKind.C", + "ACLineSegmentPhase.sequenceNumber": 3, + "PowerSystemResource.Location": "Location::'quad_Loc'" + } + ] + } + } + } + } + } + }, + "PerLengthLineParameter": { + "PerLengthImpedance": { + "PerLengthPhaseImpedance": { + "556mcm": { + "Ravens.CimObjectType": "PerLengthPhaseImpedance", + "IdentifiedObject.mRID": "310333B8-C0C0-4050-99D1-8FA4B908BCD0", + "IdentifiedObject.name": "556mcm", + "PerLengthPhaseImpedance.conductorCount": 3, + "PerLengthPhaseImpedance.PhaseImpedanceData": [ + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 1, + "PhaseImpedanceData.column": 1, + "PhaseImpedanceData.r": 0.1, + "PhaseImpedanceData.x": 0.0583, + "PhaseImpedanceData.b": 1.6e-05 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 2, + "PhaseImpedanceData.column": 1, + "PhaseImpedanceData.r": 0.04, + "PhaseImpedanceData.x": 0.0233, + "PhaseImpedanceData.b": 0 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 2, + "PhaseImpedanceData.column": 2, + "PhaseImpedanceData.r": 0.1, + "PhaseImpedanceData.x": 0.0583, + "PhaseImpedanceData.b": 1.6e-05 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 3, + "PhaseImpedanceData.column": 1, + "PhaseImpedanceData.r": 0.04, + "PhaseImpedanceData.x": 0.0233, + "PhaseImpedanceData.b": 0 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 3, + "PhaseImpedanceData.column": 2, + "PhaseImpedanceData.r": 0.04, + "PhaseImpedanceData.x": 0.0233, + "PhaseImpedanceData.b": 0 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 3, + "PhaseImpedanceData.column": 3, + "PhaseImpedanceData.r": 0.1, + "PhaseImpedanceData.x": 0.0583, + "PhaseImpedanceData.b": 1.6e-05 + } + ] + }, + "4/0quad": { + "Ravens.CimObjectType": "PerLengthPhaseImpedance", + "IdentifiedObject.mRID": "84D09E3E-40B6-4765-8C63-197B9391797E", + "IdentifiedObject.name": "4/0quad", + "PerLengthPhaseImpedance.conductorCount": 3, + "PerLengthPhaseImpedance.PhaseImpedanceData": [ + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 1, + "PhaseImpedanceData.column": 1, + "PhaseImpedanceData.r": 0.1167, + "PhaseImpedanceData.x": 0.0667, + "PhaseImpedanceData.b": 1.6e-05 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 2, + "PhaseImpedanceData.column": 1, + "PhaseImpedanceData.r": 0.0467, + "PhaseImpedanceData.x": 0.0267, + "PhaseImpedanceData.b": 0 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 2, + "PhaseImpedanceData.column": 2, + "PhaseImpedanceData.r": 0.1167, + "PhaseImpedanceData.x": 0.0667, + "PhaseImpedanceData.b": 1.6e-05 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 3, + "PhaseImpedanceData.column": 1, + "PhaseImpedanceData.r": 0.0467, + "PhaseImpedanceData.x": 0.0267, + "PhaseImpedanceData.b": 0 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 3, + "PhaseImpedanceData.column": 2, + "PhaseImpedanceData.r": 0.0467, + "PhaseImpedanceData.x": 0.0267, + "PhaseImpedanceData.b": 0 + }, + { + "Ravens.CimObjectType": "PhaseImpedanceData", + "PhaseImpedanceData.row": 3, + "PhaseImpedanceData.column": 3, + "PhaseImpedanceData.r": 0.1167, + "PhaseImpedanceData.x": 0.0667, + "PhaseImpedanceData.b": 1.6e-05 + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/examples/schema/BatteryUnit.json b/examples/schema/BatteryUnit.json index e69de29bb..ab2eaaef4 100644 --- a/examples/schema/BatteryUnit.json +++ b/examples/schema/BatteryUnit.json @@ -0,0 +1,22 @@ +{ + "PowerSystemResource": { + "Equipment": { + "ConductingEquipment": { + "EnergyConnection": { + "RegulatingCondEq": { + "PowerElectronicsConnection": { + "battery": { + "IdentifiedObject.name": "battery", + "IdentifiedObject.mRID": "8a611147-1eb3-4cc6-9df8-4b49873717b1", + "Ravens.cimObjectType": "PowerElectronicsUnit", + "PowerElectronicsConnection.PowerElectronicsUnit": { + "Ravens.cimObjectType": "BatteryUnit" + } + } + } + } + } + } + } + } +} diff --git a/examples/schema/ConnectivityNodes.json b/examples/schema/ConnectivityNodes.json index e69de29bb..89d7764cc 100644 --- a/examples/schema/ConnectivityNodes.json +++ b/examples/schema/ConnectivityNodes.json @@ -0,0 +1,19 @@ +{ + "ConnectivityNode": { + "sourcebus": { + "IdentifiedObject.name": "sourcebus", + "IdentifiedObject.mRID": "45984648-c6b4-41ef-9106-03787a523f41", + "Ravens.cimObjectType": "ConnectivityNode" + }, + "primary": { + "IdentifiedObject.name": "primary", + "IdentifiedObject.mRID": "f05ce4e3-130e-4c71-a5d1-d94cf3178334", + "Ravens.cimObjectType": "ConnectivityNode" + }, + "loadbus": { + "IdentifiedObject.name": "loadbus", + "IdentifiedObject.mRID": "197dc17e-90fd-42a5-8c5f-846bfec8c299", + "Ravens.cimObjectType": "ConnectivityNode" + } + } +} diff --git a/pyproject.toml b/pyproject.toml index 70c0f403e..96eba984e 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,23 +6,23 @@ authors = ["David M Fobes "] license = "Apache-2" readme = "README.md" packages = [{include = "ravens"}] +include = ["lib/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi", "lib/template.json"] [tool.poetry.dependencies] python = "^3.12" networkx = "^3.2" pandas = "^2.1" -pandas_access = "^0.0.1" rdflib = "^7.0" -json-schema-for-humans = "^0.46" +json-schema-for-humans = "^1" markdownify = "^0.13" lxml = "^5.3.0" opendssdirect-py = {extras = ["extras"], version = "^0.9.4"} jschon= {extras = ["requests"], version = "^0.11.1"} [tool.poetry.group.dev.dependencies] -pytest = "^7.4.0" -sphinx = "^7.2.6" -myst-parser = "^2.0.0" +pytest = "^8.3.3" +sphinx = "^8.1.3" +myst-parser = "^4.0.0" sphinx-press-theme = "^0.9.1" [build-system] diff --git a/ravens/cim_tools/common.py b/ravens/cim_tools/common.py deleted file mode 100644 index d4afd0461..000000000 --- a/ravens/cim_tools/common.py +++ /dev/null @@ -1,16 +0,0 @@ -import pandas as pd - - -def build_package_exclusions(package_data: pd.DataFrame, lambda_func) -> list: - return [pkg.Index for pkg in package_data.itertuples() if lambda_func(pkg)] - - -def build_object_exclusions(object_data: pd.DataFrame, lambda_func, exclude_packages=None) -> list: - if exclude_packages is None: - exclude_packages = [] - - return [obj.Index for obj in object_data.itertuples() if lambda_func(obj)] + [obj.Index for p in exclude_packages for obj in object_data[object_data["Package_ID"] == p].itertuples()] - - -def get_names_of_enumeration_classes(object_data): - return [o.Name for o in object_data.itertuples() if o.Object_Type == "Class" and o.Stereotype == "enumeration"] diff --git a/ravens/cim_tools/conversion.py b/ravens/cim_tools/conversion.py deleted file mode 100644 index d90b47db3..000000000 --- a/ravens/cim_tools/conversion.py +++ /dev/null @@ -1,57 +0,0 @@ -import html - -import networkx as nx - -from ravens.io import UMLData - -cimPrimativesMap = { - "Integer": "integer", - "Float": "number", - "Decimal": "number", - "Boolean": "boolean", - "String": "string", - "Time": "string", - "Date": "string", - "Duration": "string", - "DateTime": "string", - "MonthDay": "string", -} - - -def convert_cim_type(cim_type: str) -> str: - if cim_type in cimPrimativesMap: - return cimPrimativesMap[cim_type] - - return cim_type - - -def add_cim_attributes_to_properties(properties: dict, k: str, v: dict, uml_data: UMLData, GG: nx.DiGraph, AT: nx.DiGraph) -> dict: - try: - if "$objectId" in v.keys(): - object_name = v["$objectId"] - else: - object_name = k - - object_id = uml_data.objects[(uml_data.objects["Name"] == object_name) & (uml_data.objects["Object_Type"] == "Class")].iloc[0]._name - except: - raise KeyError(f"Object {object_name} Object_ID not found in CIM UML") - attribute_ids = {a for n in [object_id] + list(nx.ancestors(GG, object_id)) + list(nx.descendants(GG, object_id)) for a in list(AT.predecessors(n))} - for attr_id in attribute_ids: - attribute = uml_data.attributes.loc[attr_id] - parent = uml_data.objects.loc[attribute.Object_ID] - try: - attribute_name = f"{parent.Name}.{attribute.Name}" - - attribute_data = { - "title": str(attribute.Name), - "type": convert_cim_type(str(attribute.Type)), - "description": html.unescape(str(attribute.Notes).strip()), - } - if attribute_name in properties.keys(): - attribute_data.update(properties[attribute_name]) - - properties[attribute_name] = attribute_data - except: - raise Exception(f"Failed to add attribute {attribute.Name} to {k}") - - return properties diff --git a/ravens/cim_tools/graph.py b/ravens/cim_tools/graph.py deleted file mode 100644 index 0d8ea6175..000000000 --- a/ravens/cim_tools/graph.py +++ /dev/null @@ -1,198 +0,0 @@ -import networkx as nx -import pandas as pd - -from ravens.io import UMLData -from ravens.cim_tools.template import collect_template_node_names - - -def add_class_nodes_to_graph(G, uml_data: UMLData, exclude_packages: list = None, exclude_objects: list = None): - if exclude_packages is None: - exclude_packages = [] - - if exclude_objects is None: - exclude_objects = [] - - attrs = {obj.Index: [attr.Index for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == obj.Index].itertuples()] for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples() if pd.isnull(obj.Stereotype)} - - conns = { - obj.Index: [c.Index for c in uml_data.connectors[(uml_data.connectors["Start_Object_ID"] == obj.Index) | (uml_data.connectors["End_Object_ID"] == obj.Index)].itertuples()] - for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples() - if pd.isnull(obj.Stereotype) - } - - gens = {node: [c for c in cs if uml_data.connectors.loc[c]["Connector_Type"] == "Generalization"] for node, cs in conns.items()} - - assocs = {node: [c for c in cs if uml_data.connectors.loc[c]["Connector_Type"] == "Association"] for node, cs in conns.items()} - - G.add_nodes_from( - [ - ( - obj.Index, - { - "Name": str(obj.Name) + f" ({len(attrs[obj.Index])}+{len(gens[obj.Index])}+{len(assocs[obj.Index])})", - "Object_ID": str(obj.Index), - "Note": str(obj.Note), - "Attributes": ", ".join([uml_data.attributes.loc[i]["Name"] for i in attrs[obj.Index]]), - "Object_Type": "Class", - }, - ) - for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples() - if pd.isnull(obj.Stereotype) and obj.Package_ID not in exclude_packages and obj.Index not in exclude_objects - ] - ) - - return G - - -def build_generalization_graph(uml_data: UMLData, exclude_packages: list = None, exclude_objects: list = None) -> nx.MultiDiGraph: - if exclude_packages is None: - exclude_packages = [] - - if exclude_objects is None: - exclude_objects = [] - - GG = nx.MultiDiGraph() - GG = add_class_nodes_to_graph(GG, uml_data, exclude_packages, exclude_objects) - for c in uml_data.connectors[uml_data.connectors["Connector_Type"] == "Generalization"].itertuples(): - if ( - uml_data.objects.loc[c.Start_Object_ID]["Object_Type"] == "Class" - and uml_data.objects.loc[c.End_Object_ID]["Object_Type"] == "Class" - and uml_data.objects.loc[c.Start_Object_ID]["Package_ID"] not in exclude_packages - and uml_data.objects.loc[c.End_Object_ID]["Package_ID"] not in exclude_packages - and pd.isnull(uml_data.objects.loc[c.Start_Object_ID]["Stereotype"]) - and pd.isnull(uml_data.objects.loc[c.End_Object_ID]["Stereotype"]) - and c.Start_Object_ID not in exclude_objects - and c.End_Object_ID not in exclude_objects - ): - GG.add_edge( - c.Start_Object_ID, - c.End_Object_ID, - Start_Object_ID=str(c.Start_Object_ID), - End_Object_ID=str(c.End_Object_ID), - Connector_ID="GEN_" + str(c.Index), - Connector_Type="Generalization", - weight=10.0, - ) - - return GG - - -def build_attribute_graph(uml_data: UMLData, exclude_packages: list = None, exclude_objects: list = None) -> nx.MultiDiGraph: - if exclude_packages is None: - exclude_packages = [] - - if exclude_objects is None: - exclude_objects = [] - - AT = nx.MultiDiGraph() - AT = add_class_nodes_to_graph(AT, uml_data, exclude_packages, exclude_objects) - for n in list(AT.nodes): - for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == n].itertuples(): - AT.add_edge(attr.Index, n, Connector_Type="Attribute", Connector_ID="ATTR_" + str(attr.Index), weight=100.0) - AT.nodes[attr.Index].update({"Name": str(attr.Name), "Note": str(attr.Notes), "Object_Type": "Attribute", "Attribute_ID": str(attr.Index)}) - - return AT - - -def build_association_graph(uml_data: UMLData, exclude_packages: list = None, exclude_objects: list = None) -> nx.MultiDiGraph: - if exclude_packages is None: - exclude_packages = [] - - if exclude_objects is None: - exclude_objects = [] - - AG = nx.MultiDiGraph() - AG = add_class_nodes_to_graph(AG, uml_data, exclude_packages, exclude_objects) - for c in uml_data.connectors[(uml_data.connectors["Connector_Type"] == "Association") | (uml_data.connectors["Connector_Type"] == "Aggregation")].itertuples(): - if ( - uml_data.objects.loc[c.Start_Object_ID]["Object_Type"] == "Class" - and uml_data.objects.loc[c.End_Object_ID]["Object_Type"] == "Class" - and uml_data.objects.loc[c.Start_Object_ID]["Package_ID"] not in exclude_packages - and uml_data.objects.loc[c.End_Object_ID]["Package_ID"] not in exclude_packages - and pd.isnull(uml_data.objects.loc[c.Start_Object_ID]["Stereotype"]) - and pd.isnull(uml_data.objects.loc[c.End_Object_ID]["Stereotype"]) - and c.Start_Object_ID not in exclude_objects - and c.End_Object_ID not in exclude_objects - ): - AG.add_edge( - c.End_Object_ID, - c.Start_Object_ID, - SourceCard=str(c.DestCard), - DestCard=str(c.SourceCard), - SourceRole=str(c.DestRole) if not pd.isnull(c.DestRole) else str(uml_data.objects.loc[c.End_Object_ID]["Name"]), - DestRole=str(c.SourceRole) if not pd.isnull(c.SourceRole) else str(uml_data.objects.loc[c.Start_Object_ID]["Name"]), - Connector_ID="ASC_REV_" + str(c.Index), - End_Object_ID=str(c.Start_Object_ID), - Start_Object_ID=str(c.End_Object_ID), - Connector_Type=str(c.Connector_Type), - weight=1.0, - ) - - AG.add_edge( - c.Start_Object_ID, - c.End_Object_ID, - DestCard=str(c.DestCard), - SourceCard=str(c.SourceCard), - DestRole=str(c.DestRole) if not pd.isnull(c.DestRole) else str(uml_data.objects.loc[c.End_Object_ID]["Name"]), - SourceRole=str(c.SourceRole) if not pd.isnull(c.SourceRole) else str(uml_data.objects.loc[c.Start_Object_ID]["Name"]), - Connector_ID="ASC_FWD_" + str(c.Index), - Start_Object_ID=str(c.Start_Object_ID), - End_Object_ID=str(c.End_Object_ID), - Connector_Type=str(c.Connector_Type), - weight=1.0, - ) - - return AG - - -def build_template_cim_graphs(template, uml_data, GG, AG, AT, clean_dir=False): - id2name = { - **{obj.Index: str(obj.Name) for obj in uml_data.objects.itertuples()}, - **{attr.Index: str(attr.Name) for attr in uml_data.attributes.itertuples()}, - } - cls_name2id = {str(obj.Name): obj.Index for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples() if pd.isnull(obj.Stereotype)} - - template_names = [] - template_names = collect_template_node_names(template, template_names) - - if clean_dir: - for file in glob.glob("out/template_graphs/*"): - os.remove(file) - - for name in template_names: - obj_id = cls_name2id[name] - nodes = {at for n in [obj_id] + list(nx.ancestors(GG, obj_id)) + list(nx.descendants(GG, obj_id)) for a in list(AG.neighbors(n)) + [n] for at in [n, a] + list(AT.predecessors(a)) + list(AT.predecessors(n))} - - SG = nx.subgraph(GG_AT_AG, nodes) - nx.write_graphml(SG, f"out/template_graphs/{name}.graphml") - - -if __name__ == "__main__": - from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions - from ravens.cim_tools.template import CIMTemplate - from ravens.io import parse_uml_data - - db_filename = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - - uml_data = parse_uml_data(db_filename) - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - GG = build_generalization_graph(uml_data, exclude_packages, exclude_objects) - # nx.write_graphml(GG, "out/CIM_graphs/GG.graphml") - AT = build_attribute_graph(uml_data, exclude_packages, exclude_objects) - # nx.write_graphml(AT, "out/CIM_graphs/AT.graphml") - AG = build_association_graph(uml_data, exclude_packages, exclude_objects) - # nx.write_graphml(AG, "out/CIM_graphs/AG.graphml") - - GG_AT_AG = nx.compose_all([GG, AT, AG]) - nx.write_graphml(GG_AT_AG, "out/CIM_graphs/GG_AT_AG.graphml", named_key_ids=True, edge_id_from_attribute="Connector_ID") - # nx.write_gml(GG_AT_AG, "out/CIM_graphs/GG_AT_AG.gml") - - cim_template = CIMTemplate("ravens/cim_tools/cim_conversion_template.json") - build_template_cim_graphs(cim_template.template, uml_data, GG, AG, AT, clean_dir=True) diff --git a/ravens/cim_tools/template.py b/ravens/cim_tools/template.py deleted file mode 100644 index 0d53a7fc7..000000000 --- a/ravens/cim_tools/template.py +++ /dev/null @@ -1,24 +0,0 @@ -import json - - -class CIMTemplate: - def __init__(self, file): - with open(file, "r") as f: - self.template = json.load(f) - - -def collect_template_node_names(template: dict, nodes: list, currentParent: str = None, parentObject: str = None): - for k, v in template.items(): - if isinstance(v, dict) and v.get("type", "none") == "container": - nodes = collect_template_node_names(v.get("properties", {}), nodes, parentObject=k) - elif isinstance(v, dict) and v.get("type", "none") == "object": - if currentParent is not None: - nodes.append(f"{currentParent}::{k}") - nodes = collect_template_node_names(v.get("properties", {}), nodes, currentParent=f"{currentParent}::{k}") - else: - nodes.append(k) - nodes = collect_template_node_names(v.get("properties", {}), nodes, currentParent=k) - elif isinstance(v, dict) and v.get("type", "none") == "ref": - continue - - return nodes diff --git a/ravens/data.py b/ravens/data.py new file mode 100644 index 000000000..9a949a710 --- /dev/null +++ b/ravens/data.py @@ -0,0 +1,37 @@ +import importlib + + +_CIM_PRIMATIVES = { + "Integer": "integer", + "Float": "number", + "String": "string", + "Boolean": "boolean", + "Time": "string", + "Decimal": "number", + "Date": "string", + "Duration": "string", + "DateTime": "string", + "MonthDay": "string", + # json type passthrough + "integer": "integer", + "number": "number", + "string": "string", + "boolean": "boolean", + "null": "null", + "object": "object", + "array": "array", +} + + +_CIM_RGB_TO_HEX = {z * 65536 + y * 256 + x: "#{:02x}{:02x}{:02x}".format(x, y, z) for x in range(256) for y in range(256) for z in range(256)} + +_RAVENS_SCHEMA_BASE_URL = "https://raw.githubusercontent.com/lanl-ansi/MG-RAVENS/refs/heads/schema/schema" +_JSON_SCHEMA_URL = "https://json-schema.org/draft/2020-12/schema" + +_LIB_FILES_PATH = importlib.resources.files("ravens.lib") + +_TEMPLATE_JSON_PATH = _LIB_FILES_PATH.joinpath("template.json") + +_UML_XML_PATH = _LIB_FILES_PATH.joinpath("iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xml") + +_SVG_RENDERER_PATH = _LIB_FILES_PATH.joinpath("svgRenderer.js") diff --git a/ravens/io.py b/ravens/io.py index 51d8398da..bc354907f 100644 --- a/ravens/io.py +++ b/ravens/io.py @@ -2,14 +2,15 @@ import io import json import os +import pathlib from copy import deepcopy import xml.etree.ElementTree as ET import json_schema_for_humans.generate as JSFHGenerate -import pandas_access as mdb import pandas as pd +from ravens.data import _UML_XML_PATH _index_columns = { "t_object": "Object_ID", @@ -196,22 +197,26 @@ class UMLData: - def __init__(self, file: str, filetype: str = "auto", set_index: bool = True): - if (filetype == "auto" and file.endswith("eap")) or filetype == "eap": - self.parse_eap_file(file, set_index=set_index) - elif (filetype == "auto" and file.endswith("xmi")) or filetype == "xmi": - self.parse_xmi_file(file, set_index=set_index) - else: - if filetype == "auto": - raise Exception(f"Unable to detect filetype of '{file}'") - else: - raise Exception(f"Filetype '{filetype}' is unsupported. Use 'auto', 'eap', or 'xmi'.") + def __init__(self): + dataframes = self._create_dataframes() + for table_name, df in dataframes.items(): + setattr(self, _attr_names[table_name], df) + + @classmethod + def loadf(cls, file: pathlib.PosixPath = _UML_XML_PATH, set_index: bool = True) -> object: + dataframes = cls._create_dataframes(file, set_index=set_index) + obj = cls() + for table_name, df in dataframes.items(): + setattr(obj, _attr_names[table_name], df) + + return obj - def parse_xmi_file(self, file: str, set_index: bool = True): + @staticmethod + def _create_dataframes(file: pathlib.PosixPath = _UML_XML_PATH, set_index: bool = True) -> object: tree = ET.parse(file) root = tree.getroot() - tables_data = {} + dataframes = {} for table in root.findall("Table"): table_name = table.attrib.get("name") @@ -246,34 +251,13 @@ def parse_xmi_file(self, file: str, set_index: bool = True): if set_index: df = df.set_index(_index_columns[table_name]) - setattr(self, _attr_names[table_name], df) - - def parse_eap_file(self, file: str, set_index: bool = True): - for table_name, attr_name in _attr_names.items(): - df = mdb.read_table(file, table_name, converters_from_schema=False) - - if set_index: - df = df.set_index(_index_columns[table_name]) - - setattr(self, attr_name, df) - - -def parse_eap_data(file: str, set_index: bool = True) -> UMLData: - "Loads connector, object, attribute, package, diagram, diagramlinks, diagramobjects data from EAP file" - return UMLData(file, filetype="eap", set_index=set_index) - - -def parse_xmi_data(file: str, set_index: bool = True) -> UMLData: - "Loads connector, object, attribute, package, diagram, diagramlinks, diagramobjects data from Native XMI file" - return UMLData(file, filetype="xmi", set_index=set_index) + dataframes[table_name] = df + return dataframes -def parse_uml_data(file: str, filetype: str = "auto", set_index: bool = True) -> UMLData: - "Loads connector, object, attribute, package, diagram, diagramlinks, diagramobjects data from UML file (XMI or EAP)" - return UMLData(file, filetype=filetype, set_index=set_index) +def write_schemas(chemas: dict, models_path: str = "models", cleanup_model_dir: bool = False, flatten: bool = False): -def write_schemas(schemas: dict, models_path: str = "models", cleanup_model_dir: bool = False, flatten: bool = False): "Helper function to write schema to file(s)" if cleanup_model_dir: for file in glob.glob(os.path.join(models_path, "*.json"), recursive=True): @@ -300,6 +284,6 @@ def write_schema_docs(schema: dict, out_file: str): if __name__ == "__main__": - xmi_file = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" + uml = UMLData() - uml_data = parse_uml_data(xmi_file) + uml2 = UMLData.loadf("ravens/lib/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi") diff --git a/cim/.gitignore b/ravens/lib/.gitignore similarity index 55% rename from cim/.gitignore rename to ravens/lib/.gitignore index 7203863d5..3d1f308f2 100755 --- a/cim/.gitignore +++ b/ravens/lib/.gitignore @@ -1,4 +1,7 @@ * !.gitignore -!iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi +!__init__.py +!iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xml +!template.json +!svgRenderer.js diff --git a/ravens/cim_tools/__init__.py b/ravens/lib/__init__.py similarity index 100% rename from ravens/cim_tools/__init__.py rename to ravens/lib/__init__.py diff --git a/cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi b/ravens/lib/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xml similarity index 100% rename from cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi rename to ravens/lib/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xml diff --git a/ravens/uml/js/svgRenderer.js b/ravens/lib/svgRenderer.js similarity index 100% rename from ravens/uml/js/svgRenderer.js rename to ravens/lib/svgRenderer.js diff --git a/ravens/cim_tools/cim_conversion_template.json b/ravens/lib/template.json similarity index 100% rename from ravens/cim_tools/cim_conversion_template.json rename to ravens/lib/template.json diff --git a/ravens/schema/__init__.py b/ravens/schema/__init__.py index e69de29bb..5eca75aeb 100644 --- a/ravens/schema/__init__.py +++ b/ravens/schema/__init__.py @@ -0,0 +1,3 @@ +from .schema import RavensSchema, generate_schema_docs +from .template import SchemaTemplate +from .validate import RavensValidator diff --git a/ravens/schema/add_copyright_notice.py b/ravens/schema/add_copyright_notice.py deleted file mode 100644 index 021ae2bed..000000000 --- a/ravens/schema/add_copyright_notice.py +++ /dev/null @@ -1,23 +0,0 @@ -import html -import markdownify - -from ravens.io import UMLData - - -def get_cim_copyright_notice(uml_data: UMLData, cim_copyright_notice_object_id=29601): - return "\n".join(markdownify.markdownify(html.unescape(str(uml_data.objects.loc[cim_copyright_notice_object_id].Note).strip())).splitlines()).strip() - - -def add_copyright_notice_to_decomposed_schemas(schemas, copyright_notice: str): - for k in schemas.keys(): - schemas[k]["license"] = copyright_notice - - -def add_cim_copyright_notice_to_decomposed_schemas(schemas, uml_data: UMLData): - copyright_notice = get_cim_copyright_notice(uml_data) - for k in schemas.keys(): - schemas[k]["license"] = copyright_notice - - -if __name__ == "__main__": - pass diff --git a/ravens/schema/build_definitions.py b/ravens/schema/build_definitions.py deleted file mode 100644 index d5dc2a77a..000000000 --- a/ravens/schema/build_definitions.py +++ /dev/null @@ -1,80 +0,0 @@ -import html -import json - -import pandas as pd - -from ravens.io import UMLData - - -cim_primitives_to_json = { - "Integer": "integer", - "Float": "number", - "String": "string", - "Boolean": "boolean", - "Time": "string", - "Decimal": "number", - "Date": "string", - "Duration": "string", - "DateTime": "string", - "MonthDay": "string", - "integer": "integer", - "number": "number", - "string": "string", - "boolean": "boolean", - "null": "null", - "object": "object", - "array": "array", -} - - -def build_definitions(uml_data: UMLData) -> dict: - defs = {} - for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples(): - if str(obj.Stereotype).strip() == "enumeration": - defs[str(obj.Name).replace(" ", "")] = { - "title": str(obj.Name).replace(" ", ""), - "description": html.unescape(str(obj.Note)).strip(), - "type": "string", - "enum": [str(attr.Name) for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == obj.Index].itertuples()], - } - elif not pd.isnull(obj.Stereotype): - defs[str(obj.Name).replace(" ", "")] = { - "title": str(obj.Name).replace(" ", ""), - "description": html.unescape(str(obj.Note)).strip(), - "type": "object", - "properties": { - str(attr.Name): { - "type": str(attr.Type) if str(attr.Type) not in cim_primitives_to_json else cim_primitives_to_json[str(attr.Type)], - "description": str(attr.Notes), - "default": attr.Default if not pd.isnull(attr.Default) else "none" if str(attr.Type) == "UnitMultiplier" else None, - } - for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == obj.Index].itertuples() - }, - } - - if all(v["default"] is not None for k, v in defs[str(obj.Name).replace(" ", "")]["properties"].items() if k != "value") and "value" in defs[str(obj.Name).replace(" ", "")]["properties"]: - defs[str(obj.Name).replace(" ", "")]["type"] = [ - "object", - cim_primitives_to_json[defs[str(obj.Name).replace(" ", "")]["properties"]["value"]["type"]], - ] - - for k, v in defs.items(): - if "properties" in v: - for _k, _v in v["properties"].items(): - if "type" in _v and _v["type"] in defs: - _v["$ref"] = f"#/$defs/{_v.pop("type")}" - - return defs - - -if __name__ == "__main__": - from ravens.io import parse_uml_data - - db_filename = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - - uml_data = parse_uml_data(db_filename) - - defs = build_definitions(uml_data) - - with open("out/schema/definitions.json", "w") as f: - json.dump({"title": "CIM definitions", "type": "object", "properties": {}, "$defs": defs}, f, indent=2) diff --git a/ravens/schema/build_map.py b/ravens/schema/build_map.py deleted file mode 100644 index 7a84220a0..000000000 --- a/ravens/schema/build_map.py +++ /dev/null @@ -1,123 +0,0 @@ -import html - -from copy import deepcopy - -import networkx as nx - -from ravens.io import UMLData -from ravens.cim_tools.conversion import add_cim_attributes_to_properties - - -def add_attributes_to_template(data: dict, template: dict, uml_data: UMLData, GG: nx.MultiDiGraph, AT: nx.MultiDiGraph): - "adds attributes from UML" - if template["type"] == "object": - for k, v in template.get("properties", {}).items(): - if v["type"] == "object": - try: - object_name = k - if "$objectId" in v.keys(): - object_name = v["$objectId"] - obj = uml_data.objects[(uml_data.objects["Name"] == object_name) & (uml_data.objects["Object_Type"] == "Class")].iloc[0] - - if "description" not in v: - data["properties"][k]["description"] = html.unescape(str(obj.Note).strip()) - if "title" not in v: - data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) - - except Exception as msg: - raise Exception(f"Cannot find CIM object {object_name}: {msg}") - - if v.get("$objectType", "") == "container": - data["properties"][k] = add_attributes_to_template(data["properties"][k], v, uml_data, GG, AT) - elif v.get("$objectType", "") == "object": - if "oneOf" in v.keys(): - for i, item in enumerate(v["oneOf"]): - if item.get("$objectType", "") == "object": - object_name = item["$objectId"] - obj = uml_data.objects[(uml_data.objects["Name"] == object_name) & (uml_data.objects["Object_Type"] == "Class")].iloc[0] - if "title" not in item: - data["properties"][k]["oneOf"][i]["title"] = str(obj.Name) - if "description" not in item: - data["properties"][k]["oneOf"][i]["description"] = html.unescape(str(obj.Note).strip()) - - data["properties"][k]["oneOf"][i]["properties"] = add_cim_attributes_to_properties(data["properties"][k]["oneOf"][i]["properties"], item["$objectId"], item, uml_data, GG, AT) - - data["properties"][k]["oneOf"][i] = add_attributes_to_template(data["properties"][k]["oneOf"][i], item, uml_data, GG, AT) - else: - data["properties"][k]["properties"] = add_cim_attributes_to_properties(data["properties"][k]["properties"], k, v, uml_data, GG, AT) - - data["properties"][k] = add_attributes_to_template(data["properties"][k], v, uml_data, GG, AT) - - elif v.get("$objectType", "") == "reference": - obj = uml_data.objects[(uml_data.objects["Name"] == v["$objectId"]) & (uml_data.objects["Object_Type"] == "Class")].iloc[0] - if "title" not in v: - data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + "Pointer" - if "description" not in v: - data["properties"][k]["description"] = f"Pointer to {html.unescape(str(obj.Name).strip())} object" - elif v["type"] == "array": - try: - if v["items"].get("type", "") == "array": - # do nothing - continue - elif v["items"].get("$objectType", "") == "reference": - obj = uml_data.objects[(uml_data.objects["Name"] == v["items"].get("$objectId", k)) & (uml_data.objects["Object_Type"] == "Class")].iloc[0] - data["properties"][k]["description"] = f"Pointers to {html.unescape(str(obj.Name).strip())} objects" - data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + "PointerArray" - data["properties"][k]["items"]["title"] = html.unescape(str(obj.Name).strip()) + "Pointer" - data["properties"][k]["items"]["description"] = f"Pointer to {html.unescape(str(obj.Name).strip())} object" - else: - obj = uml_data.objects[(uml_data.objects["Name"] == v["items"].get("$objectId", k)) & (uml_data.objects["Object_Type"] == "Class")].iloc[0] - - data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + "Array" - data["properties"][k]["description"] = f"Array of {html.unescape(str(obj.Name).strip())} objects" - data["properties"][k]["items"]["title"] = html.unescape(str(obj.Name).strip()) - data["properties"][k]["items"]["description"] = html.unescape(str(obj.Note).strip()) - - if "oneOf" in v["items"]: - for i, item in enumerate(v["items"]["oneOf"]): - oneof_obj = uml_data.objects[(uml_data.objects["Name"] == item["$objectId"]) & (uml_data.objects["Object_Type"] == "Class")].iloc[0] - data["properties"][k]["items"]["oneOf"][i]["title"] = html.unescape(str(oneof_obj.Name).strip()) - data["properties"][k]["items"]["oneOf"][i]["description"] = html.unescape(str(oneof_obj.Note).strip()) - - data["properties"][k]["items"]["oneOf"][i]["properties"] = add_cim_attributes_to_properties(data["properties"][k]["items"]["oneOf"][i]["properties"], k, item, uml_data, GG, AT) - data["properties"][k]["items"]["oneOf"][i] = add_attributes_to_template(data["properties"][k]["items"]["oneOf"][i], item, uml_data, GG, AT) - else: - data["properties"][k]["items"]["properties"] = add_cim_attributes_to_properties(data["properties"][k]["items"]["properties"], k, v["items"], uml_data, GG, AT) - - data["properties"][k]["items"] = add_attributes_to_template(data["properties"][k]["items"], v["items"], uml_data, GG, AT) - - except KeyError as msg: - raise KeyError(f"Unabled to find {msg} in object {k} of type array") - else: - raise Exception(f"Object {k} of $objectType '{v.get('$objectType', '')}' not recognized") - - return data - - -if __name__ == "__main__": - import json - from ravens.io import parse_uml_data - from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions - from ravens.cim_tools.graph import build_generalization_graph, build_attribute_graph - from ravens.cim_tools.template import CIMTemplate - - db_filename = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - - uml_data = parse_uml_data(db_filename) - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - GG = build_generalization_graph(uml_data, exclude_packages, exclude_objects) - AT = build_attribute_graph(uml_data, exclude_packages, exclude_objects) - - cim_template = CIMTemplate("ravens/cim_tools/cim_conversion_template.json") - - cim_template_with_attributes = add_attributes_to_template(deepcopy(cim_template.template), cim_template.template, uml_data, GG, AT) - - with open("out/schema/cim_template_with_attributes.json", "w") as f: - json.dump(cim_template_with_attributes, f, indent=2) diff --git a/ravens/schema/build_schema.py b/ravens/schema/build_schema.py deleted file mode 100644 index 6cf6314a3..000000000 --- a/ravens/schema/build_schema.py +++ /dev/null @@ -1,99 +0,0 @@ -def build_schema_from_map(schema_map: dict) -> dict: - schema = {} - for k, v in schema_map.items(): - if k.startswith("$"): - continue - elif isinstance(v, dict): - if "type" in v: - if v["type"] == "object": - if "properties" in v: - if v.get("$primaryObjectHash", None) is None: - schema[k] = build_schema_from_map(v) - else: - schema[k] = { - "type": "object", - "title": v.get("title", "") + "Container", - "description": f"Hash table of {v.get('title', '')} objects", - "patternProperties": { - "^.+$": { - **{_k: _v for _k, _v in v.items() if not _k.startswith("$") and _k != "properties"}, - **{"properties": build_schema_from_map(v["properties"])}, - } - }, - } - - elif "oneOf" in v: - if v.get("$primaryObjectHash", None) is None: - schema[k] = build_schema_from_map(v) - else: - schema[k] = { - "type": "object", - "title": v.get("title", "") + "Container", - "description": f"Hash table of {v.get('title', '')} objects", - "patternProperties": { - "^.+$": { - **{_k: _v for _k, _v in v.items() if not _k.startswith("$") and _k != "oneOf"}, - **{"oneOf": [build_schema_from_map(item) for item in v["oneOf"]]}, - } - }, - } - - elif v["type"] == "array": - schema[k] = { - **{_k: _v for _k, _v in v.items() if not _k.startswith("$") and _k != "items"}, - **{"items": build_schema_from_map(v["items"])}, - } - else: - schema[k] = build_schema_from_map(v) - else: - schema[k] = build_schema_from_map(v) - elif k == "oneOf" and isinstance(v, list): - schema[k] = [build_schema_from_map(item) for item in v] - elif k == "type" and v not in ["object", "string", "array", "boolean", "number", "null", "integer"]: - schema["$ref"] = f"#/$defs/{v}" - else: - schema[k] = v - - return schema - - -if __name__ == "__main__": - from copy import deepcopy - import json - from ravens.io import parse_uml_data, write_schema_docs - from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions - from ravens.cim_tools.graph import build_generalization_graph, build_attribute_graph - from ravens.cim_tools.template import CIMTemplate - from ravens.schema.build_definitions import build_definitions - from ravens.schema.build_map import add_attributes_to_template - import json_schema_for_humans.generate as Gen - - db_filename = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - - uml_data = parse_uml_data(db_filename) - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - GG = build_generalization_graph(uml_data, exclude_packages, exclude_objects) - AT = build_attribute_graph(uml_data, exclude_packages, exclude_objects) - - cim_template = CIMTemplate("ravens/cim_tools/cim_conversion_template.json") - - cim_template_with_attributes = add_attributes_to_template(deepcopy(cim_template.template), cim_template.template, uml_data, GG, AT) - - with open("out/schema/cim_template_with_attributes.json", "w") as f: - json.dump(cim_template_with_attributes, f, indent=2) - - schema = build_schema_from_map(cim_template_with_attributes) - - schema["$defs"] = build_definitions(uml_data) - - with open("out/schema/test_schema_conversion.json", "w") as f: - json.dump(schema, f, indent=2) - - Gen.generate_from_filename("out/schema/test_schema_conversion.json", "out/schema/test_schema_conversion.html") diff --git a/ravens/schema/decompose_schema.py b/ravens/schema/decompose_schema.py deleted file mode 100755 index d3e91a650..000000000 --- a/ravens/schema/decompose_schema.py +++ /dev/null @@ -1,170 +0,0 @@ -from copy import deepcopy - -_default_base_uri = "https://raw.githubusercontent.com/lanl-ansi/MG-RAVENS/refs/heads/schema/schema" -_schema_url = "https://json-schema.org/draft/2020-12/schema" - - -class Schemas: - def __init__(self, schema, base_id_uri=_default_base_uri): - self.schema = deepcopy(schema) - self.schemas = {} - self.base_id_uri = base_id_uri - - self.decompose_schema(deepcopy(self.schema)) - self.decompose_defs(deepcopy(self.schema).get("$defs", {})) - self.schemas[f"{self.base_id_uri}/Root.json"].pop("$defs") - - self.insert_refs() - - def decompose_schema(self, schema: dict, debug_key: str = None) -> str: - _schema = deepcopy(schema) - - if isinstance(_schema, dict): - _schema["$schema"] = _schema_url - _schema["additionalProperties"] = False - - title = _schema.get("title", None) - if title is not None: - if "patternProperties" in _schema: - title = f"{title}_Container" - else: - print(debug_key, _schema.keys()) - - _schema["$id"] = f"{self.base_id_uri}/{title}.json" - - if _schema.get("type", None) == "object": - for n in ["properties", "patternProperties"]: - if n in _schema: - for k, v in _schema[n].items(): - ref_id = self.decompose_schema(v, debug_key=k) - if ref_id is not None: - _schema[n][k] = {"$ref": ref_id} - - if "oneOf" in _schema: - _oneOf = [] - for item in _schema["oneOf"]: - ref_id = self.decompose_schema(item, debug_key=debug_key) - if ref_id is not None: - _oneOf.append({"$ref": ref_id}) - else: - _oneOf.append(item) - - _schema["oneOf"] = _oneOf - - elif _schema.get("type", None) == "array": - ref_id = self.decompose_schema(_schema["items"], debug_key=debug_key) - if ref_id is not None: - _schema["items"] = {"$ref": ref_id} - else: - return None - - self.schemas[_schema["$id"]] = _schema - - return _schema["$id"] - - return None - - def decompose_defs(self, defs: dict): - for k, v in defs.items(): - _schema = deepcopy(v) - _schema["$schema"] = _schema_url - _schema["$id"] = f"{self.base_id_uri}/{_schema["title"]}.json" - self.schemas[_schema["$id"]] = _schema - - def insert_refs(self): - for schema_key, schema in self.schemas.items(): - # print(schema_key, schema.keys()) - if schema.get("patternProperties", None) is not None: - for pattern, json_object in schema["patternProperties"].items(): - key = json_object.get("$id", None) - if key in self.schemas.keys(): - self.schemas[schema_key]["patternProperties"][pattern] = {"$ref": key} - elif "properties" in schema.keys(): - for k, v in schema["properties"].items(): - if v.get("type", "") == "object" or (isinstance(v.get("type", ""), list) and "object" in v["type"]): - key = v.get("$id", k) - if key in self.schemas: - self.schemas[schema_key]["properties"][k] = {"$ref": key} - - elif v.get("type", "") == "array" and v["items"].get("type", "") == "object": - key = v["items"].get("$id", k) - if key in self.schemas: - self.schemas[schema_key]["properties"][k]["items"] = {"$ref": key} - elif v.get("$ref", "").startswith("#/$defs/"): - ref = v["$ref"].split("#/$defs/")[1] - if f"{self.base_id_uri}/{ref}.json" in self.schemas: - self.schemas[schema_key]["properties"][k]["$ref"] = f"{self.base_id_uri}/{ref}.json" - elif "oneOf" in schema: - for i, item in enumerate(schema["oneOf"]): - if item.get("$ref", "").startswith("#/$defs/"): - ref = v["$ref"].split("#/$defs/")[1] - if f"{self.base_id_uri}/{ref}.json" in self.schemas: - self.schemas[schema_key]["oneOf"][i]["$ref"] = f"{self.base_id_uri}/{ref}.json" - else: - key = item.get("title", "") - if key in self.schemas: - self.schemas[schema_key]["oneOf"][i] = {"$ref": f"./{key}.json"} - elif "items" in schema: - if "oneOf" in schema["items"]: - for i, item in enumerate(schema["items"]["oneOf"]): - if item.get("$ref", "").startswith("#/$defs/"): - ref = v["$ref"].split("#/$defs/")[1] - if f"{self.base_id_uri}/{ref}.json" in self.schemas: - self.schemas[schema_key]["items"]["oneOf"][i]["$ref"] = f"{self.base_id_uri}/{ref}.json" - else: - key = item.get("$id", "") - if key in self.schemas: - self.schemas[schema_key]["items"]["oneOf"][i] = {"$ref": key} - else: - key = schema["items"].get("$id", "") - if key in self.schemas: - self.schemas[schema_key]["items"] = {"$ref": key} - - -if __name__ == "__main__": - import json - import os - from ravens.io import parse_uml_data - from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions - from ravens.cim_tools.graph import build_generalization_graph, build_attribute_graph - from ravens.cim_tools.template import CIMTemplate - from ravens.schema.build_definitions import build_definitions - from ravens.schema.build_map import add_attributes_to_template - from ravens.schema.build_schema import build_schema_from_map - from ravens.schema.add_copyright_notice import add_cim_copyright_notice_to_decomposed_schemas - import json_schema_for_humans.generate as Gen - - uml_data = parse_uml_data("cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi") - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - schema = build_schema_from_map( - add_attributes_to_template( - CIMTemplate("ravens/cim_tools/cim_conversion_template.json").template, - CIMTemplate("ravens/cim_tools/cim_conversion_template.json").template, - uml_data, - build_generalization_graph(uml_data, exclude_packages, exclude_objects), - build_attribute_graph(uml_data, exclude_packages, exclude_objects), - ) - ) - - schema["$defs"] = build_definitions(uml_data) - - a = Schemas(schema, base_id_uri=f"file://{os.getcwd()}/out/schema/separate") - - add_cim_copyright_notice_to_decomposed_schemas(a.schemas, uml_data) - - with open("out/schema/test_schema.json", "w") as f: - json.dump(schema, f, indent=2) - - for k, v in a.schemas.items(): - filename = k.split("/")[-1].replace(".json", "") - with open(f"out/schema/separate/{filename}.json", "w") as f: - json.dump(v, f, indent=2) - - Gen.generate_from_filename("out/schema/separate/", "out/schema/docs/") diff --git a/ravens/schema/graph.py b/ravens/schema/graph.py deleted file mode 100644 index 281e223b5..000000000 --- a/ravens/schema/graph.py +++ /dev/null @@ -1,49 +0,0 @@ -if __name__ == "__main__": - import json - import networkx as nx - from ravens.io import parse_uml_data - from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions - from ravens.cim_tools.graph import build_generalization_graph, build_attribute_graph, build_association_graph - from ravens.cim_tools.template import CIMTemplate - from ravens.schema.build_map import add_attributes_to_template - from ravens.schema.build_schema import build_schema_from_map - from ravens.schema.build_definitions import build_definitions - from ravens.schema.decompose_schema import Schemas - - uml_data = parse_uml_data("cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi") - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - schema = build_schema_from_map( - add_attributes_to_template( - CIMTemplate("ravens/cim_tools/cim_conversion_template.json").template, - CIMTemplate("ravens/cim_tools/cim_conversion_template.json").template, - uml_data, - build_generalization_graph(uml_data, exclude_packages, exclude_objects), - build_attribute_graph(uml_data, exclude_packages, exclude_objects), - ) - ) - schema["$defs"] = build_definitions(uml_data) - a = Schemas(schema) - - # exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects_more = build_object_exclusions( - uml_data.objects, - lambda x: str(x.Name) not in a.schemas.keys(), - exclude_packages=exclude_packages, - ) - - GG = build_generalization_graph(uml_data, exclude_packages, exclude_objects_more) - # nx.write_graphml(GG, "out/CIM_graphs/GG.graphml") - AT = build_attribute_graph(uml_data, exclude_packages, exclude_objects_more) - # nx.write_graphml(AT, "out/CIM_graphs/AT.graphml") - AG = build_association_graph(uml_data, exclude_packages, exclude_objects_more) - # nx.write_graphml(AG, "out/CIM_graphs/AG.graphml") - - GG_AT_AG = nx.compose_all([GG, AT, AG]) - nx.write_graphml(GG_AT_AG, "out/CIM_graphs/RAVENS_Schema.graphml", named_key_ids=True, edge_id_from_attribute="Connector_ID") diff --git a/ravens/schema/schema.py b/ravens/schema/schema.py new file mode 100755 index 000000000..6f7152575 --- /dev/null +++ b/ravens/schema/schema.py @@ -0,0 +1,277 @@ +import html +import json +import os +import pathlib + +import json_schema_for_humans.generate as Gen +import markdownify +import pandas as pd + +from copy import deepcopy + +from ravens.data import _RAVENS_SCHEMA_BASE_URL, _JSON_SCHEMA_URL, _CIM_PRIMATIVES +from ravens.io import UMLData +from ravens.uml import UMLGraphs, UMLExclusions +from ravens.schema.template import SchemaTemplate + + +class RavensSchema: + def __init__(self, schema_template: SchemaTemplate = None, base_id_uri=_RAVENS_SCHEMA_BASE_URL, uml_data: UMLData = None, uml_graphs: UMLGraphs = None, uml_exclusions: UMLExclusions = None): + if uml_data is None: + uml_data = UMLData() + + self.uml_data = uml_data + + if schema_template is None: + schema_template = SchemaTemplate(uml_data=uml_data, uml_graphs=uml_graphs, uml_exclusions=uml_exclusions) + + self.schema_template = schema_template + + self.schema = self.build_schema_from_map(self.schema_template.template) + self.schema["$defs"] = self.build_definitions(self.uml_data) + + self.schemas = {} + self.base_id_uri = base_id_uri + + self.decompose_schema(deepcopy(self.schema)) + self.decompose_defs(deepcopy(self.schema).get("$defs", {})) + + self.schemas[f"{self.base_id_uri}/Root.json"].pop("$defs") + + self.insert_refs() + + self.add_cim_copyright_notice_to_decomposed_schemas(self.uml_data) + + def build_schema_from_map(self, schema_map: dict) -> dict: + schema = {} + for k, v in schema_map.items(): + if k.startswith("$"): + continue + elif isinstance(v, dict): + if "type" in v: + if v["type"] == "object": + if "properties" in v: + if v.get("$primaryObjectHash", None) is None: + schema[k] = self.build_schema_from_map(v) + else: + schema[k] = { + "type": "object", + "title": v.get("title", "") + "Container", + "description": f"Hash table of {v.get('title', '')} objects", + "patternProperties": { + "^.+$": { + **{_k: _v for _k, _v in v.items() if not _k.startswith("$") and _k != "properties"}, + **{"properties": self.build_schema_from_map(v["properties"])}, + } + }, + } + + elif "oneOf" in v: + if v.get("$primaryObjectHash", None) is None: + schema[k] = self.build_schema_from_map(v) + else: + schema[k] = { + "type": "object", + "title": v.get("title", "") + "Container", + "description": f"Hash table of {v.get('title', '')} objects", + "patternProperties": { + "^.+$": { + **{_k: _v for _k, _v in v.items() if not _k.startswith("$") and _k != "oneOf"}, + **{"oneOf": [self.build_schema_from_map(item) for item in v["oneOf"]]}, + } + }, + } + + elif v["type"] == "array": + schema[k] = { + **{_k: _v for _k, _v in v.items() if not _k.startswith("$") and _k != "items"}, + **{"items": self.build_schema_from_map(v["items"])}, + } + else: + schema[k] = self.build_schema_from_map(v) + else: + schema[k] = self.build_schema_from_map(v) + elif k == "oneOf" and isinstance(v, list): + schema[k] = [self.build_schema_from_map(item) for item in v] + elif k == "type" and v not in ["object", "string", "array", "boolean", "number", "null", "integer"]: + schema["$ref"] = f"#/$defs/{v}" + else: + schema[k] = v + + return schema + + def build_definitions(self, uml_data: UMLData) -> dict: + defs = {} + for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples(): + if str(obj.Stereotype).strip() == "enumeration": + defs[str(obj.Name).replace(" ", "")] = { + "title": str(obj.Name).replace(" ", ""), + "description": html.unescape(str(obj.Note)).strip(), + "type": "string", + "enum": [str(attr.Name) for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == obj.Index].itertuples()], + } + elif not pd.isnull(obj.Stereotype): + defs[str(obj.Name).replace(" ", "")] = { + "title": str(obj.Name).replace(" ", ""), + "description": html.unescape(str(obj.Note)).strip(), + "type": "object", + "properties": { + str(attr.Name): { + "type": str(attr.Type) if str(attr.Type) not in _CIM_PRIMATIVES else _CIM_PRIMATIVES[str(attr.Type)], + "description": str(attr.Notes), + "default": attr.Default if not pd.isnull(attr.Default) else "none" if str(attr.Type) == "UnitMultiplier" else None, + } + for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == obj.Index].itertuples() + }, + } + + if all(v["default"] is not None for k, v in defs[str(obj.Name).replace(" ", "")]["properties"].items() if k != "value") and "value" in defs[str(obj.Name).replace(" ", "")]["properties"]: + defs[str(obj.Name).replace(" ", "")]["type"] = [ + "object", + _CIM_PRIMATIVES[defs[str(obj.Name).replace(" ", "")]["properties"]["value"]["type"]], + ] + + for k, v in defs.items(): + if "properties" in v: + for _k, _v in v["properties"].items(): + if "type" in _v and _v["type"] in defs: + _v["$ref"] = f"#/$defs/{_v.pop("type")}" + + return defs + + def decompose_schema(self, schema: dict, debug_key: str = None) -> str: + _schema = deepcopy(schema) + + if isinstance(_schema, dict): + _schema["$schema"] = _JSON_SCHEMA_URL + _schema["additionalProperties"] = False + + title = _schema.get("title", None) + if title is not None: + if "patternProperties" in _schema: + title = f"{title}_Container" + else: + print(debug_key, _schema.keys()) + + _schema["$id"] = f"{self.base_id_uri}/{title}.json" + + if _schema.get("type", None) == "object": + for n in ["properties", "patternProperties"]: + if n in _schema: + for k, v in _schema[n].items(): + ref_id = self.decompose_schema(v, debug_key=k) + if ref_id is not None: + _schema[n][k] = {"$ref": ref_id} + + if "oneOf" in _schema: + _oneOf = [] + for item in _schema["oneOf"]: + ref_id = self.decompose_schema(item, debug_key=debug_key) + if ref_id is not None: + _oneOf.append({"$ref": ref_id}) + else: + _oneOf.append(item) + + _schema["oneOf"] = _oneOf + + elif _schema.get("type", None) == "array": + ref_id = self.decompose_schema(_schema["items"], debug_key=debug_key) + if ref_id is not None: + _schema["items"] = {"$ref": ref_id} + else: + return None + + self.schemas[_schema["$id"]] = _schema + + return _schema["$id"] + + return None + + def decompose_defs(self, defs: dict): + for k, v in defs.items(): + _schema = deepcopy(v) + _schema["$schema"] = _JSON_SCHEMA_URL + _schema["$id"] = f"{self.base_id_uri}/{_schema["title"]}.json" + self.schemas[_schema["$id"]] = _schema + + def insert_refs(self): + for schema_key, schema in self.schemas.items(): + # print(schema_key, schema.keys()) + if schema.get("patternProperties", None) is not None: + for pattern, json_object in schema["patternProperties"].items(): + key = json_object.get("$id", None) + if key in self.schemas.keys(): + self.schemas[schema_key]["patternProperties"][pattern] = {"$ref": key} + elif "properties" in schema.keys(): + for k, v in schema["properties"].items(): + if v.get("type", "") == "object" or (isinstance(v.get("type", ""), list) and "object" in v["type"]): + key = v.get("$id", k) + if key in self.schemas: + self.schemas[schema_key]["properties"][k] = {"$ref": key} + + elif v.get("type", "") == "array" and v["items"].get("type", "") == "object": + key = v["items"].get("$id", k) + if key in self.schemas: + self.schemas[schema_key]["properties"][k]["items"] = {"$ref": key} + elif v.get("$ref", "").startswith("#/$defs/"): + ref = v["$ref"].split("#/$defs/")[1] + if f"{self.base_id_uri}/{ref}.json" in self.schemas: + self.schemas[schema_key]["properties"][k]["$ref"] = f"{self.base_id_uri}/{ref}.json" + elif "oneOf" in schema: + for i, item in enumerate(schema["oneOf"]): + if item.get("$ref", "").startswith("#/$defs/"): + ref = v["$ref"].split("#/$defs/")[1] + if f"{self.base_id_uri}/{ref}.json" in self.schemas: + self.schemas[schema_key]["oneOf"][i]["$ref"] = f"{self.base_id_uri}/{ref}.json" + else: + key = item.get("title", "") + if key in self.schemas: + self.schemas[schema_key]["oneOf"][i] = {"$ref": f"./{key}.json"} + elif "items" in schema: + if "oneOf" in schema["items"]: + for i, item in enumerate(schema["items"]["oneOf"]): + if item.get("$ref", "").startswith("#/$defs/"): + ref = v["$ref"].split("#/$defs/")[1] + if f"{self.base_id_uri}/{ref}.json" in self.schemas: + self.schemas[schema_key]["items"]["oneOf"][i]["$ref"] = f"{self.base_id_uri}/{ref}.json" + else: + key = item.get("$id", "") + if key in self.schemas: + self.schemas[schema_key]["items"]["oneOf"][i] = {"$ref": key} + else: + key = schema["items"].get("$id", "") + if key in self.schemas: + self.schemas[schema_key]["items"] = {"$ref": key} + + @staticmethod + def get_cim_copyright_notice(uml_data: UMLData, cim_copyright_notice_object_id=29601): + return "\n".join(markdownify.markdownify(html.unescape(str(uml_data.objects.loc[cim_copyright_notice_object_id].Note).strip())).splitlines()).strip() + + def add_cim_copyright_notice_to_decomposed_schemas(self, uml_data: UMLData): + copyright_notice = self.get_cim_copyright_notice(uml_data) + for k in self.schemas.keys(): + self.schemas[k]["license"] = copyright_notice + + def export_schema(self, file_out: pathlib.PosixPath): + with open(file_out, "w") as f: + json.dump(self.schema, f) + + def export_schemas(self, out_dir: pathlib.PosixPath): + for k, v in self.schemas.items(): + filename = k.split("/")[-1].replace(".json", "") + with open(os.path.join(out_dir, f"{filename}.json"), "w") as f: + json.dump(v, f, indent=2) + + +def generate_schema_docs(schema_dir: pathlib.PosixPath, out_dir: pathlib.PosixPath): + Gen.generate_from_filename(schema_dir, out_dir, config=Gen.GenerationConfiguration(template_name="js")) + + +if __name__ == "__main__": + schema = RavensSchema(base_id_uri=f"file://{os.getcwd()}/out/schema/separate") + + schema.export_schema("out/schema/test_schema.json") + + schema.export_schemas("out/schema/separate/") + + generate_schema_docs("out/schema/separate", "out/schema/docs") diff --git a/ravens/schema/template.py b/ravens/schema/template.py new file mode 100644 index 000000000..c72bd6fc4 --- /dev/null +++ b/ravens/schema/template.py @@ -0,0 +1,197 @@ +import html +import json +import pathlib + +import networkx as nx +import pandas as pd + +from copy import deepcopy + +from ravens.data import _TEMPLATE_JSON_PATH, _CIM_PRIMATIVES +from ravens.io import UMLData +from ravens.uml.graph import UMLGraphs +from ravens.uml.exclusions import UMLExclusions + + +class SchemaTemplate: + def __init__(self, uml_data: UMLData = None, uml_graphs: UMLGraphs = None, uml_exclusions: UMLExclusions = None): + if uml_data is None: + uml_data = UMLData() + + self.uml_data = uml_data + self.uml_graphs = UMLGraphs(self.uml_data, exclusions=uml_exclusions) if uml_graphs is None else uml_graphs + + self.template = {} + self.raw_template = {} + self.nodes = [] + + self.loadf(_TEMPLATE_JSON_PATH) + + def loadf(self, file: pathlib.PosixPath = _TEMPLATE_JSON_PATH) -> object: + with open(file, "r") as f: + return self.loadio(f) + + def loads(self, json_str: str) -> object: + self.raw_template = json.loads(json_str) + self.template = deepcopy(self.raw_template) + self.nodes = self.collect_template_node_names(self.template) + self.template = self.add_attributes_to_template(self.template, self.raw_template) + + return self + + def loadio(self, io): + return self.loads(io.read()) + + def add_attributes_to_template(self, data: dict, template: dict): + GG = self.uml_graphs.gen_graph + AT = self.uml_graphs.attr_graph + + if template["type"] == "object": + for k, v in template.get("properties", {}).items(): + if v["type"] == "object": + try: + object_name = k + if "$objectId" in v.keys(): + object_name = v["$objectId"] + obj = self.uml_data.objects[(self.uml_data.objects["Name"] == object_name) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0] + + if "description" not in v and not pd.isnull(obj.Note): + data["properties"][k]["description"] = html.unescape(str(obj.Note).strip()) + if "title" not in v: + data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + + except Exception as msg: + raise Exception(f"Cannot find CIM object {object_name}: {msg}") + + if v.get("$objectType", "") == "container": + data["properties"][k] = self.add_attributes_to_template(data["properties"][k], v) + elif v.get("$objectType", "") == "object": + if "oneOf" in v.keys(): + for i, item in enumerate(v["oneOf"]): + if item.get("$objectType", "") == "object": + object_name = item["$objectId"] + obj = self.uml_data.objects[(self.uml_data.objects["Name"] == object_name) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0] + if "title" not in item: + data["properties"][k]["oneOf"][i]["title"] = str(obj.Name) + if "description" not in item and not pd.isnull(obj.Note): + data["properties"][k]["oneOf"][i]["description"] = html.unescape(str(obj.Note).strip()) + + data["properties"][k]["oneOf"][i]["properties"] = self.add_cim_attributes_to_properties(data["properties"][k]["oneOf"][i]["properties"], item["$objectId"], item) + + data["properties"][k]["oneOf"][i] = self.add_attributes_to_template(data["properties"][k]["oneOf"][i], item) + else: + data["properties"][k]["properties"] = self.add_cim_attributes_to_properties(data["properties"][k]["properties"], k, v) + + data["properties"][k] = self.add_attributes_to_template(data["properties"][k], v) + + elif v.get("$objectType", "") == "reference": + obj = self.uml_data.objects[(self.uml_data.objects["Name"] == v["$objectId"]) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0] + if "title" not in v: + data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + "Pointer" + if "description" not in v: + data["properties"][k]["description"] = f"Pointer to {html.unescape(str(obj.Name).strip())} object" + elif v["type"] == "array": + try: + if v["items"].get("type", "") == "array": + # do nothing + continue + elif v["items"].get("$objectType", "") == "reference": + obj = self.uml_data.objects[(self.uml_data.objects["Name"] == v["items"].get("$objectId", k)) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0] + data["properties"][k]["description"] = f"Pointers to {html.unescape(str(obj.Name).strip())} objects" + data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + "PointerArray" + data["properties"][k]["items"]["title"] = html.unescape(str(obj.Name).strip()) + "Pointer" + data["properties"][k]["items"]["description"] = f"Pointer to {html.unescape(str(obj.Name).strip())} object" + else: + obj = self.uml_data.objects[(self.uml_data.objects["Name"] == v["items"].get("$objectId", k)) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0] + + data["properties"][k]["title"] = html.unescape(str(obj.Name).strip()) + "Array" + data["properties"][k]["description"] = f"Array of {html.unescape(str(obj.Name).strip())} objects" + data["properties"][k]["items"]["title"] = html.unescape(str(obj.Name).strip()) + if not pd.isnull(obj.Note): + data["properties"][k]["items"]["description"] = html.unescape(str(obj.Note).strip()) + + if "oneOf" in v["items"]: + for i, item in enumerate(v["items"]["oneOf"]): + oneof_obj = self.uml_data.objects[(self.uml_data.objects["Name"] == item["$objectId"]) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0] + data["properties"][k]["items"]["oneOf"][i]["title"] = html.unescape(str(oneof_obj.Name).strip()) + if not pd.isnull(oneof_obj.Note): + data["properties"][k]["items"]["oneOf"][i]["description"] = html.unescape(str(oneof_obj.Note).strip()) + + data["properties"][k]["items"]["oneOf"][i]["properties"] = self.add_cim_attributes_to_properties(data["properties"][k]["items"]["oneOf"][i]["properties"], k, item) + data["properties"][k]["items"]["oneOf"][i] = self.add_attributes_to_template(data["properties"][k]["items"]["oneOf"][i], item) + else: + data["properties"][k]["items"]["properties"] = self.add_cim_attributes_to_properties(data["properties"][k]["items"]["properties"], k, v["items"]) + + data["properties"][k]["items"] = self.add_attributes_to_template(data["properties"][k]["items"], v["items"]) + + except KeyError as msg: + raise KeyError(f"Unabled to find {msg} in object {k} of type array") + else: + raise Exception(f"Object {k} of $objectType '{v.get('$objectType', '')}' not recognized") + + return data + + def add_cim_attributes_to_properties(self, properties: dict, k: str, v: dict) -> dict: + GG = self.uml_graphs.gen_graph + AT = self.uml_graphs.attr_graph + + try: + if "$objectId" in v.keys(): + object_name = v["$objectId"] + else: + object_name = k + + object_id = self.uml_data.objects[(self.uml_data.objects["Name"] == object_name) & (self.uml_data.objects["Object_Type"] == "Class")].iloc[0]._name + except: + raise KeyError(f"Object {object_name} Object_ID not found in CIM UML") + attribute_ids = {a for n in [object_id] + list(nx.ancestors(GG, object_id)) + list(nx.descendants(GG, object_id)) for a in list(AT.predecessors(n))} + for attr_id in attribute_ids: + attribute = self.uml_data.attributes.loc[attr_id] + parent = self.uml_data.objects.loc[attribute.Object_ID] + try: + attribute_name = f"{parent.Name}.{attribute.Name}" + + attribute_data = { + "title": str(attribute.Name), + "type": self.convert_cim_type(str(attribute.Type)), + } + if not pd.isnull(attribute.Notes): + attribute_data["description"] = html.unescape(str(attribute.Notes).strip()) + if attribute_name in properties.keys(): + attribute_data.update(properties[attribute_name]) + + properties[attribute_name] = attribute_data + except: + raise Exception(f"Failed to add attribute {attribute.Name} to {k}") + + return properties + + @staticmethod + def convert_cim_type(cim_type: str) -> str: + return _CIM_PRIMATIVES.get(cim_type, cim_type) + + @staticmethod + def collect_template_node_names(template: dict, nodes: list = None, currentParent: str = None, parentObject: str = None): + if nodes is None: + nodes = [] + + for k, v in template.items(): + if isinstance(v, dict) and v.get("type", "none") == "container": + nodes = self.collect_template_node_names(v.get("properties", {}), nodes, parentObject=k) + elif isinstance(v, dict) and v.get("type", "none") == "object": + if currentParent is not None: + nodes.append(f"{currentParent}::{k}") + nodes = self.collect_template_node_names(v.get("properties", {}), nodes, currentParent=f"{currentParent}::{k}") + else: + nodes.append(k) + nodes = self.collect_template_node_names(v.get("properties", {}), nodes, currentParent=k) + elif isinstance(v, dict) and v.get("type", "none") == "ref": + continue + + return nodes + + +if __name__ == "__main__": + template = SchemaTemplate() + + template.template diff --git a/ravens/schema/validate.py b/ravens/schema/validate.py index 8352586a3..125dcbfe2 100755 --- a/ravens/schema/validate.py +++ b/ravens/schema/validate.py @@ -1,11 +1,13 @@ import json import pathlib + from jschon import create_catalog, JSON, JSONSchema, URI, LocalSource, RemoteSource -from ravens.schema.decompose_schema import _default_base_uri + +from ravens.data import _RAVENS_SCHEMA_BASE_URL class RavensValidator: - def __init__(self, schema_base_uri: str = _default_base_uri, schema_url: str = None, local_path_to_schema: pathlib.PosixPath = None): + def __init__(self, schema_base_uri: str = _RAVENS_SCHEMA_BASE_URL, schema_url: str = None, local_path_to_schema: pathlib.PosixPath = None): self.catalog = create_catalog("2020-12") if schema_url is None: diff --git a/ravens/uml/__init__.py b/ravens/uml/__init__.py index e69de29bb..7767e504e 100644 --- a/ravens/uml/__init__.py +++ b/ravens/uml/__init__.py @@ -0,0 +1,4 @@ +from .data import UMLData +from .exclusions import UMLExclusions +from .graph import UMLGraphs +from .visualize import UMLVisualizer diff --git a/ravens/uml/common.py b/ravens/uml/common.py new file mode 100644 index 000000000..777ce760c --- /dev/null +++ b/ravens/uml/common.py @@ -0,0 +1,5 @@ +from ravens.io import UMLData + + +def get_names_of_enumeration_classes(uml_data: UMLData): + return [o.Name for o in uml_data.objects.itertuples() if o.Object_Type == "Class" and o.Stereotype == "enumeration"] diff --git a/ravens/uml/d3.py b/ravens/uml/d3.py deleted file mode 100755 index 334c43d5b..000000000 --- a/ravens/uml/d3.py +++ /dev/null @@ -1,274 +0,0 @@ -import json -import os -import subprocess - -import pandas as pd - -from ravens.io import parse_uml_data - -ea_rgb_dec2hex = {z * 65536 + y * 256 + x: "#{:02x}{:02x}{:02x}".format(x, y, z) for x in range(256) for y in range(256) for z in range(256)} - - -def parse_link_style(link_style_string: str): - link_style = {} - try: - if "$" in link_style_string: - pre, post = link_style_string.split("$") - if pre: - for item in pre.split(";"): - if "," in item: - link_style["link_geometry"] = [i for i in item.split(",") if i] - elif item: - key, value = item.split("=", 1) - link_style[key] = value - - if post: - for item in post.split(";"): - if item: - outer_key, values = item.split("=", 1) - link_style[outer_key] = {} - if values: - for i in values.split(":"): - key, value = i.split("=") - link_style[outer_key][key] = int(value) - else: - for item in link_style_string.split(";"): - if "," in item: - link_style["link_geometry"] = [i for i in item.split(",") if i] - elif item: - key, value = item.split("=", 1) - link_style[key] = value - - except Exception as msg: - raise Exception(link_style_string) - - return link_style - - -def parse_object_style(object_style_string: str): - object_style = {} - try: - for item in object_style_string.split(";"): - if item: - if item.count("=") == 1: - key, value = item.split("=") - object_style[key] = value - elif item.count("=") == 2: - key_outer, key_inner, value = item.split("=") - object_style[key_outer] = {key_inner: value} - else: - raise ValueError - except ValueError as msg: - print(object_style_string) - - return object_style - - -def create_svg_data(uml_data, diagram_id: int): - dobjects = uml_data.diagramobjects[uml_data.diagramobjects["Diagram_ID"] == diagram_id] - if dobjects.empty: - return {"cx": 0, "cy": 0, "nodes": [], "links": []} - - svg_data = { - "cx": max([abs(o.RectRight) for o in dobjects.itertuples()]), - "cy": max([abs(o.RectBottom) for o in dobjects.itertuples()]), - } - - boxes_data = [] - nodes = [] - objs_in_diagram = [_o.Object_ID for _o in uml_data.diagramobjects[uml_data.diagramobjects["Diagram_ID"] == diagram_id].itertuples()] - - for o in uml_data.diagramobjects[uml_data.diagramobjects["Diagram_ID"] == diagram_id].itertuples(): - object_style = parse_object_style(str(o.ObjectStyle)) - obj = uml_data.objects.loc[o.Object_ID] - - text_lines = [] - if obj.Stereotype == "enumeration": - text_lines.append({"text": f"<<{obj.Stereotype}>>", "align": "center"}) - text_lines.append({"text": f"{obj.Name}", "align": "center", "style": "bold"}) - text_lines.append({}) - text_lines.append({"text": "literals", "align": "center", "style": "italic"}) - for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == o.Object_ID].itertuples(): - text_lines.append({"text": f"{attr.Name}", "align": "left"}) - elif obj.Stereotype == "CIMDatatype": - text_lines.append({"text": f"<<{obj.Stereotype}>>", "align": "center"}) - text_lines.append({"text": f"{obj.Name}", "align": "center", "style": "bold"}) - if object_style.get("AttPub", "1") == "1": - text_lines.append({}) - for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == o.Object_ID].itertuples(): - text_lines.append({"text": f"+ {attr.Name}: {attr.Type}", "align": "left"}) - else: - gen_obj_id = None - for c in uml_data.connectors[(uml_data.connectors["Start_Object_ID"] == o.Object_ID) & (uml_data.connectors["Connector_Type"] == "Generalization")].itertuples(): - gen_obj_id = c.End_Object_ID - break - - if (gen_obj_id is not None) and (gen_obj_id not in objs_in_diagram): - text_lines.append({"text": f"{uml_data.objects.loc[gen_obj_id].Name}", "align": "right", "style": "italic"}) - - text_lines.append({"text": f"{obj.Name}", "align": "center", "style": "bold"}) - if object_style.get("AttPub", "1") == "1": - text_lines.append({}) - for attr in uml_data.attributes[uml_data.attributes["Object_ID"] == o.Object_ID].itertuples(): - text_lines.append({"text": f"+ {attr.Name}: {attr.Type}", "align": "left"}) - - box_color = int(object_style.get("BCol", "-1")) - if box_color == -1: - if obj.Stereotype == "enumeration": - box_color = 14941672 - else: - box_color = 16251645 # default color of Classes - - box_data = { - "id": o.Object_ID, - "x": o.RectLeft, - "y": -o.RectTop, - "width": abs(o.RectRight - o.RectLeft), - "height": abs(o.RectTop - o.RectBottom), - "textLines": text_lines, - "color": ea_rgb_dec2hex[box_color], - } - nodes.append(o.Object_ID) - - boxes_data.append(box_data) - - svg_data["nodes"] = boxes_data - - links_data = [] - for l in uml_data.diagramlinks[uml_data.diagramlinks["DiagramID"] == diagram_id].itertuples(): - if l.Hidden: - continue - - link_style = parse_link_style(str(l.Geometry)) - - connector = uml_data.connectors.loc[l.ConnectorID] - if connector.Start_Object_ID not in nodes or connector.End_Object_ID not in nodes: - continue - - line_color = int(connector.LineColor) - if line_color == -1: - line_color = 9204585 # default line color different than default object color - - link_data = { - "source": str(connector.Start_Object_ID), - "target": str(connector.End_Object_ID), - "type": str(connector.Connector_Type).lower(), - "textStartTop": f"+{connector.SourceRole}" if not pd.isnull(connector.SourceRole) else "", - "textStartTopHidden": link_style.get("LLT", {}).get("HDN", 0), - "textStartTopXPos": link_style.get("LLT", {}).get("CX", 0.0), - "textStartTopYPos": link_style.get("LLT", {}).get("CY", 0.0), - "textEndTop": f"+{connector.DestRole}" if not pd.isnull(connector.DestRole) else "", - "textEndTopHidden": link_style.get("LRT", {}).get("HDN", 0), - "textEndTopXPos": link_style.get("LRT", {}).get("CX", 0.0), - "textEndTopYPos": link_style.get("LRT", {}).get("CY", 0.0), - "textStartBtm": f"{connector.SourceCard}" if not pd.isnull(connector.SourceCard) else "", - "textStartBtmHidden": link_style.get("LLB", {}).get("HDN", 0), - "textStartBtmXPos": link_style.get("LLB", {}).get("CX", 0.0), - "textStartBtmYPos": link_style.get("LLB", {}).get("CY", 0.0), - "textEndBtm": f"{connector.DestCard}" if not pd.isnull(connector.DestCard) else "", - "textEndBtmHidden": link_style.get("LRB", {}).get("HDN", 0), - "textEndBtmXPos": link_style.get("LRB", {}).get("CX", 0.0), - "textEndBtmYPos": link_style.get("LRB", {}).get("CY", 0.0), - "color": ea_rgb_dec2hex[line_color], - } - - links_data.append(link_data) - - svg_data["links"] = links_data - - return svg_data - - -def create_svg(uml_data): - data_json = json.dumps(uml_data) - - current_dir = os.path.dirname(os.path.abspath(__file__)) - js_file_path = os.path.join(current_dir, "js/svgRenderer.js") - - # Call the Node.js script - result = subprocess.run(["node", js_file_path, data_json], capture_output=True, text=True) - - if result.stderr: - raise Exception(result.stderr) - - # Get the SVG output - svg_output = result.stdout - return svg_output - - -def save_svg(uml_data, filename): - uml_data["outputPath"] = filename - svg_content = create_svg(uml_data) - - -def save_uml_diagram_from_package_and_diagram_name(uml_data, package_name, diagram_name, svg_dir_path): - pkg_id = uml_data.packages[uml_data.packages["Name"] == package_name].iloc[0]._name - diagram_id = uml_data.diagrams[(uml_data.diagrams["Package_ID"] == pkg_id) & (uml_data.diagrams["Name"] == diagram_name)].iloc[0]._name - svg_data = create_svg_data(uml_data, diagram_id) - path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram_name)}.svg") - save_svg(svg_data, path) - - return path - - -def save_uml_diagrams_from_package_name(uml_data, package_name, svg_dir_path): - paths = [] - pkg_id = uml_data.packages[uml_data.packages["Name"] == package_name].iloc[0]._name - for diagram in uml_data.diagrams[uml_data.diagrams["Package_ID"] == pkg_id].itertuples(): - svg_data = create_svg_data(uml_data, diagram.Index) - path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram.Name)}.svg") - save_svg(svg_data, path) - paths.append(path) - - return paths - - -def save_uml_diagrams_from_package_id(uml_data, package_id, svg_dir_path): - paths = [] - package_name = str(uml_data.packages.loc[package_id].Name).strip() - for diagram in uml_data.diagrams[uml_data.diagrams["Package_ID"] == package_id].itertuples(): - svg_data = create_svg_data(uml_data, diagram.Index) - path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram.Name)}.svg") - save_svg(svg_data, path) - paths.append(path) - - return paths - - -def save_all_uml_diagrams(uml_data, svg_dir_path): - paths = [] - for diagram in uml_data.diagrams[uml_data.diagrams["Diagram_Type"] == "Logical"].itertuples(): - package_name = str(uml_data.packages.loc[diagram.Package_ID].Name).strip() - try: - svg_data = create_svg_data(uml_data, diagram.Index) - path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram.Name)}.svg") - save_svg(svg_data, path) - paths.append(path) - except Exception as msg: - print(f"{str(package_name)}.{str(diagram.Name)} :: {str(diagram.Index)}") - print(msg) - continue - - return paths - - -if __name__ == "__main__": - __file__ = os.path.join(os.getcwd(), "ravens/uml/d3.py") - - db_filename = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - - uml_data = parse_uml_data(db_filename, set_index=True) - - svg_data = create_svg_data(uml_data, 11103) - - create_svg(svg_data) - - save_svg(svg_data, "test.svg") - - save_uml_diagram_from_package_and_diagram_name(uml_data, "EconomicDesign", "ProposedAssetOptions", "out/uml_d3") - save_uml_diagram_from_package_and_diagram_name(uml_data, "SimplifiedDiagrams", "Faults", "out/uml_d3") - - save_uml_diagrams_from_package_name(uml_data, "EconomicDesign", "docs/source/_static/uml") - save_uml_diagrams_from_package_name(uml_data, "SimplifiedDiagrams", "docs/source/_static/uml") - - save_all_uml_diagrams(uml_data, "docs/source/_static/uml") diff --git a/ravens/uml/data.py b/ravens/uml/data.py new file mode 100644 index 000000000..8f0a971cf --- /dev/null +++ b/ravens/uml/data.py @@ -0,0 +1,255 @@ +import pathlib + +import pandas as pd +import xml.etree.ElementTree as ET + +from ravens.data import _UML_XML_PATH + +_index_columns = { + "t_object": "Object_ID", + "t_connector": "Connector_ID", + "t_attribute": "ID", + "t_package": "Package_ID", + "t_diagram": "Diagram_ID", + "t_diagramlinks": "Instance_ID", + "t_diagramobjects": "Instance_ID", +} + + +_attr_names = { + "t_object": "objects", + "t_connector": "connectors", + "t_attribute": "attributes", + "t_package": "packages", + "t_diagram": "diagrams", + "t_diagramlinks": "diagramlinks", + "t_diagramobjects": "diagramobjects", +} + +_expected_dtypes = { + "t_package": { + "Package_ID": int, + "Name": str, + "Parent_ID": int, + "CreatedDate": "datetime", + "ModifiedDate": "datetime", + "Notes": str, + "ea_guid": str, + "IsControlled": bool, + "LastLoadDate": "datetime", + "LastSaveDate": "datetime", + "Version": str, + "Protected": bool, + "UseDTD": bool, + "LogXML": bool, + "TPos": int, + "BatchSave": int, + "BatchLoad": int, + }, + "t_object": { + "Object_ID": int, + "Object_Type": str, + "Diagram_ID": int, + "Name": str, + "Author": str, + "Version": str, + "Package_ID": int, + "NType": int, + "Complexity": int, + "Effort": int, + "Backcolor": int, + "BorderStyle": int, + "BorderWidth": int, + "Fontcolor": int, + "Bordercolor": int, + "CreatedDate": "datetime", + "ModifiedDate": "datetime", + "Status": str, + "Abstract": int, + "Tagged": int, + "GenType": str, + "Phase": str, + "Scope": str, + "Classifier": int, + "ea_guid": str, + "ParentID": int, + "Classifier_guid": str, + "IsRoot": bool, + "IsLeaf": bool, + "IsSpec": bool, + "IsActive": bool, + }, + "t_connector": { + "Connector_ID": int, + "Connector_Type": str, + "SourceAccess": str, + "DestAccess": str, + "SourceIsAggregate": int, + "SourceIsOrdered": int, + "DestIsAggregate": int, + "DestIsOrdered": int, + "Start_Object_ID": int, + "End_Object_ID": int, + "Start_Edge": int, + "End_Edge": int, + "PtStartX": int, + "PtStartY": int, + "PtEndX": int, + "PtEndY": int, + "SeqNo": int, + "HeadStyle": int, + "LineStyle": int, + "RouteStyle": int, + "IsBold": int, + "LineColor": int, + "VirtualInheritance": int, + "DiagramID": int, + "ea_guid": str, + "SourceIsNavigable": bool, + "DestIsNavigable": bool, + "IsRoot": bool, + "IsLeaf": bool, + "IsSpec": bool, + "IsSignal": bool, + "IsStimulus": bool, + "Target2": int, + }, + "t_attribute": { + "Object_ID": int, + "Name": str, + "Scope": str, + "Containment": str, + "IsStatic": int, + "IsCollection": int, + "IsOrdered": int, + "AllowDuplicates": int, + "LowerBound": int, + "UpperBound": int, + "Notes": str, + "Derived": int, + "ID": int, + "Pos": int, + "Length": int, + "Precision": int, + "Scale": int, + "Const": int, + "Classifier": int, + "Type": str, + "ea_guid": str, + "StyleEx": str, + }, + "t_diagram": { + "Diagram_ID": int, + "Package_ID": int, + "ParentID": int, + "Diagram_Type": str, + "Name": str, + "Version": str, + "Author": str, + "ShowDetails": int, + "Notes": str, + "AttPub": bool, + "AttPri": bool, + "AttPro": bool, + "Orientation": str, + "cx": int, + "cy": int, + "Scale": float, + "CreatedDate": "datetime", + "ModifiedDate": "datetime", + "ShowForeign": bool, + "ShowBorder": bool, + "ShowPackageContents": bool, + "PDATA": str, + "Locked": bool, + "ea_guid": str, + "TPos": int, + "Swimlanes": str, + "StyleEx": str, + }, + "t_diagramobjects": { + "Diagram_ID": int, + "Object_ID": int, + "RectTop": int, + "RectLeft": int, + "RectRight": int, + "RectBottom": int, + "Sequence": int, + "ObjectStyle": str, + "Instance_ID": int, + }, + "t_diagramlinks": { + "DiagramID": int, + "ConnectorID": int, + "Geometry": str, + "Style": str, + "Hidden": bool, + "Instance_ID": int, + }, +} + + +class UMLData: + def __init__(self): + dataframes = self._create_dataframes() + for table_name, df in dataframes.items(): + setattr(self, _attr_names[table_name], df) + + @classmethod + def loadf(cls, file: pathlib.PosixPath = _UML_XML_PATH, set_index: bool = True) -> object: + dataframes = cls._create_dataframes(file, set_index=set_index) + obj = cls() + for table_name, df in dataframes.items(): + setattr(obj, _attr_names[table_name], df) + + return obj + + @staticmethod + def _create_dataframes(file: pathlib.PosixPath = _UML_XML_PATH, set_index: bool = True) -> object: + tree = ET.parse(file) + root = tree.getroot() + + dataframes = {} + + for table in root.findall("Table"): + table_name = table.attrib.get("name") + if table_name not in _attr_names: + continue + + rows_data = [] + + for row in table.findall("Row"): + row_data = {} + + for column in row.findall("Column"): + col_name = column.attrib.get("name") + col_value = column.attrib.get("value") + row_data[col_name] = col_value + + rows_data.append(row_data) + + df = pd.DataFrame(rows_data) + + if table_name in _expected_dtypes: + for col, dtype in _expected_dtypes[table_name].items(): + if dtype == "datetime": + df[col] = pd.to_datetime(df[col], errors="coerce") + elif dtype == bool: + df[col] = df[col].map({"TRUE": True, "FALSE": False, "true": True, "false": False}) + elif dtype == int or dtype == float: + df[col] = pd.to_numeric(df[col], errors="coerce") + else: + df[col] = df[col].astype(dtype) + + if set_index: + df = df.set_index(_index_columns[table_name]) + + dataframes[table_name] = df + + return dataframes + + +if __name__ == "__main__": + uml = UMLData() + + uml2 = UMLData.loadf(_UML_XML_PATH) diff --git a/ravens/uml/exclusions.py b/ravens/uml/exclusions.py new file mode 100644 index 000000000..3c0139a24 --- /dev/null +++ b/ravens/uml/exclusions.py @@ -0,0 +1,34 @@ +from ravens.uml import UMLData + + +class UMLExclusions: + def __init__(self, uml_data: UMLData = None): + if uml_data is None: + uml_data = UMLData() + + self.uml_data = uml_data + self.package_ids = [] + self.object_ids = [] + + def exclude_by_name_startswith(self, exclusions: list, skip_object_exclusion: bool = False, skip_package_exclusion: bool = False): + lambda_func = lambda x: any(str(x.Name).startswith(k) for k in exclusions) + + return self.exclude_by_lambda_function(lambda_func, skip_object_exclusion=skip_object_exclusion, skip_package_exclusion=skip_package_exclusion) + + def exclude_by_lambda_function(self, lambda_func, skip_object_exclusion: bool = False, skip_package_exclusion: bool = False): + if not skip_package_exclusion: + self._build_package_exclusions(lambda_func) + if not skip_object_exclusion: + self._build_object_exclusions(lambda_func) + + return self + + def _build_package_exclusions(self, lambda_func): + self.package_ids = [pkg.Index for pkg in self.uml_data.packages.itertuples() if lambda_func(pkg)] + + def _build_object_exclusions(self, lambda_func): + self.object_ids = [obj.Index for obj in self.uml_data.objects.itertuples() if lambda_func(obj)] + [obj.Index for p in self.package_ids for obj in self.uml_data.objects[self.uml_data.objects["Package_ID"] == p].itertuples()] + + +if __name__ == "__main__": + exclusions = UMLExclusions().exclude_by_name_startswith(["Inf", "Mkt"]) diff --git a/ravens/uml/graph.py b/ravens/uml/graph.py new file mode 100644 index 000000000..77add3642 --- /dev/null +++ b/ravens/uml/graph.py @@ -0,0 +1,191 @@ +import glob +import os + +import networkx as nx +import pandas as pd + +from ravens.uml import UMLData, UMLExclusions + + +class UMLGraphs: + def __init__(self, uml_data: UMLData = None, exclusions: UMLExclusions = None, schema_template=None): + if uml_data is None: + uml_data = UMLData() + + self.uml_data = uml_data + + self.exclusions = UMLExclusions() if exclusions is None else exclusions + + self.gen_graph = self.build_generalization_graph() + self.attr_graph = self.build_attribute_graph() + self.assoc_graph = self.build_association_graph() + + self.graph = nx.compose_all([self.gen_graph, self.attr_graph, self.assoc_graph]) + + self.subgraphs = {} + if schema_template is not None: + self._build_subgraphs_from_template(schema_template) + + def _add_class_nodes_to_graph(self, G): + attrs = { + obj.Index: [attr.Index for attr in self.uml_data.attributes[self.uml_data.attributes["Object_ID"] == obj.Index].itertuples()] + for obj in self.uml_data.objects[self.uml_data.objects["Object_Type"] == "Class"].itertuples() + if pd.isnull(obj.Stereotype) + } + + conns = { + obj.Index: [c.Index for c in self.uml_data.connectors[(self.uml_data.connectors["Start_Object_ID"] == obj.Index) | (self.uml_data.connectors["End_Object_ID"] == obj.Index)].itertuples()] + for obj in self.uml_data.objects[self.uml_data.objects["Object_Type"] == "Class"].itertuples() + if pd.isnull(obj.Stereotype) + } + + gens = {node: [c for c in cs if self.uml_data.connectors.loc[c]["Connector_Type"] == "Generalization"] for node, cs in conns.items()} + + assocs = {node: [c for c in cs if self.uml_data.connectors.loc[c]["Connector_Type"] == "Association"] for node, cs in conns.items()} + + G.add_nodes_from( + [ + ( + obj.Index, + { + "Name": str(obj.Name) + f" ({len(attrs[obj.Index])}+{len(gens[obj.Index])}+{len(assocs[obj.Index])})", + "Object_ID": str(obj.Index), + "Note": str(obj.Note), + "Attributes": ", ".join([self.uml_data.attributes.loc[i]["Name"] for i in attrs[obj.Index]]), + "Object_Type": "Class", + }, + ) + for obj in self.uml_data.objects[self.uml_data.objects["Object_Type"] == "Class"].itertuples() + if pd.isnull(obj.Stereotype) and obj.Package_ID not in self.exclusions.package_ids and obj.Index not in self.exclusions.object_ids + ] + ) + + return G + + def build_generalization_graph(self) -> nx.MultiDiGraph: + GG = nx.MultiDiGraph() + GG = self._add_class_nodes_to_graph(GG) + for c in self.uml_data.connectors[self.uml_data.connectors["Connector_Type"] == "Generalization"].itertuples(): + if ( + self.uml_data.objects.loc[c.Start_Object_ID]["Object_Type"] == "Class" + and self.uml_data.objects.loc[c.End_Object_ID]["Object_Type"] == "Class" + and self.uml_data.objects.loc[c.Start_Object_ID]["Package_ID"] not in self.exclusions.package_ids + and self.uml_data.objects.loc[c.End_Object_ID]["Package_ID"] not in self.exclusions.package_ids + and pd.isnull(self.uml_data.objects.loc[c.Start_Object_ID]["Stereotype"]) + and pd.isnull(self.uml_data.objects.loc[c.End_Object_ID]["Stereotype"]) + and c.Start_Object_ID not in self.exclusions.object_ids + and c.End_Object_ID not in self.exclusions.object_ids + ): + GG.add_edge( + c.Start_Object_ID, + c.End_Object_ID, + Start_Object_ID=str(c.Start_Object_ID), + End_Object_ID=str(c.End_Object_ID), + Connector_ID="GEN_" + str(c.Index), + Connector_Type="Generalization", + weight=10.0, + ) + + return GG + + def build_attribute_graph(self) -> nx.MultiDiGraph: + AT = nx.MultiDiGraph() + AT = self._add_class_nodes_to_graph(AT) + for n in list(AT.nodes): + for attr in self.uml_data.attributes[self.uml_data.attributes["Object_ID"] == n].itertuples(): + AT.add_edge(attr.Index, n, Connector_Type="Attribute", Connector_ID="ATTR_" + str(attr.Index), weight=100.0) + AT.nodes[attr.Index].update({"Name": str(attr.Name), "Note": str(attr.Notes), "Object_Type": "Attribute", "Attribute_ID": str(attr.Index)}) + + return AT + + def build_association_graph(self) -> nx.MultiDiGraph: + AG = nx.MultiDiGraph() + AG = self._add_class_nodes_to_graph(AG) + for c in self.uml_data.connectors[(self.uml_data.connectors["Connector_Type"] == "Association") | (self.uml_data.connectors["Connector_Type"] == "Aggregation")].itertuples(): + if ( + self.uml_data.objects.loc[c.Start_Object_ID]["Object_Type"] == "Class" + and self.uml_data.objects.loc[c.End_Object_ID]["Object_Type"] == "Class" + and self.uml_data.objects.loc[c.Start_Object_ID]["Package_ID"] not in self.exclusions.package_ids + and self.uml_data.objects.loc[c.End_Object_ID]["Package_ID"] not in self.exclusions.package_ids + and pd.isnull(self.uml_data.objects.loc[c.Start_Object_ID]["Stereotype"]) + and pd.isnull(self.uml_data.objects.loc[c.End_Object_ID]["Stereotype"]) + and c.Start_Object_ID not in self.exclusions.object_ids + and c.End_Object_ID not in self.exclusions.object_ids + ): + AG.add_edge( + c.End_Object_ID, + c.Start_Object_ID, + SourceCard=str(c.DestCard), + DestCard=str(c.SourceCard), + SourceRole=str(c.DestRole) if not pd.isnull(c.DestRole) else str(self.uml_data.objects.loc[c.End_Object_ID]["Name"]), + DestRole=str(c.SourceRole) if not pd.isnull(c.SourceRole) else str(self.uml_data.objects.loc[c.Start_Object_ID]["Name"]), + Connector_ID="ASC_REV_" + str(c.Index), + End_Object_ID=str(c.Start_Object_ID), + Start_Object_ID=str(c.End_Object_ID), + Connector_Type=str(c.Connector_Type), + weight=1.0, + ) + + AG.add_edge( + c.Start_Object_ID, + c.End_Object_ID, + DestCard=str(c.DestCard), + SourceCard=str(c.SourceCard), + DestRole=str(c.DestRole) if not pd.isnull(c.DestRole) else str(self.uml_data.objects.loc[c.End_Object_ID]["Name"]), + SourceRole=str(c.SourceRole) if not pd.isnull(c.SourceRole) else str(self.uml_data.objects.loc[c.Start_Object_ID]["Name"]), + Connector_ID="ASC_FWD_" + str(c.Index), + Start_Object_ID=str(c.Start_Object_ID), + End_Object_ID=str(c.End_Object_ID), + Connector_Type=str(c.Connector_Type), + weight=1.0, + ) + + return AG + + def _build_subgraphs_from_template(self, template): + id2name = { + **{obj.Index: str(obj.Name) for obj in uml_data.objects.itertuples()}, + **{attr.Index: str(attr.Name) for attr in uml_data.attributes.itertuples()}, + } + cls_name2id = {str(obj.Name): obj.Index for obj in uml_data.objects[uml_data.objects["Object_Type"] == "Class"].itertuples() if pd.isnull(obj.Stereotype)} + + template_names = template.nodes + + for name in template_names: + obj_id = cls_name2id[name] + nodes = {at for n in [obj_id] + list(nx.ancestors(GG, obj_id)) + list(nx.descendants(GG, obj_id)) for a in list(AG.neighbors(n)) + [n] for at in [n, a] + list(AT.predecessors(a)) + list(AT.predecessors(n))} + + self.subgraphs[name] = nx.subgraph(GG_AT_AG, nodes) + + def export_subgraphs(self, export_dir: str, clean_dir: bool = False): + if clean_dir: + for file in glob.glob(os.path.join(export_dir, "*")): + os.remove(file) + + for k, v in self.subgraphs.items(): + nx.export_graphml(v, os.path.join(export_dir, f"{k}.graphml")) + + def export_graph(self, file_out: str): + nx.write_graphml(self.graph, file_out, named_key_ids=True, edge_id_from_attribute="Connector_ID") + + def export_generalization_graph(self, file_out: str): + nx.write_graphml(self.gen_graph, file_out) + + def export_attribute_graph(self, file_out: str): + nx.write_graphml(self.attr_graph, file_out) + + def export_association_graph(self, file_out: str): + nx.write_graphml(self.assoc_graph, file_out) + + +if __name__ == "__main__": + pathlib.Path("out/CIM_graphs").mkdir(parents=True, exist_ok=True) + pathlib.Path("out/template_graphs").mkdir(parents=True, exist_ok=True) + + exclusions = UMLExclusions().exclude_by_name_startswith(["Inf", "Mkt"]) + + graphs = UMLGraphs(exclusions=exclusions) + graphs.export_graph("out/CIM_graphs/GG_AT_AG.graphml") + + graphs_with_subgraphs = UMLGraphs(exclusions=exclusions, schema_template=SchemaTemplate().loadf("cim/schema_template.json")) + graphs.export_subgraphs("out/template_graphs") diff --git a/ravens/uml/plantuml.py b/ravens/uml/plantuml.py index d161e1cc4..29c5d782c 100644 --- a/ravens/uml/plantuml.py +++ b/ravens/uml/plantuml.py @@ -1,9 +1,10 @@ import html import json +import pathlib import pandas as pd -from ravens.io import CoreData, DiagramData +from ravens.uml import UMLData connector_strings = { "Generalization": "-up-|>", @@ -109,11 +110,9 @@ def build_plantuml_link(uml_data: UMLData, link): if __name__ == "__main__": - from ravens.io import parse_uml_data + pathlib.Path("out/uml").mkdir(parents=True, exist_ok=True) - db_filename = "cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi" - - data = parse_uml_data(db_filename, set_index=False) + data = UMLData() plantuml_diagrams = build_all_plantuml_diagrams(uml_data) diff --git a/ravens/uml/visualize.py b/ravens/uml/visualize.py new file mode 100755 index 000000000..af9f6571c --- /dev/null +++ b/ravens/uml/visualize.py @@ -0,0 +1,283 @@ +import json +import os +import pathlib +import subprocess + +import pandas as pd + +from ravens.data import _SVG_RENDERER_PATH, _CIM_RGB_TO_HEX +from ravens.io import UMLData + + +class UMLDiagramData: + def __init__(self): + pass + + +class UMLVisualizer: + def __init__(self, uml_data: UMLData = None): + if uml_data is None: + uml_data = UMLData() + + self.uml_data = uml_data + + self._current_diagram = None + self._current_svg_data = None + self._current_svg = None + + @staticmethod + def _parse_link_style(link_style_string: str): + link_style = {} + try: + if "$" in link_style_string: + pre, post = link_style_string.split("$") + if pre: + for item in pre.split(";"): + if "," in item: + link_style["link_geometry"] = [i for i in item.split(",") if i] + elif item: + key, value = item.split("=", 1) + link_style[key] = value + + if post: + for item in post.split(";"): + if item: + outer_key, values = item.split("=", 1) + link_style[outer_key] = {} + if values: + for i in values.split(":"): + key, value = i.split("=") + link_style[outer_key][key] = int(value) + else: + for item in link_style_string.split(";"): + if "," in item: + link_style["link_geometry"] = [i for i in item.split(",") if i] + elif item: + key, value = item.split("=", 1) + link_style[key] = value + + except Exception as msg: + raise Exception(link_style_string) + + return link_style + + @staticmethod + def _parse_object_style(object_style_string: str): + object_style = {} + try: + for item in object_style_string.split(";"): + if item: + if item.count("=") == 1: + key, value = item.split("=") + object_style[key] = value + elif item.count("=") == 2: + key_outer, key_inner, value = item.split("=") + object_style[key_outer] = {key_inner: value} + else: + raise ValueError + except ValueError as msg: + print(object_style_string) + + return object_style + + def _create_svg_data(self, diagram_id: int): + dobjects = self.uml_data.diagramobjects[self.uml_data.diagramobjects["Diagram_ID"] == diagram_id] + if dobjects.empty: + return {"cx": 0, "cy": 0, "nodes": [], "links": []} + + svg_data = { + "cx": max([abs(o.RectRight) for o in dobjects.itertuples()]), + "cy": max([abs(o.RectBottom) for o in dobjects.itertuples()]), + } + + boxes_data = [] + nodes = [] + objs_in_diagram = [_o.Object_ID for _o in self.uml_data.diagramobjects[self.uml_data.diagramobjects["Diagram_ID"] == diagram_id].itertuples()] + + for o in self.uml_data.diagramobjects[self.uml_data.diagramobjects["Diagram_ID"] == diagram_id].itertuples(): + object_style = self._parse_object_style(str(o.ObjectStyle)) + obj = self.uml_data.objects.loc[o.Object_ID] + + text_lines = [] + if obj.Stereotype == "enumeration": + text_lines.append({"text": f"<<{obj.Stereotype}>>", "align": "center"}) + text_lines.append({"text": f"{obj.Name}", "align": "center", "style": "bold"}) + text_lines.append({}) + text_lines.append({"text": "literals", "align": "center", "style": "italic"}) + for attr in self.uml_data.attributes[self.uml_data.attributes["Object_ID"] == o.Object_ID].itertuples(): + text_lines.append({"text": f"{attr.Name}", "align": "left"}) + elif obj.Stereotype == "CIMDatatype": + text_lines.append({"text": f"<<{obj.Stereotype}>>", "align": "center"}) + text_lines.append({"text": f"{obj.Name}", "align": "center", "style": "bold"}) + if object_style.get("AttPub", "1") == "1": + text_lines.append({}) + for attr in self.uml_data.attributes[self.uml_data.attributes["Object_ID"] == o.Object_ID].itertuples(): + text_lines.append({"text": f"+ {attr.Name}: {attr.Type}", "align": "left"}) + else: + gen_obj_id = None + for c in self.uml_data.connectors[(self.uml_data.connectors["Start_Object_ID"] == o.Object_ID) & (self.uml_data.connectors["Connector_Type"] == "Generalization")].itertuples(): + gen_obj_id = c.End_Object_ID + break + + if (gen_obj_id is not None) and (gen_obj_id not in objs_in_diagram): + text_lines.append({"text": f"{self.uml_data.objects.loc[gen_obj_id].Name}", "align": "right", "style": "italic"}) + + text_lines.append({"text": f"{obj.Name}", "align": "center", "style": "bold"}) + if object_style.get("AttPub", "1") == "1": + text_lines.append({}) + for attr in self.uml_data.attributes[self.uml_data.attributes["Object_ID"] == o.Object_ID].itertuples(): + text_lines.append({"text": f"+ {attr.Name}: {attr.Type}", "align": "left"}) + + box_color = int(object_style.get("BCol", "-1")) + if box_color == -1: + if obj.Stereotype == "enumeration": + box_color = 14941672 + else: + box_color = 16251645 # default color of Classes + + box_data = { + "id": o.Object_ID, + "x": o.RectLeft, + "y": -o.RectTop, + "width": abs(o.RectRight - o.RectLeft), + "height": abs(o.RectTop - o.RectBottom), + "textLines": text_lines, + "color": _CIM_RGB_TO_HEX[box_color], + } + nodes.append(o.Object_ID) + + boxes_data.append(box_data) + + svg_data["nodes"] = boxes_data + + links_data = [] + for l in self.uml_data.diagramlinks[self.uml_data.diagramlinks["DiagramID"] == diagram_id].itertuples(): + if l.Hidden: + continue + + link_style = self._parse_link_style(str(l.Geometry)) + + connector = self.uml_data.connectors.loc[l.ConnectorID] + if connector.Start_Object_ID not in nodes or connector.End_Object_ID not in nodes: + continue + + line_color = int(connector.LineColor) + if line_color == -1: + line_color = 9204585 # default line color different than default object color + + link_data = { + "source": str(connector.Start_Object_ID), + "target": str(connector.End_Object_ID), + "type": str(connector.Connector_Type).lower(), + "textStartTop": f"+{connector.SourceRole}" if not pd.isnull(connector.SourceRole) else "", + "textStartTopHidden": link_style.get("LLT", {}).get("HDN", 0), + "textStartTopXPos": link_style.get("LLT", {}).get("CX", 0.0), + "textStartTopYPos": link_style.get("LLT", {}).get("CY", 0.0), + "textEndTop": f"+{connector.DestRole}" if not pd.isnull(connector.DestRole) else "", + "textEndTopHidden": link_style.get("LRT", {}).get("HDN", 0), + "textEndTopXPos": link_style.get("LRT", {}).get("CX", 0.0), + "textEndTopYPos": link_style.get("LRT", {}).get("CY", 0.0), + "textStartBtm": f"{connector.SourceCard}" if not pd.isnull(connector.SourceCard) else "", + "textStartBtmHidden": link_style.get("LLB", {}).get("HDN", 0), + "textStartBtmXPos": link_style.get("LLB", {}).get("CX", 0.0), + "textStartBtmYPos": link_style.get("LLB", {}).get("CY", 0.0), + "textEndBtm": f"{connector.DestCard}" if not pd.isnull(connector.DestCard) else "", + "textEndBtmHidden": link_style.get("LRB", {}).get("HDN", 0), + "textEndBtmXPos": link_style.get("LRB", {}).get("CX", 0.0), + "textEndBtmYPos": link_style.get("LRB", {}).get("CY", 0.0), + "color": _CIM_RGB_TO_HEX[line_color], + } + + links_data.append(link_data) + + svg_data["links"] = links_data + + self._current_diagram = diagram_id + self._current_svg_data = svg_data + + def _create_svg(self): + data_json = json.dumps(self._current_svg_data) + + # Call the Node.js script + result = subprocess.run(["node", _SVG_RENDERER_PATH, data_json], capture_output=True, text=True) + + if result.stderr: + raise Exception(result.stderr) + + # Get the SVG output + self._current_svg = result.stdout + + def _save_current_svg(self, filename: str): + self._current_svg_data["outputPath"] = filename + self._current_svg = self._create_svg() + + def save_uml_diagram_from_package_and_diagram_name(self, package_name: str, diagram_name: str, svg_dir_path: pathlib.PosixPath) -> str: + pkg_id = self.uml_data.packages[self.uml_data.packages["Name"] == package_name].iloc[0]._name + diagram_id = self.uml_data.diagrams[(self.uml_data.diagrams["Package_ID"] == pkg_id) & (self.uml_data.diagrams["Name"] == diagram_name)].iloc[0]._name + self._create_svg_data(diagram_id) + + path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram_name)}.svg") + self._save_current_svg(path) + + return path + + def save_uml_diagrams_from_package_name(self, package_name: str, svg_dir_path: pathlib.PosixPath) -> list: + paths = [] + pkg_id = self.uml_data.packages[self.uml_data.packages["Name"] == package_name].iloc[0]._name + for diagram in self.uml_data.diagrams[self.uml_data.diagrams["Package_ID"] == pkg_id].itertuples(): + self._create_svg_data(diagram.Index) + + path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram.Name)}.svg") + self._save_current_svg(path) + + paths.append(path) + + return paths + + def save_uml_diagrams_from_package_id(self, package_id: str, svg_dir_path: pathlib.PosixPath) -> list: + paths = [] + package_name = str(self.uml_data.packages.loc[package_id].Name).strip() + for diagram in self.uml_data.diagrams[self.uml_data.diagrams["Package_ID"] == package_id].itertuples(): + self._create_svg_data(self.uml_data, diagram.Index) + + path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram.Name)}.svg") + self._save_current_svg(path) + + paths.append(path) + + return paths + + def save_all_uml_diagrams(self, svg_dir_path: pathlib.PosixPath) -> list: + paths = [] + for diagram in self.uml_data.diagrams[self.uml_data.diagrams["Diagram_Type"] == "Logical"].itertuples(): + package_name = str(self.uml_data.packages.loc[diagram.Package_ID].Name).strip() + try: + self._create_svg_data(self.uml_data, diagram.Index) + + path = os.path.join(svg_dir_path, f"{str(package_name)}.{str(diagram.Name)}.svg") + self._save_current_svg(path) + + paths.append(path) + except Exception as msg: + print(f"{str(package_name)}.{str(diagram.Name)} :: {str(diagram.Index)}") + print(msg) + continue + + return paths + + +if __name__ == "__main__": + pathlib.Path("out/uml_d3").mkdir(parents=True, exist_ok=True) + + uml_vis = UMLVisualizer() + + uml_vis._create_svg_data(11103) + uml_vis._save_current_svg("out/test.svg") + + uml_vis.save_uml_diagram_from_package_and_diagram_name("EconomicDesign", "ProposedAssetOptions", "out/uml_d3") + uml_vis.save_uml_diagram_from_package_and_diagram_name("SimplifiedDiagrams", "Faults", "out/uml_d3") + + uml_vis.save_uml_diagrams_from_package_name("EconomicDesign", "out/uml_d3") + uml_vis.save_uml_diagrams_from_package_name("SimplifiedDiagrams", "out/uml_d3") + + uml_vis.save_all_uml_diagrams(uml_data, "out/uml_d3") diff --git a/ravens/xml/opendss2xml.py b/ravens/xml/opendss2xml.py index bc44a0686..4c9dc1105 100644 --- a/ravens/xml/opendss2xml.py +++ b/ravens/xml/opendss2xml.py @@ -1,4 +1,5 @@ import math +import pathlib from copy import deepcopy from uuid import uuid4 @@ -173,6 +174,21 @@ def _build_vector_group(self): self.vector_group = self.vector_group[0].upper() + self.vector_group[1::] +class TranformerInfo: + def __init__(self): + self.max_wdg = 0 + self.wdg_list = None + self.core_list = None + self.mesh_list = None + + def set_max_wdg(self, max_wdg): + if max_wdg > 0: + self.max_wdg = max_wdg + self.wdg_list = [None for i in range(max_wdg)] + self.core_list = [None for i in range(max_wdg)] + self.mesh_list = [None for i in range((max_wdg - 1) * max_wdg / 2)] + + class DssExport(object): """ Class for converting a DSS file into CIM XML @@ -225,6 +241,9 @@ def __init__(self, dss_file: str): self.dss = self.raw_dss.to_altdss() self.uuid_map = {} + + # Transformer specific properties + self.transformer_info = TransformerInfo() self.transformer_banks = {} self.graph = Graph() @@ -245,7 +264,7 @@ def _convert_dss_to_rdf(self): self._add_LinearShuntCompensators() self._add_Transformers() - def save(self, path: str): + def save(self, path: pathlib.PosixPath): self.graph.serialize(path, max_depth=1, format="pretty-xml") @staticmethod @@ -1017,5 +1036,7 @@ def _add_ShortCircuitTest(self, xfmrcode: object, subject_uris: list, seq: int, if __name__ == "__main__": + pathlib.Path("out").mkdir(parents=True, exist_ok=True) + d = DssExport("../../ronm/PowerModelsDistribution.jl/test/data/opendss/ut_trans_2w_yy.dss") d.save("out/test_opendss_convert.xml") diff --git a/ravens/xml/ravens2xml.py b/ravens/xml/ravens2xml.py index c9f0d1dd2..58f0f655f 100644 --- a/ravens/xml/ravens2xml.py +++ b/ravens/xml/ravens2xml.py @@ -1,4 +1,5 @@ import json +import pathlib import re from copy import deepcopy @@ -9,17 +10,19 @@ from rdflib.term import URIRef, Literal from rdflib import Graph, RDF -from ravens.cim_tools.common import get_names_of_enumeration_classes -from ravens.io import parse_uml_data +from ravens.uml.common import get_names_of_enumeration_classes +from ravens.uml import UMLData class RavensExport(object): - def __init__(self, data, uml_data_path=None): + def __init__(self, data, uml_data: UMLData = None): self.data = deepcopy(data) self.cim_enums = None - if uml_data_path is not None: - uml_data = parse_uml_data(uml_data_path) - self.cim_enums = get_names_of_enumeration_classes(uml_data.objects) + + if uml_data is None: + uml_data = UMLData() + + self.cim_enums = get_names_of_enumeration_classes(uml_data) self.graph = Graph() self.cim = Namespace("http://iec.ch/TC57/CIM100#") @@ -110,10 +113,15 @@ def update_enumeration_uri_refs(self): for triple in triple_to_delete: self.graph.remove(triple) + def save_cim_profile(self, file_path: pathlib.PosixPath): + self.graph.serialize(file_path, max_depth=1, format="pretty-xml") + if __name__ == "__main__": - with open("out/test_xml2json_case3.json", "r") as f: + pathlib.Path("out").mkdir(parents=True, exist_ok=True) + + with open("examples/case3_balanced.json", "r") as f: d = json.load(f) - r = RavensExport(d, uml_data_path="cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi") - r.graph.serialize("out/test_output.xml", max_depth=1, format="pretty-xml") + r = RavensExport(d) + r.save_cim_profile("out/test_output.xml") diff --git a/ravens/xml/xml2ravens.py b/ravens/xml/xml2ravens.py index 5b1023813..dd1bbfb60 100644 --- a/ravens/xml/xml2ravens.py +++ b/ravens/xml/xml2ravens.py @@ -1,13 +1,18 @@ +import pathlib import re import warnings +import networkx as nx + from ast import literal_eval from collections import namedtuple from copy import deepcopy from rdflib import Graph +from rdflib.extras.external_graph_libs import rdflib_to_networkx_multidigraph from rdflib.term import URIRef, Literal +from ravens.schema import SchemaTemplate Reference = namedtuple("Reference", ["parent", "id"]) @@ -176,20 +181,25 @@ def __iter__(self): class RAVENSData: - def __init__(self, rdf_graph, template, prune_unncessary=False): + def __init__(self, cim_profile_path: pathlib.PosixPath, schema_template: SchemaTemplate = None, prune_unncessary=False): + g = Graph() + self.rdf = g.parse("examples/IEEE13_Assets.xml", format="application/rdf+xml", publicID="urn:uuid:") + self.cim_ns = "http://iec.ch/TC57/CIM100" self.rdf_type = URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type") self.prune_unncessary = prune_unncessary self.untokenized_paths = [] self.reference_paths = {} - self.build_paths_from_template(template) + + if schema_template is None: + schema_template = SchemaTemplate() + + self.build_paths_from_template(schema_template.template) self.tokenized_paths = {} self.tokenize_paths() - self.rdf = rdf_graph - self.unique_subject_types = {s: o.split("#")[-1] for s, o in self.rdf.subject_objects(predicate=self.rdf_type)} self.paths = {s: [] for s, t in self.unique_subject_types.items()} @@ -226,7 +236,7 @@ def build_paths_from_template(self, template, current_path=None): pass else: raise Exception(f"unrecognized objectType for '{obj_id}': '{obj.get('$objectType', None)}'") - elif obj["type"] == "string" and obj["$objectType"] == "reference": + elif obj["type"] == "string" and obj.get("$objectType", None) == "reference": if obj_id not in self.reference_paths: self.reference_paths[obj_id] = set() @@ -569,46 +579,24 @@ def add_to_data(self, data, data_to_insert): else: raise Exception(f"This shouldn't happen: {path}, {self.current_resolved_path}") + def export_rdf_graphml(self, file_path: pathlib.PosixPath): + G = rdflib_to_networkx_multidigraph(self.rdf) -if __name__ == "__main__": - from ravens.cim_tools.template import CIMTemplate - from rdflib.extras.external_graph_libs import rdflib_to_networkx_multidigraph - import networkx as nx - import json + for i, e in enumerate(G.edges(keys=True)): + G.edges[e].update({"label": str(e[-1]), "id": str(i)}) + for n in G.nodes: + G.nodes[n].update({"label": str(n)}) - g = Graph() - g.parse("examples/IEEE13_Assets.xml", format="application/rdf+xml", publicID="urn:uuid:") - # g.parse("examples/case3_balanced.xml", format="application/rdf+xml", publicID="urn:uuid:") - # g.parse("examples/IEEE13_Assets.xml", format="application/rdf+xml", publicID="urn:uuid:") - # g.parse("examples/ieee8500u_fuseless_CIM100x.XML", format="application/rdf+xml", publicID="urn:uuid:") + nx.write_graphml(G, file_path, named_key_ids=True, edge_id_from_attribute="id") - # rm = set() - # for s, p, o in g.triples((None, None, None)): - # if not isinstance(o, URIRef) or str(o).startswith("http://iec.ch"): - # rm.add((s, p, o)) + def dump(self, file_path: pathlib.PosixPath, indent=None): + with open(file_path, "w") as f: + json.dump(self.data, f, indent=indent) - # for triple in rm: - # g.remove(triple) - # G = rdflib_to_networkx_multidigraph(g) - # for i, e in enumerate(G.edges(keys=True)): - # G.edges[e].update({"label": str(e[-1]), "id": str(i)}) - # for n in G.nodes: - # G.nodes[n].update({"label": str(n)}) - - # nx.write_graphml(G, "out/rdf_graphs/case3_balanced_uriref.graphml", named_key_ids=True, edge_id_from_attribute="id") - # nx.write_graphml(G, "out/rdf_graphs/ieee13_assets_uriref.graphml", named_key_ids=True, edge_id_from_attribute="id") - # nx.write_graphml(G, "out/rdf_graphs/ieee8500u_fuseless.graphml", named_key_ids=True, edge_id_from_attribute="id") - - d = RAVENSData(g, CIMTemplate("ravens/cim_tools/cim_conversion_template.json").template, prune_unncessary=False) - - with open("out/IEEE13_Assets.json", "w") as f: - json.dump(d.data, f, indent=2) - - # with open("out/test_xml2json_ieee13.json", "w") as f: - # json.dump(d.data, f, indent=2) +if __name__ == "__main__": + pathlib.Path("out").mkdir(parents=True, exist_ok=True) - # with open("out/ieee8500u_fuseless.json", "w") as f: - # json.dump(d.data, f, indent=2) + d = RAVENSData("examples/IEEE13_Assets.xml") - # g.serialize("out/rdf_graphs/case3_balanced.json", format="json-ld") + d.dump("out/IEEE13_Assets.json", indent=2) diff --git a/schema/build_schema.py b/schema/build_schema.py index 03f7a1425..41e411202 100755 --- a/schema/build_schema.py +++ b/schema/build_schema.py @@ -2,54 +2,17 @@ import json import os -import json_schema_for_humans.generate as Gen - -from ravens.io import parse_uml_data -from ravens.cim_tools.common import build_package_exclusions, build_object_exclusions -from ravens.cim_tools.graph import build_generalization_graph, build_attribute_graph -from ravens.cim_tools.template import CIMTemplate -from ravens.schema.build_definitions import build_definitions -from ravens.schema.build_map import add_attributes_to_template -from ravens.schema.build_schema import build_schema_from_map -from ravens.schema.add_copyright_notice import add_cim_copyright_notice_to_decomposed_schemas -from ravens.schema.decompose_schema import Schemas +from ravens.schema import RavensSchema +from ravens.uml import UMLExclusions def build_schema(): current_dir = os.path.dirname(os.path.abspath(__file__)) - template_path = os.path.join(current_dir, "../ravens/cim_tools/cim_conversion_template.json") - xmi_path = os.path.join(current_dir, "../cim/iec61970cim17v40_iec61968cim13v13b_iec62325cim03v17b_CIM100.1.1.1_mgravens24v1.xmi") schema_dir = os.path.join(current_dir, "../schema") - uml_data = parse_uml_data(xmi_path) - - exclude_packages = build_package_exclusions(uml_data.packages, lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"])) - exclude_objects = build_object_exclusions( - uml_data.objects, - lambda x: any(str(x.Name).startswith(k) for k in ["Inf", "Mkt"]), - exclude_packages=exclude_packages, - ) - - schema = build_schema_from_map( - add_attributes_to_template( - CIMTemplate(template_path).template, - CIMTemplate(template_path).template, - uml_data, - build_generalization_graph(uml_data, exclude_packages, exclude_objects), - build_attribute_graph(uml_data, exclude_packages, exclude_objects), - ) - ) - - schema["$defs"] = build_definitions(uml_data) - - a = Schemas(schema) - - add_cim_copyright_notice_to_decomposed_schemas(a.schemas, uml_data) + a = RavensSchema(uml_exclusions=UMLExclusions().exclude_by_name_startswith(["Inf", "Mkt"])) - for k, v in a.schemas.items(): - filename = k.split("/")[-1].replace(".json", "") - with open(os.path.join(schema_dir, f"{filename}.json"), "w") as f: - json.dump(v, f, indent=2) + a.export_schemas(schema_dir) if __name__ == "__main__": diff --git a/tests/test_build_schema.py b/tests/test_build_schema.py new file mode 100644 index 000000000..0f0cdf44a --- /dev/null +++ b/tests/test_build_schema.py @@ -0,0 +1,5 @@ +import pytest + + +def test_build_schema(): + pass diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 000000000..77ecf0189 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,10 @@ +import pytest + +from ravens.schema.validate import RavensValidator + + +def test_examples(): + validator = RavensValidator() + + for file in glob.glob("examples/schema/*.json"): + pass