Add support for jsonc files (#40)

* Add support for .jsonc in GUI
* Accept either json or jsonc input
* Generate pure json output when json is selected

Author: sveinse

.gitignore

@@ -21,6 +21,7 @@ _tmp*
 tests/od/extra-compare*
 /*.od
 /*.json
+/*.jsonc
 
 objdictgen.*
 fw-can-shared

README.md

@@ -18,8 +18,8 @@ Laerdal Medical fork for the canfestival library:
 objdictgen is a tool to parse, view and manipulate files containing object
 dictionary (OD). An object dictionary is entries with data and configuration
 in CANopen devices. The `odg` executable is installed. It supports
-reading and writing OD files in `.json` format, in legacy XML `.od` and `.eds`
-files. It can generate c code for use with the canfestival library.
+reading and writing OD files in `.json`/`.jsonc` format, in legacy XML `.od`
+and `.eds` files. It can generate c code for use with the canfestival library.
 
 
 ## Install
@@ -76,22 +76,19 @@ descriptions, help with values and validate the file.
     "json.schemas": [
       {
         "fileMatch": [
-          "**.json"
+          "**.jsonc"
         ],
         "url": "./src/objdictgen/schema/od.schema.json"
       }
     ],
-    "files.associations": {
-        "*.json": "jsonc"
-    }
 ```
 
 ## Conversion
 
 The recommended way to convert existing/legacy `.od` files to the new JSON
 format is:
 
-    $ odg generate <file.od> <file.json> --fix --drop-unused [--nosort]
+    $ odg generate <file.od> <file.jsonc> --fix --drop-unused [--nosort]
 
 The `--fix` option might be necessary if the OD-file contains internal
 inconsistencies. It is safe to run this option as it will not delete any active
@@ -102,10 +99,10 @@ parameter that might be used in the file.
 ## Motivation
 
 The biggest improvement with the new tool over the original implementation is
-the introduction of a new `.json` based format to store the object dictionary.
-The JSON format is well-known and easy to read. The tool supports jsonc,
+the introduction of a new `.jsonc` based format to store the object dictionary.
+The JSON format is well-known and easy to read. The tool use jsonc,
 allowing comments in the json file. `odg` will process the file in a repeatable
-manner, making it possible support diffing of the `.json` file output. `odg`
+manner, making it possible support diffing of the `.jsonc` file output. `odg`
 remains 100% compatible with the legacy `.od` format on both input and output.
 
 The original objdictedit and objdictgen tool were written in legacy python 2 and

src/objdictgen/__main__.py

@@ -220,7 +220,7 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
     subp.add_argument('-x', '--exclude', action="append", help="OD Index to exclude.")
     subp.add_argument('-f', '--fix', action="store_true",
                         help="Fix any inconsistency errors in OD before generate output")
-    subp.add_argument('-t', '--type', choices=['od', 'eds', 'json', 'c'],
+    subp.add_argument('-t', '--type', choices=['od', 'eds', 'json', 'jsonc', 'c'],
                         help="Select output file type")
     subp.add_argument('--drop-unused', action="store_true", help="Remove unused parameters")
     subp.add_argument('--internal', action="store_true",
@@ -351,8 +351,9 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
 
         # Write the data
         od.DumpFile(opts.out,
-            filetype=opts.type, sort=not opts.no_sort,
-            internal=opts.internal, validate=not opts.novalidate
+            filetype=opts.type,
+            # These additional options are only used for JSON output
+            sort=not opts.no_sort, internal=opts.internal, validate=not opts.novalidate
         )
 
 

src/objdictgen/jsonod.py

@@ -357,7 +357,8 @@ def compare_profile(profilename: TPath, params: ODMapping, menu: TProfileMenu|No
         return False, False
 
 
-def generate_jsonc(node: "Node", compact=False, sort=False, internal=False, validate=True) -> str:
+def generate_jsonc(node: "Node", compact=False, sort=False, internal=False,
+                   validate=True, jsonc=True) -> str:
     """ Export a JSONC string representation of the node """
 
     # Get the dict representation
@@ -374,20 +375,37 @@ def generate_jsonc(node: "Node", compact=False, sort=False, internal=False, vali
     text = json.dumps(jd, separators=(',', ': '), indent=2)
 
     # Convert the special __ fields to jsonc comments
+    # Syntax:  "__<field>: <value>"
     text = re.sub(
         r'^(\s*)"__(\w+)": "(.*)",?$',
-        r'\1// "\2": "\3"',
+        # In regular json files, __* fields are omitted from output
+        # In jsonc files, the __* entry is converted to a comment:
+        #     "// <field>: <value>"
+        r'\1// "\2": "\3"' if jsonc else '',
         text,
         flags=re.MULTILINE,
     )
 
+    if jsonc:
+        # In jsonc the field is converted to "<field>,  // <comment>"
+        repl = lambda m: m[1].replace('\\"', '"') + m[3] + m[2]
+    else:
+        # In json the field is converted to "<field>,"
+        repl = lambda m: m[1].replace('\\"', '"') + m[3]
+
     # Convert the special @@ fields to jsonc comments
+    # Syntax:  "@@<field>,  // <comment>@@"
     text = re.sub(
         r'"@@(.*?)(\s*//.*?)?@@"(.*)$',
-        lambda m: m[1].replace('\\"', '"') + m[3] + m[2],
+        repl,
         text,
         flags=re.MULTILINE,
     )
+
+    # In case the json contains empty lines, remove them
+    if not jsonc:
+        text = "\n".join(line for line in text.splitlines() if line.strip())
+
     return text
 
 

src/objdictgen/node.py

@@ -196,14 +196,14 @@ def LoadJson(contents: str) -> "Node":
         """ Import a new Node from a JSON string """
         return jsonod.generate_node(contents)
 
-    def DumpFile(self, filepath: TPath, filetype: str|None = "json", **kwargs):
+    def DumpFile(self, filepath: TPath, filetype: str|None = "jsonc", **kwargs):
         """ Save node into file """
 
         # Attempt to determine the filetype from the filepath
         if not filetype:
             filetype = Path(filepath).suffix[1:]
         if not filetype:
-            filetype = "json"
+            filetype = "jsonc"
 
         if filetype == 'od':
             log.debug("Writing XML OD '%s'", filepath)
@@ -219,9 +219,11 @@ def DumpFile(self, filepath: TPath, filetype: str|None = "json", **kwargs):
                 f.write(content)
             return
 
-        if filetype == 'json':
+        if filetype in ('json', 'jsonc'):
             log.debug("Writing JSON OD '%s'", filepath)
-            jdata = self.DumpJson(**kwargs)
+            kw = kwargs.copy()
+            kw['jsonc'] = filetype == 'jsonc'
+            jdata = self.DumpJson(**kw)
             with open(filepath, "w", encoding="utf-8") as f:
                 f.write(jdata)
             return
@@ -234,10 +236,10 @@ def DumpFile(self, filepath: TPath, filetype: str|None = "json", **kwargs):
 
         raise ValueError("Unknown file suffix, unable to write file")
 
-    def DumpJson(self, compact=False, sort=False, internal=False, validate=True) -> str:
+    def DumpJson(self, compact=False, sort=False, internal=False, validate=True, jsonc=True) -> str:
         """ Dump the node into a JSON string """
         return jsonod.generate_jsonc(
-            self, compact=compact, sort=sort, internal=internal, validate=validate
+            self, compact=compact, sort=sort, internal=internal, validate=validate, jsonc=jsonc
         )
 
     def asdict(self) -> dict[str, Any]:

src/objdictgen/ui/objdictedit.py

@@ -424,7 +424,7 @@ def OnOpenMenu(self, event):  # pylint: disable=unused-argument
 
         with wx.FileDialog(
             self, "Choose a file", directory, "",
-            "OD files (*.json;*.od;*.eds)|*.json;*.od;*.eds|All files|*.*",
+            wildcard="OD JSON file (*.jsonc;*.json)|*.jsonc;*.json|Legacy OD file (*.od)|*.od|EDS file (*.eds)|*.eds|All files|*.*",
             style=wx.FD_OPEN | wx.FD_CHANGE_DIR,
         ) as dialog:
             if dialog.ShowModal() != wx.ID_OK:
@@ -475,7 +475,7 @@ def SaveAs(self):
 
         with wx.FileDialog(
             self, "Choose a file", directory, filename,
-            wildcard="OD JSON file (*.json)|*.json;|OD file (*.od)|*.od;|EDS file (*.eds)|*.eds",
+            wildcard="OD JSON file (*.jsonc;*.json)|*.jsonc;*.json|Legacy OD file (*.od)|*.od|EDS file (*.eds)|*.eds",
             style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT | wx.FD_CHANGE_DIR,
         ) as dialog:
             if dialog.ShowModal() != wx.ID_OK:

tests/conftest.py

@@ -200,7 +200,7 @@ def pytest_generate_tests(metafunc):
     # Add "_suffix" fixture
     if "_suffix" in metafunc.fixturenames:
         metafunc.parametrize(
-            "_suffix", ['od', 'json', 'eds'], indirect=False, scope="session"
+            "_suffix", ['od', 'jsonc', 'json', 'eds'], indirect=False, scope="session"
         )
 
     # Make a list of all .od files in tests/od
@@ -211,6 +211,7 @@ def pytest_generate_tests(metafunc):
     jsonfiles = []
     for d in oddirs:
         jsonfiles += ODPath.nfactory(d.glob('*.json'))
+        jsonfiles += ODPath.nfactory(d.glob('*.jsonc'))
 
     edsfiles = []
     for d in oddirs:
@@ -228,15 +229,15 @@ def odids(odlist):
         )
 
     # Add "odjson" fixture
-    # Fixture for each of the .od and .json files in the test directory
+    # Fixture for each of the .od and .json[c] files in the test directory
     if "odjson" in metafunc.fixturenames:
         data = sorted(odfiles + jsonfiles)
         metafunc.parametrize(
             "odjson", data, ids=odids(data), indirect=False, scope="session"
         )
 
     # Add "odjsoneds" fixture
-    # Fixture for each of the .od, .json, and .eds files in the test directory
+    # Fixture for each of the .od, .json[c], and .eds files in the test directory
     if "odjsoneds" in metafunc.fixturenames:
         data = sorted(odfiles + jsonfiles + edsfiles)
         metafunc.parametrize(

tests/od/jsonod-comments.json

@@ -12,15 +12,15 @@
   "default_string_size": 24,
   "dictionary": [
     {
-      "index": "0x1003",  // 4099
+      "index": "0x1003",
       "name": "Pre-defined Error Field",
       "struct": "array",
       "group": "built-in",
       "mandatory": false,
       "profile_callback": true,
       "each": {
         "name": "Standard Error Field",
-        "type": "UNSIGNED32",  // 7
+        "type": "UNSIGNED32",
         "access": "ro",
         "pdo": false,
         "nbmin": 1,
@@ -29,23 +29,17 @@
       "sub": [
         {
           "name": "Number of Errors",
-          "type": "UNSIGNED8",  // 5
+          "type": "UNSIGNED8",
           "access": "rw",
           "pdo": false
         },
         {
-          // "name": "Standard Error Field"
-          // "type": "UNSIGNED32"  // 7
           "value": 0
         },
         {
-          // "name": "Standard Error Field"
-          // "type": "UNSIGNED32"  // 7
           "value": 0
         },
         {
-          // "name": "Standard Error Field"
-          // "type": "UNSIGNED32"  // 7
           "value": 0
         }
       ]

tests/od/jsonod-comments.jsonc

@@ -0,0 +1,54 @@
+{
+  "$id": "od data",
+  "$version": "1",
+  "$description": "Canfestival object dictionary data",
+  "$tool": "odg 3.5",
+  "$date": "2024-08-13T11:36:24.609168",
+  "name": "JSON comments",
+  "description": "Test for JSON comments",
+  "type": "slave",
+  "id": 0,
+  "profile": "None",
+  "default_string_size": 24,
+  "dictionary": [
+    {
+      "index": "0x1003",  // 4099
+      "name": "Pre-defined Error Field",
+      "struct": "array",
+      "group": "built-in",
+      "mandatory": false,
+      "profile_callback": true,
+      "each": {
+        "name": "Standard Error Field",
+        "type": "UNSIGNED32",  // 7
+        "access": "ro",
+        "pdo": false,
+        "nbmin": 1,
+        "nbmax": 254
+      },
+      "sub": [
+        {
+          "name": "Number of Errors",
+          "type": "UNSIGNED8",  // 5
+          "access": "rw",
+          "pdo": false
+        },
+        {
+          // "name": "Standard Error Field"
+          // "type": "UNSIGNED32"  // 7
+          "value": 0
+        },
+        {
+          // "name": "Standard Error Field"
+          // "type": "UNSIGNED32"  // 7
+          "value": 0
+        },
+        {
+          // "name": "Standard Error Field"
+          // "type": "UNSIGNED32"  // 7
+          "value": 0
+        }
+      ]
+    }
+  ]
+}

tests/test_jsonod.py

@@ -135,15 +135,28 @@ def test_jsonod_timezone():
 def test_jsonod_comments(odpath):
     """ Test that the json file exports comments correctly. """
 
-    fname = odpath / "jsonod-comments.json"
-    m1 = Node.LoadFile(fname)
-    with open(fname, "r") as f:
-        od = f.read()
+    fname_jsonc = odpath / "jsonod-comments.jsonc"
+    fname_json = odpath / "jsonod-comments.json"
 
-    out = generate_jsonc(m1, compact=False, sort=False, internal=False, validate=True)
+    m1 = Node.LoadFile(fname_jsonc)
+
+    with open(fname_jsonc, "r") as f:
+        jsonc_data = f.read()
+    with open(fname_json, "r") as f:
+        json_data = f.read()
+
+    out = generate_jsonc(m1, compact=False, sort=False, internal=False, validate=True, jsonc=True)
+
+    # Compare the jsonc data with the generated data
+    for a, b in zip(jsonc_data.splitlines(), out.splitlines()):
+        if '"$date"' in a or '"$tool"' in a:
+            continue
+        assert a == b
+
+    out = generate_jsonc(m1, compact=False, sort=False, internal=False, validate=True, jsonc=False)
 
-    for a, b in zip(od.splitlines(), out.splitlines()):
-        print(a)
+    # Compare the json data with the generated data
+    for a, b in zip(json_data.splitlines(), out.splitlines()):
         if '"$date"' in a or '"$tool"' in a:
             continue
         assert a == b