Skip to content

Commit 75c5c39

Browse files
carlesarnalcarlesarnal
authored
Port json schema objects, asyncapi and openapi dereferencing (#5315)
* Add test for json schema object and property level reference * Add object and property level tests for json schema * Add test with multiple references to a single file * Port openapi dereference
1 parent ef1c9cc commit 75c5c39

17 files changed

+1013
-9
lines changed

‎docs/modules/ROOT/partials/getting-started/proc-managing-artifact-references-using-rest-api.adoc

+5-2
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,13 @@ $ curl -H "Authorization: Bearer $ACCESS_TOKEN" MY-REGISTRY-URL/apis/registry/v3
156156
{"type":"record","name":"Item","namespace":"com.example.common","fields":[{"name":"itemId","type":{"type":"record","name":"ItemId","fields":[{"name":"id","type":"int"}]}}]}
157157
----
158158

159-
#In Protobuf dereferencing content is only supported when all the schemas in the try belong to the same package.#
159+
This support is currently implemented only for Avro, Protobuf, OpenAPI, AsyncAPI, and JSON Schema artifacts when the `dereference` parameter is specified in the API operation. This parameter is not supported for any other artifact types.
160160

161+
NOTE: For Protobuf artifacts, dereferencing content is supported only when all the schemas belong to the same package.
162+
163+
NOTE: Circular dependencies are allowed by some artifact types (e.g. JSON Schema) but are not supported by {registry}.
161164

162165
[role="_additional-resources"]
163166
.Additional resources
164167
* For more details, see the {registry-rest-api}.
165-
* For more examples of artifact references, see the section on configuring each artifact type in {registry-client-serdes-config}.
168+
* For more examples of artifact references, see the section on configuring each artifact type in {registry-client-serdes-config}.

‎schema-util/openapi/src/main/java/io/apicurio/registry/content/dereference/ApicurioDataModelsContentDereferencer.java

+5-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import io.apicurio.datamodels.Library;
66
import io.apicurio.datamodels.TraverserDirection;
77
import io.apicurio.datamodels.models.Document;
8-
import io.apicurio.datamodels.refs.IReferenceResolver;
98
import io.apicurio.registry.content.ContentHandle;
109
import io.apicurio.registry.content.TypedContent;
1110
import io.apicurio.registry.content.util.ContentTypeUtil;
@@ -20,12 +19,11 @@ public class ApicurioDataModelsContentDereferencer implements ContentDereference
2019
public TypedContent dereference(TypedContent content, Map<String, TypedContent> resolvedReferences) {
2120
try {
2221
JsonNode node = ContentTypeUtil.parseJsonOrYaml(content);
23-
Document document = Library.readDocument((ObjectNode) node);
24-
IReferenceResolver resolver = new RegistryReferenceResolver(resolvedReferences);
25-
Document dereferencedDoc = Library.dereferenceDocument(document, resolver, false);
26-
String dereferencedContentStr = Library.writeDocumentToJSONString(dereferencedDoc);
27-
return TypedContent.create(ContentHandle.create(dereferencedContentStr),
28-
ContentTypes.APPLICATION_JSON);
22+
Document doc = Library.readDocument((ObjectNode) node);
23+
ReferenceInliner inliner = new ReferenceInliner(resolvedReferences);
24+
Library.visitTree(doc, inliner, TraverserDirection.down);
25+
String dereferencedContent = Library.writeDocumentToJSONString(doc);
26+
return TypedContent.create(ContentHandle.create(dereferencedContent), content.getContentType());
2927
} catch (IOException e) {
3028
throw new RuntimeException(e);
3129
}

‎schema-util/openapi/src/main/java/io/apicurio/registry/content/dereference/ReferenceInliner.java

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Copyright 2023 Red Hat Inc
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.apicurio.registry.content.dereference;
18+
19+
import com.fasterxml.jackson.core.JsonPointer;
20+
import com.fasterxml.jackson.databind.JsonNode;
21+
import com.fasterxml.jackson.databind.ObjectMapper;
22+
import com.fasterxml.jackson.databind.node.ObjectNode;
23+
import io.apicurio.datamodels.Library;
24+
import io.apicurio.datamodels.models.Node;
25+
import io.apicurio.datamodels.models.Referenceable;
26+
import io.apicurio.datamodels.models.visitors.AllNodeVisitor;
27+
import io.apicurio.registry.content.ContentHandle;
28+
import io.apicurio.registry.content.TypedContent;
29+
import io.apicurio.registry.content.refs.JsonPointerExternalReference;
30+
import org.slf4j.Logger;
31+
import org.slf4j.LoggerFactory;
32+
33+
import java.util.Map;
34+
35+
/**
36+
* Inlines all references found in the data model.
37+
*
38+
* @author eric.wittmann@gmail.com
39+
*/
40+
public class ReferenceInliner extends AllNodeVisitor {
41+
private static final Logger logger = LoggerFactory.getLogger(ReferenceInliner.class);
42+
private static final ObjectMapper mapper = new ObjectMapper();
43+
private final Map<String, TypedContent> resolvedReferences;
44+
45+
/**
46+
* Constructor.
47+
*
48+
* @param resolvedReferences
49+
*/
50+
public ReferenceInliner(Map<String, TypedContent> resolvedReferences) {
51+
this.resolvedReferences = resolvedReferences;
52+
}
53+
54+
/**
55+
* @see AllNodeVisitor#visitNode(Node)
56+
*/
57+
@Override
58+
protected void visitNode(Node node) {
59+
if (node instanceof Referenceable) {
60+
String $ref = ((Referenceable) node).get$ref();
61+
if ($ref != null && resolvedReferences.containsKey($ref)) {
62+
inlineRef((Referenceable) node);
63+
}
64+
}
65+
}
66+
67+
/**
68+
* Inlines the given reference.
69+
*
70+
* @param refNode
71+
*/
72+
private void inlineRef(Referenceable refNode) {
73+
String $ref = refNode.get$ref();
74+
75+
JsonPointerExternalReference refPointer = new JsonPointerExternalReference($ref);
76+
ContentHandle refContent = resolvedReferences.get($ref).getContent();
77+
78+
// Get the specific node within the content that this $ref points to
79+
ObjectNode refContentNode = getRefNodeFromContent(refContent, refPointer.getComponent());
80+
if (refContentNode != null) {
81+
// Read that content *into* the current node
82+
Library.readNode(refContentNode, (Node) refNode);
83+
84+
// Set the $ref to null (now that we've inlined the referenced node)
85+
refNode.set$ref(null);
86+
}
87+
}
88+
89+
private ObjectNode getRefNodeFromContent(ContentHandle refContent, String refComponent) {
90+
try {
91+
ObjectNode refContentRootNode = (ObjectNode) mapper.readTree(refContent.content());
92+
if (refComponent != null) {
93+
JsonPointer pointer = JsonPointer.compile(refComponent.substring(1));
94+
JsonNode nodePointedTo = refContentRootNode.at(pointer);
95+
if (!nodePointedTo.isMissingNode() && nodePointedTo.isObject()) {
96+
return (ObjectNode) nodePointedTo;
97+
}
98+
} else {
99+
return refContentRootNode;
100+
}
101+
} catch (Exception e) {
102+
logger.error("Failed to get referenced node from $ref content.", e);
103+
}
104+
return null;
105+
}
106+
107+
}

‎schema-util/util-provider/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,11 @@
6868
<artifactId>junit-jupiter</artifactId>
6969
<scope>test</scope>
7070
</dependency>
71+
<dependency>
72+
<groupId>io.apicurio</groupId>
73+
<artifactId>apicurio-registry-utils-tests</artifactId>
74+
<scope>test</scope>
75+
</dependency>
7176
</dependencies>
7277

7378
<build>

‎schema-util/util-provider/src/test/java/io/apicurio/registry/content/dereference/JsonSchemaContentDereferencerTest.java

+80
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,16 @@
66
import io.apicurio.registry.content.refs.JsonSchemaReferenceFinder;
77
import io.apicurio.registry.content.refs.ReferenceFinder;
88
import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase;
9+
import io.apicurio.registry.types.ContentTypes;
910
import org.junit.jupiter.api.Assertions;
1011
import org.junit.jupiter.api.Test;
1112

13+
import java.util.LinkedHashMap;
1214
import java.util.Map;
1315
import java.util.Set;
1416

17+
import static io.apicurio.registry.utils.tests.TestUtils.normalizeMultiLineString;
18+
1519
public class JsonSchemaContentDereferencerTest extends ArtifactUtilProviderTestBase {
1620

1721
@Test
@@ -30,4 +34,80 @@ public void testRewriteReferences() {
3034
.contains(new JsonPointerExternalReference("https://www.example.org/schemas/ssn.json")));
3135
}
3236

37+
@Test
38+
public void testDereferenceObjectLevel() throws Exception {
39+
TypedContent content = TypedContent.create(
40+
resourceToContentHandle("json-schema-to-deref-object-level.json"),
41+
ContentTypes.APPLICATION_JSON);
42+
JsonSchemaDereferencer dereferencer = new JsonSchemaDereferencer();
43+
// Note: order is important. The JSON schema dereferencer needs to convert the ContentHandle Map
44+
// to a JSONSchema map. So it *must* resolve the leaves of the dependency tree before the branches.
45+
Map<String, TypedContent> resolvedReferences = new LinkedHashMap<>();
46+
resolvedReferences.put("types/city/qualification.json", TypedContent.create(
47+
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
48+
resolvedReferences.put("city/qualification.json", TypedContent.create(
49+
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
50+
resolvedReferences.put("identifier/qualification.json",
51+
TypedContent.create(resourceToContentHandle("types/identifier/qualification.json"),
52+
ContentTypes.APPLICATION_JSON));
53+
resolvedReferences.put("types/all-types.json#/definitions/City", TypedContent
54+
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
55+
resolvedReferences.put("types/all-types.json#/definitions/Identifier", TypedContent
56+
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
57+
TypedContent modifiedContent = dereferencer.dereference(content, resolvedReferences);
58+
String expectedContent = resourceToString("expected-testDereference-object-level-json.json");
59+
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
60+
normalizeMultiLineString(modifiedContent.getContent().content()));
61+
}
62+
63+
@Test
64+
public void testDereferencePropertyLevel() throws Exception {
65+
TypedContent content = TypedContent.create(
66+
resourceToContentHandle("json-schema-to-deref-property-level.json"),
67+
ContentTypes.APPLICATION_JSON);
68+
JsonSchemaDereferencer dereferencer = new JsonSchemaDereferencer();
69+
// Note: order is important. The JSON schema dereferencer needs to convert the ContentHandle Map
70+
// to a JSONSchema map. So it *must* resolve the leaves of the dependency tree before the branches.
71+
Map<String, TypedContent> resolvedReferences = new LinkedHashMap<>();
72+
resolvedReferences.put("types/city/qualification.json", TypedContent.create(
73+
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
74+
resolvedReferences.put("city/qualification.json", TypedContent.create(
75+
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
76+
resolvedReferences.put("identifier/qualification.json",
77+
TypedContent.create(resourceToContentHandle("types/identifier/qualification.json"),
78+
ContentTypes.APPLICATION_JSON));
79+
resolvedReferences.put("types/all-types.json#/definitions/City/properties/name", TypedContent
80+
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
81+
resolvedReferences.put("types/all-types.json#/definitions/Identifier", TypedContent
82+
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
83+
TypedContent modifiedContent = dereferencer.dereference(content, resolvedReferences);
84+
String expectedContent = resourceToString("expected-testDereference-property-level-json.json");
85+
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
86+
normalizeMultiLineString(modifiedContent.getContent().content()));
87+
}
88+
89+
// Resolves multiple $refs using a single reference to a file with multiple definitions
90+
@Test
91+
public void testReferenceSingleFile() throws Exception {
92+
TypedContent content = TypedContent.create(
93+
resourceToContentHandle("json-schema-to-deref-property-level.json"),
94+
ContentTypes.APPLICATION_JSON);
95+
JsonSchemaDereferencer dereferencer = new JsonSchemaDereferencer();
96+
// Note: order is important. The JSON schema dereferencer needs to convert the ContentHandle Map
97+
// to a JSONSchema map. So it *must* resolve the leaves of the dependency tree before the branches.
98+
Map<String, TypedContent> resolvedReferences = new LinkedHashMap<>();
99+
resolvedReferences.put("types/city/qualification.json", TypedContent.create(
100+
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
101+
resolvedReferences.put("city/qualification.json", TypedContent.create(
102+
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
103+
resolvedReferences.put("identifier/qualification.json",
104+
TypedContent.create(resourceToContentHandle("types/identifier/qualification.json"),
105+
ContentTypes.APPLICATION_JSON));
106+
resolvedReferences.put("types/all-types.json", TypedContent
107+
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
108+
TypedContent modifiedContent = dereferencer.dereference(content, resolvedReferences);
109+
String expectedContent = resourceToString("expected-testDereference-property-level-json.json");
110+
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
111+
normalizeMultiLineString(modifiedContent.getContent().content()));
112+
}
33113
}

‎schema-util/util-provider/src/test/java/io/apicurio/registry/content/dereference/OpenApiContentDereferencerTest.java

+22
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
11
package io.apicurio.registry.content.dereference;
22

3+
import io.apicurio.registry.content.ContentHandle;
34
import io.apicurio.registry.content.TypedContent;
45
import io.apicurio.registry.content.refs.ExternalReference;
56
import io.apicurio.registry.content.refs.JsonPointerExternalReference;
67
import io.apicurio.registry.content.refs.OpenApiReferenceFinder;
78
import io.apicurio.registry.content.refs.ReferenceFinder;
89
import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase;
10+
import io.apicurio.registry.types.ContentTypes;
911
import org.junit.jupiter.api.Assertions;
1012
import org.junit.jupiter.api.Test;
1113

1214
import java.util.Map;
1315
import java.util.Set;
1416

17+
import static io.apicurio.registry.utils.tests.TestUtils.normalizeMultiLineString;
18+
1519
public class OpenApiContentDereferencerTest extends ArtifactUtilProviderTestBase {
1620

1721
@Test
@@ -32,4 +36,22 @@ public void testRewriteReferences() {
3236
"https://www.example.org/schemas/foo-types.json#/components/schemas/Foo")));
3337
}
3438

39+
@Test
40+
public void testDereference() throws Exception {
41+
ContentHandle content = resourceToContentHandle("openapi-to-deref.json");
42+
OpenApiDereferencer dereferencer = new OpenApiDereferencer();
43+
Map<String, TypedContent> resolvedReferences = Map.of(
44+
"http://types.example.org/all-types.json#/components/schemas/Foo",
45+
TypedContent.create(resourceToContentHandle("all-types.json"), ContentTypes.APPLICATION_JSON),
46+
"http://types.example.org/all-types.json#/components/schemas/Bar",
47+
TypedContent.create(resourceToContentHandle("all-types.json"), ContentTypes.APPLICATION_JSON),
48+
"http://types.example.org/address.json#/components/schemas/Address",
49+
TypedContent.create(resourceToContentHandle("address.json"), ContentTypes.APPLICATION_JSON));
50+
TypedContent modifiedContent = dereferencer
51+
.dereference(TypedContent.create(content, ContentTypes.APPLICATION_JSON), resolvedReferences);
52+
String expectedContent = resourceToString("expected-testDereference-openapi.json");
53+
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
54+
normalizeMultiLineString(modifiedContent.getContent().content()));
55+
}
56+
3557
}

‎schema-util/util-provider/src/test/resources/io/apicurio/registry/content/dereference/address.json

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"openapi": "3.0.2",
3+
"info": {
4+
"title": "address.json",
5+
"version": "1.0.0",
6+
"description": ""
7+
},
8+
"paths": {
9+
"/": {}
10+
},
11+
"components": {
12+
"schemas": {
13+
"Address": {
14+
"title": "Root Type for Address",
15+
"description": "",
16+
"type": "object",
17+
"properties": {
18+
"address1": {
19+
"type": "string"
20+
},
21+
"city": {
22+
"type": "string"
23+
},
24+
"state": {
25+
"type": "string"
26+
},
27+
"zip": {
28+
"type": "string"
29+
}
30+
},
31+
"example": {
32+
"address1": "225 West South St",
33+
"city": "Springfield",
34+
"state": "Massiginia",
35+
"zip": "11556"
36+
}
37+
}
38+
}
39+
}
40+
}

0 commit comments

Comments
 (0)