Skip to content

Commit 23c8293

Browse files
EricWittmannEricWittmann
authored
Content handling updates to properly support YAML (#4739)
* Added a new /search/versions endpoint * Remove test generated script.sql file * Fix examples due to mojo property rename from "type" to "artifactType" * Lots of changes to properly support YAML for openapi/asyncapi * Ensure content canonicalization works for JSON/YAML in openapi/asyncapi * Remove debug println statement from test * Fix yaml content handling in UI * Fixed several broken tests
1 parent 576ea8d commit 23c8293

File tree

130 files changed

+1966
-1396
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

130 files changed

+1966
-1396
lines changed

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/AbstractResource.java

+19-12
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import io.apicurio.registry.ccompat.rest.error.ConflictException;
66
import io.apicurio.registry.ccompat.rest.error.UnprocessableEntityException;
77
import io.apicurio.registry.content.ContentHandle;
8+
import io.apicurio.registry.content.TypedContent;
89
import io.apicurio.registry.model.BranchId;
910
import io.apicurio.registry.model.GA;
1011
import io.apicurio.registry.model.GAV;
@@ -30,7 +31,6 @@
3031
import io.apicurio.registry.types.VersionState;
3132
import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
3233
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
33-
import io.apicurio.registry.util.ContentTypeUtil;
3434
import jakarta.inject.Inject;
3535
import org.apache.avro.AvroTypeException;
3636
import org.apache.avro.SchemaParseException;
@@ -68,19 +68,18 @@ protected ArtifactVersionMetaDataDto createOrUpdateArtifact(String subject, Stri
6868
ArtifactVersionMetaDataDto res;
6969
final List<ArtifactReferenceDto> parsedReferences = parseReferences(references, groupId);
7070
final List<ArtifactReference> artifactReferences = parsedReferences.stream().map(dto -> ArtifactReference.builder().name(dto.getName()).groupId(dto.getGroupId()).artifactId(dto.getArtifactId()).version(dto.getVersion()).build()).collect(Collectors.toList());
71-
final Map<String, ContentHandle> resolvedReferences = storage.resolveReferences(parsedReferences);
71+
final Map<String, TypedContent> resolvedReferences = storage.resolveReferences(parsedReferences);
7272
try {
7373
ContentHandle schemaContent;
7474
schemaContent = ContentHandle.create(schema);
7575
String contentType = ContentTypes.APPLICATION_JSON;
7676
if (artifactType.equals(ArtifactType.PROTOBUF)) {
7777
contentType = ContentTypes.APPLICATION_PROTOBUF;
78-
} else if (ContentTypeUtil.isParsableYaml(schemaContent)) {
79-
contentType = ContentTypes.APPLICATION_YAML;
8078
}
8179

8280
if (!doesArtifactExist(subject, groupId)) {
83-
rulesService.applyRules(groupId, subject, artifactType, schemaContent, RuleApplicationType.CREATE, artifactReferences, resolvedReferences);
81+
TypedContent typedSchemaContent = TypedContent.create(schemaContent, contentType);
82+
rulesService.applyRules(groupId, subject, artifactType, typedSchemaContent, RuleApplicationType.CREATE, artifactReferences, resolvedReferences);
8483

8584
EditableArtifactMetaDataDto artifactMetaData = EditableArtifactMetaDataDto.builder().build();
8685
EditableVersionMetaDataDto firstVersionMetaData = EditableVersionMetaDataDto.builder().build();
@@ -93,7 +92,8 @@ protected ArtifactVersionMetaDataDto createOrUpdateArtifact(String subject, Stri
9392
res = storage.createArtifact(groupId, subject, artifactType, artifactMetaData, null,
9493
firstVersionContent, firstVersionMetaData, null).getValue();
9594
} else {
96-
rulesService.applyRules(groupId, subject, artifactType, schemaContent, RuleApplicationType.UPDATE, artifactReferences, resolvedReferences);
95+
TypedContent typedSchemaContent = TypedContent.create(schemaContent, contentType);
96+
rulesService.applyRules(groupId, subject, artifactType, typedSchemaContent, RuleApplicationType.UPDATE, artifactReferences, resolvedReferences);
9797
ContentWrapperDto versionContent = ContentWrapperDto.builder()
9898
.content(schemaContent)
9999
.contentType(contentType)
@@ -116,13 +116,15 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String subject
116116
//FIXME simplify logic
117117
try {
118118
final String type = schemaType == null ? ArtifactType.AVRO : schemaType;
119+
final String contentType = type.equals(ArtifactType.PROTOBUF) ? ContentTypes.APPLICATION_PROTOBUF : ContentTypes.APPLICATION_JSON;
120+
TypedContent typedSchemaContent = TypedContent.create(ContentHandle.create(schema), contentType);
119121
final List<ArtifactReferenceDto> artifactReferences = parseReferences(schemaReferences, groupId);
120122
ArtifactTypeUtilProvider artifactTypeProvider = factory.getArtifactTypeProvider(type);
121123
ArtifactVersionMetaDataDto amd;
122124

123125
if (cconfig.canonicalHashModeEnabled.get() || normalize) {
124126
try {
125-
amd = storage.getArtifactVersionMetaDataByContent(groupId, subject, true, ContentHandle.create(schema), artifactReferences);
127+
amd = storage.getArtifactVersionMetaDataByContent(groupId, subject, true, typedSchemaContent, artifactReferences);
126128
} catch (ArtifactNotFoundException ex) {
127129
if (type.equals(ArtifactType.AVRO)) {
128130
//When comparing using content, sometimes the references might be inlined into the content, try to dereference the existing content and compare as a fallback. See https://github.com/Apicurio/apicurio-registry/issues/3588 for more information.
@@ -131,8 +133,13 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String subject
131133
amd = storage.getArtifactVersions(groupId, subject)
132134
.stream().filter(version -> {
133135
StoredArtifactVersionDto artifactVersion = storage.getArtifactVersionContent(groupId, subject, version);
134-
Map<String, ContentHandle> artifactVersionReferences = storage.resolveReferences(artifactVersion.getReferences());
135-
String dereferencedExistingContentSha = DigestUtils.sha256Hex(artifactTypeProvider.getContentDereferencer().dereference(artifactVersion.getContent(), artifactVersionReferences).content());
136+
TypedContent typedArtifactVersion = TypedContent.create(artifactVersion.getContent(), artifactVersion.getContentType());
137+
Map<String, TypedContent> artifactVersionReferences = storage.resolveReferences(artifactVersion.getReferences());
138+
String dereferencedExistingContentSha = DigestUtils.sha256Hex(
139+
artifactTypeProvider.getContentDereferencer().dereference(
140+
typedArtifactVersion, artifactVersionReferences
141+
).getContent().content()
142+
);
136143
return dereferencedExistingContentSha.equals(DigestUtils.sha256Hex(schema));
137144
})
138145
.findAny()
@@ -144,7 +151,7 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String subject
144151
}
145152

146153
} else {
147-
amd = storage.getArtifactVersionMetaDataByContent(groupId, subject, false, ContentHandle.create(schema), artifactReferences);
154+
amd = storage.getArtifactVersionMetaDataByContent(groupId, subject, false, typedSchemaContent, artifactReferences);
148155
}
149156

150157
return amd;
@@ -153,8 +160,8 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String subject
153160
}
154161
}
155162

156-
protected Map<String, ContentHandle> resolveReferences(List<SchemaReference> references) {
157-
Map<String, ContentHandle> resolvedReferences = Collections.emptyMap();
163+
protected Map<String, TypedContent> resolveReferences(List<SchemaReference> references) {
164+
Map<String, TypedContent> resolvedReferences = Collections.emptyMap();
158165
if (references != null && !references.isEmpty()) {
159166
//Transform the given references into dtos and set the contentId, this will also detect if any of the passed references does not exist.
160167
final List<ArtifactReferenceDto> referencesAsDtos = references.stream().map(schemaReference -> {

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/CompatibilityResourceImpl.java

+19-2
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
import io.apicurio.registry.ccompat.rest.error.UnprocessableEntityException;
1010
import io.apicurio.registry.ccompat.rest.v7.CompatibilityResource;
1111
import io.apicurio.registry.content.ContentHandle;
12+
import io.apicurio.registry.content.TypedContent;
1213
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
1314
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
1415
import io.apicurio.registry.rules.RuleViolationException;
1516
import io.apicurio.registry.rules.UnprocessableSchemaException;
1617
import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto;
18+
import io.apicurio.registry.types.ArtifactType;
19+
import io.apicurio.registry.types.ContentTypes;
1720
import jakarta.interceptor.Interceptors;
1821

1922
import java.util.Collections;
@@ -31,7 +34,14 @@ public CompatibilityCheckResponse testCompatibilityBySubjectName(String subject,
3134
final List<String> versions = storage.getArtifactVersions(groupId, subject);
3235
for (String version : versions) {
3336
final ArtifactVersionMetaDataDto artifactVersionMetaData = storage.getArtifactVersionMetaData(groupId, subject, version);
34-
rulesService.applyRules(groupId, subject, version, artifactVersionMetaData.getArtifactType(), ContentHandle.create(request.getSchema()), Collections.emptyList(), Collections.emptyMap());
37+
// Assume the content type of the SchemaContent is the same as the previous version.
38+
String contentType = ContentTypes.APPLICATION_JSON;
39+
if (artifactVersionMetaData.getArtifactType().equals(ArtifactType.PROTOBUF)) {
40+
contentType = ContentTypes.APPLICATION_PROTOBUF;
41+
}
42+
TypedContent typedContent = TypedContent.create(ContentHandle.create(request.getSchema()), contentType);
43+
rulesService.applyRules(groupId, subject, version, artifactVersionMetaData.getArtifactType(),
44+
typedContent, Collections.emptyList(), Collections.emptyMap());
3545
}
3646
return CompatibilityCheckResponse.IS_COMPATIBLE;
3747
} catch (RuleViolationException ex) {
@@ -53,7 +63,14 @@ public CompatibilityCheckResponse testCompatibilityByVersion(String subject, Str
5363
return parseVersionString(subject, versionString, groupId, v -> {
5464
try {
5565
final ArtifactVersionMetaDataDto artifact = storage.getArtifactVersionMetaData(groupId, subject, v);
56-
rulesService.applyRules(groupId, subject, v, artifact.getArtifactType(), ContentHandle.create(request.getSchema()), Collections.emptyList(), Collections.emptyMap());
66+
// Assume the content type of the SchemaContent is correct based on the artifact type.
67+
String contentType = ContentTypes.APPLICATION_JSON;
68+
if (artifact.getArtifactType().equals(ArtifactType.PROTOBUF)) {
69+
contentType = ContentTypes.APPLICATION_PROTOBUF;
70+
}
71+
TypedContent typedContent = TypedContent.create(ContentHandle.create(request.getSchema()), contentType);
72+
rulesService.applyRules(groupId, subject, v, artifact.getArtifactType(),
73+
typedContent, Collections.emptyList(), Collections.emptyMap());
5774
return CompatibilityCheckResponse.IS_COMPATIBLE;
5875
} catch (RuleViolationException ex) {
5976
if (fverbose) {

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SchemasResourceImpl.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import io.apicurio.registry.ccompat.dto.SubjectVersion;
99
import io.apicurio.registry.ccompat.rest.v7.SchemasResource;
1010
import io.apicurio.registry.content.ContentHandle;
11+
import io.apicurio.registry.content.TypedContent;
1112
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
1213
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
1314
import io.apicurio.registry.storage.dto.ArtifactReferenceDto;
@@ -45,8 +46,9 @@ public SchemaInfo getSchema(int id, String subject, String groupId) {
4546
contentType = contentWrapper.getContentType();
4647
references = contentWrapper.getReferences();
4748
}
48-
return converter.convert(contentHandle, ArtifactTypeUtil.determineArtifactType(contentHandle, null, contentType,
49-
storage.resolveReferences(references), factory.getAllArtifactTypes()), references);
49+
TypedContent typedContent = TypedContent.create(contentHandle, contentType);
50+
return converter.convert(contentHandle, ArtifactTypeUtil.determineArtifactType(typedContent, null,
51+
storage.resolveReferences(references), factory), references);
5052
}
5153

5254
@Override

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SubjectVersionsResourceImpl.java

+7-5
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import io.apicurio.registry.ccompat.rest.error.UnprocessableEntityException;
1515
import io.apicurio.registry.ccompat.rest.v7.SubjectVersionsResource;
1616
import io.apicurio.registry.content.ContentHandle;
17+
import io.apicurio.registry.content.TypedContent;
1718
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
1819
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
1920
import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto;
@@ -24,8 +25,8 @@
2425
import io.apicurio.registry.storage.error.VersionNotFoundException;
2526
import io.apicurio.registry.types.VersionState;
2627
import io.apicurio.registry.util.ArtifactTypeUtil;
27-
import io.apicurio.registry.util.ContentTypeUtil;
28-
import io.apicurio.registry.util.VersionUtil;
28+
import io.apicurio.registry.content.util.ContentTypeUtil;
29+
import io.apicurio.registry.utils.VersionUtil;
2930
import jakarta.inject.Inject;
3031
import jakarta.interceptor.Interceptors;
3132
import jakarta.ws.rs.BadRequestException;
@@ -71,7 +72,7 @@ public SchemaId register(String subject, SchemaInfo request, Boolean normalize,
7172
throw new UnprocessableEntityException("The schema provided is null.");
7273
}
7374

74-
final Map<String, ContentHandle> resolvedReferences = resolveReferences(request.getReferences());
75+
final Map<String, TypedContent> resolvedReferences = resolveReferences(request.getReferences());
7576

7677
try {
7778
ArtifactVersionMetaDataDto dto = lookupSchema(groupId, subject, request.getSchema(), request.getReferences(), request.getSchemaType(), fnormalize);
@@ -88,10 +89,11 @@ public SchemaId register(String subject, SchemaInfo request, Boolean normalize,
8889
try {
8990
ContentHandle schemaContent = ContentHandle.create(request.getSchema());
9091
String contentType = ContentTypeUtil.determineContentType(schemaContent);
92+
TypedContent typedSchemaContent = TypedContent.create(schemaContent, contentType);
9193

9294
// We validate the schema at creation time by inferring the type from the content
93-
final String artifactType = ArtifactTypeUtil.determineArtifactType(ContentHandle.create(request.getSchema()),
94-
null, contentType, resolvedReferences, factory.getAllArtifactTypes());
95+
final String artifactType = ArtifactTypeUtil.determineArtifactType(typedSchemaContent,
96+
null, resolvedReferences, factory);
9597
if (request.getSchemaType() != null && !artifactType.equals(request.getSchemaType())) {
9698
throw new UnprocessableEntityException(String.format("Given schema is not from type: %s", request.getSchemaType()));
9799
}

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SubjectsResourceImpl.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import io.apicurio.registry.storage.error.InvalidArtifactStateException;
3333
import io.apicurio.registry.storage.error.InvalidVersionStateException;
3434
import io.apicurio.registry.types.VersionState;
35-
import io.apicurio.registry.util.VersionUtil;
35+
import io.apicurio.registry.utils.VersionUtil;
3636
import jakarta.interceptor.Interceptors;
3737

3838
@Interceptors({ResponseErrorLivenessCheck.class, ResponseTimeoutReadinessCheck.class})

‎app/src/main/java/io/apicurio/registry/rest/v2/AbstractResourceImpl.java

+17-21
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,7 @@
11
package io.apicurio.registry.rest.v2;
22

3-
import java.net.URI;
4-
import java.net.URISyntaxException;
5-
import java.net.URLEncoder;
6-
import java.nio.charset.StandardCharsets;
7-
import java.util.HashMap;
8-
import java.util.List;
9-
import java.util.Map;
10-
11-
import jakarta.inject.Inject;
12-
import jakarta.servlet.http.HttpServletRequest;
13-
import jakarta.ws.rs.core.Context;
14-
15-
import org.eclipse.microprofile.config.inject.ConfigProperty;
16-
import org.slf4j.Logger;
17-
183
import io.apicurio.common.apps.config.Info;
19-
import io.apicurio.registry.content.ContentHandle;
4+
import io.apicurio.registry.content.TypedContent;
205
import io.apicurio.registry.content.dereference.ContentDereferencer;
216
import io.apicurio.registry.content.refs.JsonPointerExternalReference;
227
import io.apicurio.registry.storage.RegistryStorage;
@@ -25,6 +10,19 @@
2510
import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
2611
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
2712
import io.apicurio.registry.utils.StringUtil;
13+
import jakarta.inject.Inject;
14+
import jakarta.servlet.http.HttpServletRequest;
15+
import jakarta.ws.rs.core.Context;
16+
import org.eclipse.microprofile.config.inject.ConfigProperty;
17+
import org.slf4j.Logger;
18+
19+
import java.net.URI;
20+
import java.net.URISyntaxException;
21+
import java.net.URLEncoder;
22+
import java.nio.charset.StandardCharsets;
23+
import java.util.HashMap;
24+
import java.util.List;
25+
import java.util.Map;
2826

2927
public abstract class AbstractResourceImpl {
3028

@@ -51,13 +49,13 @@ public abstract class AbstractResourceImpl {
5149
* @param dereference
5250
* @param content
5351
*/
54-
protected ContentHandle handleContentReferences(boolean dereference, String artifactType,
55-
ContentHandle content, List<ArtifactReferenceDto> references) {
52+
protected TypedContent handleContentReferences(boolean dereference, String artifactType,
53+
TypedContent content, List<ArtifactReferenceDto> references) {
5654
// Dereference or rewrite references
5755
if (!references.isEmpty() && dereference) {
5856
ArtifactTypeUtilProvider artifactTypeProvider = factory.getArtifactTypeProvider(artifactType);
5957
ContentDereferencer contentDereferencer = artifactTypeProvider.getContentDereferencer();
60-
Map<String, ContentHandle> resolvedReferences = storage.resolveReferences(references);
58+
Map<String, TypedContent> resolvedReferences = storage.resolveReferences(references);
6159
content = contentDereferencer.dereference(content, resolvedReferences);
6260
}
6361
return content;
@@ -119,7 +117,6 @@ protected String resolveReferenceUrl(ArtifactReferenceDto reference) {
119117

120118
/**
121119
* Resolves a host name from the information found in X-Forwarded-Host and X-Forwarded-Proto.
122-
* @param path
123120
*/
124121
private static URI getApiBaseHrefFromXForwarded(HttpServletRequest request) throws URISyntaxException {
125122
String fproto = request.getHeader("X-Forwarded-Proto");
@@ -133,7 +130,6 @@ private static URI getApiBaseHrefFromXForwarded(HttpServletRequest request) thro
133130

134131
/**
135132
* Resolves a host name from the request information.
136-
* @param path
137133
*/
138134
private static URI getApiBaseHrefFromRequest(HttpServletRequest request) throws URISyntaxException {
139135
String requestUrl = request.getRequestURL().toString();

‎app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@
5252
import java.util.zip.ZipInputStream;
5353

5454
import static io.apicurio.common.apps.logging.audit.AuditingConstants.*;
55-
import static io.apicurio.registry.util.DtoUtil.appAuthPropertyToRegistry;
56-
import static io.apicurio.registry.util.DtoUtil.registryAuthPropertyToApp;
55+
import static io.apicurio.registry.utils.DtoUtil.appAuthPropertyToRegistry;
56+
import static io.apicurio.registry.utils.DtoUtil.registryAuthPropertyToApp;
5757

5858
@ApplicationScoped
5959
@Interceptors({ResponseErrorLivenessCheck.class, ResponseTimeoutReadinessCheck.class})

0 commit comments

Comments
 (0)