Skip to content

Commit e057544

Browse files
EricWittmannEricWittmann
authored
[REST API + KafkaSQL] Significant updates to the REST API to change how artifact and version metadata behave (#4410)
* Multiple changes to REST API related to artifacts, versions, state, and metadata * Rewrote the kafkasql layer to use an RPC based approach (will require snapshotting) * Fix conflict with main * Fix two failing integration tests. * Fix a failing integration test (take 2) * Fix for some failing tests * Attempts to fix broken tests * Fix python tests * Remove some prints in python test * Fix a kafkasql import bug * Fix v2 api compatibility * Fix unused import * Fix errors in the UI due to REST API changes. * Fix failing UI tests * Fix updating the owner of an artifact
1 parent ee640b6 commit e057544

File tree

253 files changed

+6296
-9420
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

253 files changed

+6296
-9420
lines changed

‎.github/workflows/integration-tests.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ jobs:
324324
echo "Starting Registry App (In Memory)"
325325
docker run -it -p 8181:8080 -d ttl.sh/${{ github.sha }}/apicurio/apicurio-registry:1d
326326
cd registry-v2
327-
./mvnw -T 1.5C -Pintegration-tests clean install -DskipTests=true -Dmaven.javadoc.skip=true
327+
./mvnw -T 1.5C -Pintegration-tests clean install -DskipTests=true -DskipUiBuild=true -Dmaven.javadoc.skip=true
328328
cd integration-tests
329329
../mvnw -T 1.5C verify -Pregression -Dmaven.javadoc.skip=true -Dquarkus.http.test-host=localhost -Dquarkus.http.test-port=8181
330330

‎app/src/main/java/io/apicurio/registry/auth/AbstractAccessController.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,14 @@
22

33
import io.apicurio.registry.storage.RegistryStorage;
44
import io.apicurio.registry.storage.dto.ArtifactMetaDataDto;
5+
import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto;
56
import io.apicurio.registry.storage.dto.GroupMetaDataDto;
67
import io.apicurio.registry.storage.error.NotFoundException;
78
import io.apicurio.registry.types.Current;
89
import io.quarkus.security.identity.SecurityIdentity;
910
import jakarta.inject.Inject;
1011
import jakarta.interceptor.InvocationContext;
1112

12-
import static io.apicurio.registry.storage.RegistryStorage.ArtifactRetrievalBehavior.DEFAULT;
13-
1413
public abstract class AbstractAccessController implements IAccessController {
1514

1615
@Inject
@@ -58,7 +57,7 @@ private boolean verifyGroupOwner(String groupId) {
5857

5958
private boolean verifyArtifactOwner(String groupId, String artifactId) {
6059
try {
61-
ArtifactMetaDataDto dto = storage.getArtifactMetaData(groupId, artifactId, DEFAULT);
60+
ArtifactMetaDataDto dto = storage.getArtifactMetaData(groupId, artifactId);
6261
String owner = dto.getOwner();
6362
return owner == null || owner.equals(securityIdentity.getPrincipal().getName());
6463
} catch (NotFoundException nfe) {
@@ -71,7 +70,8 @@ private boolean verifyArtifactOwner(String groupId, String artifactId) {
7170

7271
private boolean verifyArtifactOwner(long globalId) {
7372
try {
74-
ArtifactMetaDataDto dto = storage.getArtifactMetaData(globalId);
73+
ArtifactVersionMetaDataDto versionMetaData = storage.getArtifactVersionMetaData(globalId);
74+
ArtifactMetaDataDto dto = storage.getArtifactMetaData(versionMetaData.getGroupId(), versionMetaData.getArtifactId());
7575
String owner = dto.getOwner();
7676
return owner == null || owner.equals(securityIdentity.getPrincipal().getName());
7777
} catch (NotFoundException nfe) {

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/AbstractResource.java

+33-37
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,44 @@
11
package io.apicurio.registry.ccompat.rest.v7.impl;
22

33

4+
import java.util.Collections;
5+
import java.util.List;
6+
import java.util.Map;
7+
import java.util.function.Function;
8+
import java.util.stream.Collectors;
9+
10+
import org.apache.avro.AvroTypeException;
11+
import org.apache.avro.SchemaParseException;
12+
import org.apache.commons.codec.digest.DigestUtils;
13+
import org.slf4j.Logger;
14+
415
import io.apicurio.registry.ccompat.dto.SchemaReference;
516
import io.apicurio.registry.ccompat.rest.error.ConflictException;
617
import io.apicurio.registry.ccompat.rest.error.UnprocessableEntityException;
718
import io.apicurio.registry.content.ContentHandle;
19+
import io.apicurio.registry.model.BranchId;
20+
import io.apicurio.registry.model.GA;
21+
import io.apicurio.registry.model.GAV;
822
import io.apicurio.registry.rest.v3.beans.ArtifactReference;
923
import io.apicurio.registry.rules.RuleApplicationType;
1024
import io.apicurio.registry.rules.RuleViolationException;
1125
import io.apicurio.registry.rules.RulesService;
1226
import io.apicurio.registry.storage.RegistryStorage;
13-
import io.apicurio.registry.storage.dto.ArtifactMetaDataDto;
27+
import io.apicurio.registry.storage.RegistryStorage.ArtifactRetrievalBehavior;
1428
import io.apicurio.registry.storage.dto.ArtifactReferenceDto;
1529
import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto;
16-
import io.apicurio.registry.storage.dto.StoredArtifactDto;
30+
import io.apicurio.registry.storage.dto.StoredArtifactVersionDto;
1731
import io.apicurio.registry.storage.error.ArtifactNotFoundException;
1832
import io.apicurio.registry.storage.error.RuleNotFoundException;
1933
import io.apicurio.registry.storage.error.VersionNotFoundException;
20-
import io.apicurio.registry.types.ArtifactState;
2134
import io.apicurio.registry.types.ArtifactType;
2235
import io.apicurio.registry.types.Current;
2336
import io.apicurio.registry.types.RuleType;
37+
import io.apicurio.registry.types.VersionState;
2438
import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
2539
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
26-
import org.apache.avro.AvroTypeException;
27-
import org.apache.avro.SchemaParseException;
28-
import org.apache.commons.codec.digest.DigestUtils;
29-
import org.slf4j.Logger;
30-
3140
import jakarta.inject.Inject;
3241

33-
import java.util.Collections;
34-
import java.util.List;
35-
import java.util.Map;
36-
import java.util.function.Function;
37-
import java.util.stream.Collectors;
38-
39-
import static io.apicurio.registry.storage.RegistryStorage.ArtifactRetrievalBehavior.DEFAULT;
40-
4142
public abstract class AbstractResource {
4243

4344
@Inject
@@ -59,8 +60,8 @@ public abstract class AbstractResource {
5960
@Inject
6061
ArtifactTypeUtilProviderFactory factory;
6162

62-
protected ArtifactMetaDataDto createOrUpdateArtifact(String subject, String schema, String artifactType, List<SchemaReference> references, String groupId) {
63-
ArtifactMetaDataDto res;
63+
protected ArtifactVersionMetaDataDto createOrUpdateArtifact(String subject, String schema, String artifactType, List<SchemaReference> references, String groupId) {
64+
ArtifactVersionMetaDataDto res;
6465
final List<ArtifactReferenceDto> parsedReferences = parseReferences(references, groupId);
6566
final List<ArtifactReference> artifactReferences = parsedReferences.stream().map(dto -> ArtifactReference.builder().name(dto.getName()).groupId(dto.getGroupId()).artifactId(dto.getArtifactId()).version(dto.getVersion()).build()).collect(Collectors.toList());
6667
final Map<String, ContentHandle> resolvedReferences = storage.resolveReferences(parsedReferences);
@@ -73,7 +74,7 @@ protected ArtifactMetaDataDto createOrUpdateArtifact(String subject, String sche
7374
res = storage.createArtifact(groupId, subject, null, artifactType, schemaContent, parsedReferences);
7475
} else {
7576
rulesService.applyRules(groupId, subject, artifactType, schemaContent, RuleApplicationType.UPDATE, artifactReferences, resolvedReferences);
76-
res = storage.updateArtifact(groupId, subject, null, artifactType, schemaContent, parsedReferences);
77+
res = storage.createArtifactVersion(groupId, subject, null, artifactType, schemaContent, parsedReferences);
7778
}
7879
} catch (RuleViolationException ex) {
7980
if (ex.getRuleType() == RuleType.VALIDITY) {
@@ -95,15 +96,15 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String subject
9596

9697
if (cconfig.canonicalHashModeEnabled.get() || normalize) {
9798
try {
98-
amd = storage.getArtifactVersionMetaData(groupId, subject, true, ContentHandle.create(schema), artifactReferences);
99+
amd = storage.getArtifactVersionMetaDataByContent(groupId, subject, true, ContentHandle.create(schema), artifactReferences);
99100
} catch (ArtifactNotFoundException ex) {
100101
if (type.equals(ArtifactType.AVRO)) {
101102
//When comparing using content, sometimes the references might be inlined into the content, try to dereference the existing content and compare as a fallback. See https://github.com/Apicurio/apicurio-registry/issues/3588 for more information.
102103
//If using this method there is no matching content either, just re-throw the exception.
103104
//This approach only works for schema types with dereference support (for now, only Avro in the ccompat API).
104105
amd = storage.getArtifactVersions(groupId, subject)
105106
.stream().filter(version -> {
106-
StoredArtifactDto artifactVersion = storage.getArtifactVersion(groupId, subject, version);
107+
StoredArtifactVersionDto artifactVersion = storage.getArtifactVersionContent(groupId, subject, version);
107108
Map<String, ContentHandle> artifactVersionReferences = storage.resolveReferences(artifactVersion.getReferences());
108109
String dereferencedExistingContentSha = DigestUtils.sha256Hex(artifactTypeProvider.getContentDereferencer().dereference(artifactVersion.getContent(), artifactVersionReferences).content());
109110
return dereferencedExistingContentSha.equals(DigestUtils.sha256Hex(schema));
@@ -117,7 +118,7 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String subject
117118
}
118119

119120
} else {
120-
amd = storage.getArtifactVersionMetaData(groupId, subject, false, ContentHandle.create(schema), artifactReferences);
121+
amd = storage.getArtifactVersionMetaDataByContent(groupId, subject, false, ContentHandle.create(schema), artifactReferences);
121122
}
122123

123124
return amd;
@@ -150,43 +151,38 @@ protected Map<String, ContentHandle> resolveReferences(List<SchemaReference> ref
150151
return resolvedReferences;
151152
}
152153

153-
protected boolean isArtifactActive(String subject, String groupId, RegistryStorage.ArtifactRetrievalBehavior retrievalBehavior) {
154-
final ArtifactState state = storage.getArtifactMetaData(groupId, subject, retrievalBehavior).getState();
155-
return storage.isArtifactExists(groupId, subject) && (state.equals(ArtifactState.ENABLED) || state.equals(ArtifactState.DEPRECATED));
154+
protected boolean isArtifactActive(String subject, String groupId) {
155+
long count = storage.countActiveArtifactVersions(groupId, subject);
156+
return count > 0;
156157
}
157158

158159
protected String getLatestArtifactVersionForSubject(String subject, String groupId) {
159160
try {
160-
ArtifactMetaDataDto latest = storage.getArtifactMetaData(groupId, subject);
161-
return latest.getVersion();
161+
GAV latestGAV = storage.getArtifactBranchTip(new GA(groupId, subject), BranchId.LATEST, ArtifactRetrievalBehavior.SKIP_DISABLED_LATEST);
162+
return latestGAV.getRawVersionId();
162163
} catch (ArtifactNotFoundException ex) {
163164
throw new VersionNotFoundException(groupId, subject, "latest");
164165
}
165166
}
166167

167-
protected boolean shouldFilterState(boolean deleted, ArtifactState state) {
168+
protected boolean shouldFilterState(boolean deleted, VersionState state) {
168169
if (deleted) {
169170
//if deleted is enabled, just return all states
170171
return true;
171172
} else {
172-
return state.equals(ArtifactState.ENABLED);
173+
return state.equals(VersionState.ENABLED);
173174
}
174175
}
175176

176177
protected boolean areAllSchemasDisabled(List<Long> globalIds) {
177178
return globalIds.stream().anyMatch(globalId -> {
178-
ArtifactState state = storage.getArtifactMetaData(globalId).getState();
179-
return state.equals(ArtifactState.DISABLED);
179+
VersionState state = storage.getArtifactVersionMetaData(globalId).getState();
180+
return state.equals(VersionState.DISABLED);
180181
});
181182
}
182183

183184
protected boolean doesArtifactExist(String artifactId, String groupId) {
184-
try {
185-
storage.getArtifact(groupId, artifactId, DEFAULT);
186-
return true;
187-
} catch (ArtifactNotFoundException ignored) {
188-
return false;
189-
}
185+
return storage.isArtifactExists(groupId, artifactId);
190186
}
191187

192188
protected boolean doesArtifactRuleExist(String artifactId, RuleType type, String groupId) {

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/ApiConverter.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import io.apicurio.registry.ccompat.dto.SubjectVersion;
1010
import io.apicurio.registry.content.ContentHandle;
1111
import io.apicurio.registry.storage.dto.ArtifactReferenceDto;
12-
import io.apicurio.registry.storage.dto.StoredArtifactDto;
12+
import io.apicurio.registry.storage.dto.StoredArtifactVersionDto;
1313

1414
import java.util.List;
1515
import java.util.stream.Collectors;
@@ -27,11 +27,11 @@ public int convertUnsigned(long value) {
2727
return (int) value;
2828
}
2929

30-
public Schema convert(String subject, StoredArtifactDto storedArtifact) {
30+
public Schema convert(String subject, StoredArtifactVersionDto storedArtifact) {
3131
return convert(subject, storedArtifact, null);
3232
}
3333

34-
public Schema convert(String subject, StoredArtifactDto storedArtifact, String artifactType) {
34+
public Schema convert(String subject, StoredArtifactVersionDto storedArtifact, String artifactType) {
3535
return new Schema(
3636
convertUnsigned(cconfig.legacyIdModeEnabled.get() ? storedArtifact.getGlobalId() : storedArtifact.getContentId()),
3737
subject,

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SchemasResourceImpl.java

+16-17
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
package io.apicurio.registry.ccompat.rest.v7.impl;
22

3+
import java.util.Arrays;
4+
import java.util.Collections;
5+
import java.util.List;
6+
import java.util.stream.Collectors;
7+
38
import io.apicurio.common.apps.logging.Logged;
49
import io.apicurio.registry.auth.Authorized;
510
import io.apicurio.registry.auth.AuthorizedLevel;
@@ -10,21 +15,15 @@
1015
import io.apicurio.registry.content.ContentHandle;
1116
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
1217
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
13-
import io.apicurio.registry.storage.dto.ArtifactMetaDataDto;
1418
import io.apicurio.registry.storage.dto.ArtifactReferenceDto;
19+
import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto;
1520
import io.apicurio.registry.storage.dto.ContentWrapperDto;
16-
import io.apicurio.registry.storage.dto.StoredArtifactDto;
21+
import io.apicurio.registry.storage.dto.StoredArtifactVersionDto;
1722
import io.apicurio.registry.types.ArtifactType;
23+
import io.apicurio.registry.types.VersionState;
1824
import io.apicurio.registry.util.ArtifactTypeUtil;
1925
import jakarta.interceptor.Interceptors;
2026

21-
import java.util.Arrays;
22-
import java.util.Collections;
23-
import java.util.List;
24-
import java.util.stream.Collectors;
25-
26-
import static io.apicurio.registry.storage.RegistryStorage.ArtifactRetrievalBehavior.DEFAULT;
27-
2827
@Interceptors({ResponseErrorLivenessCheck.class, ResponseTimeoutReadinessCheck.class})
2928
@Logged
3029
public class SchemasResourceImpl extends AbstractResource implements SchemasResource {
@@ -35,15 +34,16 @@ public SchemaInfo getSchema(int id, String subject, String groupId) {
3534
ContentHandle contentHandle;
3635
List<ArtifactReferenceDto> references;
3736
if (cconfig.legacyIdModeEnabled.get()) {
38-
StoredArtifactDto artifactVersion = storage.getArtifactVersion(id);
37+
StoredArtifactVersionDto artifactVersion = storage.getArtifactVersionContent(id);
3938
contentHandle = artifactVersion.getContent();
4039
references = artifactVersion.getReferences();
4140
} else {
42-
ContentWrapperDto contentWrapper = storage.getArtifactByContentId(id);
41+
ContentWrapperDto contentWrapper = storage.getContentById(id);
4342
contentHandle = contentWrapper.getContent();
4443
references = contentWrapper.getReferences();
4544
}
46-
return converter.convert(contentHandle, ArtifactTypeUtil.determineArtifactType(contentHandle, null, null, storage.resolveReferences(references), factory.getAllArtifactTypes()), references);
45+
return converter.convert(contentHandle, ArtifactTypeUtil.determineArtifactType(contentHandle, null, null,
46+
storage.resolveReferences(references), factory.getAllArtifactTypes()), references);
4747
}
4848

4949
@Override
@@ -57,14 +57,13 @@ public List<String> getRegisteredTypes() {
5757
public List<SubjectVersion> getSubjectVersions(int id, Boolean fdeleted) {
5858
boolean deleted = fdeleted != null && fdeleted;
5959
if (cconfig.legacyIdModeEnabled.get()) {
60-
ArtifactMetaDataDto artifactMetaData = storage.getArtifactMetaData(id);
61-
return Collections.singletonList(converter.convert(artifactMetaData.getId(), artifactMetaData.getVersionOrder()));
60+
ArtifactVersionMetaDataDto metaData = storage.getArtifactVersionMetaData((long) id);
61+
return Collections.singletonList(converter.convert(metaData.getArtifactId(), metaData.getVersionOrder()));
6262
}
63-
6463
return storage.getArtifactVersionsByContentId(id)
6564
.stream()
66-
.filter(artifactMetaData -> deleted || isArtifactActive(artifactMetaData.getId(), artifactMetaData.getGroupId(), DEFAULT))
67-
.map(artifactMetaData -> converter.convert(artifactMetaData.getId(), artifactMetaData.getVersionOrder()))
65+
.filter(versionMetaData -> deleted || versionMetaData.getState() != VersionState.DISABLED)
66+
.map(versionMetaData -> converter.convert(versionMetaData.getArtifactId(), versionMetaData.getVersionOrder()))
6867
.collect(Collectors.toList());
6968
}
7069
}

0 commit comments

Comments
 (0)