Skip to content

Commit 6d7388c

Browse files
carlesarnalAndreaScarselli
carlesarnal
and
AndreaScarselli
authored
Backport avro canonicalizer (#4278)
* Change content canonicalizer for AVRO to improve schema evolution and equality checks #300 * Backport enhanced avro canonicalizer * Update db vesion in ddls * Send content in kafka message to prevent log compaction bug * Fix local kafka test data preparation * Fix local test execution and wrong column being fetched * Fix avro upgrader * Refactor tests to use different versions according to the upgrade needs * Fix sql upgrade tests * Add migration step to kubernetes tests * Do not deploy external resource when they are already available * Do not deploy tenant manager when it is already available * Bump version used for upgrade test * Add avro type to references upgrader * Increase db version so the new upgraders are executed for databases already on version 14 --------- Co-authored-by: Andrea Scarselli <andreascarse@gmail.com>
1 parent a7d38f6 commit 6d7388c

File tree

52 files changed

+2216
-471
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+2216
-471
lines changed

‎app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/CCompatConfig.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public class CCompatConfig {
3636
@Info(category = "ccompat", description = "Legacy ID mode (compatibility API)", availableSince = "2.0.2.Final")
3737
Supplier<Boolean> legacyIdModeEnabled;
3838

39-
@Dynamic(label = "Canonical hash mode (compatibility API)", description = "When selected, the Schema Registy compatibility API uses the canonical hash instead of the regular hash of the content.")
39+
@Dynamic(label = "Canonical hash mode (compatibility API)", description = "When selected, the Schema Registry compatibility API uses the canonical hash instead of the regular hash of the content.")
4040
@ConfigProperty(name = "registry.ccompat.use-canonical-hash", defaultValue = "false")
4141
@Info(category = "ccompat", description = "Canonical hash mode (compatibility API)", availableSince = "2.3.0.Final")
4242
Supplier<Boolean> canonicalHashModeEnabled;

‎app/src/main/java/io/apicurio/registry/rules/RulesService.java

+5
Original file line numberDiff line numberDiff line change
@@ -80,4 +80,9 @@ public void applyRule(String groupId, String artifactId, String artifactType, Co
8080
public void applyRules(String groupId, String artifactId, String artifactVersion, String artifactType,
8181
ContentHandle updatedContent, List<ArtifactReference> references, Map<String, ContentHandle> resolvedReferences)
8282
throws RuleViolationException;
83+
84+
85+
public void applyRulesCompat(String groupId, String artifactId, String artifactVersion, String artifactType,
86+
ContentHandle updatedContent, List<ArtifactReference> references,
87+
Map<String, ContentHandle> resolvedReferences) throws RuleViolationException;
8388
}

‎app/src/main/java/io/apicurio/registry/rules/RulesServiceImpl.java

+35-25
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,19 @@
1919

2020
import io.apicurio.registry.content.ContentHandle;
2121
import io.apicurio.registry.rest.v2.beans.ArtifactReference;
22+
import io.apicurio.registry.content.canon.ContentCanonicalizer;
2223
import io.apicurio.registry.storage.RegistryStorage;
2324
import io.apicurio.registry.storage.dto.LazyContentList;
2425
import io.apicurio.registry.storage.dto.RuleConfigurationDto;
2526
import io.apicurio.registry.storage.dto.StoredArtifactDto;
2627
import io.apicurio.registry.types.Current;
2728
import io.apicurio.registry.types.RuleType;
29+
import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
30+
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
2831

2932
import jakarta.enterprise.context.ApplicationScoped;
3033
import jakarta.inject.Inject;
34+
3135
import java.util.Collections;
3236
import java.util.List;
3337
import java.util.Map;
@@ -52,13 +56,16 @@ public class RulesServiceImpl implements RulesService {
5256
@Inject
5357
RulesProperties rulesProperties;
5458

59+
@Inject
60+
ArtifactTypeUtilProviderFactory providerFactory;
61+
5562
/**
5663
* @see io.apicurio.registry.rules.RulesService#applyRules(java.lang.String, java.lang.String, java.lang.String, io.apicurio.registry.content.ContentHandle, io.apicurio.registry.rules.RuleApplicationType, java.util.List, java.util.Map)
5764
*/
5865
@Override
5966
public void applyRules(String groupId, String artifactId, String artifactType, ContentHandle artifactContent,
60-
RuleApplicationType ruleApplicationType, List<ArtifactReference> references,
61-
Map<String, ContentHandle> resolvedReferences) throws RuleViolationException {
67+
RuleApplicationType ruleApplicationType, List<ArtifactReference> references,
68+
Map<String, ContentHandle> resolvedReferences) throws RuleViolationException {
6269
@SuppressWarnings("unchecked")
6370
List<RuleType> rules = Collections.EMPTY_LIST;
6471
if (ruleApplicationType == RuleApplicationType.UPDATE) {
@@ -75,20 +82,20 @@ public void applyRules(String groupId, String artifactId, String artifactType, C
7582
}
7683

7784
private void applyGlobalAndArtifactRules(String groupId, String artifactId, String artifactType,
78-
List<ContentHandle> currentArtifactContent, ContentHandle updatedArtifactContent,
79-
List<RuleType> artifactRules, List<ArtifactReference> references, Map<String, ContentHandle> resolvedReferences) {
85+
List<ContentHandle> currentArtifactContent, ContentHandle updatedArtifactContent,
86+
List<RuleType> artifactRules, List<ArtifactReference> references, Map<String, ContentHandle> resolvedReferences) {
8087

8188
Map<RuleType, RuleConfigurationDto> globalOrArtifactRulesMap = artifactRules.stream()
82-
.collect(Collectors.toMap(ruleType -> ruleType, ruleType -> storage.getArtifactRule(groupId, artifactId, ruleType)));
89+
.collect(Collectors.toMap(ruleType -> ruleType, ruleType -> storage.getArtifactRule(groupId, artifactId, ruleType)));
8390

8491
if (globalOrArtifactRulesMap.isEmpty()) {
8592
List<RuleType> globalRules = storage.getGlobalRules();
8693
globalOrArtifactRulesMap = globalRules.stream()
87-
.collect(Collectors.toMap(ruleType -> ruleType, storage::getGlobalRule));
94+
.collect(Collectors.toMap(ruleType -> ruleType, storage::getGlobalRule));
8895

8996
// Add any default global rules to the map (after filtering out any global rules from artifactStore)
90-
Map<RuleType, RuleConfigurationDto> filteredDefaultGlobalRulesMap = rulesProperties.getFilteredDefaultGlobalRules(globalRules).stream()
91-
.collect(Collectors.toMap(ruleType -> ruleType, rulesProperties::getDefaultGlobalRuleConfiguration));
97+
Map<RuleType, RuleConfigurationDto> filteredDefaultGlobalRulesMap = rulesProperties.getFilteredDefaultGlobalRules(globalRules).stream()
98+
.collect(Collectors.toMap(ruleType -> ruleType, rulesProperties::getDefaultGlobalRuleConfiguration));
9299
globalOrArtifactRulesMap.putAll(filteredDefaultGlobalRulesMap);
93100
}
94101

@@ -107,34 +114,25 @@ private void applyGlobalAndArtifactRules(String groupId, String artifactId, Stri
107114
*/
108115
@Override
109116
public void applyRule(String groupId, String artifactId, String artifactType, ContentHandle artifactContent,
110-
RuleType ruleType, String ruleConfiguration, RuleApplicationType ruleApplicationType,
111-
List<ArtifactReference> references, Map<String, ContentHandle> resolvedReferences)
117+
RuleType ruleType, String ruleConfiguration, RuleApplicationType ruleApplicationType,
118+
List<ArtifactReference> references, Map<String, ContentHandle> resolvedReferences)
112119
throws RuleViolationException {
113120
LazyContentList currentContent = null;
114121
if (ruleApplicationType == RuleApplicationType.UPDATE) {
115122
currentContent = new LazyContentList(storage, storage.getEnabledArtifactContentIds(groupId, artifactId));
116123
}
117-
applyRule(groupId, artifactId, artifactType, currentContent, artifactContent, ruleType, ruleConfiguration,
124+
applyRule(groupId, artifactId, artifactType, currentContent, artifactContent, ruleType, ruleConfiguration,
118125
references, resolvedReferences);
119126
}
120127

121128
/**
122129
* Applies a single rule. Throws an exception if the rule is violated.
123-
* @param groupId
124-
* @param artifactId
125-
* @param artifactType
126-
* @param currentContent
127-
* @param updatedContent
128-
* @param ruleType
129-
* @param ruleConfiguration
130-
* @param references
131-
* @param resolvedReferences
132130
*/
133131
private void applyRule(String groupId, String artifactId, String artifactType, List<ContentHandle> currentContent,
134-
ContentHandle updatedContent, RuleType ruleType, String ruleConfiguration,
132+
ContentHandle updatedContent, RuleType ruleType, String ruleConfiguration,
135133
List<ArtifactReference> references, Map<String, ContentHandle> resolvedReferences) {
136134
RuleExecutor executor = factory.createExecutor(ruleType);
137-
RuleContext context = new RuleContext(groupId, artifactId, artifactType, ruleConfiguration, currentContent,
135+
RuleContext context = new RuleContext(groupId, artifactId, artifactType, ruleConfiguration, currentContent,
138136
updatedContent, references, resolvedReferences);
139137
executor.execute(context);
140138
}
@@ -143,11 +141,23 @@ private void applyRule(String groupId, String artifactId, String artifactType, L
143141
* @see io.apicurio.registry.rules.RulesService#applyRules(java.lang.String, java.lang.String, java.lang.String, java.lang.String, io.apicurio.registry.content.ContentHandle, java.util.List, java.util.Map)
144142
*/
145143
@Override
146-
public void applyRules(String groupId, String artifactId, String artifactVersion, String artifactType,
147-
ContentHandle updatedContent, List<ArtifactReference> references,
148-
Map<String, ContentHandle> resolvedReferences) throws RuleViolationException {
144+
public void applyRules(String groupId, String artifactId, String artifactVersion, String artifactType,
145+
ContentHandle updatedContent, List<ArtifactReference> references,
146+
Map<String, ContentHandle> resolvedReferences) throws RuleViolationException {
149147
StoredArtifactDto versionContent = storage.getArtifactVersion(groupId, artifactId, artifactVersion);
150148
applyGlobalAndArtifactRules(groupId, artifactId, artifactType, Collections.singletonList(versionContent.getContent()),
151149
updatedContent, storage.getArtifactRules(groupId, artifactId), references, resolvedReferences);
152150
}
151+
152+
@Override
153+
public void applyRulesCompat(String groupId, String artifactId, String artifactVersion, String artifactType,
154+
ContentHandle updatedContent, List<ArtifactReference> references,
155+
Map<String, ContentHandle> resolvedReferences) throws RuleViolationException {
156+
ArtifactTypeUtilProvider artifactTypeProvider = providerFactory.getArtifactTypeProvider(artifactType);
157+
ContentCanonicalizer contentCanonicalizer = artifactTypeProvider.getContentCanonicalizer();
158+
StoredArtifactDto versionContent = storage.getArtifactVersion(groupId, artifactId, artifactVersion);
159+
applyGlobalAndArtifactRules(groupId, artifactId, artifactType,
160+
Collections.singletonList(contentCanonicalizer.canonicalize(versionContent.getContent(), Map.of())),
161+
updatedContent, storage.getArtifactRules(groupId, artifactId), references, resolvedReferences);
162+
}
153163
}

‎app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ private void applyUpgrader(Handle handle, String cname) {
286286
@SuppressWarnings("unchecked")
287287
Class<IDbUpgrader> upgraderClass = (Class<IDbUpgrader>) Class.forName(cname);
288288
IDbUpgrader upgrader = upgraderClass.getConstructor().newInstance();
289-
upgrader.upgrade(handle);
289+
upgrader.upgrade( handle);
290290
} catch (Exception e) {
291291
throw new RuntimeException(e);
292292
}

‎app/src/main/java/io/apicurio/registry/storage/impl/sql/IDbUpgrader.java

+5
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,9 @@ public interface IDbUpgrader {
2929
*/
3030
public void upgrade(Handle dbHandle) throws Exception;
3131

32+
/**
33+
* Called by the {@link AbstractSqlRegistryStorage} class when upgrading the database.
34+
* @param registryStorage
35+
* @param dbHandle
36+
*/
3237
}

‎app/src/main/java/io/apicurio/registry/storage/impl/sql/upgrader/AbstractReferencesCanonicalHashUpgrader.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public void upgrade(Handle handle) throws Exception {
6969
private void updateEntity(Handle handle, ExtendedContentEntity entity) {
7070
try {
7171
beforeEach();
72-
if (entity.contentEntity.serializedReferences != null || ArtifactType.PROTOBUF.equals(entity.type) /* Replaces ProtobufCanonicalHashUpgrader */) {
72+
if (entityHasToBeUpgraded(entity) /* Replaces ProtobufCanonicalHashUpgrader */) {
7373

7474
var newCanonicalHash = RegistryContentUtils.canonicalContentHash(
7575
entity.type,
@@ -91,6 +91,10 @@ private void updateEntity(Handle handle, ExtendedContentEntity entity) {
9191
}
9292
}
9393

94+
private static boolean entityHasToBeUpgraded(ExtendedContentEntity entity) {
95+
return entity.contentEntity.serializedReferences != null || ArtifactType.PROTOBUF.equals(entity.type) || ArtifactType.AVRO.equals(entity.type);
96+
}
97+
9498
protected abstract void beforeEach();
9599

96100
protected abstract void applyUpgrade(Handle handle, ExtendedContentEntity entity);

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/db-version

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
14
1+
15

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/h2.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
CREATE TABLE apicurio (prop_name VARCHAR(255) NOT NULL, prop_value VARCHAR(255));
66
ALTER TABLE apicurio ADD PRIMARY KEY (prop_name);
7-
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 14);
7+
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 15);
88

99
CREATE TABLE sequences (tenantId VARCHAR(128) NOT NULL, name VARCHAR(32) NOT NULL, seq_value BIGINT NOT NULL);
1010
ALTER TABLE sequences ADD PRIMARY KEY (tenantId, name);

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/mssql.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
CREATE TABLE apicurio (prop_name VARCHAR(255) NOT NULL, prop_value VARCHAR(255));
66
ALTER TABLE apicurio ADD PRIMARY KEY (prop_name);
7-
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 14);
7+
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 15);
88

99
CREATE TABLE sequences (tenantId VARCHAR(128) NOT NULL, name VARCHAR(32) NOT NULL, value BIGINT NOT NULL);
1010
ALTER TABLE sequences ADD PRIMARY KEY (tenantId, name);

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/mysql.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
CREATE TABLE apicurio (prop_name VARCHAR(255) NOT NULL, prop_value VARCHAR(255));
66
ALTER TABLE apicurio ADD PRIMARY KEY (prop_name);
7-
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 14);
7+
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 15);
88

99
CREATE TABLE sequences (tenantId VARCHAR(128) NOT NULL, name VARCHAR(32) NOT NULL, value BIGINT NOT NULL);
1010
ALTER TABLE sequences ADD PRIMARY KEY (tenantId, name);

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/postgresql.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
CREATE TABLE apicurio (prop_name VARCHAR(255) NOT NULL, prop_value VARCHAR(255));
66
ALTER TABLE apicurio ADD PRIMARY KEY (prop_name);
7-
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 14);
7+
INSERT INTO apicurio (prop_name, prop_value) VALUES ('db_version', 15);
88

99
CREATE TABLE sequences (tenantId VARCHAR(128) NOT NULL, name VARCHAR(32) NOT NULL, value BIGINT NOT NULL);
1010
ALTER TABLE sequences ADD PRIMARY KEY (tenantId, name);

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/upgrades/14/h2.upgrade.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55

66
UPDATE apicurio SET prop_value = 14 WHERE prop_name = 'db_version';
77

8-
UPGRADER:io.apicurio.registry.storage.impl.sql.upgrader.SqlReferencesCanonicalHashUpgrader;
8+
-- This upgrade script left intentionally blank.

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/upgrades/14/mssql.upgrade.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55

66
UPDATE apicurio SET prop_value = 14 WHERE prop_name = 'db_version';
77

8-
UPGRADER:io.apicurio.registry.storage.impl.sql.upgrader.SqlReferencesCanonicalHashUpgrader;
8+
-- This upgrade script left intentionally blank.

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/upgrades/14/postgresql.upgrade.ddl

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55

66
UPDATE apicurio SET prop_value = 14 WHERE prop_name = 'db_version';
77

8-
UPGRADER:io.apicurio.registry.storage.impl.sql.upgrader.SqlReferencesCanonicalHashUpgrader;
8+
-- This upgrade script left intentionally blank.

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/upgrades/15/h2.upgrade.ddl

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
-- *********************************************************************
2+
-- DDL for the Apicurio Registry - Database: H2
3+
-- Upgrades the DB schema from version 14 to version 15.
4+
-- *********************************************************************
5+
6+
UPDATE apicurio SET prop_value = 15 WHERE prop_name = 'db_version';
7+
8+
UPGRADER:io.apicurio.registry.storage.impl.sql.upgrader.SqlReferencesCanonicalHashUpgrader;

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/upgrades/15/mssql.upgrade.ddl

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
-- *********************************************************************
2+
-- DDL for the Apicurio Registry - Database: mssql
3+
-- Upgrades the DB schema from version 14 to version 15.
4+
-- *********************************************************************
5+
6+
UPDATE apicurio SET prop_value = 15 WHERE prop_name = 'db_version';
7+
8+
UPGRADER:io.apicurio.registry.storage.impl.sql.upgrader.SqlReferencesCanonicalHashUpgrader;

‎app/src/main/resources/io/apicurio/registry/storage/impl/sql/upgrades/15/postgresql.upgrade.ddl

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
-- *********************************************************************
2+
-- DDL for the Apicurio Registry - Database: PostgreSQL
3+
-- Upgrades the DB schema from version 14 to version 15.
4+
-- *********************************************************************
5+
6+
UPDATE apicurio SET prop_value = 15 WHERE prop_name = 'db_version';
7+
8+
UPGRADER:io.apicurio.registry.storage.impl.sql.upgrader.SqlReferencesCanonicalHashUpgrader;

‎app/src/test/java/io/apicurio/registry/noprofile/content/ContentCanonicalizerTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ void testAvro() {
7979
" { \"name\": \"last\", \"type\": \"string\" }\r\n" +
8080
" ]\r\n" +
8181
"} ";
82-
String expected = "{\"fields\":[{\"name\":\"first\",\"type\":\"string\"},{\"name\":\"last\",\"type\":\"string\"},{\"name\":\"middle\",\"type\":\"string\"}],\"name\":\"FullName\",\"namespace\":\"com.example\",\"type\":\"record\"}";
82+
String expected = "{\"type\":\"record\",\"name\":\"FullName\",\"namespace\":\"com.example\",\"doc\":\"\",\"fields\":[{\"name\":\"first\",\"type\":\"string\",\"doc\":\"\"},{\"name\":\"middle\",\"type\":\"string\",\"doc\":\"\"},{\"name\":\"last\",\"type\":\"string\",\"doc\":\"\"}]}";
8383

8484
ContentHandle content = ContentHandle.create(before);
8585
String actual = canonicalizer.canonicalize(content, Collections.emptyMap()).content();

0 commit comments

Comments
 (0)