Skip to content

Commit 4ea84ba

Browse files
carlesarnalcarlesarnal
committed
Add kafkasql snapshotting integration tests
1 parent 94654a0 commit 4ea84ba

File tree

15 files changed

+454
-29
lines changed

15 files changed

+454
-29
lines changed

‎.github/workflows/integration-tests.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,9 @@ jobs:
234234
- name: Run Integration Tests - migration - Kafkasql
235235
run: ./mvnw -T 1.5C verify -am --no-transfer-progress -Pintegration-tests -Pmigration -Dregistry-kafkasql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry:1d -Premote-kafka -pl integration-tests -Dmaven.javadoc.skip=true
236236

237+
- name: Run Integration Tests - snapshotting - Kafkasql
238+
run: ./mvnw -T 1.5C verify -am --no-transfer-progress -Pintegration-tests -Pkafkasql-snapshotting -Dregistry-kafkasql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry:1d -Premote-kafka -pl integration-tests -Dmaven.javadoc.skip=true
239+
237240
- name: Collect logs
238241
if: failure()
239242
run: sh ./.github/scripts/collect_logs.sh

‎app/src/test/java/io/apicurio/registry/storage/impl/kafkasql/KafkasqlRecoverFromSnapshotTest.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,11 @@ public void init() {
5252
String artifactId = UUID.randomUUID().toString();
5353
CreateArtifact createArtifact = TestUtils.clientCreateArtifact(artifactId, ArtifactType.AVRO, simpleAvro,
5454
ContentTypes.APPLICATION_JSON);
55-
clientV3.groups().byGroupId(NEW_ARTIFACTS_SNAPSHOT_TEST_GROUP_ID).artifacts()
56-
.post(createArtifact, config -> config.headers.add("X-Registry-ArtifactId", artifactId));
55+
clientV3.groups().byGroupId(NEW_ARTIFACTS_SNAPSHOT_TEST_GROUP_ID).artifacts().post(createArtifact, config -> config.headers.add("X-Registry-ArtifactId", artifactId));
5756
Rule rule = new Rule();
5857
rule.setType(RuleType.VALIDITY);
5958
rule.setConfig("SYNTAX_ONLY");
60-
clientV3.groups().byGroupId(NEW_ARTIFACTS_SNAPSHOT_TEST_GROUP_ID).artifacts().byArtifactId(artifactId).rules().post(rule);
61-
}
59+
clientV3.groups().byGroupId(NEW_ARTIFACTS_SNAPSHOT_TEST_GROUP_ID).artifacts().byArtifactId(artifactId).rules().post(rule); }
6260
}
6361

6462
@Test

‎examples/rest-client/src/main/java/io/apicurio/registry/examples/RegistryLoader.java

+13-9
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
public class RegistryLoader {
3434

3535
public static void main(String[] args) throws Exception {
36-
String registryUrl = "http://localhost:8080/apis/registry/v2";
36+
String registryUrl = "http://localhost:8080/apis/registry/v3";
3737

3838
VertXRequestAdapter vertXRequestAdapter = new VertXRequestAdapter(VertXAuthFactory.defaultVertx);
3939
vertXRequestAdapter.setBaseUrl(registryUrl);
@@ -48,7 +48,7 @@ public static void main(String[] args) throws Exception {
4848
}""";
4949

5050
for (int i = 0; i < 600; i++) {
51-
Task task = new Task(simpleAvro, client, 1000, i + 800000);
51+
Task task = new Task(simpleAvro, client, 100, i + 100);
5252
task.start();
5353
}
5454
}
@@ -72,16 +72,20 @@ public void run() {
7272
for (int idx = 0; idx < numArtifacts; idx++) {
7373
System.out.println("Iteration: " + idx);
7474
String artifactId = UUID.randomUUID().toString();
75+
7576
CreateArtifact createArtifact = new CreateArtifact();
76-
createArtifact.setArtifactId("city");
77-
createArtifact.setType("JSON");
78-
createArtifact.setFirstVersion(new CreateVersion());
79-
createArtifact.getFirstVersion().setContent(new VersionContent());
80-
createArtifact.getFirstVersion().getContent().setContent(simpleAvro.replace("userInfo", "userInfo" + threadId + numArtifacts));
81-
createArtifact.getFirstVersion().getContent().setContentType("application/json");
77+
createArtifact.setType("AVRO");
78+
createArtifact.setArtifactId(artifactId);
79+
CreateVersion createVersion = new CreateVersion();
80+
createArtifact.setFirstVersion(createVersion);
81+
VersionContent versionContent = new VersionContent();
82+
createVersion.setContent(versionContent);
83+
versionContent.setContent(simpleAvro.replace("userInfo", "userInfo" + threadId + numArtifacts));
84+
versionContent.setContentType("application/json");
85+
8286
client.groups().byGroupId("default").artifacts().post(createArtifact, config -> {
83-
config.headers.add("X-Registry-ArtifactId", artifactId);
8487
});
88+
8589
Rule rule = new Rule();
8690
rule.setType(RuleType.VALIDITY);
8791
rule.setConfig("SYNTAX_ONLY");

‎go-sdk/pkg/registryclient-v2/models/new_comment_escaped.go

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package models
2+
3+
import (
4+
i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91 "github.com/microsoft/kiota-abstractions-go/serialization"
5+
)
6+
7+
// NewCommentEscaped
8+
type NewCommentEscaped struct {
9+
// Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
10+
additionalData map[string]any
11+
// The value property
12+
value *string
13+
}
14+
15+
// NewNewCommentEscaped instantiates a new NewCommentEscaped and sets the default values.
16+
func NewNewCommentEscaped() *NewCommentEscaped {
17+
m := &NewCommentEscaped{}
18+
m.SetAdditionalData(make(map[string]any))
19+
return m
20+
}
21+
22+
// CreateNewCommentEscapedFromDiscriminatorValue creates a new instance of the appropriate class based on discriminator value
23+
func CreateNewCommentEscapedFromDiscriminatorValue(parseNode i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) (i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.Parsable, error) {
24+
return NewNewCommentEscaped(), nil
25+
}
26+
27+
// GetAdditionalData gets the AdditionalData property value. Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
28+
func (m *NewCommentEscaped) GetAdditionalData() map[string]any {
29+
return m.additionalData
30+
}
31+
32+
// GetFieldDeserializers the deserialization information for the current model
33+
func (m *NewCommentEscaped) GetFieldDeserializers() map[string]func(i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) error {
34+
res := make(map[string]func(i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) error)
35+
res["value"] = func(n i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) error {
36+
val, err := n.GetStringValue()
37+
if err != nil {
38+
return err
39+
}
40+
if val != nil {
41+
m.SetValue(val)
42+
}
43+
return nil
44+
}
45+
return res
46+
}
47+
48+
// GetValue gets the value property value. The value property
49+
func (m *NewCommentEscaped) GetValue() *string {
50+
return m.value
51+
}
52+
53+
// Serialize serializes information the current object
54+
func (m *NewCommentEscaped) Serialize(writer i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.SerializationWriter) error {
55+
{
56+
err := writer.WriteStringValue("value", m.GetValue())
57+
if err != nil {
58+
return err
59+
}
60+
}
61+
{
62+
err := writer.WriteAdditionalData(m.GetAdditionalData())
63+
if err != nil {
64+
return err
65+
}
66+
}
67+
return nil
68+
}
69+
70+
// SetAdditionalData sets the AdditionalData property value. Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
71+
func (m *NewCommentEscaped) SetAdditionalData(value map[string]any) {
72+
m.additionalData = value
73+
}
74+
75+
// SetValue sets the value property value. The value property
76+
func (m *NewCommentEscaped) SetValue(value *string) {
77+
m.value = value
78+
}
79+
80+
// NewCommentEscapedable
81+
type NewCommentEscapedable interface {
82+
i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.AdditionalDataHolder
83+
i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.Parsable
84+
GetValue() *string
85+
SetValue(value *string)
86+
}

‎go-sdk/pkg/registryclient-v3/admin/config_trigger_snapshot_request_builder.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package admin
2+
3+
import (
4+
"context"
5+
i00eb2e63d156923d00d8e86fe16b5d74daf30e363c9f185a8165cb42aa2f2c71 "github.com/apicurio/apicurio-registry/go-sdk/pkg/registryclient-v3/models"
6+
i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f "github.com/microsoft/kiota-abstractions-go"
7+
)
8+
9+
// ConfigTriggerSnapshotRequestBuilder triggers a snapshot of the Registry storage. Only supported in KafkaSQL storage
10+
type ConfigTriggerSnapshotRequestBuilder struct {
11+
i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.BaseRequestBuilder
12+
}
13+
14+
// ConfigTriggerSnapshotRequestBuilderGetRequestConfiguration configuration for the request such as headers, query parameters, and middleware options.
15+
type ConfigTriggerSnapshotRequestBuilderGetRequestConfiguration struct {
16+
// Request headers
17+
Headers *i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestHeaders
18+
// Request options
19+
Options []i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestOption
20+
}
21+
22+
// NewConfigTriggerSnapshotRequestBuilderInternal instantiates a new TriggerSnapshotRequestBuilder and sets the default values.
23+
func NewConfigTriggerSnapshotRequestBuilderInternal(pathParameters map[string]string, requestAdapter i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestAdapter) *ConfigTriggerSnapshotRequestBuilder {
24+
m := &ConfigTriggerSnapshotRequestBuilder{
25+
BaseRequestBuilder: *i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.NewBaseRequestBuilder(requestAdapter, "{+baseurl}/admin/config/triggerSnapshot", pathParameters),
26+
}
27+
return m
28+
}
29+
30+
// NewConfigTriggerSnapshotRequestBuilder instantiates a new TriggerSnapshotRequestBuilder and sets the default values.
31+
func NewConfigTriggerSnapshotRequestBuilder(rawUrl string, requestAdapter i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestAdapter) *ConfigTriggerSnapshotRequestBuilder {
32+
urlParams := make(map[string]string)
33+
urlParams["request-raw-url"] = rawUrl
34+
return NewConfigTriggerSnapshotRequestBuilderInternal(urlParams, requestAdapter)
35+
}
36+
37+
// Get triggers the creation of a snapshot of the internal database for compatible storages.This operation can fail for the following reasons:* A server error occurred (HTTP error `500`)
38+
func (m *ConfigTriggerSnapshotRequestBuilder) Get(ctx context.Context, requestConfiguration *ConfigTriggerSnapshotRequestBuilderGetRequestConfiguration) ([]byte, error) {
39+
requestInfo, err := m.ToGetRequestInformation(ctx, requestConfiguration)
40+
if err != nil {
41+
return nil, err
42+
}
43+
errorMapping := i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.ErrorMappings{
44+
"500": i00eb2e63d156923d00d8e86fe16b5d74daf30e363c9f185a8165cb42aa2f2c71.CreateErrorFromDiscriminatorValue,
45+
}
46+
res, err := m.BaseRequestBuilder.RequestAdapter.SendPrimitive(ctx, requestInfo, "[]byte", errorMapping)
47+
if err != nil {
48+
return nil, err
49+
}
50+
if res == nil {
51+
return nil, nil
52+
}
53+
return res.([]byte), nil
54+
}
55+
56+
// ToGetRequestInformation triggers the creation of a snapshot of the internal database for compatible storages.This operation can fail for the following reasons:* A server error occurred (HTTP error `500`)
57+
func (m *ConfigTriggerSnapshotRequestBuilder) ToGetRequestInformation(ctx context.Context, requestConfiguration *ConfigTriggerSnapshotRequestBuilderGetRequestConfiguration) (*i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestInformation, error) {
58+
requestInfo := i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.NewRequestInformationWithMethodAndUrlTemplateAndPathParameters(i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.GET, m.BaseRequestBuilder.UrlTemplate, m.BaseRequestBuilder.PathParameters)
59+
if requestConfiguration != nil {
60+
requestInfo.Headers.AddAll(requestConfiguration.Headers)
61+
requestInfo.AddRequestOptions(requestConfiguration.Options)
62+
}
63+
requestInfo.Headers.TryAdd("Accept", "application/json")
64+
return requestInfo, nil
65+
}
66+
67+
// WithUrl returns a request builder with the provided arbitrary URL. Using this method means any other path or query parameters are ignored.
68+
func (m *ConfigTriggerSnapshotRequestBuilder) WithUrl(rawUrl string) *ConfigTriggerSnapshotRequestBuilder {
69+
return NewConfigTriggerSnapshotRequestBuilder(rawUrl, m.BaseRequestBuilder.RequestAdapter)
70+
}

‎go-sdk/pkg/registryclient-v3/models/new_comment_escaped.go

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package models
2+
3+
import (
4+
i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91 "github.com/microsoft/kiota-abstractions-go/serialization"
5+
)
6+
7+
// NewCommentEscaped
8+
type NewCommentEscaped struct {
9+
// Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
10+
additionalData map[string]any
11+
// The value property
12+
value *string
13+
}
14+
15+
// NewNewCommentEscaped instantiates a new NewCommentEscaped and sets the default values.
16+
func NewNewCommentEscaped() *NewCommentEscaped {
17+
m := &NewCommentEscaped{}
18+
m.SetAdditionalData(make(map[string]any))
19+
return m
20+
}
21+
22+
// CreateNewCommentEscapedFromDiscriminatorValue creates a new instance of the appropriate class based on discriminator value
23+
func CreateNewCommentEscapedFromDiscriminatorValue(parseNode i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) (i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.Parsable, error) {
24+
return NewNewCommentEscaped(), nil
25+
}
26+
27+
// GetAdditionalData gets the AdditionalData property value. Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
28+
func (m *NewCommentEscaped) GetAdditionalData() map[string]any {
29+
return m.additionalData
30+
}
31+
32+
// GetFieldDeserializers the deserialization information for the current model
33+
func (m *NewCommentEscaped) GetFieldDeserializers() map[string]func(i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) error {
34+
res := make(map[string]func(i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) error)
35+
res["value"] = func(n i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.ParseNode) error {
36+
val, err := n.GetStringValue()
37+
if err != nil {
38+
return err
39+
}
40+
if val != nil {
41+
m.SetValue(val)
42+
}
43+
return nil
44+
}
45+
return res
46+
}
47+
48+
// GetValue gets the value property value. The value property
49+
func (m *NewCommentEscaped) GetValue() *string {
50+
return m.value
51+
}
52+
53+
// Serialize serializes information the current object
54+
func (m *NewCommentEscaped) Serialize(writer i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.SerializationWriter) error {
55+
{
56+
err := writer.WriteStringValue("value", m.GetValue())
57+
if err != nil {
58+
return err
59+
}
60+
}
61+
{
62+
err := writer.WriteAdditionalData(m.GetAdditionalData())
63+
if err != nil {
64+
return err
65+
}
66+
}
67+
return nil
68+
}
69+
70+
// SetAdditionalData sets the AdditionalData property value. Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
71+
func (m *NewCommentEscaped) SetAdditionalData(value map[string]any) {
72+
m.additionalData = value
73+
}
74+
75+
// SetValue sets the value property value. The value property
76+
func (m *NewCommentEscaped) SetValue(value *string) {
77+
m.value = value
78+
}
79+
80+
// NewCommentEscapedable
81+
type NewCommentEscapedable interface {
82+
i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.AdditionalDataHolder
83+
i878a80d2330e89d26896388a3f487eef27b0a0e6c010c493bf80be1452208f91.Parsable
84+
GetValue() *string
85+
SetValue(value *string)
86+
}

‎integration-tests/pom.xml

+12
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,11 @@
9191
<artifactId>apicurio-registry-utils-converter</artifactId>
9292
<scope>test</scope>
9393
</dependency>
94+
<dependency>
95+
<groupId>io.apicurio</groupId>
96+
<artifactId>apicurio-registry-utils-kafka</artifactId>
97+
<scope>test</scope>
98+
</dependency>
9499
<!-- Test Only -->
95100
<dependency>
96101
<groupId>io.confluent</groupId>
@@ -425,6 +430,13 @@
425430
<skipTests>false</skipTests>
426431
</properties>
427432
</profile>
433+
<profile>
434+
<id>kafkasql-snapshotting</id>
435+
<properties>
436+
<groups>kafkasql-snapshotting</groups>
437+
<skipTests>false</skipTests>
438+
</properties>
439+
</profile>
428440
<profile>
429441
<id>migration</id>
430442
<properties>

‎integration-tests/src/main/java/io/apicurio/deployment/Constants.java

+5
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ public class Constants {
1919
*/
2020
static final String KAFKA_SQL = "kafkasqlit";
2121

22+
/**
23+
* Tag for sql db upgrade tests profile.
24+
*/
25+
static final String KAFKA_SQL_SNAPSHOTTING = "kafkasql-snapshotting";
26+
2227
/**
2328
* Tag for sql db upgrade tests profile.
2429
*/

0 commit comments

Comments
 (0)