Skip to content

Commit 5eec328

Browse files
EricWittmannEricWittmann
authored
Several improvements to the Import/Export process (#5037)
* Import now writes ZIP to disk first. All import options now support both v2 and v3 ZIP files. Fails on others. * By default, require a registry to be empty before allowing an import * Add debugging to migration test * code formatting * Make sure to read artifact rules after versions so that importing from v2 works
1 parent 6a58724 commit 5eec328

File tree

49 files changed

+823
-758
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+823
-758
lines changed

‎app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java

+19-36
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,22 @@
11
package io.apicurio.registry;
22

3-
import io.apicurio.common.apps.config.Info;
3+
import io.apicurio.registry.rest.ConflictException;
4+
import io.apicurio.registry.rest.v3.AdminResourceImpl;
45
import io.apicurio.registry.storage.RegistryStorage;
56
import io.apicurio.registry.storage.StorageEvent;
67
import io.apicurio.registry.storage.StorageEventType;
78
import io.apicurio.registry.storage.error.ReadOnlyStorageException;
8-
import io.apicurio.registry.storage.impexp.EntityInputStream;
9+
import io.apicurio.registry.storage.importing.ImportExportConfigProperties;
910
import io.apicurio.registry.types.Current;
10-
import io.apicurio.registry.utils.impexp.Entity;
11-
import io.apicurio.registry.utils.impexp.v3.EntityReader;
1211
import jakarta.enterprise.context.ApplicationScoped;
1312
import jakarta.enterprise.event.ObservesAsync;
1413
import jakarta.inject.Inject;
15-
import org.eclipse.microprofile.config.inject.ConfigProperty;
1614
import org.slf4j.Logger;
1715

1816
import java.io.BufferedInputStream;
1917
import java.io.IOException;
2018
import java.io.InputStream;
2119
import java.net.URL;
22-
import java.nio.charset.StandardCharsets;
23-
import java.util.Optional;
24-
import java.util.zip.ZipInputStream;
2520

2621
@ApplicationScoped
2722
public class ImportLifecycleBean {
@@ -33,42 +28,30 @@ public class ImportLifecycleBean {
3328
@Current
3429
RegistryStorage storage;
3530

36-
@ConfigProperty(name = "apicurio.import.url")
37-
@Info(category = "import", description = "The import URL", availableSince = "2.1.0.Final")
38-
Optional<URL> registryImportUrlProp;
31+
@Inject
32+
ImportExportConfigProperties importExportProps;
33+
34+
@Inject
35+
AdminResourceImpl v3Admin;
3936

4037
void onStorageReady(@ObservesAsync StorageEvent ev) {
41-
if (StorageEventType.READY.equals(ev.getType()) && registryImportUrlProp.isPresent()) {
38+
if (StorageEventType.READY.equals(ev.getType())
39+
&& importExportProps.registryImportUrlProp.isPresent()) {
4240
log.info("Import URL exists.");
43-
final URL registryImportUrl = registryImportUrlProp.get();
41+
final URL registryImportUrl = importExportProps.registryImportUrlProp.get();
4442
try (final InputStream registryImportZip = new BufferedInputStream(
4543
registryImportUrl.openStream())) {
4644
log.info("Importing {} on startup.", registryImportUrl);
47-
final ZipInputStream zip = new ZipInputStream(registryImportZip, StandardCharsets.UTF_8);
48-
final EntityReader reader = new EntityReader(zip);
49-
try (EntityInputStream stream = new EntityInputStream() {
50-
@Override
51-
public Entity nextEntity() {
52-
try {
53-
return reader.readEntity();
54-
} catch (Exception e) {
55-
log.error("Error reading data from import ZIP file {}.", registryImportUrl, e);
56-
return null;
57-
}
58-
}
59-
60-
@Override
61-
public void close() throws IOException {
62-
zip.close();
63-
}
64-
}) {
65-
storage.importData(stream, true, true);
66-
log.info("Registry successfully imported from {}", registryImportUrl);
67-
} catch (ReadOnlyStorageException e) {
68-
log.error("Registry import failed, because the storage is in read-only mode.");
69-
}
45+
v3Admin.importData(null, null, null, registryImportZip);
46+
log.info("Registry successfully imported from {}", registryImportUrl);
7047
} catch (IOException ioe) {
7148
log.error("Registry import from {} failed", registryImportUrl, ioe);
49+
} catch (ReadOnlyStorageException rose) {
50+
log.error("Registry import failed, because the storage is in read-only mode.");
51+
} catch (ConflictException ce) {
52+
log.info("Import skipped, registry not empty.");
53+
} catch (Exception e) {
54+
log.error("Registry import failed", e);
7255
}
7356
}
7457
}

‎app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java

+9-62
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
package io.apicurio.registry.rest.v2;
22

3-
import io.apicurio.common.apps.config.Dynamic;
43
import io.apicurio.common.apps.config.DynamicConfigPropertyDef;
54
import io.apicurio.common.apps.config.DynamicConfigPropertyDto;
65
import io.apicurio.common.apps.config.DynamicConfigPropertyIndex;
7-
import io.apicurio.common.apps.config.Info;
86
import io.apicurio.common.apps.logging.Logged;
97
import io.apicurio.common.apps.logging.audit.Audited;
108
import io.apicurio.registry.auth.Authorized;
@@ -16,51 +14,37 @@
1614
import io.apicurio.registry.rest.MissingRequiredParameterException;
1715
import io.apicurio.registry.rest.v2.beans.ArtifactTypeInfo;
1816
import io.apicurio.registry.rest.v2.beans.ConfigurationProperty;
19-
import io.apicurio.registry.rest.v2.beans.DownloadRef;
2017
import io.apicurio.registry.rest.v2.beans.RoleMapping;
2118
import io.apicurio.registry.rest.v2.beans.Rule;
2219
import io.apicurio.registry.rest.v2.beans.UpdateConfigurationProperty;
2320
import io.apicurio.registry.rest.v2.beans.UpdateRole;
24-
import io.apicurio.registry.rest.v2.shared.DataExporter;
2521
import io.apicurio.registry.rules.DefaultRuleDeletionException;
2622
import io.apicurio.registry.rules.RulesProperties;
2723
import io.apicurio.registry.storage.RegistryStorage;
28-
import io.apicurio.registry.storage.dto.DownloadContextDto;
29-
import io.apicurio.registry.storage.dto.DownloadContextType;
3024
import io.apicurio.registry.storage.dto.RoleMappingDto;
3125
import io.apicurio.registry.storage.dto.RuleConfigurationDto;
3226
import io.apicurio.registry.storage.error.ConfigPropertyNotFoundException;
3327
import io.apicurio.registry.storage.error.InvalidPropertyValueException;
3428
import io.apicurio.registry.storage.error.RuleNotFoundException;
35-
import io.apicurio.registry.storage.impexp.EntityInputStream;
29+
import io.apicurio.registry.storage.importing.ImportExportConfigProperties;
3630
import io.apicurio.registry.types.Current;
3731
import io.apicurio.registry.types.RoleType;
3832
import io.apicurio.registry.types.RuleType;
3933
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
40-
import io.apicurio.registry.utils.impexp.Entity;
41-
import io.apicurio.registry.utils.impexp.v2.EntityReader;
4234
import jakarta.enterprise.context.ApplicationScoped;
4335
import jakarta.inject.Inject;
4436
import jakarta.interceptor.Interceptors;
45-
import jakarta.servlet.http.HttpServletRequest;
46-
import jakarta.ws.rs.core.Context;
47-
import jakarta.ws.rs.core.MediaType;
4837
import jakarta.ws.rs.core.Response;
4938
import org.eclipse.microprofile.config.Config;
50-
import org.eclipse.microprofile.config.inject.ConfigProperty;
5139
import org.slf4j.Logger;
5240

53-
import java.io.IOException;
5441
import java.io.InputStream;
55-
import java.nio.charset.StandardCharsets;
5642
import java.util.HashMap;
5743
import java.util.List;
5844
import java.util.Map;
5945
import java.util.Set;
60-
import java.util.function.Supplier;
6146
import java.util.stream.Collectors;
6247
import java.util.stream.Stream;
63-
import java.util.zip.ZipInputStream;
6448

6549
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_FOR_BROWSER;
6650
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_NAME;
@@ -97,17 +81,12 @@ public class AdminResourceImpl implements AdminResource {
9781
Config config;
9882

9983
@Inject
100-
DataExporter exporter;
84+
ImportExportConfigProperties importExportProps;
10185

102-
@Context
103-
HttpServletRequest request;
104-
105-
@Dynamic(label = "Download link expiry", description = "The number of seconds that a generated link to a .zip download file is active before expiring.")
106-
@ConfigProperty(name = "apicurio.download.href.ttl.seconds", defaultValue = "30")
107-
@Info(category = "download", description = "Download link expiry", availableSince = "2.1.2.Final")
108-
Supplier<Long> downloadHrefTtl;
86+
@Inject
87+
io.apicurio.registry.rest.v3.AdminResourceImpl v3Admin;
10988

110-
private static final void requireParameter(String parameterName, Object parameterValue) {
89+
private static void requireParameter(String parameterName, Object parameterValue) {
11190
if (parameterValue == null) {
11291
throw new MissingRequiredParameterException(parameterName);
11392
}
@@ -124,7 +103,6 @@ public List<ArtifactTypeInfo> listArtifactTypes() {
124103
ati.setName(t);
125104
return ati;
126105
}).collect(Collectors.toList());
127-
128106
}
129107

130108
/**
@@ -246,26 +224,7 @@ public void deleteGlobalRule(RuleType rule) {
246224
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin)
247225
public void importData(Boolean xRegistryPreserveGlobalId, Boolean xRegistryPreserveContentId,
248226
InputStream data) {
249-
final ZipInputStream zip = new ZipInputStream(data, StandardCharsets.UTF_8);
250-
final EntityReader reader = new EntityReader(zip);
251-
EntityInputStream stream = new EntityInputStream() {
252-
@Override
253-
public Entity nextEntity() throws IOException {
254-
try {
255-
return reader.readEntity();
256-
} catch (Exception e) {
257-
log.error("Error reading data from import ZIP file.", e);
258-
return null;
259-
}
260-
}
261-
262-
@Override
263-
public void close() throws IOException {
264-
zip.close();
265-
}
266-
};
267-
this.storage.upgradeData(stream, isNullOrTrue(xRegistryPreserveGlobalId),
268-
isNullOrTrue(xRegistryPreserveContentId));
227+
v3Admin.importData(xRegistryPreserveGlobalId, xRegistryPreserveContentId, false, data);
269228
}
270229

271230
/**
@@ -275,20 +234,8 @@ public void close() throws IOException {
275234
@Audited(extractParameters = { "0", KEY_FOR_BROWSER })
276235
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin)
277236
public Response exportData(Boolean forBrowser) {
278-
String acceptHeader = request.getHeader("Accept");
279-
if (Boolean.TRUE.equals(forBrowser) || MediaType.APPLICATION_JSON.equals(acceptHeader)) {
280-
long expires = System.currentTimeMillis() + (downloadHrefTtl.get() * 1000);
281-
DownloadContextDto downloadCtx = DownloadContextDto.builder().type(DownloadContextType.EXPORT)
282-
.expires(expires).build();
283-
String downloadId = storage.createDownload(downloadCtx);
284-
String downloadHref = createDownloadHref(downloadId);
285-
DownloadRef downloadRef = new DownloadRef();
286-
downloadRef.setDownloadId(downloadId);
287-
downloadRef.setHref(downloadHref);
288-
return Response.ok(downloadRef).type(MediaType.APPLICATION_JSON_TYPE).build();
289-
} else {
290-
return exporter.exportData();
291-
}
237+
throw new UnsupportedOperationException(
238+
"Exporting data using the Registry Core v2 API is no longer supported. Use the v3 API.");
292239
}
293240

294241
/**
@@ -463,7 +410,7 @@ private ConfigurationProperty defToConfigurationProperty(DynamicConfigPropertyDe
463410

464411
/**
465412
* Lookup the dynamic configuration property being set. Ensure that it exists (throws a
466-
* {@link NotFoundException} if it does not.
413+
* {@link io.apicurio.registry.storage.error.NotFoundException} if it does not.
467414
*
468415
* @param propertyName the name of the dynamic property
469416
* @return the dynamic config property definition

‎app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java

-20
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,8 @@
66
import io.apicurio.registry.auth.AuthorizedStyle;
77
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
88
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
9-
import io.apicurio.registry.rest.v2.shared.DataExporter;
10-
import io.apicurio.registry.storage.RegistryStorage;
11-
import io.apicurio.registry.storage.dto.DownloadContextDto;
12-
import io.apicurio.registry.storage.dto.DownloadContextType;
139
import io.apicurio.registry.storage.error.DownloadNotFoundException;
14-
import io.apicurio.registry.types.Current;
1510
import jakarta.enterprise.context.ApplicationScoped;
16-
import jakarta.inject.Inject;
1711
import jakarta.interceptor.Interceptors;
1812
import jakarta.ws.rs.GET;
1913
import jakarta.ws.rs.Path;
@@ -27,25 +21,11 @@
2721
@Path("/apis/registry/v2/downloads")
2822
public class DownloadsResourceImpl {
2923

30-
@Inject
31-
@Current
32-
RegistryStorage storage;
33-
34-
@Inject
35-
DataExporter exporter;
36-
3724
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.None)
3825
@GET
3926
@Path("{downloadId}")
4027
@Produces("*/*")
4128
public Response download(@PathParam("downloadId") String downloadId) {
42-
DownloadContextDto downloadContext = storage.consumeDownload(downloadId);
43-
if (downloadContext.getType() == DownloadContextType.EXPORT) {
44-
return exporter.exportData();
45-
}
46-
47-
// TODO support other types of downloads (e.g. download content by contentId)
48-
4929
throw new DownloadNotFoundException();
5030
}
5131

‎app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java

-61
This file was deleted.

0 commit comments

Comments
 (0)