Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Several improvements to the Import/Export process #5037

Merged
merged 5 commits into from
Aug 16, 2024
Merged

Several improvements to the Import/Export process #5037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eeae909
Import now writes ZIP to disk first. All import options now support b…
EricWittmann Aug 15, 2024
440df86
By default, require a registry to be empty before allowing an import
EricWittmann Aug 15, 2024
c3dff8b
Add debugging to migration test
EricWittmann Aug 16, 2024
3c143e2
code formatting
EricWittmann Aug 16, 2024
cdc438f
Make sure to read artifact rules after versions so that importing fro…
EricWittmann Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 19 additions & 36 deletions app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,22 @@
package io.apicurio.registry;

import io.apicurio.common.apps.config.Info;
import io.apicurio.registry.rest.ConflictException;
import io.apicurio.registry.rest.v3.AdminResourceImpl;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.StorageEvent;
import io.apicurio.registry.storage.StorageEventType;
import io.apicurio.registry.storage.error.ReadOnlyStorageException;
import io.apicurio.registry.storage.impexp.EntityInputStream;
import io.apicurio.registry.storage.importing.ImportExportConfigProperties;
import io.apicurio.registry.types.Current;
import io.apicurio.registry.utils.impexp.Entity;
import io.apicurio.registry.utils.impexp.v3.EntityReader;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.ObservesAsync;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.slf4j.Logger;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.zip.ZipInputStream;

@ApplicationScoped
public class ImportLifecycleBean {
Expand All @@ -33,42 +28,30 @@ public class ImportLifecycleBean {
@Current
RegistryStorage storage;

@ConfigProperty(name = "apicurio.import.url")
@Info(category = "import", description = "The import URL", availableSince = "2.1.0.Final")
Optional<URL> registryImportUrlProp;
@Inject
ImportExportConfigProperties importExportProps;

@Inject
AdminResourceImpl v3Admin;

void onStorageReady(@ObservesAsync StorageEvent ev) {
if (StorageEventType.READY.equals(ev.getType()) && registryImportUrlProp.isPresent()) {
if (StorageEventType.READY.equals(ev.getType())
&& importExportProps.registryImportUrlProp.isPresent()) {
log.info("Import URL exists.");
final URL registryImportUrl = registryImportUrlProp.get();
final URL registryImportUrl = importExportProps.registryImportUrlProp.get();
try (final InputStream registryImportZip = new BufferedInputStream(
registryImportUrl.openStream())) {
log.info("Importing {} on startup.", registryImportUrl);
final ZipInputStream zip = new ZipInputStream(registryImportZip, StandardCharsets.UTF_8);
final EntityReader reader = new EntityReader(zip);
try (EntityInputStream stream = new EntityInputStream() {
@Override
public Entity nextEntity() {
try {
return reader.readEntity();
} catch (Exception e) {
log.error("Error reading data from import ZIP file {}.", registryImportUrl, e);
return null;
}
}

@Override
public void close() throws IOException {
zip.close();
}
}) {
storage.importData(stream, true, true);
log.info("Registry successfully imported from {}", registryImportUrl);
} catch (ReadOnlyStorageException e) {
log.error("Registry import failed, because the storage is in read-only mode.");
}
v3Admin.importData(null, null, null, registryImportZip);
log.info("Registry successfully imported from {}", registryImportUrl);
} catch (IOException ioe) {
log.error("Registry import from {} failed", registryImportUrl, ioe);
} catch (ReadOnlyStorageException rose) {
log.error("Registry import failed, because the storage is in read-only mode.");
} catch (ConflictException ce) {
log.info("Import skipped, registry not empty.");
} catch (Exception e) {
log.error("Registry import failed", e);
}
}
}
Expand Down
71 changes: 9 additions & 62 deletions app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package io.apicurio.registry.rest.v2;

import io.apicurio.common.apps.config.Dynamic;
import io.apicurio.common.apps.config.DynamicConfigPropertyDef;
import io.apicurio.common.apps.config.DynamicConfigPropertyDto;
import io.apicurio.common.apps.config.DynamicConfigPropertyIndex;
import io.apicurio.common.apps.config.Info;
import io.apicurio.common.apps.logging.Logged;
import io.apicurio.common.apps.logging.audit.Audited;
import io.apicurio.registry.auth.Authorized;
Expand All @@ -16,51 +14,37 @@
import io.apicurio.registry.rest.MissingRequiredParameterException;
import io.apicurio.registry.rest.v2.beans.ArtifactTypeInfo;
import io.apicurio.registry.rest.v2.beans.ConfigurationProperty;
import io.apicurio.registry.rest.v2.beans.DownloadRef;
import io.apicurio.registry.rest.v2.beans.RoleMapping;
import io.apicurio.registry.rest.v2.beans.Rule;
import io.apicurio.registry.rest.v2.beans.UpdateConfigurationProperty;
import io.apicurio.registry.rest.v2.beans.UpdateRole;
import io.apicurio.registry.rest.v2.shared.DataExporter;
import io.apicurio.registry.rules.DefaultRuleDeletionException;
import io.apicurio.registry.rules.RulesProperties;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.dto.DownloadContextDto;
import io.apicurio.registry.storage.dto.DownloadContextType;
import io.apicurio.registry.storage.dto.RoleMappingDto;
import io.apicurio.registry.storage.dto.RuleConfigurationDto;
import io.apicurio.registry.storage.error.ConfigPropertyNotFoundException;
import io.apicurio.registry.storage.error.InvalidPropertyValueException;
import io.apicurio.registry.storage.error.RuleNotFoundException;
import io.apicurio.registry.storage.impexp.EntityInputStream;
import io.apicurio.registry.storage.importing.ImportExportConfigProperties;
import io.apicurio.registry.types.Current;
import io.apicurio.registry.types.RoleType;
import io.apicurio.registry.types.RuleType;
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
import io.apicurio.registry.utils.impexp.Entity;
import io.apicurio.registry.utils.impexp.v2.EntityReader;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.interceptor.Interceptors;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.eclipse.microprofile.config.Config;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.slf4j.Logger;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipInputStream;

import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_FOR_BROWSER;
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_NAME;
Expand Down Expand Up @@ -97,17 +81,12 @@ public class AdminResourceImpl implements AdminResource {
Config config;

@Inject
DataExporter exporter;
ImportExportConfigProperties importExportProps;

@Context
HttpServletRequest request;

@Dynamic(label = "Download link expiry", description = "The number of seconds that a generated link to a .zip download file is active before expiring.")
@ConfigProperty(name = "apicurio.download.href.ttl.seconds", defaultValue = "30")
@Info(category = "download", description = "Download link expiry", availableSince = "2.1.2.Final")
Supplier<Long> downloadHrefTtl;
@Inject
io.apicurio.registry.rest.v3.AdminResourceImpl v3Admin;

private static final void requireParameter(String parameterName, Object parameterValue) {
private static void requireParameter(String parameterName, Object parameterValue) {
if (parameterValue == null) {
throw new MissingRequiredParameterException(parameterName);
}
Expand All @@ -124,7 +103,6 @@ public List<ArtifactTypeInfo> listArtifactTypes() {
ati.setName(t);
return ati;
}).collect(Collectors.toList());

}

/**
Expand Down Expand Up @@ -246,26 +224,7 @@ public void deleteGlobalRule(RuleType rule) {
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin)
public void importData(Boolean xRegistryPreserveGlobalId, Boolean xRegistryPreserveContentId,
InputStream data) {
final ZipInputStream zip = new ZipInputStream(data, StandardCharsets.UTF_8);
final EntityReader reader = new EntityReader(zip);
EntityInputStream stream = new EntityInputStream() {
@Override
public Entity nextEntity() throws IOException {
try {
return reader.readEntity();
} catch (Exception e) {
log.error("Error reading data from import ZIP file.", e);
return null;
}
}

@Override
public void close() throws IOException {
zip.close();
}
};
this.storage.upgradeData(stream, isNullOrTrue(xRegistryPreserveGlobalId),
isNullOrTrue(xRegistryPreserveContentId));
v3Admin.importData(xRegistryPreserveGlobalId, xRegistryPreserveContentId, false, data);
}

/**
Expand All @@ -275,20 +234,8 @@ public void close() throws IOException {
@Audited(extractParameters = { "0", KEY_FOR_BROWSER })
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin)
public Response exportData(Boolean forBrowser) {
String acceptHeader = request.getHeader("Accept");
if (Boolean.TRUE.equals(forBrowser) || MediaType.APPLICATION_JSON.equals(acceptHeader)) {
long expires = System.currentTimeMillis() + (downloadHrefTtl.get() * 1000);
DownloadContextDto downloadCtx = DownloadContextDto.builder().type(DownloadContextType.EXPORT)
.expires(expires).build();
String downloadId = storage.createDownload(downloadCtx);
String downloadHref = createDownloadHref(downloadId);
DownloadRef downloadRef = new DownloadRef();
downloadRef.setDownloadId(downloadId);
downloadRef.setHref(downloadHref);
return Response.ok(downloadRef).type(MediaType.APPLICATION_JSON_TYPE).build();
} else {
return exporter.exportData();
}
throw new UnsupportedOperationException(
"Exporting data using the Registry Core v2 API is no longer supported. Use the v3 API.");
}

/**
Expand Down Expand Up @@ -463,7 +410,7 @@ private ConfigurationProperty defToConfigurationProperty(DynamicConfigPropertyDe

/**
* Lookup the dynamic configuration property being set. Ensure that it exists (throws a
* {@link NotFoundException} if it does not.
* {@link io.apicurio.registry.storage.error.NotFoundException} if it does not.
*
* @param propertyName the name of the dynamic property
* @return the dynamic config property definition
Expand Down
20 changes: 0 additions & 20 deletions app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,8 @@
import io.apicurio.registry.auth.AuthorizedStyle;
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
import io.apicurio.registry.rest.v2.shared.DataExporter;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.dto.DownloadContextDto;
import io.apicurio.registry.storage.dto.DownloadContextType;
import io.apicurio.registry.storage.error.DownloadNotFoundException;
import io.apicurio.registry.types.Current;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.interceptor.Interceptors;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
Expand All @@ -27,25 +21,11 @@
@Path("/apis/registry/v2/downloads")
public class DownloadsResourceImpl {

@Inject
@Current
RegistryStorage storage;

@Inject
DataExporter exporter;

@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.None)
@GET
@Path("{downloadId}")
@Produces("*/*")
public Response download(@PathParam("downloadId") String downloadId) {
DownloadContextDto downloadContext = storage.consumeDownload(downloadId);
if (downloadContext.getType() == DownloadContextType.EXPORT) {
return exporter.exportData();
}

// TODO support other types of downloads (e.g. download content by contentId)

throw new DownloadNotFoundException();
}

Expand Down
61 changes: 0 additions & 61 deletions app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java
View file
Open in desktop

This file was deleted.

Loading
Loading