Skip to content

Commit 9af2a8e

Browse files
andreaTPandreaTP
authored
Create artifacts from URL (#2660)
* create artifacts from URL * revert apicurio-common-app-components bump to let CI do it's job * fix testsuite compilation error * add rest tests * improve tests * minor * fix failing tests * bump apicurio-common-app-components.version and remove todos * review comments * revert unrelated changes * Better handling for yaml files * more strict parsable yaml * fixing the ci * better fixes * more fixes * Isolate the groupId for ProtobufSerdeTest * review
1 parent 1a3676e commit 9af2a8e

File tree

25 files changed

+519
-104
lines changed

25 files changed

+519
-104
lines changed

‎app/pom.xml

+8
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,14 @@
118118
<groupId>io.quarkus</groupId>
119119
<artifactId>quarkus-smallrye-context-propagation</artifactId>
120120
</dependency>
121+
<dependency>
122+
<groupId>io.quarkus</groupId>
123+
<artifactId>quarkus-rest-client</artifactId>
124+
</dependency>
125+
<dependency>
126+
<groupId>io.quarkus</groupId>
127+
<artifactId>quarkus-rest-client-jackson</artifactId>
128+
</dependency>
121129

122130
<dependency>
123131
<groupId>io.quarkus</groupId>

‎app/src/main/java/io/apicurio/registry/rest/RestConfig.java

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package io.apicurio.registry.rest;
2+
3+
import org.eclipse.microprofile.config.inject.ConfigProperty;
4+
5+
import javax.inject.Singleton;
6+
7+
@Singleton
8+
public class RestConfig {
9+
10+
@ConfigProperty(name = "registry.rest.artifact.download.maxSize", defaultValue = "1000000")
11+
int downloadMaxSize;
12+
13+
@ConfigProperty(name = "registry.rest.artifact.download.skipSSLValidation", defaultValue = "false")
14+
boolean downloadSkipSSLValidation;
15+
16+
public int getDownloadMaxSize() { return this.downloadMaxSize; }
17+
18+
public boolean getDownloadSkipSSLValidation() { return this.downloadSkipSSLValidation; }
19+
20+
}

‎app/src/main/java/io/apicurio/registry/rest/v2/GroupsResourceImpl.java

+111-14
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package io.apicurio.registry.rest.v2;
1818

19+
import com.google.common.hash.Hashing;
1920
import io.apicurio.registry.auth.Authorized;
2021
import io.apicurio.registry.auth.AuthorizedLevel;
2122
import io.apicurio.registry.auth.AuthorizedStyle;
@@ -27,6 +28,7 @@
2728
import io.apicurio.registry.rest.HeadersHack;
2829
import io.apicurio.registry.rest.MissingRequiredParameterException;
2930
import io.apicurio.registry.rest.ParametersConflictException;
31+
import io.apicurio.registry.rest.RestConfig;
3032
import io.apicurio.registry.rest.v2.beans.ArtifactMetaData;
3133
import io.apicurio.registry.rest.v2.beans.ArtifactReference;
3234
import io.apicurio.registry.rest.v2.beans.ArtifactSearchResults;
@@ -69,17 +71,27 @@
6971
import io.apicurio.registry.util.ContentTypeUtil;
7072
import io.apicurio.registry.utils.ArtifactIdValidator;
7173
import io.apicurio.registry.utils.IoUtil;
74+
import io.apicurio.registry.utils.JAXRSClientUtil;
7275
import org.jose4j.base64url.Base64;
7376

7477
import javax.enterprise.context.ApplicationScoped;
7578
import javax.inject.Inject;
7679
import javax.interceptor.Interceptors;
7780
import javax.servlet.http.HttpServletRequest;
7881
import javax.ws.rs.BadRequestException;
82+
import javax.ws.rs.client.Client;
7983
import javax.ws.rs.core.Context;
8084
import javax.ws.rs.core.MediaType;
8185
import javax.ws.rs.core.Response;
86+
import java.io.BufferedInputStream;
8287
import java.io.InputStream;
88+
import java.net.MalformedURLException;
89+
import java.net.URI;
90+
import java.net.URISyntaxException;
91+
import java.net.URL;
92+
import java.nio.charset.StandardCharsets;
93+
import java.security.KeyManagementException;
94+
import java.security.NoSuchAlgorithmException;
8395
import java.util.Collections;
8496
import java.util.HashSet;
8597
import java.util.List;
@@ -94,12 +106,14 @@
94106
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_DESCRIPTION;
95107
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_DESCRIPTION_ENCODED;
96108
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_EDITABLE_METADATA;
109+
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_FROM_URL;
97110
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_GROUP_ID;
98111
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_IF_EXISTS;
99112
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_NAME;
100113
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_NAME_ENCODED;
101114
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_RULE;
102115
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_RULE_TYPE;
116+
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_SHA;
103117
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_UPDATE_STATE;
104118
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_VERSION;
105119

@@ -131,6 +145,9 @@ public class GroupsResourceImpl implements GroupsResource {
131145
@Inject
132146
ArtifactTypeUtilProviderFactory factory;
133147

148+
@Inject
149+
RestConfig restConfig;
150+
134151
/**
135152
* @see io.apicurio.registry.rest.v2.GroupsResource#getLatestArtifact(java.lang.String, java.lang.String, java.lang.Boolean)
136153
*/
@@ -585,30 +602,86 @@ public void deleteArtifactsInGroup(String groupId) {
585602
}
586603

587604
/**
588-
* @see io.apicurio.registry.rest.v2.GroupsResource#createArtifact(String, ArtifactType, String, String, IfExists, Boolean, String, String, String, String, InputStream)
605+
* @see io.apicurio.registry.rest.v2.GroupsResource#createArtifact(String, ArtifactType, String, String, IfExists, Boolean, String, String, String, String, String, String, InputStream)
589606
*/
590607
@Override
591-
@Audited(extractParameters = {"0", KEY_GROUP_ID, "1", KEY_ARTIFACT_TYPE, "2", KEY_ARTIFACT_ID, "3", KEY_VERSION, "4", KEY_IF_EXISTS, "5", KEY_CANONICAL, "6", KEY_DESCRIPTION, "7", KEY_DESCRIPTION_ENCODED, "8", KEY_NAME, "9", KEY_NAME_ENCODED})
608+
@Audited(extractParameters = {"0", KEY_GROUP_ID, "1", KEY_ARTIFACT_TYPE, "2", KEY_ARTIFACT_ID, "3", KEY_VERSION, "4", KEY_IF_EXISTS, "5", KEY_CANONICAL, "6", KEY_DESCRIPTION, "7", KEY_DESCRIPTION_ENCODED, "8", KEY_NAME, "9", KEY_NAME_ENCODED, "10", KEY_FROM_URL, "11", KEY_SHA})
592609
@Authorized(style = AuthorizedStyle.GroupOnly, level = AuthorizedLevel.Write)
593610
public ArtifactMetaData createArtifact(String groupId, ArtifactType xRegistryArtifactType, String xRegistryArtifactId,
594611
String xRegistryVersion, IfExists ifExists, Boolean canonical,
595612
String xRegistryDescription, String xRegistryDescriptionEncoded,
596-
String xRegistryName, String xRegistryNameEncoded, InputStream data) {
597-
598-
return this.createArtifactWithRefs(groupId, xRegistryArtifactType, xRegistryArtifactId, xRegistryVersion, ifExists, canonical, xRegistryDescription, xRegistryDescriptionEncoded, xRegistryName, xRegistryNameEncoded, data, Collections.emptyList());
613+
String xRegistryName, String xRegistryNameEncoded,
614+
String xRegistryContentHash, String xRegistryHashAlgorithm, InputStream data) {
615+
return this.createArtifactWithRefs(groupId, xRegistryArtifactType, xRegistryArtifactId, xRegistryVersion, ifExists, canonical, xRegistryDescription, xRegistryDescriptionEncoded, xRegistryName, xRegistryNameEncoded, xRegistryContentHash, xRegistryHashAlgorithm, data, Collections.emptyList());
599616
}
600617

601618
/**
602-
* @see io.apicurio.registry.rest.v2.GroupsResource#createArtifact(java.lang.String, io.apicurio.registry.types.ArtifactType, java.lang.String, java.lang.String, io.apicurio.registry.rest.v2.beans.IfExists, java.lang.Boolean, java.lang.String, java.lang.String, java.lang.String, java.lang.String, io.apicurio.registry.rest.v2.beans.ContentCreateRequest)
619+
* @see io.apicurio.registry.rest.v2.GroupsResource#createArtifact(String, ArtifactType, String, String, IfExists, Boolean, String, String, String, String, String, String, ContentCreateRequest)
603620
*/
604621
@Override
605-
@Audited(extractParameters = {"0", KEY_GROUP_ID, "1", KEY_ARTIFACT_TYPE, "2", KEY_ARTIFACT_ID, "3", KEY_VERSION, "4", KEY_IF_EXISTS, "5", KEY_CANONICAL, "6", KEY_DESCRIPTION, "7", KEY_DESCRIPTION_ENCODED, "8", KEY_NAME, "9", KEY_NAME_ENCODED})
622+
@Audited(extractParameters = {"0", KEY_GROUP_ID, "1", KEY_ARTIFACT_TYPE, "2", KEY_ARTIFACT_ID, "3", KEY_VERSION, "4", KEY_IF_EXISTS, "5", KEY_CANONICAL, "6", KEY_DESCRIPTION, "7", KEY_DESCRIPTION_ENCODED, "8", KEY_NAME, "9", KEY_NAME_ENCODED, "10", "from_url" /*KEY_FROM_URL*/, "11", "artifact_sha" /*KEY_SHA*/})
606623
@Authorized(style = AuthorizedStyle.GroupOnly, level = AuthorizedLevel.Write)
607-
public ArtifactMetaData createArtifact(String groupId, ArtifactType xRegistryArtifactType,
608-
String xRegistryArtifactId, String xRegistryVersion, IfExists ifExists, Boolean canonical,
609-
String xRegistryDescription, String xRegistryDescriptionEncoded, String xRegistryName,
610-
String xRegistryNameEncoded, ContentCreateRequest data) {
611-
return this.createArtifactWithRefs(groupId, xRegistryArtifactType, xRegistryArtifactId, xRegistryVersion, ifExists, canonical, xRegistryDescription, xRegistryDescriptionEncoded, xRegistryName, xRegistryNameEncoded, IoUtil.toStream(data.getContent()), data.getReferences());
624+
public ArtifactMetaData createArtifact(String groupId, ArtifactType xRegistryArtifactType, String xRegistryArtifactId,
625+
String xRegistryVersion, IfExists ifExists, Boolean canonical,
626+
String xRegistryDescription, String xRegistryDescriptionEncoded,
627+
String xRegistryName, String xRegistryNameEncoded,
628+
String xRegistryContentHash, String xRegistryHashAlgorithm, ContentCreateRequest data) {
629+
InputStream content = null;
630+
try {
631+
URL url = new URL(data.getContent());
632+
content = fetchContentFromURL(url.toURI());
633+
} catch (MalformedURLException | URISyntaxException e) {
634+
content = IoUtil.toStream(data.getContent());
635+
}
636+
637+
return this.createArtifactWithRefs(groupId, xRegistryArtifactType, xRegistryArtifactId, xRegistryVersion, ifExists, canonical, xRegistryDescription, xRegistryDescriptionEncoded, xRegistryName, xRegistryNameEncoded, xRegistryContentHash, xRegistryHashAlgorithm, content, data.getReferences());
638+
}
639+
640+
public enum RegistryHashAlgorithm {
641+
SHA256,
642+
MD5
643+
}
644+
645+
/**
646+
* Return an InputStream for the resource to be downloaded
647+
* @param url
648+
*/
649+
private InputStream fetchContentFromURL(URI url) {
650+
Client client = null;
651+
try {
652+
client = JAXRSClientUtil.getJAXRSClient(restConfig.getDownloadSkipSSLValidation());
653+
} catch (KeyManagementException kme) {
654+
throw new RuntimeException(kme);
655+
} catch (NoSuchAlgorithmException nsae) {
656+
throw new RuntimeException(nsae);
657+
}
658+
659+
// 1. Registry issues HTTP HEAD request to the target URL.
660+
List<Object> contentLengthHeaders = client
661+
.target(url)
662+
.request()
663+
.head()
664+
.getHeaders()
665+
.get("Content-Length");
666+
667+
if (contentLengthHeaders == null || contentLengthHeaders.size() < 1) {
668+
throw new BadRequestException("Requested resource URL does not provide 'Content-Length' in the headers");
669+
}
670+
671+
// 2. According to HTTP specification, target server must return Content-Length header.
672+
int contentLength = Integer.parseInt(contentLengthHeaders.get(0).toString());
673+
674+
// 3. Registry analyzes value of Content-Length to check if file with declared size could be processed securely.
675+
if (contentLength > restConfig.getDownloadMaxSize()) {
676+
throw new BadRequestException("Requested resource is bigger than " + restConfig.getDownloadMaxSize() + " and cannot be downloaded.");
677+
}
678+
679+
// 4. Finally, registry issues HTTP GET to the target URL and fetches only amount of bytes specified by HTTP HEAD from step 1.
680+
return new BufferedInputStream(client
681+
.target(url)
682+
.request()
683+
.get()
684+
.readEntity(InputStream.class), contentLength);
612685
}
613686

614687
/**
@@ -623,13 +696,17 @@ public ArtifactMetaData createArtifact(String groupId, ArtifactType xRegistryArt
623696
* @param xRegistryDescriptionEncoded
624697
* @param xRegistryName
625698
* @param xRegistryNameEncoded
699+
* @param xRegistryContentHash
700+
* @param xRegistryHashAlgorithm
626701
* @param data
627702
* @param references
628703
*/
629704
private ArtifactMetaData createArtifactWithRefs(String groupId, ArtifactType xRegistryArtifactType, String xRegistryArtifactId,
630705
String xRegistryVersion, IfExists ifExists, Boolean canonical,
631706
String xRegistryDescription, String xRegistryDescriptionEncoded,
632-
String xRegistryName, String xRegistryNameEncoded, InputStream data, List<ArtifactReference> references) {
707+
String xRegistryName, String xRegistryNameEncoded,
708+
String xRegistryContentHash, String xRegistryHashAlgorithm,
709+
InputStream data, List<ArtifactReference> references) {
633710

634711
requireParameter("groupId", groupId);
635712

@@ -649,6 +726,25 @@ private ArtifactMetaData createArtifactWithRefs(String groupId, ArtifactType xRe
649726
if (content.bytes().length == 0) {
650727
throw new BadRequestException(EMPTY_CONTENT_ERROR_MESSAGE);
651728
}
729+
730+
// Mitigation for MITM attacks, verify that the artifact is the expected one
731+
if (xRegistryContentHash != null) {
732+
String calculatedSha = null;
733+
RegistryHashAlgorithm algorithm = (xRegistryHashAlgorithm == null) ? RegistryHashAlgorithm.SHA256 : RegistryHashAlgorithm.valueOf(xRegistryHashAlgorithm);
734+
switch (algorithm) {
735+
case MD5:
736+
calculatedSha = Hashing.md5().hashString(content.content(), StandardCharsets.UTF_8).toString();
737+
break;
738+
case SHA256:
739+
calculatedSha = Hashing.sha256().hashString(content.content(), StandardCharsets.UTF_8).toString();
740+
break;
741+
}
742+
743+
if (!calculatedSha.equals(xRegistryContentHash.trim())) {
744+
throw new BadRequestException("Provided Artifact Hash doesn't match with the content");
745+
}
746+
}
747+
652748
final boolean fcanonical = canonical == null ? Boolean.FALSE : canonical;
653749

654750
String ct = getContentType();
@@ -660,7 +756,8 @@ private ArtifactMetaData createArtifactWithRefs(String groupId, ArtifactType xRe
660756
} else if (!ArtifactIdValidator.isArtifactIdAllowed(artifactId)) {
661757
throw new InvalidArtifactIdException(ArtifactIdValidator.ARTIFACT_ID_ERROR_MESSAGE);
662758
}
663-
if (ContentTypeUtil.isApplicationYaml(ct)) {
759+
if (ContentTypeUtil.isApplicationYaml(ct) ||
760+
(ContentTypeUtil.isApplicationCreateExtended(ct) && ContentTypeUtil.isParsableYaml(content))) {
664761
content = ContentTypeUtil.yamlToJson(content);
665762
}
666763

‎app/src/main/java/io/apicurio/registry/util/ArtifactTypeUtil.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ private ArtifactTypeUtil() {
6464
*
6565
* @param content the content
6666
* @param xArtifactType the artifact type
67-
* @param ct content type from request API
67+
* @param contentType content type from request API
6868
*/
6969
//FIXME:references artifact must be dereferenced here otherwise this will fail to discover the type
7070
public static ArtifactType determineArtifactType(ContentHandle content, ArtifactType xArtifactType, String contentType) {

‎app/src/main/java/io/apicurio/registry/util/ContentTypeUtil.java

+31
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
public final class ContentTypeUtil {
2828

2929
public static final String CT_APPLICATION_JSON = "application/json";
30+
public static final String CT_APPLICATION_CREATE_EXTENDED = "application/create.extended+json";
3031
public static final String CT_APPLICATION_YAML = "application/x-yaml";
3132
public static final String CT_APPLICATION_XML = "application/xml";
3233

@@ -57,6 +58,36 @@ public static boolean isApplicationYaml(String ct) {
5758
return ct.contains(CT_APPLICATION_YAML);
5859
}
5960

61+
/**
62+
* Returns true if the Content-Type of the inbound request is "application/create.extended+json".
63+
*
64+
* @param ct content type
65+
*/
66+
public static boolean isApplicationCreateExtended(String ct) {
67+
if (ct == null) {
68+
return false;
69+
}
70+
return ct.contains(CT_APPLICATION_CREATE_EXTENDED);
71+
}
72+
73+
/**
74+
* Returns true if the content can be parsed as yaml.
75+
*
76+
*/
77+
public static boolean isParsableYaml(ContentHandle yaml) {
78+
try {
79+
String content = yaml.content().trim();
80+
// it's Json or Xml
81+
if (content.startsWith("{") || content.startsWith("<")) {
82+
return false;
83+
}
84+
JsonNode root = yamlMapper.readTree(yaml.stream());
85+
return root != null && root.elements().hasNext();
86+
} catch (Throwable t) {
87+
return false;
88+
}
89+
}
90+
6091
public static ContentHandle yamlToJson(ContentHandle yaml) {
6192
try {
6293
JsonNode root = yamlMapper.readTree(yaml.stream());

‎app/src/main/resources/application.properties

+6-2
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ registry.api.errors.include-stack-in-response=${REGISTRY_API_ERRORS_INCLUDE_STAC
137137
quarkus.http.cors=true
138138
quarkus.http.cors.origins=${CORS_ALLOWED_ORIGINS:}
139139
quarkus.http.cors.methods=${CORS_ALLOWED_METHODS:GET,PUT,POST,PATCH,DELETE,OPTIONS}
140-
quarkus.http.cors.headers=${CORS_ALLOWED_HEADERS:x-registry-name,x-registry-name-encoded,x-registry-description,x-registry-description-encoded,x-registry-version,x-registry-artifactid,x-registry-artifacttype,access-control-request-method,access-control-allow-credentials,access-control-allow-origin,access-control-allow-headers,authorization,content-type}
140+
quarkus.http.cors.headers=${CORS_ALLOWED_HEADERS:x-registry-name,x-registry-name-encoded,x-registry-description,x-registry-description-encoded,x-registry-version,x-registry-artifactid,x-registry-artifacttype,x-registry-hash-algorithm,x-registry-content-hash,access-control-request-method,access-control-allow-credentials,access-control-allow-origin,access-control-allow-headers,authorization,content-type}
141141

142142
## Disable OpenAPI class scanning
143143
mp.openapi.scan.disable=true
@@ -160,4 +160,8 @@ registry.storage.metrics.cache.check-period=30000
160160
registry.limits.config.cache.check-period=30000
161161
registry.downloads.reaper.every=60s
162162

163-
quarkus.native.additional-build-args=--initialize-at-run-time=org.apache.kafka.common.security.authenticator.SaslClientAuthenticator
163+
quarkus.native.additional-build-args=--initialize-at-run-time=org.apache.kafka.common.security.authenticator.SaslClientAuthenticator
164+
165+
# Artifact download
166+
registry.rest.artifact.download.maxSize=1000000
167+
registry.rest.artifact.download.skipSSLValidation=false

0 commit comments

Comments
 (0)