NIFIREG-216 - S3BundlePersistenceProvider
- Add optional S3 Endpoint URL override property
- Add Bundle Persistence Docker configuration via environment
This closes #169.
diff --git a/nifi-registry-core/nifi-registry-docker/dockerhub/DockerImage.txt b/nifi-registry-core/nifi-registry-docker/dockerhub/DockerImage.txt
index 07d6e8d..fe17934 100644
--- a/nifi-registry-core/nifi-registry-docker/dockerhub/DockerImage.txt
+++ b/nifi-registry-core/nifi-registry-docker/dockerhub/DockerImage.txt
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-apache/nifi-registry:0.3.0
+apache/nifi-registry:0.4.0
diff --git a/nifi-registry-core/nifi-registry-docker/dockerhub/README.md b/nifi-registry-core/nifi-registry-docker/dockerhub/README.md
index 9ecae2e..9c41024 100644
--- a/nifi-registry-core/nifi-registry-docker/dockerhub/README.md
+++ b/nifi-registry-core/nifi-registry-docker/dockerhub/README.md
@@ -42,7 +42,7 @@
## Running a container
-### Standalone Instance, Unsecured
+### Unsecured
The minimum to run a NiFi Registry instance is as follows:
docker run --name nifi-registry \
@@ -63,7 +63,7 @@
For a list of the environment variables recognised in this build, look into the .sh/secure.sh and .sh/start.sh scripts
-### Standalone Instance, Two-Way SSL
+### Secured with Two-Way TLS
In this configuration, the user will need to provide certificates and the associated configuration information.
Of particular note, is the `AUTH` environment variable which is set to `tls`. Additionally, the user must provide an
the DN as provided by an accessing client certificate in the `INITIAL_ADMIN_IDENTITY` environment variable.
@@ -84,7 +84,7 @@
-d \
apache/nifi-registry:latest
-### Standalone Instance, LDAP
+### Secured with LDAP
In this configuration, the user will need to provide certificates and the associated configuration information. Optionally,
if the LDAP provider of interest is operating in LDAPS or START_TLS modes, certificates will additionally be needed.
Of particular note, is the `AUTH` environment variable which is set to `ldap`. Additionally, the user must provide a
@@ -92,7 +92,7 @@
used to seed the instance with an initial user with administrative privileges. Finally, this command makes use of a
volume to provide certificates on the host system to the container instance.
-#### For a minimal, connection to an LDAP server using SIMPLE authentication:
+For a minimal, connection to an LDAP server using SIMPLE authentication:
docker run --name nifi-registry \
-v /path/to/tls/certs/localhost:/opt/certs \
@@ -115,7 +115,7 @@
-d \
apache/nifi-registry:latest
-#### The following, optional environment variables may be added to the above command when connecting to a secure LDAP server configured with START_TLS or LDAPS
+The following, optional environment variables may be added to the above command when connecting to a secure LDAP server configured with START_TLS or LDAPS
-e LDAP_TLS_KEYSTORE: ''
-e LDAP_TLS_KEYSTORE_PASSWORD: ''
@@ -124,7 +124,11 @@
-e LDAP_TLS_TRUSTSTORE_PASSWORD: ''
-e LDAP_TLS_TRUSTSTORE_TYPE: ''
-### The following, optional environment variables can be used to configure the database
+### Additional Configuration Options
+
+#### Database Configuration
+
+The following, optional environment variables can be used to configure the database.
| nifi-registry.properties entry | Variable |
|----------------------------------------|----------------------------|
@@ -136,7 +140,9 @@
| nifi.registry.db.maxConnections | NIFI_REGISTRY_DB_MAX_CONNS |
| nifi.registry.db.sql.debug | NIFI_REGISTRY_DB_DEBUG_SQL |
-#### The following, optional environment variables may be added to configure flow persistence provider.
+#### Flow Persistence Configuration
+
+The following, optional environment variables may be added to configure flow persistence provider.
| Environment Variable | Configuration Property |
|--------------------------------|--------------------------------------|
@@ -146,3 +152,19 @@
| NIFI_REGISTRY_GIT_USER | Remote Access User |
| NIFI_REGISTRY_GIT_PASSWORD | Remote Access Password |
+#### Extension Bundle Persistence Configuration
+
+The following, optional environment variables may be added to configure extension bundle persistence provider.
+
+| Environment Variable | Configuration Property |
+|---------------------------------------|-------------------------------------|
+| NIFI_REGISTRY_BUNDLE_STORAGE_DIR | Extension Bundle Storage Directory |
+| NIFI_REGISTRY_BUNDLE_PROVIDER | (Class tag); valid values: file, s3 |
+| NIFI_REGISTRY_S3_REGION | Region |
+| NIFI_REGISTRY_S3_BUCKET_NAME | Bucket Name |
+| NIFI_REGISTRY_S3_KEY_PREFIX | Key Prefix |
+| NIFI_REGISTRY_S3_CREDENTIALS_PROVIDER | Credentials Provider |
+| NIFI_REGISTRY_S3_ACCESS_KEY | Access Key |
+| NIFI_REGISTRY_S3_SECRET_ACCESS_KEY | Secret Access Key |
+| NIFI_REGISTRY_S3_ENDPOINT_URL | Endpoint URL |
+
diff --git a/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh b/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh
index d281490..c65f3ea 100755
--- a/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh
+++ b/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh
@@ -43,6 +43,7 @@
esac
. "${scripts_dir}/update_flow_provider.sh"
+. "${scripts_dir}/update_bundle_provider.sh"
# Continuously provide logs so that 'docker logs' can produce them
tail -F "${NIFI_REGISTRY_HOME}/logs/nifi-registry-app.log" &
diff --git a/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_bundle_provider.sh b/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_bundle_provider.sh
new file mode 100644
index 0000000..27d5c94
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_bundle_provider.sh
@@ -0,0 +1,48 @@
+#!/bin/sh -e
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+providers_file=${NIFI_REGISTRY_HOME}/conf/providers.xml
+property_xpath='/providers/extensionBundlePersistenceProvider'
+
+add_property() {
+ property_name=$1
+ property_value=$2
+
+ if [ -n "${property_value}" ]; then
+ xmlstarlet ed --inplace --subnode "${property_xpath}" --type elem -n property -v "${property_value}" \
+ -i \$prev --type attr -n name -v "${property_name}" \
+ "${providers_file}"
+ fi
+}
+
+xmlstarlet ed --inplace -u "${property_xpath}/property[@name='Extension Bundle Storage Directory']" -v "${NIFI_REGISTRY_BUNDLE_STORAGE_DIR:-./extension_bundles}" "${providers_file}"
+
+case ${NIFI_REGISTRY_BUNDLE_PROVIDER} in
+ file)
+ xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.extension.FileSystemBundlePersistenceProvider" "${providers_file}"
+ ;;
+ s3)
+ xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.aws.S3BundlePersistenceProvider" "${providers_file}"
+ add_property "Region" "${NIFI_REGISTRY_S3_REGION:-}"
+ add_property "Bucket Name" "${NIFI_REGISTRY_S3_BUCKET_NAME:-}"
+ add_property "Key Prefix" "${NIFI_REGISTRY_S3_KEY_PREFIX:-}"
+ add_property "Credentials Provider" "${NIFI_REGISTRY_S3_CREDENTIALS_PROVIDER:-DEFAULT_CHAIN}"
+ add_property "Access Key" "${NIFI_REGISTRY_S3_ACCESS_KEY:-}"
+ add_property "Secret Access Key" "${NIFI_REGISTRY_S3_SECRET_ACCESS_KEY:-}"
+ add_property "Endpoint URL" "${NIFI_REGISTRY_S3_ENDPOINT_URL:-}"
+ ;;
+esac
diff --git a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml
index 4252d70..f41eee8 100644
--- a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml
+++ b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml
@@ -74,6 +74,8 @@
- STATIC requires that "Access Key" and "Secret Access Key" be specified directly in this file
- "Access Key" - The access key to use when using STATIC credentials provider
- "Secret Access Key" - The secret access key to use when using STATIC credentials provider
+ - "Endpoint URL" - An optional URL that overrides the default AWS S3 endpoint URL.
+ Set this when using an AWS S3 API compatible service hosted at a different URL.
-->
<!--
<extensionBundlePersistenceProvider>
@@ -84,6 +86,7 @@
<property name="Credentials Provider">DEFAULT_CHAIN</property>
<property name="Access Key"></property>
<property name="Secret Access Key"></property>
+ <property name="Endpoint URL"></property>
</extensionBundlePersistenceProvider>
-->
diff --git a/nifi-registry-core/pom.xml b/nifi-registry-core/pom.xml
index e09a698..47fe429 100644
--- a/nifi-registry-core/pom.xml
+++ b/nifi-registry-core/pom.xml
@@ -36,16 +36,16 @@
<module>nifi-registry-runtime</module>
<module>nifi-registry-security-api</module>
<module>nifi-registry-security-utils</module>
- <module>nifi-registry-framework</module>
+ <module>nifi-registry-framework</module>
<module>nifi-registry-provider-api</module>
<module>nifi-registry-web-api</module>
<module>nifi-registry-web-ui</module>
<module>nifi-registry-web-docs</module>
- <module>nifi-registry-bootstrap</module>
+ <module>nifi-registry-bootstrap</module>
<module>nifi-registry-docs</module>
- <module>nifi-registry-client</module>
+ <module>nifi-registry-client</module>
<module>nifi-registry-docker</module>
- <module>nifi-registry-bundle-utils</module>
+ <module>nifi-registry-bundle-utils</module>
</modules>
<dependencyManagement>
diff --git a/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-assembly/README.md b/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-assembly/README.md
index 008a850..84af605 100644
--- a/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-assembly/README.md
+++ b/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-assembly/README.md
@@ -78,6 +78,7 @@
<property name="Credentials Provider">DEFAULT_CHAIN</property>
<property name="Access Key"></property>
<property name="Secret Access Key"></property>
+ <property name="Endpoint URL"></property>
</extensionBundlePersistenceProvider>
-->
```
diff --git a/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/main/java/org/apache/nifi/registry/aws/S3BundlePersistenceProvider.java b/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/main/java/org/apache/nifi/registry/aws/S3BundlePersistenceProvider.java
index 6254649..8eb64e2 100644
--- a/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/main/java/org/apache/nifi/registry/aws/S3BundlePersistenceProvider.java
+++ b/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/main/java/org/apache/nifi/registry/aws/S3BundlePersistenceProvider.java
@@ -36,6 +36,7 @@
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
@@ -48,6 +49,7 @@
import java.io.InputStream;
import java.io.OutputStream;
+import java.net.URI;
/**
* An {@link BundlePersistenceProvider} that uses AWS S3 for storage.
@@ -62,6 +64,7 @@
public static final String CREDENTIALS_PROVIDER_PROP = "Credentials Provider";
public static final String ACCESS_KEY_PROP = "Access Key";
public static final String SECRET_ACCESS_KEY_PROP = "Secret Access Key";
+ public static final String ENDPOINT_URL_PROP = "Endpoint URL";
public static final String NAR_EXTENSION = ".nar";
public static final String CPP_EXTENSION = ".cpp";
@@ -90,10 +93,17 @@
protected S3Client createS3Client(final ProviderConfigurationContext configurationContext) {
- return S3Client.builder()
+ final S3ClientBuilder builder = S3Client.builder()
.region(getRegion(configurationContext))
- .credentialsProvider(getCredentialsProvider(configurationContext))
- .build();
+ .credentialsProvider(getCredentialsProvider(configurationContext));
+
+ final URI s3EndpointOverride = getS3EndpointOverride(configurationContext);
+ if (s3EndpointOverride != null) {
+ builder.endpointOverride(s3EndpointOverride);
+ }
+
+ return builder.build();
+
}
private Region getRegion(final ProviderConfigurationContext configurationContext) {
@@ -153,6 +163,21 @@
}
}
+ private URI getS3EndpointOverride(final ProviderConfigurationContext configurationContext) {
+ final URI s3EndpointOverride;
+ final String endpointUrlValue = configurationContext.getProperties().get(ENDPOINT_URL_PROP);
+ try {
+ s3EndpointOverride = StringUtils.isBlank(endpointUrlValue) ? null : URI.create(endpointUrlValue);
+ } catch (IllegalArgumentException e) {
+ final String errMessage = "The optional property '" + ENDPOINT_URL_PROP + "' must be a valid URL if set. " +
+ "URI Syntax Exception is: " + e.getLocalizedMessage();
+ LOGGER.error(errMessage);
+ LOGGER.debug("", e);
+ throw new ProviderCreationException(errMessage, e);
+ }
+ return s3EndpointOverride;
+ }
+
@Override
public synchronized void createBundleVersion(final BundlePersistenceContext context, final InputStream contentStream)
throws BundlePersistenceException {
diff --git a/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/test/java/org/apache/nifi/registry/aws/S3BundlePersistenceProviderIT.java b/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/test/java/org/apache/nifi/registry/aws/S3BundlePersistenceProviderIT.java
index 491dbb7..478ee9d 100644
--- a/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/test/java/org/apache/nifi/registry/aws/S3BundlePersistenceProviderIT.java
+++ b/nifi-registry-extensions/nifi-registry-aws/nifi-registry-aws-extensions/src/test/java/org/apache/nifi/registry/aws/S3BundlePersistenceProviderIT.java
@@ -26,7 +26,6 @@
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
-import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
@@ -56,17 +55,9 @@
public void setup() {
final Region region = Region.US_EAST_1;
final String bucketName = "integration-test-" + System.currentTimeMillis();
-
- // Create a separate client just for the IT test so we can setup a new bucket
- s3Client = S3Client.builder().region(region)
- .credentialsProvider(DefaultCredentialsProvider.create())
- .build();
-
- final CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
- .bucket(bucketName)
- .build();
-
- s3Client.createBucket(createBucketRequest);
+ final String endpointUrl =
+ //null; // When using AWS S3
+ "http://localhost:9000"; // When using Docker: docker run -it -p 9000:9000 minio/minio server /data
// Create config context and provider, and call onConfigured
final Map<String,String> properties = new HashMap<>();
@@ -74,12 +65,23 @@
properties.put(S3BundlePersistenceProvider.BUCKET_NAME_PROP, bucketName);
properties.put(S3BundlePersistenceProvider.CREDENTIALS_PROVIDER_PROP,
S3BundlePersistenceProvider.CredentialProvider.DEFAULT_CHAIN.name());
+ properties.put(S3BundlePersistenceProvider.ENDPOINT_URL_PROP, endpointUrl);
configurationContext = mock(ProviderConfigurationContext.class);
when(configurationContext.getProperties()).thenReturn(properties);
provider = new S3BundlePersistenceProvider();
provider.onConfigured(configurationContext);
+
+ // Create a separate client just for the IT test so we can setup a new bucket
+ s3Client = ((S3BundlePersistenceProvider)provider).createS3Client(configurationContext);
+
+ final CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
+ .bucket(bucketName)
+ .build();
+
+ s3Client.createBucket(createBucketRequest);
+
}
@After
diff --git a/nifi-registry-extensions/pom.xml b/nifi-registry-extensions/pom.xml
index 90a2ace..40f55ea 100644
--- a/nifi-registry-extensions/pom.xml
+++ b/nifi-registry-extensions/pom.xml
@@ -26,7 +26,7 @@
<modules>
<module>nifi-registry-ranger</module>
- <module>nifi-registry-aws</module>
+ <module>nifi-registry-aws</module>
</modules>
