AMBARI-24918 - Infra Manager: ssl support (#17)
diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java
index 7bc952a..f149cd8 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java
@@ -86,7 +86,7 @@
public void createSolrCollection(String collectionName) {
logger.info("Creating collection");
- runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"});
+ runCommand(new String[]{"docker", "exec", "solr", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"});
}
public QueryResponse query(SolrQuery query) {
diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java
index c400aee..6f17442 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java
@@ -52,7 +52,7 @@
logger.info("Creating new docker containers for testing Ambari Infra Solr Metrics plugin ...");
runCommand(new String[]{shellScriptLocation, "start"});
- Solr solr = new Solr("/usr/lib/ambari-infra-solr/server/solr");
+ Solr solr = new Solr();
solr.waitUntilSolrIsUp();
solr.createSolrCollection(HADOOP_LOGS_COLLECTION);
diff --git a/ambari-infra-manager/docker/docker-compose.yml b/ambari-infra-manager/docker/docker-compose.yml
index 3fa21b2..5051820 100644
--- a/ambari-infra-manager/docker/docker-compose.yml
+++ b/ambari-infra-manager/docker/docker-compose.yml
@@ -15,6 +15,7 @@
version: '3.3'
services:
zookeeper:
+ container_name: zookeeper
image: zookeeper:${ZOOKEEPER_VERSION:-3.4.10}
restart: always
hostname: zookeeper
@@ -27,6 +28,7 @@
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
# TODO: use infra-solr
+ container_name: solr
image: solr:${SOLR_VERSION:-7.5.0}
restart: always
hostname: solr
@@ -47,6 +49,7 @@
volumes:
- $AMBARI_INFRA_LOCATION/ambari-infra-manager/docker/configsets:/opt/solr/configsets
fakes3:
+ container_name: fakes3
image: localstack/localstack
hostname: fakes3
ports:
@@ -61,6 +64,7 @@
env_file:
- Profile
namenode:
+ container_name: hdfs_namenode
image: flokkr/hadoop-hdfs-namenode:${HADOOP_VERSION:-3.0.0}
hostname: namenode
ports:
@@ -73,6 +77,7 @@
networks:
- infra-network
datanode:
+ container_name: hdfs_datanode
image: flokkr/hadoop-hdfs-datanode:${HADOOP_VERSION:-3.0.0}
links:
- namenode
@@ -81,6 +86,7 @@
networks:
- infra-network
inframanager:
+ container_name: inframanager
image: ambari-infra-manager:v1.0
restart: always
hostname: infra-manager.apache.org
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java
index 06174a0..e560ae9 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java
@@ -18,13 +18,17 @@
*/
package org.apache.ambari.infra.conf;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
import java.time.Duration;
import javax.inject.Inject;
+import org.apache.ambari.infra.conf.security.SslSecrets;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.web.ServerProperties;
import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory;
+import org.springframework.boot.web.server.Ssl;
import org.springframework.boot.web.server.WebServerFactoryCustomizer;
import org.springframework.stereotype.Component;
@@ -34,18 +38,47 @@
@Value("${infra-manager.server.port:61890}")
private int port;
+ @Value("${infra-manager.server.ssl.enabled:false}")
+ private boolean sslEnabled;
+
@Inject
private ServerProperties serverProperties;
+ @Inject
+ private SslSecrets sslSecrets;
+
private static final Integer SESSION_TIMEOUT = 60 * 30;
- private static final String INFRA_MANAGER_SESSIONID = "INFRAMANAGER_SESSIONID";
+ private static final String INFRA_MANAGER_SESSION_ID = "INFRAMANAGER_SESSIONID";
private static final String INFRA_MANAGER_APPLICATION_NAME = "infra-manager";
@Override
public void customize(JettyServletWebServerFactory factory) {
factory.setPort(port);
factory.setDisplayName(INFRA_MANAGER_APPLICATION_NAME);
- factory.getSession().getCookie().setName(INFRA_MANAGER_SESSIONID);
+ factory.getSession().getCookie().setName(INFRA_MANAGER_SESSION_ID);
factory.getSession().setTimeout(Duration.ofSeconds(SESSION_TIMEOUT));
+
+ Ssl ssl = new Ssl();
+ String keyStore = System.getProperty("javax.net.ssl.keyStore");
+ if (isNotBlank(keyStore)) {
+ ssl.setKeyStore(keyStore);
+ ssl.setKeyStoreType(System.getProperty("javax.net.ssl.keyStoreType"));
+ String keyStorePassword = sslSecrets.getKeyStorePassword().get().orElseThrow(() -> new IllegalStateException("Password for keystore is not set!"));
+ ssl.setKeyStorePassword(keyStorePassword);
+ System.setProperty("javax.net.ssl.keyStorePassword", keyStorePassword);
+ }
+
+ String trustStore = System.getProperty("javax.net.ssl.trustStore");
+ if (isNotBlank(trustStore)) {
+ ssl.setTrustStore(trustStore);
+ ssl.setTrustStoreType(System.getProperty("javax.net.ssl.trustStoreType"));
+ String trustStorePassword = sslSecrets.getTrustStorePassword().get().orElseThrow(() -> new IllegalStateException("Password for truststore is not set!"));
+ ssl.setKeyStorePassword(trustStorePassword);
+ System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
+ }
+
+ ssl.setEnabled(sslEnabled);
+
+ factory.setSsl(ssl);
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java
similarity index 70%
rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java
index 6d32963..e8ab52e 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java
@@ -20,17 +20,17 @@
import java.util.Optional;
-public class CompositePasswordStore implements PasswordStore {
- private PasswordStore[] passwordStores;
+public class CompositeSecret implements Secret {
+ private Secret[] secrets;
- public CompositePasswordStore(PasswordStore... passwordStores) {
- this.passwordStores = passwordStores;
+ public CompositeSecret(Secret... secrets) {
+ this.secrets = secrets;
}
@Override
- public Optional<String> getPassword(String propertyName) {
- for (PasswordStore passwordStore : passwordStores) {
- Optional<String> optionalPassword = passwordStore.getPassword(propertyName);
+ public Optional<String> get() {
+ for (Secret secret : secrets) {
+ Optional<String> optionalPassword = secret.get();
if (optionalPassword.isPresent())
return optionalPassword;
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java
similarity index 72%
rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java
index 8e3387b..887767b 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java
@@ -20,9 +20,16 @@
import java.util.Optional;
-public class SecurityEnvironment implements PasswordStore {
+public class EnvironmentalSecret implements Secret {
+
+ private final String environmentalVariableName;
+
+ public EnvironmentalSecret(String environmentalVariableName) {
+ this.environmentalVariableName = environmentalVariableName;
+ }
+
@Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(System.getenv(propertyName));
+ public Optional<String> get() {
+ return Optional.ofNullable(System.getenv(environmentalVariableName));
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java
similarity index 63%
copy from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
copy to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java
index 8e3387b..8fba08a 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java
@@ -20,9 +20,22 @@
import java.util.Optional;
-public class SecurityEnvironment implements PasswordStore {
+public class HadoopCredential implements Secret {
+
+ private final HadoopCredentialStore hadoopCredentialStore;
+ private final String propertyName;
+
+ public HadoopCredential(HadoopCredentialStore hadoopCredentialStore, String propertyName) {
+ this.propertyName = propertyName;
+ this.hadoopCredentialStore = hadoopCredentialStore;
+ }
+
@Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(System.getenv(propertyName));
+ public Optional<String> get() {
+ if (hadoopCredentialStore == null) {
+ return Optional.empty();
+ }
+
+ return hadoopCredentialStore.get(propertyName).map(String::new);
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
index 957a45d..08a8804 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
@@ -21,13 +21,11 @@
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang3.ArrayUtils.isNotEmpty;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.Optional;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class HadoopCredentialStore implements PasswordStore {
- private static final Logger logger = LogManager.getLogger(InfraManagerSecurityConfig.class);
+public class HadoopCredentialStore {
public static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path";
private final String credentialStoreProviderPath;
@@ -36,8 +34,7 @@
this.credentialStoreProviderPath = credentialStoreProviderPath;
}
- @Override
- public Optional<String> getPassword(String propertyName) {
+ public Optional<char[]> get(String key) {
try {
if (isBlank(credentialStoreProviderPath)) {
return Optional.empty();
@@ -45,11 +42,14 @@
org.apache.hadoop.conf.Configuration config = new org.apache.hadoop.conf.Configuration();
config.set(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, credentialStoreProviderPath);
- char[] passwordChars = config.getPassword(propertyName);
- return (isNotEmpty(passwordChars)) ? Optional.of(new String(passwordChars)) : Optional.empty();
- } catch (Exception e) {
- logger.warn("Could not load password {} from credential store.", propertyName);
- return Optional.empty();
+ char[] passwordChars = config.getPassword(key);
+ return (isNotEmpty(passwordChars)) ? Optional.of(passwordChars) : Optional.empty();
+ } catch (IOException e) {
+ throw new UncheckedIOException(String.format("Could not load password %s from credential store.", key), e);
}
}
+
+ public Secret getSecret(String key) {
+ return new HadoopCredential(this, key);
+ }
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
index 45b79b3..0e5196d 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
@@ -18,21 +18,44 @@
*/
package org.apache.ambari.infra.conf.security;
+import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY;
+
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY;
-
@Configuration
public class InfraManagerSecurityConfig {
@Value("${"+ CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY + ":}")
private String credentialStoreProviderPath;
+ @Bean
+ public HadoopCredentialStore hadoopCredentialStore() {
+ return new HadoopCredentialStore(credentialStoreProviderPath);
+ }
@Bean
- public PasswordStore passwords() {
- return new CompositePasswordStore(new HadoopCredentialStore(credentialStoreProviderPath), new SecurityEnvironment());
+ public S3Secrets s3SecretStore(HadoopCredentialStore hadoopCredentialStore) {
+ return new S3Secrets(s3AccessKeyId(hadoopCredentialStore), s3SecretKeyId(hadoopCredentialStore));
+ }
+
+ private Secret s3AccessKeyId(HadoopCredentialStore hadoopCredentialStore) {
+ return new CompositeSecret(
+ hadoopCredentialStore.getSecret( "AWS_ACCESS_KEY_ID"),
+ new EnvironmentalSecret("AWS_ACCESS_KEY_ID"));
+ }
+
+ private Secret s3SecretKeyId(HadoopCredentialStore hadoopCredentialStore) {
+ return new CompositeSecret(
+ hadoopCredentialStore.getSecret( "AWS_SECRET_ACCESS_KEY"),
+ new EnvironmentalSecret("AWS_SECRET_ACCESS_KEY"));
+ }
+
+ @Bean
+ public SslSecrets sslSecrets(HadoopCredentialStore hadoopCredentialStore) {
+ return new SslSecrets(
+ hadoopCredentialStore.getSecret("infra_manager_keystore_password"),
+ hadoopCredentialStore.getSecret("infra_manager_truststore_password"));
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java
similarity index 67%
copy from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
copy to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java
index 8e3387b..30a1ca9 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java
@@ -18,11 +18,21 @@
*/
package org.apache.ambari.infra.conf.security;
-import java.util.Optional;
+public class S3Secrets {
+ private final Secret s3AccessKeyId;
+ private final Secret s3SecretAccessKey;
-public class SecurityEnvironment implements PasswordStore {
- @Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(System.getenv(propertyName));
+ public S3Secrets(Secret s3AccessKeyId, Secret s3SecretAccessKey) {
+ this.s3AccessKeyId = s3AccessKeyId;
+ this.s3SecretAccessKey = s3SecretAccessKey;
+ }
+
+
+ public Secret getS3AccessKeyId() {
+ return s3AccessKeyId;
+ }
+
+ public Secret getS3SecretAccessKey() {
+ return s3SecretAccessKey;
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java
similarity index 90%
rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java
index 19848fe..e4f54a3 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java
@@ -20,6 +20,6 @@
import java.util.Optional;
-public interface PasswordStore {
- Optional<String> getPassword(String propertyName);
+public interface Secret {
+ Optional<String> get();
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java
similarity index 66%
copy from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
copy to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java
index 8e3387b..6323e95 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java
@@ -18,11 +18,20 @@
*/
package org.apache.ambari.infra.conf.security;
-import java.util.Optional;
+public class SslSecrets {
+ private final Secret keyStorePassword;
+ private final Secret trustStorePassword;
-public class SecurityEnvironment implements PasswordStore {
- @Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(System.getenv(propertyName));
+ public SslSecrets(Secret keyStorePassword, Secret trustStorePassword) {
+ this.keyStorePassword = keyStorePassword;
+ this.trustStorePassword = trustStorePassword;
+ }
+
+ public Secret getKeyStorePassword() {
+ return keyStorePassword;
+ }
+
+ public Secret getTrustStorePassword() {
+ return trustStorePassword;
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
index 319cc5b..b35c1d2 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
@@ -27,12 +27,11 @@
import javax.inject.Inject;
import org.apache.ambari.infra.conf.InfraManagerDataConfig;
-import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.conf.security.S3Secrets;
import org.apache.ambari.infra.job.AbstractJobsConfiguration;
import org.apache.ambari.infra.job.JobContextRepository;
import org.apache.ambari.infra.job.JobScheduler;
import org.apache.ambari.infra.job.ObjectSource;
-import org.apache.hadoop.fs.Path;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.batch.core.Job;
@@ -91,7 +90,7 @@
@Value("#{jobParameters[end]}") String intervalEnd,
DocumentWiper documentWiper,
JobContextRepository jobContextRepository,
- PasswordStore passwordStore) {
+ S3Secrets s3Secrets) {
File baseDir = new File(infraManagerDataConfig.getDataFolder(), "exporting");
CompositeFileAction fileAction = new CompositeFileAction(new BZip2Compressor());
@@ -99,7 +98,7 @@
case S3:
fileAction.add(new S3Uploader(
parameters.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!")),
- passwordStore));
+ s3Secrets));
break;
case HDFS:
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
index 7c4de52..3f541b8 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
@@ -25,31 +25,40 @@
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
-import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.conf.security.Secret;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-public class S3AccessCsv implements PasswordStore {
+public class S3AccessCsv implements Secret {
private static final Logger logger = LogManager.getLogger(S3AccessCsv.class);
+ public static final String ACCESS_KEY_ID = "Access key ID";
+ public static final String SECRET_ACCESS_KEY = "Secret access key";
- public static S3AccessCsv file(String path) {
+
+ public static S3AccessCsv file(String path, String propertyName) {
try {
- return new S3AccessCsv(new FileReader(path));
+ return new S3AccessCsv(new FileReader(path), propertyName);
} catch (FileNotFoundException e) {
throw new UncheckedIOException(e);
}
}
- private Map<String, String> passwordMap = new HashMap<>();
+ private final Reader reader;
+ private final String propertyName;
- public S3AccessCsv(Reader reader) {
+ S3AccessCsv(Reader reader, String propertyName) {
+ this.reader = reader;
+ this.propertyName = propertyName;
+ }
+
+ @Override
+ public Optional<String> get() {
try (CSVParser csvParser = CSVParser.parse(reader, DEFAULT.withHeader(
S3AccessKeyNames.AccessKeyId.getCsvName(), S3AccessKeyNames.SecretAccessKey.getCsvName()))) {
Iterator<CSVRecord> iterator = csvParser.iterator();
@@ -62,8 +71,8 @@
throw new S3AccessCsvFormatException("Csv file contains less than 2 columns!");
}
- checkColumnExists(record, S3AccessKeyNames.AccessKeyId);
- checkColumnExists(record, S3AccessKeyNames.SecretAccessKey);
+ checkColumnExists(record, ACCESS_KEY_ID);
+ checkColumnExists(record, SECRET_ACCESS_KEY);
if (!iterator.hasNext()) {
throw new S3AccessCsvFormatException("Csv file contains header only!");
@@ -72,23 +81,15 @@
record = iterator.next();
Map<String, Integer> header = csvParser.getHeaderMap();
- for (S3AccessKeyNames keyNames : S3AccessKeyNames.values())
- passwordMap.put(keyNames.getEnvVariableName(), record.get(header.get(keyNames.getCsvName())));
+ return Optional.ofNullable(record.get(header.get(propertyName)));
} catch (IOException e) {
throw new UncheckedIOException(e);
- } catch (S3AccessCsvFormatException e) {
- logger.warn("Unable to parse csv file: {}", e.getMessage());
}
}
- private void checkColumnExists(CSVRecord record, S3AccessKeyNames s3AccessKeyName) {
- if (!s3AccessKeyName.getCsvName().equals(record.get(s3AccessKeyName.getCsvName()))) {
- throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName.getCsvName()));
+ private void checkColumnExists(CSVRecord record, String s3AccessKeyName) {
+ if (!s3AccessKeyName.equals(record.get(s3AccessKeyName))) {
+ throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName));
}
}
-
- @Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(passwordMap.get(propertyName));
- }
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
index 3e1310a..f3c92b7 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
@@ -9,8 +9,9 @@
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
-import org.apache.ambari.infra.conf.security.CompositePasswordStore;
-import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.conf.security.CompositeSecret;
+import org.apache.ambari.infra.conf.security.S3Secrets;
+import org.apache.ambari.infra.conf.security.Secret;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.xmlpull.v1.XmlPullParserException;
@@ -51,22 +52,28 @@
private final MinioClient client;
private final String keyPrefix;
private final String bucketName;
+ private final Secret s3AccessKey;
+ private final Secret s3SecretKey;
- public S3Uploader(S3Properties s3Properties, PasswordStore passwordStore) {
+ public S3Uploader(S3Properties s3Properties, S3Secrets s3Secrets) {
logger.info("Initializing S3 client with " + s3Properties);
this.keyPrefix = s3Properties.getS3KeyPrefix();
this.bucketName = s3Properties.getS3BucketName();
- PasswordStore compositePasswordStore = passwordStore;
- if (isNotBlank((s3Properties.getS3AccessFile())))
- compositePasswordStore = new CompositePasswordStore(passwordStore, S3AccessCsv.file(s3Properties.getS3AccessFile()));
+ if (isNotBlank(s3Properties.getS3AccessFile())) {
+ this.s3AccessKey = new CompositeSecret(s3Secrets.getS3AccessKeyId(), S3AccessCsv.file(s3Properties.getS3AccessFile(), "Access key ID"));
+ this.s3SecretKey = new CompositeSecret(s3Secrets.getS3SecretAccessKey(), S3AccessCsv.file(s3Properties.getS3AccessFile(), "Secret access key"));
+ }
+ else {
+ this.s3AccessKey = s3Secrets.getS3AccessKeyId();
+ this.s3SecretKey = s3Secrets.getS3SecretAccessKey();
+ }
try {
- client = new MinioClient(s3Properties.getS3EndPoint(), compositePasswordStore.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName())
- .orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")),
- compositePasswordStore.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName())
- .orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!")));
+ client = new MinioClient(s3Properties.getS3EndPoint(),
+ s3AccessKey.get().orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")),
+ s3SecretKey.get().orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!")));
if (!client.bucketExists(bucketName))
client.makeBucket(bucketName);
diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java
similarity index 70%
rename from ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
rename to ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java
index 26a6953..78b0269 100644
--- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
+++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java
@@ -1,11 +1,11 @@
package org.apache.ambari.infra.conf.security;
-import org.junit.Test;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
import java.util.Optional;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -25,24 +25,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-public class CompositePasswordStoreTest {
+public class CompositeSecretTest {
@Test
public void testGetPasswordReturnNullIfNoPasswordStoresWereAdded() {
- assertThat(new CompositePasswordStore().getPassword("any").isPresent(), is(false));
+ assertThat(new CompositeSecret().get().isPresent(), is(false));
}
@Test
public void testGetPasswordReturnNullIfPasswordNotFoundInAnyStore() {
- assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.empty()).getPassword("any").isPresent(), is(false));
+ assertThat(new CompositeSecret(Optional::empty, Optional::empty).get().isPresent(), is(false));
}
@Test
public void testGetPasswordReturnPasswordFromFirstStoreIfExists() {
- assertThat(new CompositePasswordStore((prop) -> Optional.of("Pass"), (prop) -> Optional.empty()).getPassword("any").get(), is("Pass"));
+ assertThat(new CompositeSecret(() -> Optional.of("Pass"), Optional::empty).get().get(), is("Pass"));
}
@Test
public void testGetPasswordReturnPasswordFromSecondStoreIfNotExistsInFirst() {
- assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.of("Pass")).getPassword("any").get(), is("Pass"));
+ assertThat(new CompositeSecret(Optional::empty, () -> Optional.of("Pass")).get().get(), is("Pass"));
}
}
\ No newline at end of file
diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
index e34a222..eb4b011 100644
--- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
+++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
@@ -1,11 +1,12 @@
package org.apache.ambari.infra.job.archive;
-import org.junit.Test;
+import static org.apache.ambari.infra.job.archive.S3AccessCsv.ACCESS_KEY_ID;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
import java.io.StringReader;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -33,38 +34,33 @@
private static final String ANY_CSV_FILE = "Column1,Column2\n" +
"Foo,Bar\n";
- @Test
+ @Test(expected = S3AccessCsvFormatException.class)
public void testGetPasswordReturnsNullIfInputIsEmpty() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
@Test
public void testGetPasswordReturnsAccessAndSecretKeyIfInputIsAValidS3AccessFile() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).get(), is("someKey"));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).get(), is("someSecret"));
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().get(), is("someKey"));
}
- @Test
- public void testGetPasswordReturnsNullIfNotAValidS3AccessFileProvided() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ @Test(expected = S3AccessCsvFormatException.class)
+ public void testGetPasswordThrowsExceptionIfNotAValidS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
- @Test
- public void testGetPasswordReturnsNullIfAHeaderOnlyS3AccessFileProvided() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ @Test(expected = S3AccessCsvFormatException.class)
+ public void testGetPasswordThrowsExceptionIfAHeaderOnlyS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
- @Test
- public void testGetPasswordReturnsNullIfOnlyOneValidColumnProvided() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ @Test(expected = S3AccessCsvFormatException.class)
+ public void testGetPasswordThrowsExceptionIfOnlyOneValidColumnProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
}
\ No newline at end of file
diff --git a/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh b/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
index 7ddb757..a3df897 100755
--- a/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
+++ b/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
@@ -101,8 +101,8 @@
pushd $sdir/../
local AMBARI_SOLR_MANAGER_LOCATION=$(pwd)
cd $AMBARI_SOLR_MANAGER_LOCATION/docker
- docker exec docker_solr_1 solr create_collection -force -c hadoop_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/hadoop_logs/conf -n hadoop_logs_conf
- docker exec docker_solr_1 solr create_collection -force -c audit_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/audit_logs/conf -n audit_logs_conf
+ docker exec solr solr create_collection -force -c hadoop_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/hadoop_logs/conf -n hadoop_logs_conf
+ docker exec solr solr create_collection -force -c audit_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/audit_logs/conf -n audit_logs_conf
popd
}
