KNOX-2160 - Introducing Hadoop XML type descriptor format (#236)
diff --git a/gateway-cm-integration/pom.xml b/gateway-cm-integration/pom.xml
new file mode 100644
index 0000000..ebe282b
--- /dev/null
+++ b/gateway-cm-integration/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>gateway-cm-integration</artifactId>
+ <name>gateway-cm-integration</name>
+ <description>Cloudera Manager integration related resources</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-i18n</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-i18n-logging-log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-spi</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-topology-simple</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-util-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/gateway-cm-integration/src/main/java/org/apache/knox/gateway/ClouderaManagerIntegrationMessages.java b/gateway-cm-integration/src/main/java/org/apache/knox/gateway/ClouderaManagerIntegrationMessages.java
new file mode 100644
index 0000000..1312bc8
--- /dev/null
+++ b/gateway-cm-integration/src/main/java/org/apache/knox/gateway/ClouderaManagerIntegrationMessages.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway;
+
+import org.apache.knox.gateway.i18n.messages.Message;
+import org.apache.knox.gateway.i18n.messages.MessageLevel;
+import org.apache.knox.gateway.i18n.messages.Messages;
+import org.apache.knox.gateway.i18n.messages.StackTrace;
+
+@Messages(logger = "org.apache.knox.gateway")
+public interface ClouderaManagerIntegrationMessages {
+
+ @Message(level = MessageLevel.INFO, text = "Monitoring Cloudera Manager descriptors in {0} ...")
+ void monitoringClouderaManagerDescriptor(String path);
+
+ @Message(level = MessageLevel.INFO, text = "Parsing Cloudera Manager descriptor {0}")
+ void parseClouderaManagerDescriptor(String path);
+
+ @Message(level = MessageLevel.INFO, text = "Found Knox descriptors {0} in {1}")
+ void parsedClouderaManagerDescriptor(String descriptorList, String path);
+
+ @Message(level = MessageLevel.ERROR, text = "Parsing Knox descriptor {0} failed: {1}")
+ void failedToParseDescriptor(String name, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+
+ @Message(level = MessageLevel.ERROR, text = "Parsing XML configuration {0} failed: {1}")
+ void failedToParseXmlConfiguration(String path, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+
+ @Message(level = MessageLevel.ERROR, text = "Error while monitoring CM descriptor {0}: {1}")
+ void failedToMonitorClouderaManagerDescriptor(String descriptorPath, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+
+ @Message(level = MessageLevel.ERROR, text = "Error while producing Knox descriptor: {0}")
+ void failedToProduceKnoxDescriptor(String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+}
diff --git a/gateway-cm-integration/src/main/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorMonitor.java b/gateway-cm-integration/src/main/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorMonitor.java
new file mode 100644
index 0000000..a727839
--- /dev/null
+++ b/gateway-cm-integration/src/main/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorMonitor.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.cm.descriptor;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.knox.gateway.ClouderaManagerIntegrationMessages;
+import org.apache.knox.gateway.config.GatewayConfig;
+import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+import org.apache.knox.gateway.util.JsonUtils;
+
+/**
+ * Monitoring KNOX_DESCRIPTOR_DIR for *.cm files - which is a Hadoop XML configuration - and processing those files if they were modified
+ * since the last time it they were processed
+ */
+public class ClouderaManagerDescriptorMonitor {
+
+ private static final String CM_DESCRIPTOR_FILE_EXTENSION = ".cm";
+ private static final ClouderaManagerIntegrationMessages LOG = MessagesFactory.get(ClouderaManagerIntegrationMessages.class);
+ private final String descriptorsDir;
+ private final long monitoringInterval;
+ private final ScheduledExecutorService executorService;
+ private FileTime lastReloadTime;
+
+ public ClouderaManagerDescriptorMonitor(GatewayConfig gatewayConfig) {
+ this.descriptorsDir = gatewayConfig.getGatewayDescriptorsDir();
+ this.monitoringInterval = gatewayConfig.getClouderaManagerDescriptorsMonitoringInterval();
+ this.executorService = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("ClouderaManagerDescriptorMonitor-%d").build());
+ }
+
+ public void setupMonitor() {
+ if (monitoringInterval > 0) {
+ executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(), 0, monitoringInterval, TimeUnit.MILLISECONDS);
+ LOG.monitoringClouderaManagerDescriptor(descriptorsDir);
+ }
+ }
+
+ private void monitorClouderaManagerDescriptors() {
+ final File[] clouderaManagerDescriptorFiles = new File(descriptorsDir).listFiles((FileFilter) new SuffixFileFilter(CM_DESCRIPTOR_FILE_EXTENSION));
+ for (File clouderaManagerDescriptorFile : clouderaManagerDescriptorFiles) {
+ monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath()));
+ }
+ }
+
+ private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile) {
+ try {
+ if (Files.isReadable(clouderaManagerDescriptorFile)) {
+ final FileTime lastModifiedTime = Files.getLastModifiedTime(clouderaManagerDescriptorFile);
+ if (lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) {
+ lastReloadTime = lastModifiedTime;
+ processClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString());
+ }
+ } else {
+ LOG.failedToMonitorClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString(), "File is not readable!", null);
+ }
+ } catch (IOException e) {
+ LOG.failedToMonitorClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString(), e.getMessage(), e);
+ }
+ }
+
+ private void processClouderaManagerDescriptor(String descriptorFilePath) {
+ ClouderaManagerDescriptorParser.parse(descriptorFilePath).forEach(simpleDescriptor -> {
+ try {
+ final File knoxDescriptorFile = new File(descriptorsDir, simpleDescriptor.getName() + ".json");
+ FileUtils.writeStringToFile(knoxDescriptorFile, JsonUtils.renderAsJsonString(simpleDescriptor), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ LOG.failedToProduceKnoxDescriptor(e.getMessage(), e);
+ }
+ });
+ }
+}
diff --git a/gateway-cm-integration/src/main/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorParser.java b/gateway-cm-integration/src/main/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorParser.java
new file mode 100644
index 0000000..44075fa
--- /dev/null
+++ b/gateway-cm-integration/src/main/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorParser.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.cm.descriptor;
+
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.knox.gateway.ClouderaManagerIntegrationMessages;
+import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+import org.apache.knox.gateway.topology.simple.SimpleDescriptor;
+import org.apache.knox.gateway.topology.simple.SimpleDescriptorImpl;
+import org.apache.knox.gateway.topology.simple.SimpleDescriptorImpl.ApplicationImpl;
+import org.apache.knox.gateway.topology.simple.SimpleDescriptorImpl.ServiceImpl;
+
+public class ClouderaManagerDescriptorParser {
+ private static final ClouderaManagerIntegrationMessages log = MessagesFactory.get(ClouderaManagerIntegrationMessages.class);
+ private static final String CONFIG_NAME_DISCOVERY_TYPE = "discoveryType";
+ private static final String CONFIG_NAME_DISCOVERY_ADDRESS = "discoveryAddress";
+ private static final String CONFIG_NAME_DISCOVERY_USER = "discoveryUser";
+ private static final String CONFIG_NAME_DISCOVERY_PASSWORD_ALIAS = "discoveryPasswordAlias";
+ private static final String CONFIG_NAME_DISCOVERY_CLUSTER = "cluster";
+ private static final String CONFIG_NAME_PROVIDER_CONFIG_REFERENCE = "providerConfigRef";
+ private static final String CONFIG_NAME_APPLICATION_PREFIX = "app";
+ private static final String CONFIG_NAME_SERVICE_URL = "url";
+ private static final String CONFIG_NAME_SERVICE_VERSION = "version";
+
+ /**
+ * Produces a set of {@link SimpleDescriptor}s from the specified file.
+ *
+ * @param path
+ * The path to the configuration file which holds descriptor information in a pre-defined format.
+ * @return A SimpleDescriptor based on the contents of the given file.
+ */
+ public static Set<SimpleDescriptor> parse(String path) {
+ try {
+ log.parseClouderaManagerDescriptor(path);
+ final Configuration xmlConfiguration = new Configuration(false);
+ xmlConfiguration.addResource(Paths.get(path).toUri().toURL());
+ xmlConfiguration.reloadConfiguration();
+ final Set<SimpleDescriptor> descriptors = parseXmlConfig(xmlConfiguration);
+ log.parsedClouderaManagerDescriptor(String.join(", ", descriptors.stream().map(descriptor -> descriptor.getName()).collect(Collectors.toSet())), path);
+ return descriptors;
+ } catch (Exception e) {
+ log.failedToParseXmlConfiguration(path, e.getMessage(), e);
+ return Collections.emptySet();
+ }
+ }
+
+ private static Set<SimpleDescriptor> parseXmlConfig(Configuration xmlConfiguration) {
+ final Set<SimpleDescriptor> descriptors = new LinkedHashSet<>();
+ xmlConfiguration.forEach(xmlDescriptor -> {
+ SimpleDescriptor descriptor = parseXmlDescriptor(xmlDescriptor.getKey(), xmlDescriptor.getValue());
+ if (descriptor != null) {
+ descriptors.add(descriptor);
+ }
+ });
+ return descriptors;
+ }
+
+ private static SimpleDescriptor parseXmlDescriptor(String name, String xmlValue) {
+ try {
+ final SimpleDescriptorImpl descriptor = new SimpleDescriptorImpl();
+ descriptor.setName(name);
+ final String[] configurationPairs = xmlValue.split(";");
+ for (String configurationPair : configurationPairs) {
+ String[] parameterPairParts = configurationPair.trim().split("=", 2);
+ String parameterName = parameterPairParts[0].trim();
+ switch (parameterName) {
+ case CONFIG_NAME_DISCOVERY_TYPE:
+ descriptor.setDiscoveryType(parameterPairParts[1].trim());
+ break;
+ case CONFIG_NAME_DISCOVERY_ADDRESS:
+ descriptor.setDiscoveryAddress(parameterPairParts[1].trim());
+ break;
+ case CONFIG_NAME_DISCOVERY_USER:
+ descriptor.setDiscoveryUser(parameterPairParts[1].trim());
+ break;
+ case CONFIG_NAME_DISCOVERY_PASSWORD_ALIAS:
+ descriptor.setDiscoveryPasswordAlias(parameterPairParts[1].trim());
+ break;
+ case CONFIG_NAME_DISCOVERY_CLUSTER:
+ descriptor.setCluster(parameterPairParts[1].trim());
+ break;
+ case CONFIG_NAME_PROVIDER_CONFIG_REFERENCE:
+ descriptor.setProviderConfig(parameterPairParts[1].trim());
+ break;
+ default:
+ if (parameterName.startsWith(CONFIG_NAME_APPLICATION_PREFIX)) {
+ parseApplication(descriptor, configurationPair.trim());
+ } else {
+ parseService(descriptor, configurationPair.trim());
+ }
+ break;
+ }
+ }
+ return descriptor;
+ } catch(Exception e) {
+ log.failedToParseDescriptor(name, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * An application consists of the following parts: <code>app:$APPLICATION_NAME[:$PARAMETER_NAME=$PARAMETER_VALUE]</code>. Parameters are
+ * optional. <br>
+ * Sample application configurations:
+ * <ul>
+ * <li>app:KNOX</li>
+ * <li>app:knoxauth:param1.name=param1.value</li>
+ * </ul>
+ */
+ private static void parseApplication(SimpleDescriptorImpl descriptor, String configurationPair) {
+ final String[] applicationParts = configurationPair.split(":");
+ final String applicationName = applicationParts[1].trim();
+ ApplicationImpl application = (ApplicationImpl) descriptor.getApplication(applicationName);
+ if (application == null) {
+ application = new ApplicationImpl();
+ descriptor.addApplication(application);
+ application.setName(applicationParts[1]);
+ }
+
+ if (applicationParts.length > 2) {
+ // parameter value may contain ":" (for instance http://host:port) -> considering a parameter name/value pair everything after 'app:$APPLICATION_NAME:'
+ final String applicationParameters = configurationPair.substring(applicationName.length() + 5); // 'app:' and trailing colon takes 5 chars
+ final String[] applicationParameterParts = applicationParameters.split("=", 2);
+ application.addParam(applicationParameterParts[0], applicationParameterParts[1]);
+ }
+ }
+
+ /**
+ * A service consists of the following parts:
+ * <ul>
+ * <li><code>$SERVICE_NAME:url=$URL</code></li>
+ * <li><code>$SERVICE_NAME:version=$VERSION</code> (optional)</li>
+ * <li><code>$SERVICE_NAME[:$PARAMETER_NAME=$PARAMETER_VALUE] (optional)</code></li>
+ * </ul>
+ * Sample application configurations:
+ * <ul>
+ * <li>HIVE:url=http://localhost:123</li>
+ * <li>HIVE:version=1.0</li>
+ * <li>HIVE:param1.name=param1.value</li>
+ * </ul>
+ */
+ private static void parseService(SimpleDescriptorImpl descriptor, String configurationPair) {
+ final String[] serviceParts = configurationPair.split(":");
+ final String serviceName = serviceParts[0].trim();
+ ServiceImpl service = (ServiceImpl) descriptor.getService(serviceName);
+ if (service == null) {
+ service = new ServiceImpl();
+ service.setName(serviceName);
+ descriptor.addService(service);
+ }
+
+ // configuration value may contain ":" (for instance http://host:port) -> considering a configuration name/value pair everything after '$SERVICE_NAME:'
+ final String serviceConfiguration = configurationPair.substring(serviceName.length() + 1).trim();
+ final String[] serviceConfigurationParts = serviceConfiguration.split("=", 2);
+ final String serviceConfigurationName = serviceConfigurationParts[0].trim();
+ final String serviceConfigurationValue = serviceConfigurationParts[1].trim();
+ switch (serviceConfigurationName) {
+ case CONFIG_NAME_SERVICE_URL:
+ service.addUrl(serviceConfigurationValue);
+ break;
+ case CONFIG_NAME_SERVICE_VERSION:
+ service.setVersion(serviceConfigurationValue);
+ break;
+ default:
+ service.addParam(serviceConfigurationName, serviceConfigurationValue);
+ break;
+ }
+ }
+
+}
diff --git a/gateway-cm-integration/src/test/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorParserTest.java b/gateway-cm-integration/src/test/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorParserTest.java
new file mode 100644
index 0000000..84d825f
--- /dev/null
+++ b/gateway-cm-integration/src/test/java/org/apache/knox/gateway/cm/descriptor/ClouderaManagerDescriptorParserTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.cm.descriptor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.knox.gateway.topology.simple.SimpleDescriptor;
+import org.apache.knox.gateway.topology.simple.SimpleDescriptor.Application;
+import org.apache.knox.gateway.topology.simple.SimpleDescriptor.Service;
+import org.junit.Test;
+
+public class ClouderaManagerDescriptorParserTest {
+
+ @Test
+ public void testXmlParser() throws Exception {
+ final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptor.xml").getPath();
+ final Set<SimpleDescriptor> descriptors = ClouderaManagerDescriptorParser.parse(testConfigPath);
+ assertEquals(2, descriptors.size());
+ final Iterator<SimpleDescriptor> descriptorsIterator = descriptors.iterator();
+ validateTopology1(descriptorsIterator.next());
+ validateTopology2(descriptorsIterator.next());
+ }
+
+ @Test
+ public void testXmlParserWrongDescriptorContent() throws Exception {
+ final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorConfigurationWithWrongDescriptor.xml").getPath();
+ final Set<SimpleDescriptor> descriptors = ClouderaManagerDescriptorParser.parse(testConfigPath);
+ assertEquals(1, descriptors.size());
+ final Iterator<SimpleDescriptor> descriptorsIterator = descriptors.iterator();
+ validateTopology1(descriptorsIterator.next());
+ }
+
+ @Test
+ public void testXmlParserWrongXMLContent() throws Exception {
+ final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorConfigurationWithNonHadoopStyleConfiguration.xml").getPath();
+ final Set<SimpleDescriptor> descriptors = ClouderaManagerDescriptorParser.parse(testConfigPath);
+ assertTrue(descriptors.isEmpty());
+ }
+
+ private void validateTopology1(SimpleDescriptor descriptor) {
+ assertEquals("topology1", descriptor.getName());
+ assertEquals("ClouderaManager", descriptor.getDiscoveryType());
+ assertEquals("http://host:123", descriptor.getDiscoveryAddress());
+ assertEquals("user", descriptor.getDiscoveryUser());
+ assertEquals("alias", descriptor.getDiscoveryPasswordAlias());
+ assertEquals("Cluster 1", descriptor.getCluster());
+ assertEquals("topology1-provider", descriptor.getProviderConfig());
+ assertEquals(2, descriptor.getApplications().size());
+
+ assertApplication(descriptor, "knoxauth", Collections.singletonMap("param1.name", "param1.value"));
+ assertApplication(descriptor, "admin-ui", null);
+
+ final Map<String, String> expectedServiceParameters = Stream.of(new String[][] { { "httpclient.connectionTimeout", "5m" }, { "httpclient.socketTimeout", "100m" }, })
+ .collect(Collectors.toMap(data -> data[0], data -> data[1]));
+ assertService(descriptor, "HIVE", "1.0", Collections.singletonList("http://localhost:456"), expectedServiceParameters);
+ }
+
+ private void validateTopology2(SimpleDescriptor descriptor) {
+ assertEquals("topology2", descriptor.getName());
+ assertEquals("Ambari", descriptor.getDiscoveryType());
+ assertEquals("http://host:456", descriptor.getDiscoveryAddress());
+ assertEquals("Cluster 2", descriptor.getCluster());
+ assertEquals("topology2-provider", descriptor.getProviderConfig());
+ assertTrue(descriptor.getApplications().isEmpty());
+
+ final Map<String, String> expectedServiceParameters = Stream.of(new String[][] { { "httpclient.connectionTimeout", "5m" }, { "httpclient.socketTimeout", "100m" }, })
+ .collect(Collectors.toMap(data -> data[0], data -> data[1]));
+ assertService(descriptor, "HDFS", null, Collections.singletonList("http://localhost:456"), expectedServiceParameters);
+ }
+
+ private void assertApplication(SimpleDescriptor descriptor, String expectedApplicationName, Map<String, String> expectedParams) {
+ final Application application = descriptor.getApplication(expectedApplicationName);
+ assertNotNull(application);
+ if (expectedParams != null) {
+ assertTrue(application.getParams().entrySet().containsAll(expectedParams.entrySet()));
+ } else {
+ assertNull(application.getParams());
+ }
+ }
+
+ private void assertService(SimpleDescriptor descriptor, String expectedServiceName, String expectedVersion, List<String> expectedUrls, Map<String, String> expectedParams) {
+ final Service service = descriptor.getService(expectedServiceName);
+ assertNotNull(service);
+ if (expectedVersion != null) {
+ assertEquals(expectedVersion, service.getVersion());
+ } else {
+ assertNull(service.getVersion());
+ }
+
+ if (expectedUrls != null) {
+ assertTrue(service.getURLs().containsAll(expectedUrls));
+ }
+
+ if (expectedParams != null) {
+ assertTrue(service.getParams().entrySet().containsAll(expectedParams.entrySet()));
+ } else {
+ assertNull(service.getParams());
+ }
+ }
+
+}
diff --git a/gateway-cm-integration/src/test/resources/testDescriptor.xml b/gateway-cm-integration/src/test/resources/testDescriptor.xml
new file mode 100644
index 0000000..c853600
--- /dev/null
+++ b/gateway-cm-integration/src/test/resources/testDescriptor.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>topology1</name>
+ <value>
+ discoveryType=ClouderaManager;
+ discoveryAddress=http://host:123;
+ discoveryUser=user;
+ discoveryPasswordAlias=alias;
+ cluster=Cluster 1;
+ providerConfigRef=topology1-provider;
+ app:knoxauth:param1.name=param1.value;
+ app:admin-ui;
+ HIVE:url=http://localhost:456;
+ HIVE:version=1.0;
+ HIVE:httpclient.connectionTimeout=5m;
+ HIVE:httpclient.socketTimeout=100m
+ </value>
+ </property>
+ <property>
+ <name>topology2</name>
+ <value>
+ discoveryType=Ambari;
+ discoveryAddress=http://host:456;
+ cluster=Cluster 2;
+ providerConfigRef=topology2-provider;
+ HDFS:url=http://localhost:456;
+ HDFS:httpclient.connectionTimeout=5m;
+ HDFS:httpclient.socketTimeout=100m
+ </value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/gateway-cm-integration/src/test/resources/testDescriptorConfigurationWithNonHadoopStyleConfiguration.xml b/gateway-cm-integration/src/test/resources/testDescriptorConfigurationWithNonHadoopStyleConfiguration.xml
new file mode 100644
index 0000000..43deb1e
--- /dev/null
+++ b/gateway-cm-integration/src/test/resources/testDescriptorConfigurationWithNonHadoopStyleConfiguration.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration>
+ <config> <!-- should be property -->
+ <name>topology1</name>
+ <value>
+ discoveryType=ClouderaManager;
+ discoveryAddress=http://host:123;
+ discoveryUser=user;
+ discoveryPasswordAlias=alias;
+ cluster=Cluster 1;
+ providerConfigRef=topology1-provider;
+ app:knoxauth:param1.name=param1.value;
+ app:admin-ui;
+ HIVE:url=http://localhost:456;
+ HIVE:version=1.0;
+ HIVE:httpclient.connectionTimeout=5m;
+ HIVE:httpclient.socketTimeout=100m
+ </value>
+ </config>
+</configuration>
\ No newline at end of file
diff --git a/gateway-cm-integration/src/test/resources/testDescriptorConfigurationWithWrongDescriptor.xml b/gateway-cm-integration/src/test/resources/testDescriptorConfigurationWithWrongDescriptor.xml
new file mode 100644
index 0000000..ffa8c25
--- /dev/null
+++ b/gateway-cm-integration/src/test/resources/testDescriptorConfigurationWithWrongDescriptor.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>topology1</name>
+ <value>
+ discoveryType=ClouderaManager;
+ discoveryAddress=http://host:123;
+ discoveryUser=user;
+ discoveryPasswordAlias=alias;
+ cluster=Cluster 1;
+ providerConfigRef=topology1-provider;
+ app:knoxauth:param1.name=param1.value;
+ app:admin-ui;
+ HIVE:url=http://localhost:456;
+ HIVE:version=1.0;
+ HIVE:httpclient.connectionTimeout=5m;
+ HIVE:httpclient.socketTimeout=100m
+ </value>
+ </property>
+ <property>
+ <name>topology2</name>
+ <value>
+ discoveryType=Ambari;
+ discoveryAddress=http://host:456;
+ cluster=Cluster 2;
+ providerConfigRef=topology2-provider;
+ HDFS:url=http://localhost:456;
+ HDFS <!-- can not be parsed -->
+ </value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml
index 2306a3d..24cf45a 100644
--- a/gateway-release/pom.xml
+++ b/gateway-release/pom.xml
@@ -425,5 +425,13 @@
<groupId>org.apache.knox</groupId>
<artifactId>gateway-service-hashicorp-vault</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-topology-simple</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-cm-integration</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml
index 2924c22..ed3b7d6 100644
--- a/gateway-server/pom.xml
+++ b/gateway-server/pom.xml
@@ -67,6 +67,10 @@
</dependency>
<dependency>
<groupId>org.apache.knox</groupId>
+ <artifactId>gateway-cm-integration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
<artifactId>gateway-i18n</artifactId>
</dependency>
<dependency>
@@ -91,6 +95,10 @@
</dependency>
<dependency>
<groupId>org.apache.knox</groupId>
+ <artifactId>gateway-topology-simple</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
<artifactId>gateway-util-common</artifactId>
</dependency>
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
index 7628cf7..82c8bbb 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
@@ -27,6 +27,7 @@
import org.apache.knox.gateway.audit.api.Auditor;
import org.apache.knox.gateway.audit.api.ResourceType;
import org.apache.knox.gateway.audit.log4j.audit.AuditConstants;
+import org.apache.knox.gateway.cm.descriptor.ClouderaManagerDescriptorMonitor;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.config.GatewayConfigurationException;
import org.apache.knox.gateway.config.impl.GatewayConfigImpl;
@@ -621,6 +622,9 @@
"org.eclipse.jetty.webapp.JettyWebXmlConfiguration",
"org.eclipse.jetty.annotations.AnnotationConfiguration" );
+ final ClouderaManagerDescriptorMonitor cmDescriptorMonitor = new ClouderaManagerDescriptorMonitor(config);
+ cmDescriptorMonitor.setupMonitor();
+
// Load the current topologies.
// Redeploy autodeploy topologies.
File topologiesDir = calculateAbsoluteTopologiesDir();
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
index 19741c8..6c1ccaa 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
@@ -240,6 +240,9 @@
/* property that specifies list of services for which we need to append service name to the X-Forward-Context header */
public static final String X_FORWARD_CONTEXT_HEADER_APPEND_SERVICES = GATEWAY_CONFIG_FILE_PREFIX + ".xforwarded.header.context.append.servicename";
+ private static final String CLOUDERA_MANAGER_DESCRIPTORS_MONITOR_INTERVAL = GATEWAY_CONFIG_FILE_PREFIX + ".cloudera.manager.descriptors.monitor.interval";
+ private static final long DEFAULT_CLOUDERA_MANAGER_DESCRIPTORS_MONITOR_INTERVAL = 30000L;
+
public GatewayConfigImpl() {
init();
}
@@ -1094,4 +1097,9 @@
return set;
}
+
+ @Override
+ public long getClouderaManagerDescriptorsMonitoringInterval() {
+ return getLong(CLOUDERA_MANAGER_DESCRIPTORS_MONITOR_INTERVAL, DEFAULT_CLOUDERA_MANAGER_DESCRIPTORS_MONITOR_INTERVAL);
+ }
}
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java
deleted file mode 100644
index 535e1a9..0000000
--- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.knox.gateway.topology.simple;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-class SimpleDescriptorImpl implements SimpleDescriptor {
-
- @JsonProperty("discovery-type")
- private String discoveryType;
-
- @JsonProperty("discovery-address")
- private String discoveryAddress;
-
- @JsonProperty("discovery-user")
- private String discoveryUser;
-
- @JsonProperty("discovery-pwd-alias")
- private String discoveryPasswordAlias;
-
- @JsonProperty("provider-config-ref")
- private String providerConfig;
-
- @JsonProperty("cluster")
- private String cluster;
-
- @JsonProperty("services")
- private List<ServiceImpl> services;
-
- @JsonProperty("applications")
- private List<ApplicationImpl> applications;
-
- private String name;
-
- void setName(String name) {
- this.name = name;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getDiscoveryType() {
- return discoveryType;
- }
-
- @Override
- public String getDiscoveryAddress() {
- return discoveryAddress;
- }
-
- @Override
- public String getDiscoveryUser() {
- return discoveryUser;
- }
-
- @Override
- public String getDiscoveryPasswordAlias() {
- return discoveryPasswordAlias;
- }
-
- @Override
- public String getClusterName() {
- return cluster;
- }
-
- @Override
- public String getProviderConfig() {
- return providerConfig;
- }
-
- @Override
- public List<Service> getServices() {
- List<Service> result = new ArrayList<>();
- if (services != null) {
- result.addAll(services);
- }
- return result;
- }
-
- @Override
- public List<Application> getApplications() {
- List<Application> result = new ArrayList<>();
- if (applications != null) {
- result.addAll(applications);
- }
- return result;
- }
-
- public static class ServiceImpl implements Service {
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("version")
- private String version;
-
- @JsonProperty("params")
- private Map<String, String> params;
-
- @JsonProperty("urls")
- private List<String> urls;
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getVersion() {
- return version;
- }
-
- @Override
- public Map<String, String> getParams() {
- return params;
- }
-
- @Override
- public List<String> getURLs() {
- return urls;
- }
- }
-
- public static class ApplicationImpl implements Application {
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("params")
- private Map<String, String> params;
-
- @JsonProperty("urls")
- private List<String> urls;
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Map<String, String> getParams() {
- return params;
- }
-
- @Override
- public List<String> getURLs() {
- return urls;
- }
- }
-
-}
diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java
index 52cc6ee..cca10e0 100644
--- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java
+++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java
@@ -755,7 +755,7 @@
assertEquals(discoveryType, sd.getDiscoveryType());
assertEquals(discoveryAddress, sd.getDiscoveryAddress());
assertEquals(providerConfig, sd.getProviderConfig());
- assertEquals(clusterName, sd.getClusterName());
+ assertEquals(clusterName, sd.getCluster());
List<SimpleDescriptor.Service> actualServices = sd.getServices();
diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
index f5aca6a..4a6bc13 100644
--- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
+++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
@@ -199,7 +199,7 @@
EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(type).anyTimes();
EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
- EasyMock.expect(testDescriptor.getClusterName()).andReturn(clusterName).anyTimes();
+ EasyMock.expect(testDescriptor.getCluster()).andReturn(clusterName).anyTimes();
List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
for (String serviceName : serviceURLs.keySet()) {
SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
@@ -377,7 +377,7 @@
EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(type).anyTimes();
EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
- EasyMock.expect(testDescriptor.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
+ EasyMock.expect(testDescriptor.getCluster()).andReturn(CLUSTER_NAME).anyTimes();
List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
for (String serviceName : serviceURLs.keySet()) {
SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
@@ -580,7 +580,7 @@
EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(type).anyTimes();
EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
- EasyMock.expect(testDescriptor.getClusterName()).andReturn(clusterName).anyTimes();
+ EasyMock.expect(testDescriptor.getCluster()).andReturn(clusterName).anyTimes();
List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
for (String serviceName : serviceURLs.keySet()) {
SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
@@ -781,7 +781,7 @@
EasyMock.expect(testDescriptor.getDiscoveryAddress()).andReturn(discoveryConfig.getAbsolutePath()).anyTimes();
EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn("PROPERTIES_FILE").anyTimes();
EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
- EasyMock.expect(testDescriptor.getClusterName()).andReturn(clusterName).anyTimes();
+ EasyMock.expect(testDescriptor.getCluster()).andReturn(clusterName).anyTimes();
final SimpleDescriptor.Service knoxService = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
EasyMock.expect(knoxService.getName()).andReturn("KNOX").anyTimes();
diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
index 52a62cc..94e7642 100644
--- a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
+++ b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
@@ -642,4 +642,9 @@
* @return a set of service principal names that indicate which services to ignore doAs request
*/
Set<String> getServicesToIgnoreDoAs();
+
+ /**
+ * @return the monitoring interval (in milliseconds) of Cloudera Manager descriptors
+ */
+ long getClouderaManagerDescriptorsMonitoringInterval();
}
diff --git a/gateway-test-release-utils/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java b/gateway-test-release-utils/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
index 4a2c754..6170550 100644
--- a/gateway-test-release-utils/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
+++ b/gateway-test-release-utils/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
@@ -764,4 +764,9 @@
public Set<String> getServicesToIgnoreDoAs() {
return null;
}
+
+ @Override
+ public long getClouderaManagerDescriptorsMonitoringInterval() {
+ return 0;
+ }
}
diff --git a/gateway-test/pom.xml b/gateway-test/pom.xml
index a215525..257e5a6 100644
--- a/gateway-test/pom.xml
+++ b/gateway-test/pom.xml
@@ -102,6 +102,11 @@
</dependency>
<dependency>
<groupId>org.apache.knox</groupId>
+ <artifactId>gateway-topology-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
<artifactId>gateway-release</artifactId>
<scope>test</scope>
</dependency>
diff --git a/gateway-test/src/test/java/org/apache/knox/gateway/SimpleDescriptorHandlerFuncTest.java b/gateway-test/src/test/java/org/apache/knox/gateway/SimpleDescriptorHandlerFuncTest.java
index 87eea0b..3c53699 100644
--- a/gateway-test/src/test/java/org/apache/knox/gateway/SimpleDescriptorHandlerFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/knox/gateway/SimpleDescriptorHandlerFuncTest.java
@@ -150,7 +150,7 @@
EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(discoveryType).anyTimes();
EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
- EasyMock.expect(testDescriptor.getClusterName()).andReturn(clusterName).anyTimes();
+ EasyMock.expect(testDescriptor.getCluster()).andReturn(clusterName).anyTimes();
List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
for (String serviceName : serviceURLs.keySet()) {
SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
@@ -231,7 +231,8 @@
Map<String, File> files = SimpleDescriptorHandler.handle(config,
testDescriptor,
providerConfig.getParentFile(),
- destDir);
+ destDir,
+ GatewayServer.getGatewayServices());
topologyFile = files.get("topology");
// Validate the AliasService interaction
diff --git a/gateway-topology-simple/pom.xml b/gateway-topology-simple/pom.xml
new file mode 100644
index 0000000..53f5c97
--- /dev/null
+++ b/gateway-topology-simple/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>gateway-topology-simple</artifactId>
+ <name>gateway-topology-simple</name>
+ <description>Simple topology generation (based on provider/descriptor pairs) related resources</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-i18n</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-spi</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
similarity index 100%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfiguration.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfiguration.java
similarity index 100%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfiguration.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfiguration.java
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfigurationParser.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfigurationParser.java
similarity index 100%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfigurationParser.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/ProviderConfigurationParser.java
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java
similarity index 81%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java
index 51a121c..5b32662 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java
+++ b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java
@@ -19,6 +19,9 @@
import java.util.List;
import java.util.Map;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+
public interface SimpleDescriptor {
String getName();
@@ -31,15 +34,19 @@
String getDiscoveryPasswordAlias();
- String getClusterName();
+ String getCluster();
String getProviderConfig();
List<Service> getServices();
+ Service getService(String serviceName);
+
List<Application> getApplications();
+ Application getApplication(String applicationName);
+ @JsonDeserialize(as = SimpleDescriptorImpl.ServiceImpl.class)
interface Service {
String getName();
@@ -50,6 +57,7 @@
List<String> getURLs();
}
+ @JsonDeserialize(as = SimpleDescriptorImpl.ApplicationImpl.class)
interface Application {
String getName();
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactory.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactory.java
similarity index 100%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactory.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactory.java
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
similarity index 96%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
index 70bf3a2..5d68313 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
+++ b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
@@ -16,7 +16,6 @@
*/
package org.apache.knox.gateway.topology.simple;
-import org.apache.knox.gateway.GatewayServer;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.i18n.messages.MessagesFactory;
import org.apache.knox.gateway.services.ServiceType;
@@ -87,18 +86,6 @@
private static final Set<String> ALLOWED_SERVICES_WITHOUT_URLS_AND_PARAMS = Collections.singleton("KNOX");
- public static Map<String, File> handle(GatewayConfig config, File desc) throws IOException {
- return handle(config, desc, NO_GATEWAY_SERVICES);
- }
-
- public static Map<String, File> handle(GatewayConfig config, File desc, Service...gatewayServices) throws IOException {
- return handle(config, desc, desc.getParentFile(), gatewayServices);
- }
-
- public static Map<String, File> handle(GatewayConfig config, File desc, File destDirectory) throws IOException {
- return handle(config, desc, destDirectory, NO_GATEWAY_SERVICES);
- }
-
public static Map<String, File> handle(GatewayConfig config, File desc, File destDirectory, Service...gatewayServices) throws IOException {
return handle(config, SimpleDescriptorFactory.parse(desc.getAbsolutePath()), desc.getParentFile(), destDirectory, gatewayServices);
}
@@ -185,7 +172,7 @@
// when the topology is deployed. This is to support Knox HA deployments, where multiple Knox instances are
// generating topologies based on a shared remote descriptor, and they must all be able to encrypt/decrypt
// query params with the same credentials. (KNOX-1136)
- if (!provisionQueryParamEncryptionCredential(desc.getName())) {
+ if (!provisionQueryParamEncryptionCredential(desc.getName(), getGatewayServices(gatewayServices))) {
log.unableCreatePasswordForEncryption(desc.getName());
}
@@ -201,6 +188,14 @@
serviceParams);
}
+ private static GatewayServices getGatewayServices(Service... services) {
+ for (Service service : services) {
+ if (service instanceof GatewayServices) {
+ return (GatewayServices) service;
+ }
+ }
+ return null;
+ }
private static ServiceDiscovery.Cluster performDiscovery(GatewayConfig config, SimpleDescriptor desc, Service...gatewayServices) {
DefaultServiceDiscoveryConfig sdc = new DefaultServiceDiscoveryConfig(desc.getDiscoveryAddress());
@@ -220,7 +215,7 @@
discoveryInstances.put(discoveryType, sd);
}
- return sd.discover(config, sdc, desc.getClusterName());
+ return sd.discover(config, sdc, desc.getCluster());
}
@@ -253,11 +248,10 @@
*
* @return true if the credential was successfully provisioned; otherwise, false.
*/
- private static boolean provisionQueryParamEncryptionCredential(final String topologyName) {
+ private static boolean provisionQueryParamEncryptionCredential(final String topologyName, final GatewayServices services) {
boolean result = false;
try {
- GatewayServices services = GatewayServer.getGatewayServices();
if (services != null) {
MasterService ms = services.getService(ServiceType.MASTER_SERVICE);
if (ms != null) {
@@ -574,7 +568,7 @@
// Write the generated content to a file
String topologyFilename = desc.getName();
if (topologyFilename == null) {
- topologyFilename = desc.getClusterName();
+ topologyFilename = desc.getCluster();
}
topologyDescriptor = new File(destDirectory, topologyFilename + ".xml");
diff --git a/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java
new file mode 100644
index 0000000..e8ed738
--- /dev/null
+++ b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.topology.simple;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class SimpleDescriptorImpl implements SimpleDescriptor {
+
+ @JsonProperty("discovery-type")
+ private String discoveryType;
+
+ @JsonProperty("discovery-address")
+ private String discoveryAddress;
+
+ @JsonProperty("discovery-user")
+ private String discoveryUser;
+
+ @JsonProperty("discovery-pwd-alias")
+ private String discoveryPasswordAlias;
+
+ @JsonProperty("provider-config-ref")
+ private String providerConfig;
+
+ @JsonProperty("cluster")
+ private String cluster;
+
+ @JsonProperty("services")
+ private List<Service> services;
+
+ @JsonProperty("applications")
+ private List<Application> applications;
+
+ private String name;
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ public void setDiscoveryType(String discoveryType) {
+ this.discoveryType = discoveryType;
+ }
+
+ @Override
+ public String getDiscoveryType() {
+ return discoveryType;
+ }
+
+ public void setDiscoveryAddress(String discoveryAddress) {
+ this.discoveryAddress = discoveryAddress;
+ }
+
+ @Override
+ public String getDiscoveryAddress() {
+ return discoveryAddress;
+ }
+
+ public void setDiscoveryUser(String discoveryUser) {
+ this.discoveryUser = discoveryUser;
+ }
+
+ @Override
+ public String getDiscoveryUser() {
+ return discoveryUser;
+ }
+
+ public void setDiscoveryPasswordAlias(String discoveryPasswordAlias) {
+ this.discoveryPasswordAlias = discoveryPasswordAlias;
+ }
+
+ @Override
+ public String getDiscoveryPasswordAlias() {
+ return discoveryPasswordAlias;
+ }
+
+ public void setCluster(String cluster) {
+ this.cluster = cluster;
+ }
+
+ @Override
+ public String getCluster() {
+ return cluster;
+ }
+
+ public void setProviderConfig(String providerConfig) {
+ this.providerConfig = providerConfig;
+ }
+
+ @Override
+ public String getProviderConfig() {
+ return providerConfig;
+ }
+
+ public void addService(Service service) {
+ if (services == null) {
+ services = new ArrayList<>();
+ }
+ services.add(service);
+ }
+
+ @Override
+ public List<Service> getServices() {
+ List<Service> result = new ArrayList<>();
+ if (services != null) {
+ result.addAll(services);
+ }
+ return result;
+ }
+
+ @Override
+ public Service getService(String serviceName) {
+ return getServices().stream().filter(service -> service.getName().equals(serviceName)).findFirst().orElse(null);
+ }
+
+ public void addApplication(Application application) {
+ if (applications == null) {
+ applications = new ArrayList<>();
+ }
+ applications.add(application);
+ }
+
+ @Override
+ public List<Application> getApplications() {
+ List<Application> result = new ArrayList<>();
+ if (applications != null) {
+ result.addAll(applications);
+ }
+ return result;
+ }
+
+ @Override
+ public Application getApplication(String applicationName) {
+ return getApplications().stream().filter(application -> application.getName().equals(applicationName)).findFirst().orElse(null);
+ }
+
+ public static class ServiceImpl implements Service {
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("version")
+ private String version;
+
+ @JsonProperty("params")
+ private Map<String, String> params;
+
+ @JsonProperty("urls")
+ private List<String> urls;
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getVersion() {
+ return version;
+ }
+
+ @Override
+ public Map<String, String> getParams() {
+ return params;
+ }
+
+ @Override
+ public List<String> getURLs() {
+ return urls;
+ }
+
+ public void addUrl(String url) {
+ if (this.urls == null) {
+ this.urls = new ArrayList<>();
+ }
+ this.urls.add(url);
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public void addParam(String name, String value) {
+ if (this.params == null) {
+ this.params = new TreeMap<>();
+ }
+ this.params.put(name, value);
+ }
+ }
+
+ public static class ApplicationImpl implements Application {
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("params")
+ private Map<String, String> params;
+
+ @JsonProperty("urls")
+ private List<String> urls;
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Map<String, String> getParams() {
+ return params;
+ }
+
+ @Override
+ public List<String> getURLs() {
+ return urls;
+ }
+
+ public void addUrl(String url) {
+ if (this.urls == null) {
+ this.urls = new ArrayList<>();
+ }
+ this.urls.add(url);
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void addParam(String name, String value) {
+ if (this.params == null) {
+ this.params = new TreeMap<>();
+ }
+ this.params.put(name, value);
+ }
+ }
+
+}
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorMessages.java b/gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorMessages.java
similarity index 100%
rename from gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorMessages.java
rename to gateway-topology-simple/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorMessages.java
diff --git a/pom.xml b/pom.xml
index 8b8f673..6ebae0f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,8 @@
<module>gateway-docker</module>
<module>gateway-service-hashicorp-vault</module>
<module>gateway-release-common</module>
+ <module>gateway-topology-simple</module>
+ <module>gateway-cm-integration</module>
</modules>
<properties>
@@ -1153,6 +1155,16 @@
<artifactId>gateway-release-common</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-topology-simple</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway-cm-integration</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
