AMBARI-24725 - Infra Solr: manage autoscaling properties in Ambari (#2)
diff --git a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
index b0c7781..cc69985 100644
--- a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
+++ b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
@@ -56,6 +56,10 @@
private static final String TRANSFER_ZNODE_COMMAND = "transfer-znode";
private static final String DELETE_ZNODE_COMMAND = "delete-znode";
private static final String DUMP_COLLECTIONS_DATA_COMMAND = "dump-collections";
+ private static final String SET_AUTO_SCALING_COMMAND = "set-autoscaling";
+ private static final String SET_AUTO_SCALING_COMMAND_SHORT = "as";
+ private static final String AUTO_SCALING_JSON_LOCATION = "autoscaling-json-location";
+ private static final String AUTO_SCALING_JSON_LOCATION_SHORT = "ajl";
private static final String CMD_LINE_SYNTAX =
"\n./solrCloudCli.sh --create-collection -z host1:2181,host2:2181/ambari-solr -c collection -cs conf_set"
+ "\n./solrCloudCli.sh --upload-config -z host1:2181,host2:2181/ambari-solr -d /tmp/myconfig_dir -cs config_set"
@@ -72,7 +76,8 @@
+ "\n./solrCloudCli.sh --secure-znode -z host1:2181,host2:2181 -zn /ambari-solr -su logsearch,atlas,ranger --jaas-file /etc/myconf/jaas_file"
+ "\n./solrCloudCli.sh --unsecure-znode -z host1:2181,host2:2181 -zn /ambari-solr --jaas-file /etc/myconf/jaas_file"
+ "\n./solrCloudCli.sh --secure-solr-znode -z host1:2181,host2:2181 -zn /ambari-solr -su logsearch,atlas,ranger --jaas-file /etc/myconf/jaas_file"
- + "\n./solrCloudCli.sh --setup-kerberos-plugin -z host1:2181,host2:2181 -zn /ambari-solr --security-json-location /etc/infra-solr/conf/security.json\n ";
+ + "\n./solrCloudCli.sh --setup-kerberos-plugin -z host1:2181,host2:2181 -zn /ambari-solr --security-json-location /etc/infra-solr/conf/security.json"
+ + "\n./solrCloudCli.sh --" + SET_AUTO_SCALING_COMMAND + " -z host1:2181,host2:2181 -zn /ambari-solr [--" + AUTO_SCALING_JSON_LOCATION + "|--" + AUTO_SCALING_JSON_LOCATION_SHORT + "] /etc/infra-solr/conf/autoscaling.json\n ";
public static void main(String[] args) {
Options options = new Options();
@@ -379,6 +384,18 @@
.desc("Include the number of docs as well in collection dump")
.build();
+ final Option setAutoScaling = Option.builder(SET_AUTO_SCALING_COMMAND_SHORT)
+ .longOpt(SET_AUTO_SCALING_COMMAND)
+ .desc("Upload and set the specified autoscaling.json to znode")
+ .build();
+
+ final Option autoScalingJsonLocationOption = Option.builder(AUTO_SCALING_JSON_LOCATION_SHORT)
+ .longOpt(AUTO_SCALING_JSON_LOCATION)
+ .desc("Local autoscaling.json path")
+ .numberOfArgs(1)
+ .argName("autoscaling.json location")
+ .build();
+
options.addOption(helpOption);
options.addOption(retryOption);
options.addOption(removeAdminHandlerOption);
@@ -427,6 +444,8 @@
options.addOption(securityJsonLocationOption);
options.addOption(outputOption);
options.addOption(includeDocNumberOption);
+ options.addOption(setAutoScaling);
+ options.addOption(autoScalingJsonLocationOption);
AmbariSolrCloudClient solrCloudClient = null;
@@ -487,11 +506,14 @@
} else if (cli.hasOption("dcd")) {
command = DUMP_COLLECTIONS_DATA_COMMAND;
validateRequiredOptions(cli, command, zkConnectStringOption, outputOption);
+ } else if (cli.hasOption(SET_AUTO_SCALING_COMMAND_SHORT)) {
+ command = SET_AUTO_SCALING_COMMAND;
+ validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption, autoScalingJsonLocationOption);
} else {
List<String> commands = Arrays.asList(CREATE_COLLECTION_COMMAND, CREATE_SHARD_COMMAND, UPLOAD_CONFIG_COMMAND,
DOWNLOAD_CONFIG_COMMAND, CONFIG_CHECK_COMMAND, SET_CLUSTER_PROP, CREATE_ZNODE, SECURE_ZNODE_COMMAND, UNSECURE_ZNODE_COMMAND,
SECURE_SOLR_ZNODE_COMMAND, CHECK_ZNODE, SETUP_KERBEROS_PLUGIN, REMOVE_ADMIN_HANDLERS, TRANSFER_ZNODE_COMMAND, DELETE_ZNODE_COMMAND,
- DUMP_COLLECTIONS_DATA_COMMAND);
+ DUMP_COLLECTIONS_DATA_COMMAND, SET_AUTO_SCALING_COMMAND);
helpFormatter.printHelp(CMD_LINE_SYNTAX, options);
exit(1, String.format("One of the supported commands is required (%s)", StringUtils.join(commands, "|")));
}
@@ -527,6 +549,7 @@
String transferMode = cli.hasOption("tm") ? cli.getOptionValue("tm") : "NONE";
String output = cli.hasOption("o") ? cli.getOptionValue("o") : null;
boolean includeDocNumber = cli.hasOption("idn");
+ String autoScalingJsonLocation = cli.hasOption(AUTO_SCALING_JSON_LOCATION_SHORT) ? cli.getOptionValue(AUTO_SCALING_JSON_LOCATION_SHORT) : "";
AmbariSolrCloudClientBuilder clientBuilder = new AmbariSolrCloudClientBuilder()
.withZkConnectString(zkConnectString)
@@ -558,7 +581,8 @@
.withSecurityJsonLocation(securityJsonLocation)
.withZnode(znode)
.withSecure(isSecure)
- .withSaslUsers(saslUsers);
+ .withSaslUsers(saslUsers)
+ .withAutoScalingJsonLocation(autoScalingJsonLocation);
switch (command) {
case CREATE_COLLECTION_COMMAND:
@@ -640,6 +664,10 @@
.withSolrCloudClient().build();
solrCloudClient.outputCollectionData();
break;
+ case SET_AUTO_SCALING_COMMAND:
+ solrCloudClient = clientBuilder.withSolrCloudClient().build();
+ solrCloudClient.setAutoScaling();
+ break;
default:
throw new AmbariSolrCloudClientException(String.format("Not found command: '%s'", command));
}
diff --git a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
index 7571c99..4ef629d 100644
--- a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
+++ b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
@@ -18,7 +18,11 @@
*/
package org.apache.ambari.infra.solr;
+import java.util.Collection;
+import java.util.List;
+
import org.apache.ambari.infra.solr.commands.CheckConfigZkCommand;
+import org.apache.ambari.infra.solr.commands.CheckZnodeZkCommand;
import org.apache.ambari.infra.solr.commands.CreateCollectionCommand;
import org.apache.ambari.infra.solr.commands.CreateShardCommand;
import org.apache.ambari.infra.solr.commands.CreateSolrZnodeZkCommand;
@@ -32,11 +36,11 @@
import org.apache.ambari.infra.solr.commands.RemoveAdminHandlersCommand;
import org.apache.ambari.infra.solr.commands.SecureSolrZNodeZkCommand;
import org.apache.ambari.infra.solr.commands.SecureZNodeZkCommand;
+import org.apache.ambari.infra.solr.commands.SetAutoScalingZkCommand;
import org.apache.ambari.infra.solr.commands.SetClusterPropertyZkCommand;
import org.apache.ambari.infra.solr.commands.TransferZnodeZkCommand;
import org.apache.ambari.infra.solr.commands.UnsecureZNodeZkCommand;
import org.apache.ambari.infra.solr.commands.UploadConfigZkCommand;
-import org.apache.ambari.infra.solr.commands.CheckZnodeZkCommand;
import org.apache.ambari.infra.solr.util.ShardUtils;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.cloud.Slice;
@@ -44,9 +48,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-import java.util.List;
-
/**
* Client for communicate with Solr (and Zookeeper)
*/
@@ -80,6 +81,7 @@
private final String copyDest;
private final String output;
private final boolean includeDocNumber;
+ private final String autoScalingJsonLocation;
public AmbariSolrCloudClient(AmbariSolrCloudClientBuilder builder) {
this.zkConnectString = builder.zkConnectString;
@@ -108,6 +110,7 @@
this.copyDest = builder.copyDest;
this.output = builder.output;
this.includeDocNumber = builder.includeDocNumber;
+ this.autoScalingJsonLocation = builder.autoScalingJsonLocation;
}
/**
@@ -299,6 +302,10 @@
return new DeleteZnodeZkCommand(getRetryTimes(), getInterval()).run(this);
}
+ public void setAutoScaling() throws Exception {
+ new SetAutoScalingZkCommand(getRetryTimes(), getInterval(), autoScalingJsonLocation).run(this);
+ }
+
public String getZkConnectString() {
return zkConnectString;
}
diff --git a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java
index db4396b..834b87a 100644
--- a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java
+++ b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java
@@ -19,6 +19,10 @@
package org.apache.ambari.infra.solr;
+import static java.util.Collections.singletonList;
+
+import java.util.Optional;
+
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -58,7 +62,8 @@
String copySrc;
String copyDest;
String output;
- public boolean includeDocNumber;
+ boolean includeDocNumber;
+ String autoScalingJsonLocation;
public AmbariSolrCloudClient build() {
return new AmbariSolrCloudClient(this);
@@ -131,7 +136,7 @@
}
public AmbariSolrCloudClientBuilder withSolrCloudClient() {
- this.solrCloudClient = new CloudSolrClient.Builder().withZkHost(this.zkConnectString).build();
+ this.solrCloudClient = new CloudSolrClient.Builder(singletonList(this.zkConnectString), Optional.empty()).build();
return this;
}
@@ -243,4 +248,9 @@
System.setProperty(SOLR_HTTPCLIENT_BUILDER_FACTORY, Krb5HttpClientBuilder.class.getCanonicalName());
}
}
+
+ public AmbariSolrCloudClientBuilder withAutoScalingJsonLocation(String autoScalingJsonLocation) {
+ this.autoScalingJsonLocation = autoScalingJsonLocation;
+ return this;
+ }
}
diff --git a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetAutoScalingZkCommand.java b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetAutoScalingZkCommand.java
new file mode 100644
index 0000000..ccf89fe
--- /dev/null
+++ b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetAutoScalingZkCommand.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.solr.commands;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.commons.lang.StringUtils.isBlank;
+
+import java.io.File;
+import java.util.Optional;
+
+import org.apache.ambari.infra.solr.AmbariSolrCloudClient;
+import org.apache.ambari.infra.solr.domain.ZookeeperClient;
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.SolrZooKeeper;
+
+public class SetAutoScalingZkCommand extends AbstractZookeeperRetryCommand<String> {
+ private static final String AUTO_SCALING_JSON = "/autoscaling.json";
+
+ private final String autoScalingJsonLocation;
+
+ public SetAutoScalingZkCommand(int maxRetries, int interval, String autoScalingJsonLocation) {
+ super(maxRetries, interval);
+ this.autoScalingJsonLocation = autoScalingJsonLocation;
+ }
+
+ @Override
+ protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception {
+ if (isBlank(autoScalingJsonLocation))
+ return "";
+
+ File fileToUpload = new File(autoScalingJsonLocation);
+ if (!fileToUpload.exists())
+ return "";
+
+ String contentToUpload = FileUtils.readFileToString(fileToUpload, UTF_8);
+ if (isBlank(contentToUpload))
+ return "";
+
+ String zFilePath = client.getZnode() + AUTO_SCALING_JSON;
+ ZookeeperClient zookeeperClient = new ZookeeperClient(zkClient);
+ Optional<String> fileContent = zookeeperClient.getFileContent(zFilePath);
+ if (!fileContent.isPresent() || !contentToUpload.equals(fileContent.get()))
+ zookeeperClient.putFileContent(zFilePath, contentToUpload);
+
+ return contentToUpload;
+ }
+}
diff --git a/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/domain/ZookeeperClient.java b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/domain/ZookeeperClient.java
new file mode 100644
index 0000000..97be0a1
--- /dev/null
+++ b/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/domain/ZookeeperClient.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.solr.domain;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.zookeeper.CreateMode.PERSISTENT;
+
+import java.util.Optional;
+
+import org.apache.solr.common.cloud.SolrZkClient;
+
+public class ZookeeperClient {
+ private final SolrZkClient zkClient;
+
+ public ZookeeperClient(SolrZkClient zkClient) {
+ this.zkClient = zkClient;
+ }
+
+ public void putFileContent(String fileName, String content) throws Exception {
+ if (zkClient.exists(fileName, true)) {
+ zkClient.setData(fileName, content.getBytes(UTF_8), true);
+ } else {
+ zkClient.create(fileName, content.getBytes(UTF_8), PERSISTENT, true);
+ }
+ }
+
+ public Optional<String> getFileContent(String fileName) throws Exception {
+ if (!zkClient.exists(fileName, true))
+ return Optional.empty();
+
+ byte[] data = zkClient.getData(fileName, null, null, true);
+ return Optional.of(new String(data, UTF_8));
+ }
+}
