solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.solr.cloud;

 import java.io.File;
 import java.io.FilenameFilter;
 import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;

 import org.apache.commons.cli.CommandLine;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.util.EntityUtils;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.request.StreamingUpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.util.ExternalPaths;
 import org.apache.solr.util.SolrCLI;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import static java.util.Arrays.asList;
 import static org.apache.solr.common.util.Utils.fromJSONString;
 import static org.apache.solr.common.util.Utils.getObjectByPath;

 /**
  * Emulates bin/solr -e cloud -noprompt; bin/post -c gettingstarted example/exampledocs/*.xml;
  * this test is useful for catching regressions in indexing the example docs in collections that
  * use data driven functionality and managed schema features of the default configset
  * (configsets/_default).
  */
 public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {

   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

   public SolrCloudExampleTest() {
     super();
     sliceCount = 2;
   }

   @Test
   // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
   public void testLoadDocsIntoGettingStartedCollection() throws Exception {
     waitForThingsToLevelOut(30000);

     log.info("testLoadDocsIntoGettingStartedCollection initialized OK ... running test logic");

     String testCollectionName = "gettingstarted";
     File defaultConfigs = new File(ExternalPaths.DEFAULT_CONFIGSET);
     assertTrue(defaultConfigs.getAbsolutePath()+" not found!", defaultConfigs.isDirectory());

     Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
     if (liveNodes.isEmpty())
       fail("No live nodes found! Cannot create a collection until there is at least 1 live node in the cluster.");
     String firstLiveNode = liveNodes.iterator().next();
     String solrUrl = cloudClient.getZkStateReader().getBaseUrlForNodeName(firstLiveNode);

     // create the gettingstarted collection just like the bin/solr script would do
     String[] args = new String[] {
         "-name", testCollectionName,
         "-shards", "2",
         "-replicationFactor", "2",
         "-confname", testCollectionName,
         "-confdir", "_default",
         "-configsetsDir", defaultConfigs.getParentFile().getParentFile().getAbsolutePath(),
         "-solrUrl", solrUrl
     };

     // NOTE: not calling SolrCLI.main as the script does because it calls System.exit which is a no-no in a JUnit test

     SolrCLI.CreateCollectionTool tool = new SolrCLI.CreateCollectionTool();
     CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
     log.info("Creating the '{}' collection using SolrCLI with: {}", testCollectionName, solrUrl);
     tool.runTool(cli);
     assertTrue("Collection '" + testCollectionName + "' doesn't exist after trying to create it!",
         cloudClient.getZkStateReader().getClusterState().hasCollection(testCollectionName));

     // verify the collection is usable ...
     ensureAllReplicasAreActive(testCollectionName, "shard1", 2, 2, 20);
     ensureAllReplicasAreActive(testCollectionName, "shard2", 2, 2, 10);
     cloudClient.setDefaultCollection(testCollectionName);

     int invalidToolExitStatus = 1;
     assertEquals("Collection '" + testCollectionName + "' created even though it already existed",
         invalidToolExitStatus, tool.runTool(cli));

     // now index docs like bin/post would do but we can't use SimplePostTool because it uses System.exit when
     // it encounters an error, which JUnit doesn't like ...
     log.info("Created collection, now posting example docs!");
     File exampleDocsDir = new File(ExternalPaths.SOURCE_HOME, "example/exampledocs");
     assertTrue(exampleDocsDir.getAbsolutePath()+" not found!", exampleDocsDir.isDirectory());

     List<File> xmlFiles = Arrays.asList(exampleDocsDir.listFiles(new FilenameFilter() {
       @Override
       public boolean accept(File dir, String name) {
         return name.endsWith(".xml");
       }
     }));

     // force a deterministic random ordering of the files so seeds reproduce regardless of platform/filesystem
     Collections.sort(xmlFiles, (o1, o2) -> {
       // don't rely on File.compareTo, it's behavior varies by OS
       return o1.getName().compareTo(o2.getName());
     });
     Collections.shuffle(xmlFiles, new Random(random().nextLong()));

     // if you add/remove example XML docs, you'll have to fix these expected values
     int expectedXmlFileCount = 14;
     int expectedXmlDocCount = 32;

     assertEquals("Unexpected # of example XML files in "+exampleDocsDir.getAbsolutePath(),
                  expectedXmlFileCount, xmlFiles.size());

     for (File xml : xmlFiles) {
       if (log.isInfoEnabled()) {
         log.info("POSTing {}", xml.getAbsolutePath());
       }
       cloudClient.request(new StreamingUpdateRequest("/update",xml,"application/xml"));
     }
     cloudClient.commit();

     int numFound = 0;

     // give the update a chance to take effect.
     for (int idx = 0; idx < 100; ++idx) {
       QueryResponse qr = cloudClient.query(new SolrQuery("*:*"));
       numFound = (int) qr.getResults().getNumFound();
       if (numFound == expectedXmlDocCount) break;
       Thread.sleep(100);
     }
     assertEquals("*:* found unexpected number of documents", expectedXmlDocCount, numFound);

     log.info("Updating Config for {}", testCollectionName);
     doTestConfigUpdate(testCollectionName, solrUrl);

     log.info("Running healthcheck for {}", testCollectionName);
     doTestHealthcheck(testCollectionName, cloudClient.getZkHost());

     // verify the delete action works too
     log.info("Running delete for {}", testCollectionName);
     doTestDeleteAction(testCollectionName, solrUrl);

     log.info("testLoadDocsIntoGettingStartedCollection succeeded ... shutting down now!");
   }

   protected void doTestHealthcheck(String testCollectionName, String zkHost) throws Exception {
     String[] args = new String[]{
         "-collection", testCollectionName,
         "-zkHost", zkHost
     };
     SolrCLI.HealthcheckTool tool = new SolrCLI.HealthcheckTool();
     CommandLine cli =
         SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
     assertTrue("Healthcheck action failed!", tool.runTool(cli) == 0);
   }

   protected void doTestDeleteAction(String testCollectionName, String solrUrl) throws Exception {
     String[] args = new String[] {
         "-name", testCollectionName,
         "-solrUrl", solrUrl
     };
     SolrCLI.DeleteTool tool = new SolrCLI.DeleteTool();
     CommandLine cli =
         SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
     assertTrue("Delete action failed!", tool.runTool(cli) == 0);
     assertTrue(!SolrCLI.safeCheckCollectionExists(solrUrl, testCollectionName)); // it should not exist anymore
   }

   /**
    * Uses the SolrCLI config action to activate soft auto-commits for the getting started collection.
    */
   protected void doTestConfigUpdate(String testCollectionName, String solrUrl) throws Exception {
     if (!solrUrl.endsWith("/"))
       solrUrl += "/";
     String configUrl = solrUrl + testCollectionName + "/config";

     Map<String, Object> configJson = SolrCLI.getJson(configUrl);
     Object maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson);
     assertNotNull(maxTimeFromConfig);
     assertEquals(-1L, maxTimeFromConfig);

     String prop = "updateHandler.autoSoftCommit.maxTime";
     Long maxTime = 3000L;
     String[] args = new String[]{
         "-collection", testCollectionName,
         "-property", prop,
         "-value", maxTime.toString(),
         "-solrUrl", solrUrl
     };

     Map<String, Long> startTimes = getSoftAutocommitInterval(testCollectionName);

     SolrCLI.ConfigTool tool = new SolrCLI.ConfigTool();
     CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
     log.info("Sending set-property '{}'={} to SolrCLI.ConfigTool.", prop, maxTime);
     assertTrue("Set config property failed!", tool.runTool(cli) == 0);

     configJson = SolrCLI.getJson(configUrl);
     maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson);
     assertNotNull(maxTimeFromConfig);
     assertEquals(maxTime, maxTimeFromConfig);

     // Just check that we can access paths with slashes in them both through an intermediate method and explicitly
     // using atPath.
     assertEquals("Should have been able to get a value from the /query request handler",
         "explicit", SolrCLI.asString("/config/requestHandler/\\/query/defaults/echoParams", configJson));

     assertEquals("Should have been able to get a value from the /query request handler",
         "explicit", SolrCLI.atPath("/config/requestHandler/\\/query/defaults/echoParams", configJson));

     if (log.isInfoEnabled()) {
       log.info("live_nodes_count :  {}", cloudClient.getZkStateReader().getClusterState().getLiveNodes());
     }

     // Since it takes some time for this command to complete we need to make sure all the reloads for
     // all the cores have been done.
     boolean allGood = false;
     Map<String, Long> curSoftCommitInterval = null;
     for (int idx = 0; idx < 600 && allGood == false; ++idx) {
       curSoftCommitInterval = getSoftAutocommitInterval(testCollectionName);
       if (curSoftCommitInterval.size() > 0 && curSoftCommitInterval.size() == startTimes.size()) { // no point in even trying if they're not the same size!
         allGood = true;
         for (Map.Entry<String, Long> currEntry : curSoftCommitInterval.entrySet()) {
           if (currEntry.getValue().equals(maxTime) == false) {
             allGood = false;
           }
         }
       }
       if (allGood == false) {
         Thread.sleep(100);
       }
     }
     assertTrue("All cores should have been reloaded within 60 seconds!!!", allGood);
   }

   // Collect all of the autoSoftCommit intervals.
   private Map<String, Long> getSoftAutocommitInterval(String collection) throws Exception {
     Map<String, Long> ret = new HashMap<>();
     DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection(collection);
     for (Slice slice : coll.getActiveSlices()) {
       for (Replica replica : slice.getReplicas()) {
         String uri = "" + replica.get(ZkStateReader.BASE_URL_PROP) + "/" + replica.get(ZkStateReader.CORE_NAME_PROP) + "/config";
         @SuppressWarnings({"rawtypes"})
         Map respMap = getAsMap(cloudClient, uri);
         Long maxTime = (Long) (getObjectByPath(respMap, true, asList("config", "updateHandler", "autoSoftCommit", "maxTime")));
         ret.put(replica.getCoreName(), maxTime);
       }
     }
     return ret;
   }

   @SuppressWarnings({"rawtypes"})
   private Map getAsMap(CloudSolrClient cloudClient, String uri) throws Exception {
     HttpGet get = new HttpGet(uri);
     HttpEntity entity = null;
     try {
       entity = cloudClient.getLbClient().getHttpClient().execute(get).getEntity();
       String response = EntityUtils.toString(entity, StandardCharsets.UTF_8);
       return (Map) fromJSONString(response);
     } finally {
       EntityUtils.consumeQuietly(entity);
     }
   }

 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.solr.cloud;

	import java.io.File;
	import java.io.FilenameFilter;
	import java.lang.invoke.MethodHandles;
	import java.nio.charset.StandardCharsets;
	import java.util.Arrays;
	import java.util.Collections;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	import java.util.Random;
	import java.util.Set;

	import org.apache.commons.cli.CommandLine;
	import org.apache.http.HttpEntity;
	import org.apache.http.client.methods.HttpGet;
	import org.apache.http.util.EntityUtils;
	import org.apache.solr.client.solrj.SolrQuery;
	import org.apache.solr.client.solrj.impl.CloudSolrClient;
	import org.apache.solr.client.solrj.request.StreamingUpdateRequest;
	import org.apache.solr.client.solrj.response.QueryResponse;
	import org.apache.solr.common.cloud.DocCollection;
	import org.apache.solr.common.cloud.Replica;
	import org.apache.solr.common.cloud.Slice;
	import org.apache.solr.common.cloud.ZkStateReader;
	import org.apache.solr.util.ExternalPaths;
	import org.apache.solr.util.SolrCLI;
	import org.junit.Test;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	import static java.util.Arrays.asList;
	import static org.apache.solr.common.util.Utils.fromJSONString;
	import static org.apache.solr.common.util.Utils.getObjectByPath;

	/**
	* Emulates bin/solr -e cloud -noprompt; bin/post -c gettingstarted example/exampledocs/*.xml;
	* this test is useful for catching regressions in indexing the example docs in collections that
	* use data driven functionality and managed schema features of the default configset
	* (configsets/_default).
	*/
	public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {

	private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

	public SolrCloudExampleTest() {
	super();
	sliceCount = 2;
	}

	@Test
	// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
	public void testLoadDocsIntoGettingStartedCollection() throws Exception {
	waitForThingsToLevelOut(30000);

	log.info("testLoadDocsIntoGettingStartedCollection initialized OK ... running test logic");

	String testCollectionName = "gettingstarted";
	File defaultConfigs = new File(ExternalPaths.DEFAULT_CONFIGSET);
	assertTrue(defaultConfigs.getAbsolutePath()+" not found!", defaultConfigs.isDirectory());

	Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
	if (liveNodes.isEmpty())
	fail("No live nodes found! Cannot create a collection until there is at least 1 live node in the cluster.");
	String firstLiveNode = liveNodes.iterator().next();
	String solrUrl = cloudClient.getZkStateReader().getBaseUrlForNodeName(firstLiveNode);

	// create the gettingstarted collection just like the bin/solr script would do
	String[] args = new String[] {
	"-name", testCollectionName,
	"-shards", "2",
	"-replicationFactor", "2",
	"-confname", testCollectionName,
	"-confdir", "_default",
	"-configsetsDir", defaultConfigs.getParentFile().getParentFile().getAbsolutePath(),
	"-solrUrl", solrUrl
	};

	// NOTE: not calling SolrCLI.main as the script does because it calls System.exit which is a no-no in a JUnit test

	SolrCLI.CreateCollectionTool tool = new SolrCLI.CreateCollectionTool();
	CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
	log.info("Creating the '{}' collection using SolrCLI with: {}", testCollectionName, solrUrl);
	tool.runTool(cli);
	assertTrue("Collection '" + testCollectionName + "' doesn't exist after trying to create it!",
	cloudClient.getZkStateReader().getClusterState().hasCollection(testCollectionName));

	// verify the collection is usable ...
	ensureAllReplicasAreActive(testCollectionName, "shard1", 2, 2, 20);
	ensureAllReplicasAreActive(testCollectionName, "shard2", 2, 2, 10);
	cloudClient.setDefaultCollection(testCollectionName);

	int invalidToolExitStatus = 1;
	assertEquals("Collection '" + testCollectionName + "' created even though it already existed",
	invalidToolExitStatus, tool.runTool(cli));

	// now index docs like bin/post would do but we can't use SimplePostTool because it uses System.exit when
	// it encounters an error, which JUnit doesn't like ...
	log.info("Created collection, now posting example docs!");
	File exampleDocsDir = new File(ExternalPaths.SOURCE_HOME, "example/exampledocs");
	assertTrue(exampleDocsDir.getAbsolutePath()+" not found!", exampleDocsDir.isDirectory());

	List<File> xmlFiles = Arrays.asList(exampleDocsDir.listFiles(new FilenameFilter() {
	@Override
	public boolean accept(File dir, String name) {
	return name.endsWith(".xml");
	}
	}));

	// force a deterministic random ordering of the files so seeds reproduce regardless of platform/filesystem
	Collections.sort(xmlFiles, (o1, o2) -> {
	// don't rely on File.compareTo, it's behavior varies by OS
	return o1.getName().compareTo(o2.getName());
	});
	Collections.shuffle(xmlFiles, new Random(random().nextLong()));

	// if you add/remove example XML docs, you'll have to fix these expected values
	int expectedXmlFileCount = 14;
	int expectedXmlDocCount = 32;

	assertEquals("Unexpected # of example XML files in "+exampleDocsDir.getAbsolutePath(),
	expectedXmlFileCount, xmlFiles.size());

	for (File xml : xmlFiles) {
	if (log.isInfoEnabled()) {
	log.info("POSTing {}", xml.getAbsolutePath());
	}
	cloudClient.request(new StreamingUpdateRequest("/update",xml,"application/xml"));
	}
	cloudClient.commit();

	int numFound = 0;

	// give the update a chance to take effect.
	for (int idx = 0; idx < 100; ++idx) {
	QueryResponse qr = cloudClient.query(new SolrQuery(":"));
	numFound = (int) qr.getResults().getNumFound();
	if (numFound == expectedXmlDocCount) break;
	Thread.sleep(100);
	}
	assertEquals(": found unexpected number of documents", expectedXmlDocCount, numFound);

	log.info("Updating Config for {}", testCollectionName);
	doTestConfigUpdate(testCollectionName, solrUrl);

	log.info("Running healthcheck for {}", testCollectionName);
	doTestHealthcheck(testCollectionName, cloudClient.getZkHost());

	// verify the delete action works too
	log.info("Running delete for {}", testCollectionName);
	doTestDeleteAction(testCollectionName, solrUrl);

	log.info("testLoadDocsIntoGettingStartedCollection succeeded ... shutting down now!");
	}

	protected void doTestHealthcheck(String testCollectionName, String zkHost) throws Exception {
	String[] args = new String[]{
	"-collection", testCollectionName,
	"-zkHost", zkHost
	};
	SolrCLI.HealthcheckTool tool = new SolrCLI.HealthcheckTool();
	CommandLine cli =
	SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
	assertTrue("Healthcheck action failed!", tool.runTool(cli) == 0);
	}

	protected void doTestDeleteAction(String testCollectionName, String solrUrl) throws Exception {
	String[] args = new String[] {
	"-name", testCollectionName,
	"-solrUrl", solrUrl
	};
	SolrCLI.DeleteTool tool = new SolrCLI.DeleteTool();
	CommandLine cli =
	SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
	assertTrue("Delete action failed!", tool.runTool(cli) == 0);
	assertTrue(!SolrCLI.safeCheckCollectionExists(solrUrl, testCollectionName)); // it should not exist anymore
	}

	/**
	* Uses the SolrCLI config action to activate soft auto-commits for the getting started collection.
	*/
	protected void doTestConfigUpdate(String testCollectionName, String solrUrl) throws Exception {
	if (!solrUrl.endsWith("/"))
	solrUrl += "/";
	String configUrl = solrUrl + testCollectionName + "/config";

	Map<String, Object> configJson = SolrCLI.getJson(configUrl);
	Object maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson);
	assertNotNull(maxTimeFromConfig);
	assertEquals(-1L, maxTimeFromConfig);

	String prop = "updateHandler.autoSoftCommit.maxTime";
	Long maxTime = 3000L;
	String[] args = new String[]{
	"-collection", testCollectionName,
	"-property", prop,
	"-value", maxTime.toString(),
	"-solrUrl", solrUrl
	};

	Map<String, Long> startTimes = getSoftAutocommitInterval(testCollectionName);

	SolrCLI.ConfigTool tool = new SolrCLI.ConfigTool();
	CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
	log.info("Sending set-property '{}'={} to SolrCLI.ConfigTool.", prop, maxTime);
	assertTrue("Set config property failed!", tool.runTool(cli) == 0);

	configJson = SolrCLI.getJson(configUrl);
	maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson);
	assertNotNull(maxTimeFromConfig);
	assertEquals(maxTime, maxTimeFromConfig);

	// Just check that we can access paths with slashes in them both through an intermediate method and explicitly
	// using atPath.
	assertEquals("Should have been able to get a value from the /query request handler",
	"explicit", SolrCLI.asString("/config/requestHandler/\\/query/defaults/echoParams", configJson));

	assertEquals("Should have been able to get a value from the /query request handler",
	"explicit", SolrCLI.atPath("/config/requestHandler/\\/query/defaults/echoParams", configJson));

	if (log.isInfoEnabled()) {
	log.info("live_nodes_count : {}", cloudClient.getZkStateReader().getClusterState().getLiveNodes());
	}

	// Since it takes some time for this command to complete we need to make sure all the reloads for
	// all the cores have been done.
	boolean allGood = false;
	Map<String, Long> curSoftCommitInterval = null;
	for (int idx = 0; idx < 600 && allGood == false; ++idx) {
	curSoftCommitInterval = getSoftAutocommitInterval(testCollectionName);
	if (curSoftCommitInterval.size() > 0 && curSoftCommitInterval.size() == startTimes.size()) { // no point in even trying if they're not the same size!
	allGood = true;
	for (Map.Entry<String, Long> currEntry : curSoftCommitInterval.entrySet()) {
	if (currEntry.getValue().equals(maxTime) == false) {
	allGood = false;
	}
	}
	}
	if (allGood == false) {
	Thread.sleep(100);
	}
	}
	assertTrue("All cores should have been reloaded within 60 seconds!!!", allGood);
	}

	// Collect all of the autoSoftCommit intervals.
	private Map<String, Long> getSoftAutocommitInterval(String collection) throws Exception {
	Map<String, Long> ret = new HashMap<>();
	DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection(collection);
	for (Slice slice : coll.getActiveSlices()) {
	for (Replica replica : slice.getReplicas()) {
	String uri = "" + replica.get(ZkStateReader.BASE_URL_PROP) + "/" + replica.get(ZkStateReader.CORE_NAME_PROP) + "/config";
	@SuppressWarnings({"rawtypes"})
	Map respMap = getAsMap(cloudClient, uri);
	Long maxTime = (Long) (getObjectByPath(respMap, true, asList("config", "updateHandler", "autoSoftCommit", "maxTime")));
	ret.put(replica.getCoreName(), maxTime);
	}
	}
	return ret;
	}

	@SuppressWarnings({"rawtypes"})
	private Map getAsMap(CloudSolrClient cloudClient, String uri) throws Exception {
	HttpGet get = new HttpGet(uri);
	HttpEntity entity = null;
	try {
	entity = cloudClient.getLbClient().getHttpClient().execute(get).getEntity();
	String response = EntityUtils.toString(entity, StandardCharsets.UTF_8);
	return (Map) fromJSONString(response);
	} finally {
	EntityUtils.consumeQuietly(entity);
	}
	}

	}