solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.solr.cloud;

 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.concurrent.atomic.AtomicReference;

 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.response.CollectionAdminResponse;
 import org.apache.solr.common.util.IOUtils;
 import org.apache.zookeeper.KeeperException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 @LuceneTestCase.Slow
 @LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12386") // "Can't find resource"
 public class ConcurrentCreateRoutedAliasTest extends SolrTestCaseJ4 {

   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

   private MiniSolrCloudCluster solrCluster;

   // to avoid having to delete stuff...
   volatile int num = 0;

   @Override
   @Before
   public void setUp() throws Exception {
     super.setUp();
     solrCluster = new MiniSolrCloudCluster(4, createTempDir(), buildJettyConfig("/solr"));
   }

   @Override
   @After
   public void tearDown() throws Exception {
     solrCluster.shutdown();
     super.tearDown();
   }

   @Test
   public void testConcurrentCreateRoutedAliasMinimal() throws IOException, KeeperException.NoNodeException {
     // this is the test where be blow out a bunch of create commands all out at once.
     // other tests are more functionality based, and just use a single thread.

     // Failure of this test very occasionally due to overseer overload would not be worrisome (just bothersome).
     // Any use case creating large numbers of time routed aliases concurrently would be an EXTREMELY odd
     // if not fundamentally broken use case. This test method is just here to guard against any race
     // conditions in the code that could crop up rarely in lower volume usage.

     // That said any failures involving about NPE's or missing parameters or oddities other than overwhelming
     // the overseer queue with retry races emanating from this test should be investigated. Also if it fails
     // frequently that needs to be investigated of course.


     final AtomicReference<Exception> failure = new AtomicReference<>();

     // Note: this number of threads seems to work regularly with the up-tweaked number of retries (50) in
     // org.apache.solr.common.cloud.ZkStateReader.AliasesManager.applyModificationAndExportToZk()
     // with the original 5 retries this wouldn't reliably pass with 10 threads, but with 50 retries it seems
     // to handle 50 threads about a dozen times without any failure (on a 32 thread processor)
     // it also passed 3/3 at 150 threads and 2/3 with 250 threads on both 1 node and 4 nodes...
     // the failure mode seems to be overseer tasks that are not found. I suspect this happens when enough
     // threads get into retry races and the spam overwhelms the overseer. (that this can happen might imply
     // an issue over there, but I'm not sure, since there is an intentional hard limit on the overseer queue
     // and I haven't tried to count the retries up and figure out if the requests are actually exceeding that
     // limit or not, but the speed of retries might indicate an effectively hot loop, but again, a separate issue.

     // The hope is that the level of concurrency supported by create routed alias and the code it uses is such
     // that this test wouldn't spuriously fail more than once a year. If that's true users should never see
     // an issue in the wild unless they are doing something we probably don't want to support anyway

     final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[4];
     int numStart = num;
     for (; num < threads.length + numStart; num++) {
       final String aliasName = "testAlias" + num;
       final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString();
       final SolrClient solrClient = getHttpSolrClient(baseUrl);


       int i = num - numStart;
       threads[i] = new CreateRoutedAliasThread("create-delete-search-" + i, aliasName, "NOW/HOUR",
           solrClient, failure, false);
     }

     startAll(threads);
     joinAll(threads);

     assertNull("concurrent alias creation failed " + failure.get(), failure.get());
   }


   @Test
   public void testConcurrentCreateRoutedAliasComplex() {
     final AtomicReference<Exception> failure = new AtomicReference<>();

     final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[1];
     int numStart = num;
     System.out.println("NUM ==> " +num);
     for (; num < threads.length + numStart; num++) {
       final String aliasName = "testAliasCplx" + num;
       final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString();
       final SolrClient solrClient = getHttpSolrClient(baseUrl);

       int i = num - numStart;
       threads[i] = new CreateRoutedAliasThread("create-routed-alias-cplx-" + i,
           aliasName, "2017-12-25T23:24:25Z",
           solrClient, failure, true);
     }

     startAll(threads);
     joinAll(threads);

     assertNull("concurrent alias creation failed " + failure.get(), failure.get());
   }

   private void joinAll(final CreateRoutedAliasThread[] threads) {
     for (CreateRoutedAliasThread t : threads) {
       try {
         t.joinAndClose();
       } catch (InterruptedException e) {
         Thread.interrupted();
         throw new RuntimeException(e);
       }
     }
   }

   private void startAll(final Thread[] threads) {
     for (Thread t : threads) {
       t.start();
     }
   }

   private static class CreateRoutedAliasThread extends Thread {
     final String aliasName;
     protected final String start;
     protected final SolrClient solrClient;
     protected final AtomicReference<Exception> failure;

     CreateRoutedAliasThread(
         String name, String aliasName, String start, SolrClient solrClient,
         AtomicReference<Exception> failure, boolean v2) {
       super(name);
       this.aliasName = aliasName;
       this.start = start;
       this.solrClient = solrClient;
       this.failure = failure;
     }

     @Override
     public void run() {
         doWork();
     }

     void doWork() {
       createAlias();
     }

     void addFailure(Exception e) {
       log.error("Add Failure", e);
       synchronized (failure) {
         if (failure.get() != null) {
           failure.get().addSuppressed(e);
         } else {
           failure.set(e);
         }
       }
     }

     private void createAlias() {
       try {
         CollectionAdminRequest.CreateTimeRoutedAlias rq = CollectionAdminRequest
             .createTimeRoutedAlias(
                 aliasName,
                 start,
                 "+12HOUR",
                 "routedFoo_dt",
                 CollectionAdminRequest.createCollection("_ignored_", "_default", 1, 1)
             );

         final CollectionAdminResponse response = rq.process(solrClient);
         if (response.getStatus() != 0) {
           addFailure(new RuntimeException("failed to create collection " + aliasName));
         }
       } catch (Exception e) {
         addFailure(e);
       }

     }


     void joinAndClose() throws InterruptedException {
       try {
         super.join(60000);
       } finally {
         IOUtils.closeQuietly(solrClient);
       }
     }
   }


 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.solr.cloud;

	import java.io.IOException;
	import java.lang.invoke.MethodHandles;
	import java.util.concurrent.atomic.AtomicReference;

	import org.apache.lucene.util.LuceneTestCase;
	import org.apache.solr.SolrTestCaseJ4;
	import org.apache.solr.client.solrj.SolrClient;
	import org.apache.solr.client.solrj.request.CollectionAdminRequest;
	import org.apache.solr.client.solrj.response.CollectionAdminResponse;
	import org.apache.solr.common.util.IOUtils;
	import org.apache.zookeeper.KeeperException;
	import org.junit.After;
	import org.junit.Before;
	import org.junit.Test;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	@LuceneTestCase.Slow
	@LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12386") // "Can't find resource"
	public class ConcurrentCreateRoutedAliasTest extends SolrTestCaseJ4 {

	private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

	private MiniSolrCloudCluster solrCluster;

	// to avoid having to delete stuff...
	volatile int num = 0;

	@Override
	@Before
	public void setUp() throws Exception {
	super.setUp();
	solrCluster = new MiniSolrCloudCluster(4, createTempDir(), buildJettyConfig("/solr"));
	}

	@Override
	@After
	public void tearDown() throws Exception {
	solrCluster.shutdown();
	super.tearDown();
	}

	@Test
	public void testConcurrentCreateRoutedAliasMinimal() throws IOException, KeeperException.NoNodeException {
	// this is the test where be blow out a bunch of create commands all out at once.
	// other tests are more functionality based, and just use a single thread.

	// Failure of this test very occasionally due to overseer overload would not be worrisome (just bothersome).
	// Any use case creating large numbers of time routed aliases concurrently would be an EXTREMELY odd
	// if not fundamentally broken use case. This test method is just here to guard against any race
	// conditions in the code that could crop up rarely in lower volume usage.

	// That said any failures involving about NPE's or missing parameters or oddities other than overwhelming
	// the overseer queue with retry races emanating from this test should be investigated. Also if it fails
	// frequently that needs to be investigated of course.


	final AtomicReference<Exception> failure = new AtomicReference<>();

	// Note: this number of threads seems to work regularly with the up-tweaked number of retries (50) in
	// org.apache.solr.common.cloud.ZkStateReader.AliasesManager.applyModificationAndExportToZk()
	// with the original 5 retries this wouldn't reliably pass with 10 threads, but with 50 retries it seems
	// to handle 50 threads about a dozen times without any failure (on a 32 thread processor)
	// it also passed 3/3 at 150 threads and 2/3 with 250 threads on both 1 node and 4 nodes...
	// the failure mode seems to be overseer tasks that are not found. I suspect this happens when enough
	// threads get into retry races and the spam overwhelms the overseer. (that this can happen might imply
	// an issue over there, but I'm not sure, since there is an intentional hard limit on the overseer queue
	// and I haven't tried to count the retries up and figure out if the requests are actually exceeding that
	// limit or not, but the speed of retries might indicate an effectively hot loop, but again, a separate issue.

	// The hope is that the level of concurrency supported by create routed alias and the code it uses is such
	// that this test wouldn't spuriously fail more than once a year. If that's true users should never see
	// an issue in the wild unless they are doing something we probably don't want to support anyway

	final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[4];
	int numStart = num;
	for (; num < threads.length + numStart; num++) {
	final String aliasName = "testAlias" + num;
	final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString();
	final SolrClient solrClient = getHttpSolrClient(baseUrl);


	int i = num - numStart;
	threads[i] = new CreateRoutedAliasThread("create-delete-search-" + i, aliasName, "NOW/HOUR",
	solrClient, failure, false);
	}

	startAll(threads);
	joinAll(threads);

	assertNull("concurrent alias creation failed " + failure.get(), failure.get());
	}


	@Test
	public void testConcurrentCreateRoutedAliasComplex() {
	final AtomicReference<Exception> failure = new AtomicReference<>();

	final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[1];
	int numStart = num;
	System.out.println("NUM ==> " +num);
	for (; num < threads.length + numStart; num++) {
	final String aliasName = "testAliasCplx" + num;
	final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString();
	final SolrClient solrClient = getHttpSolrClient(baseUrl);

	int i = num - numStart;
	threads[i] = new CreateRoutedAliasThread("create-routed-alias-cplx-" + i,
	aliasName, "2017-12-25T23:24:25Z",
	solrClient, failure, true);
	}

	startAll(threads);
	joinAll(threads);

	assertNull("concurrent alias creation failed " + failure.get(), failure.get());
	}

	private void joinAll(final CreateRoutedAliasThread[] threads) {
	for (CreateRoutedAliasThread t : threads) {
	try {
	t.joinAndClose();
	} catch (InterruptedException e) {
	Thread.interrupted();
	throw new RuntimeException(e);
	}
	}
	}

	private void startAll(final Thread[] threads) {
	for (Thread t : threads) {
	t.start();
	}
	}

	private static class CreateRoutedAliasThread extends Thread {
	final String aliasName;
	protected final String start;
	protected final SolrClient solrClient;
	protected final AtomicReference<Exception> failure;

	CreateRoutedAliasThread(
	String name, String aliasName, String start, SolrClient solrClient,
	AtomicReference<Exception> failure, boolean v2) {
	super(name);
	this.aliasName = aliasName;
	this.start = start;
	this.solrClient = solrClient;
	this.failure = failure;
	}

	@Override
	public void run() {
	doWork();
	}

	void doWork() {
	createAlias();
	}

	void addFailure(Exception e) {
	log.error("Add Failure", e);
	synchronized (failure) {
	if (failure.get() != null) {
	failure.get().addSuppressed(e);
	} else {
	failure.set(e);
	}
	}
	}

	private void createAlias() {
	try {
	CollectionAdminRequest.CreateTimeRoutedAlias rq = CollectionAdminRequest
	.createTimeRoutedAlias(
	aliasName,
	start,
	"+12HOUR",
	"routedFoo_dt",
	CollectionAdminRequest.createCollection("_ignored_", "_default", 1, 1)
	);

	final CollectionAdminResponse response = rq.process(solrClient);
	if (response.getStatus() != 0) {
	addFailure(new RuntimeException("failed to create collection " + aliasName));
	}
	} catch (Exception e) {
	addFailure(e);
	}

	}


	void joinAndClose() throws InterruptedException {
	try {
	super.join(60000);
	} finally {
	IOUtils.closeQuietly(solrClient);
	}
	}
	}


	}