solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java - manifoldcf-integration-solr-3.x - Git at Google

 package org.apache.solr;

 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 import junit.framework.TestCase;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.schema.TrieDateField;
 import org.apache.solr.util.AbstractSolrTestCase;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import java.io.File;
 import java.io.IOException;
 import java.util.*;

 /**
  * Helper base class for distributed search test cases
  *
  * @since solr 1.5
  */
 public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
   public static Random r = random;

   protected int shardCount = 4;
   /**
    * Sub classes can set this flag in their constructor to true if they
    * want to fix the number of shards to 'shardCount'
    *
    * The default is false which means that test will be executed with
    * 1, 2, 3, ....shardCount number of shards repeatedly
    */
   protected boolean fixShardCount = false;

   protected JettySolrRunner controlJetty;
   protected List<SolrServer> clients = new ArrayList<SolrServer>();
   protected List<JettySolrRunner> jettys = new ArrayList<JettySolrRunner>();
   protected String context = "/solr";
   protected String shards;
   protected File testDir;
   protected SolrServer controlClient;

   // to stress with higher thread counts and requests, make sure the junit
   // xml formatter is not being used (all output will be buffered before
   // transformation to xml and cause an OOM exception).
   protected int stress = 2;
   protected boolean verifyStress = true;
   protected int nThreads = 3;


   public static int ORDERED = 1;
   public static int SKIP = 2;
   public static int SKIPVAL = 4;
   public static int UNORDERED = 8;

   protected int flags;
   protected Map<String, Integer> handle = new HashMap<String, Integer>();

   protected String id = "id";
   public static Logger log = LoggerFactory.getLogger(BaseDistributedSearchTestCase.class);

   public static RandVal rint = new RandVal() {
     @Override
     public Object val() {
       return r.nextInt();
     }
   };

   public static RandVal rlong = new RandVal() {
     @Override
     public Object val() {
       return r.nextLong();
     }
   };

   public static RandVal rfloat = new RandVal() {
     @Override
     public Object val() {
       return r.nextFloat();
     }
   };

   public static RandVal rdouble = new RandVal() {
     @Override
     public Object val() {
       return r.nextDouble();
     }
   };

   public static RandVal rdate = new RandDate();

   /**
    * Perform the actual tests here
    *
    * @throws Exception on error
    */
   public abstract void doTest() throws Exception;

   public static String[] fieldNames = new String[]{"n_ti1", "n_f1", "n_tf1", "n_d1", "n_td1", "n_l1", "n_tl1", "n_dt1", "n_tdt1"};
   public static RandVal[] randVals = new RandVal[]{rint, rfloat, rfloat, rdouble, rdouble, rlong, rlong, rdate, rdate};

   protected String[] getFieldNames() {
     return fieldNames;
   }

   protected RandVal[] getRandValues() {
     return randVals;
   }

   /**
    * Subclasses can override this to change a test's solr home
    * (default is in test-files)
    */
   public String getSolrHome() {
     return SolrTestCaseJ4.TEST_HOME();
   }

   @Override
   public void setUp() throws Exception {
     SolrTestCaseJ4.resetExceptionIgnores();  // ignore anything with ignore_exception in it
     super.setUp();
     System.setProperty("solr.test.sys.prop1", "propone");
     System.setProperty("solr.test.sys.prop2", "proptwo");
     System.setProperty("solr.solr.home", getSolrHome());
     testDir = new File(TEMP_DIR,
             getClass().getName() + "-" + System.currentTimeMillis());
     testDir.mkdirs();
   }

   @Override
   public void tearDown() throws Exception {
     destroyServers();
     if (!AbstractSolrTestCase.recurseDelete(testDir)) {
       System.err.println("!!!! WARNING: best effort to remove " + testDir.getAbsolutePath() + " FAILED !!!!!");
     }
     super.tearDown();
   }

   private void createServers(int numShards) throws Exception {
     controlJetty = createJetty(testDir, "control");
     controlClient = createNewSolrServer(controlJetty.getLocalPort());

     StringBuilder sb = new StringBuilder();
     for (int i = 1; i <= numShards; i++) {
       if (sb.length() > 0) sb.append(',');
       JettySolrRunner j = createJetty(testDir, "shard" + i);
       jettys.add(j);
       clients.add(createNewSolrServer(j.getLocalPort()));
       sb.append("localhost:").append(j.getLocalPort()).append(context);
     }

     shards = sb.toString();
   }

   protected void destroyServers() throws Exception {
     controlJetty.stop();
     for (JettySolrRunner jetty : jettys) jetty.stop();
     clients.clear();
     jettys.clear();
   }

   public static JettySolrRunner createJetty(File baseDir, String dataDirName) throws Exception {
     File subDir = new File(baseDir, dataDirName);
     subDir.mkdirs();
     System.setProperty("solr.data.dir", subDir.toString());

     JettySolrRunner jetty = new JettySolrRunner("/solr", 0);

     jetty.start();
     return jetty;
   }

   protected SolrServer createNewSolrServer(int port) {
     try {
       // setup the server...
       String url = "http://localhost:" + port + context;
       CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
       s.setConnectionTimeout(100); // 1/10th sec
       s.setDefaultMaxConnectionsPerHost(100);
       s.setMaxTotalConnections(100);
       return s;
     }
     catch (Exception ex) {
       throw new RuntimeException(ex);
     }
   }

   protected void addFields(SolrInputDocument doc, Object... fields) {
     for (int i = 0; i < fields.length; i += 2) {
       doc.addField((String) (fields[i]), fields[i + 1]);
     }
   }// add random fields to the documet before indexing

   protected void indexr(Object... fields) throws Exception {
     SolrInputDocument doc = new SolrInputDocument();
     addFields(doc, fields);
     addFields(doc, "rnd_b", true);
     addFields(doc, getRandFields(getFieldNames(), getRandValues()));
     indexDoc(doc);
   }

   protected void index(Object... fields) throws Exception {
     SolrInputDocument doc = new SolrInputDocument();
     addFields(doc, fields);
     indexDoc(doc);
   }

   protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {
     controlClient.add(doc);

     int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size();
     SolrServer client = clients.get(which);
     client.add(doc);
   }

   protected void index_specific(int serverNumber, Object... fields) throws Exception {
     SolrInputDocument doc = new SolrInputDocument();
     for (int i = 0; i < fields.length; i += 2) {
       doc.addField((String) (fields[i]), fields[i + 1]);
     }
     controlClient.add(doc);

     SolrServer client = clients.get(serverNumber);
     client.add(doc);
   }

   protected void del(String q) throws Exception {
     controlClient.deleteByQuery(q);
     for (SolrServer client : clients) {
       client.deleteByQuery(q);
     }
   }// serial commit...

   protected void commit() throws Exception {
     controlClient.commit();
     for (SolrServer client : clients) client.commit();
   }

   protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
     // query a random server
     int which = r.nextInt(clients.size());
     SolrServer client = clients.get(which);
     QueryResponse rsp = client.query(params);
     return rsp;
   }

   protected void query(Object... q) throws Exception {
     final ModifiableSolrParams params = new ModifiableSolrParams();
     params.add("reqid",Integer.toString(random.nextInt())); // just to help correlate top-level requests w/ sub requests

     for (int i = 0; i < q.length; i += 2) {
       params.add(q[i].toString(), q[i + 1].toString());
     }

     params.add("controlClient","true"); // just to enable easier sorting through log files
     final QueryResponse controlRsp = controlClient.query(params);
     params.remove("controlClient");

     // query a random server
     params.set("shards", shards);
     QueryResponse rsp = queryServer(params);

     compareResponses(rsp, controlRsp);

     if (stress > 0) {
       log.info("starting stress...");
       Thread[] threads = new Thread[nThreads];
       for (int i = 0; i < threads.length; i++) {
         threads[i] = new Thread() {
           @Override
           public void run() {
             for (int j = 0; j < stress; j++) {
               int which = r.nextInt(clients.size());
               SolrServer client = clients.get(which);
               try {
                 QueryResponse rsp = client.query(new ModifiableSolrParams(params));
                 if (verifyStress) {
                   compareResponses(rsp, controlRsp);
                 }
               } catch (SolrServerException e) {
                 throw new RuntimeException(e);
               }
             }
           }
         };
         threads[i].start();
       }

       for (Thread thread : threads) {
         thread.join();
       }
     }
   }

   public static boolean eq(String a, String b) {
     return a == b || (a != null && a.equals(b));
   }

   public static int flags(Map<String, Integer> handle, Object key) {
     if (handle == null) return 0;
     Integer f = handle.get(key);
     return f == null ? 0 : f;
   }

   public static String compare(NamedList a, NamedList b, int flags, Map<String, Integer> handle) {
     boolean ordered = (flags & UNORDERED) == 0;

     int posa = 0, posb = 0;
     int aSkipped = 0, bSkipped = 0;

     for (; ;) {
       if (posa >= a.size() || posb >= b.size()) {
         break;
       }

       String namea, nameb;
       Object vala, valb = null;

       int flagsa, flagsb;
       for (; ;) {
         namea = a.getName(posa);
         vala = a.getVal(posa);
         posa++;
         flagsa = flags(handle, namea);
         if ((flagsa & SKIP) != 0) {
           aSkipped++;
           continue;
         }
         break;
       }

       if (!ordered) posb = 0;  // reset if not ordered

       while (posb < b.size()) {
         nameb = b.getName(posb);
         valb = b.getVal(posb);
         posb++;
         flagsb = flags(handle, nameb);
         if ((flagsb & SKIP) != 0) {
           bSkipped++;
           continue;
         }
         if (eq(namea, nameb)) {
           break;
         }
         if (ordered) {
           return err("." + namea + "!=" + nameb + " (unordered or missing)");
         }
         // if unordered, continue until we find the right field.
       }

       // ok, namea and nameb should be equal here already.
       if ((flagsa & SKIPVAL) != 0) continue;  // keys matching is enough

       String cmp = compare(vala, valb, flagsa, handle);
       if (cmp != null) return "." + namea + cmp;
     }


     if (a.size() - aSkipped != b.size() - bSkipped) {
       return err(".size()==" + a.size() + "," + b.size() + "skipped=" + aSkipped + "," + bSkipped);
     }

     return null;
   }

   public static String compare1(Map a, Map b, int flags, Map<String, Integer> handle) {
     String cmp;

     for (Object keya : a.keySet()) {
       Object vala = a.get(keya);
       int flagsa = flags(handle, keya);
       if ((flagsa & SKIP) != 0) continue;
       if (!b.containsKey(keya)) {
         return err("[" + keya + "]==null");
       }
       if ((flagsa & SKIPVAL) != 0) continue;
       Object valb = b.get(keya);
       cmp = compare(vala, valb, flagsa, handle);
       if (cmp != null) return "[" + keya + "]" + cmp;
     }
     return null;
   }

   public static String compare(Map a, Map b, int flags, Map<String, Integer> handle) {
     String cmp;
     cmp = compare1(a, b, flags, handle);
     if (cmp != null) return cmp;
     return compare1(b, a, flags, handle);
   }

   public static String compare(SolrDocument a, SolrDocument b, int flags, Map<String, Integer> handle) {
     return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle);
   }

   public static String compare(SolrDocumentList a, SolrDocumentList b, int flags, Map<String, Integer> handle) {
     boolean ordered = (flags & UNORDERED) == 0;

     String cmp;
     int f = flags(handle, "maxScore");
     if ((f & SKIPVAL) == 0) {
       cmp = compare(a.getMaxScore(), b.getMaxScore(), 0, handle);
       if (cmp != null) return ".maxScore" + cmp;
     } else {
       if (b.getMaxScore() != null) {
         if (a.getMaxScore() == null) {
           return err(".maxScore missing");
         }
       }
     }

     cmp = compare(a.getNumFound(), b.getNumFound(), 0, handle);
     if (cmp != null) return ".numFound" + cmp;

     cmp = compare(a.getStart(), b.getStart(), 0, handle);
     if (cmp != null) return ".start" + cmp;

     cmp = compare(a.size(), b.size(), 0, handle);
     if (cmp != null) return ".size()" + cmp;

     // only for completely ordered results (ties might be in a different order)
     if (ordered) {
       for (int i = 0; i < a.size(); i++) {
         cmp = compare(a.get(i), b.get(i), 0, handle);
         if (cmp != null) return "[" + i + "]" + cmp;
       }
       return null;
     }

     // unordered case
     for (int i = 0; i < a.size(); i++) {
       SolrDocument doc = a.get(i);
       Object key = doc.getFirstValue("id");
       SolrDocument docb = null;
       if (key == null) {
         // no id field to correlate... must compare ordered
         docb = b.get(i);
       } else {
         for (int j = 0; j < b.size(); j++) {
           docb = b.get(j);
           if (key.equals(docb.getFirstValue("id"))) break;
         }
       }
       // if (docb == null) return "[id="+key+"]";
       cmp = compare(doc, docb, 0, handle);
       if (cmp != null) return "[id=" + key + "]" + cmp;
     }
     return null;
   }

   public static String compare(Object[] a, Object[] b, int flags, Map<String, Integer> handle) {
     if (a.length != b.length) {
       return err(".length:" + a.length + "!=" + b.length);
     }
     for (int i = 0; i < a.length; i++) {
       String cmp = compare(a[i], b[i], flags, handle);
       if (cmp != null) return "[" + i + "]" + cmp;
     }
     return null;
   }

   public static String compare(Object a, Object b, int flags, Map<String, Integer> handle) {
     if (a == b) return null;
     if (a == null || b == null) return err(":" + a + "!=" + b);

     if (a instanceof NamedList && b instanceof NamedList) {
       return compare((NamedList) a, (NamedList) b, flags, handle);
     }

     if (a instanceof SolrDocumentList && b instanceof SolrDocumentList) {
       return compare((SolrDocumentList) a, (SolrDocumentList) b, flags, handle);
     }

     if (a instanceof SolrDocument && b instanceof SolrDocument) {
       return compare((SolrDocument) a, (SolrDocument) b, flags, handle);
     }

     if (a instanceof Map && b instanceof Map) {
       return compare((Map) a, (Map) b, flags, handle);
     }

     if (a instanceof Object[] && b instanceof Object[]) {
       return compare((Object[]) a, (Object[]) b, flags, handle);
     }

     if (a instanceof byte[] && b instanceof byte[]) {
       if (!Arrays.equals((byte[]) a, (byte[]) b)) {
         return err(":" + a + "!=" + b);
       }
       return null;
     }

     if (a instanceof List && b instanceof List) {
       return compare(((List) a).toArray(), ((List) b).toArray(), flags, handle);

     }

     if (!(a.equals(b))) {
       return err(":" + a + "!=" + b);
     }

     return null;
   }

   /** This method is called for root level comparison errors and can be helpful to set a
    * breakpoint in to debug comparison failures.
    */
   public static String err(String msg) {
     return msg;
   }

   protected void compareResponses(QueryResponse a, QueryResponse b) {
     String cmp;
     cmp = compare(a.getResponse(), b.getResponse(), flags, handle);
     if (cmp != null) {
       log.info("Mismatched responses:\n" + a + "\n" + b);
       TestCase.fail(cmp);
     }
   }

   @Test
   public void testDistribSearch() throws Exception {
     if (fixShardCount) {
       createServers(shardCount);
       RandVal.uniqueValues = new HashSet(); //reset random values
       doTest();
       destroyServers();
     } else {
       for (int nServers = 1; nServers < shardCount; nServers++) {
         createServers(nServers);
         RandVal.uniqueValues = new HashSet(); //reset random values
         doTest();
         destroyServers();
       }
     }
   }

   public static Object[] getRandFields(String[] fields, RandVal[] randVals) {
     Object[] o = new Object[fields.length * 2];
     for (int i = 0; i < fields.length; i++) {
       o[i * 2] = fields[i];
       o[i * 2 + 1] = randVals[i].uval();
     }
     return o;
   }

   public static abstract class RandVal {
     public static Random r = random;
     public static Set uniqueValues = new HashSet();

     public abstract Object val();

     public Object uval() {
       for (; ;) {
         Object v = val();
         if (uniqueValues.add(v)) return v;
       }
     }
   }

   public static class RandDate extends RandVal {
     public static TrieDateField df = new TrieDateField();

     @Override
     public Object val() {
       long v = r.nextLong();
       Date d = new Date(v);
       return df.toExternal(d);
     }
   }
 }
	package org.apache.solr;

	/**
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	import junit.framework.TestCase;
	import org.apache.solr.client.solrj.SolrServer;
	import org.apache.solr.client.solrj.SolrServerException;
	import org.apache.solr.client.solrj.embedded.JettySolrRunner;
	import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
	import org.apache.solr.client.solrj.response.QueryResponse;
	import org.apache.solr.common.SolrDocument;
	import org.apache.solr.common.SolrDocumentList;
	import org.apache.solr.common.SolrInputDocument;
	import org.apache.solr.common.params.ModifiableSolrParams;
	import org.apache.solr.common.util.NamedList;
	import org.apache.solr.schema.TrieDateField;
	import org.apache.solr.util.AbstractSolrTestCase;
	import org.junit.Test;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	import java.io.File;
	import java.io.IOException;
	import java.util.*;

	/**
	* Helper base class for distributed search test cases
	*
	* @since solr 1.5
	*/
	public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
	public static Random r = random;

	protected int shardCount = 4;
	/**
	* Sub classes can set this flag in their constructor to true if they
	* want to fix the number of shards to 'shardCount'
	*
	* The default is false which means that test will be executed with
	* 1, 2, 3, ....shardCount number of shards repeatedly
	*/
	protected boolean fixShardCount = false;

	protected JettySolrRunner controlJetty;
	protected List<SolrServer> clients = new ArrayList<SolrServer>();
	protected List<JettySolrRunner> jettys = new ArrayList<JettySolrRunner>();
	protected String context = "/solr";
	protected String shards;
	protected File testDir;
	protected SolrServer controlClient;

	// to stress with higher thread counts and requests, make sure the junit
	// xml formatter is not being used (all output will be buffered before
	// transformation to xml and cause an OOM exception).
	protected int stress = 2;
	protected boolean verifyStress = true;
	protected int nThreads = 3;


	public static int ORDERED = 1;
	public static int SKIP = 2;
	public static int SKIPVAL = 4;
	public static int UNORDERED = 8;

	protected int flags;
	protected Map<String, Integer> handle = new HashMap<String, Integer>();

	protected String id = "id";
	public static Logger log = LoggerFactory.getLogger(BaseDistributedSearchTestCase.class);

	public static RandVal rint = new RandVal() {
	@Override
	public Object val() {
	return r.nextInt();
	}
	};

	public static RandVal rlong = new RandVal() {
	@Override
	public Object val() {
	return r.nextLong();
	}
	};

	public static RandVal rfloat = new RandVal() {
	@Override
	public Object val() {
	return r.nextFloat();
	}
	};

	public static RandVal rdouble = new RandVal() {
	@Override
	public Object val() {
	return r.nextDouble();
	}
	};

	public static RandVal rdate = new RandDate();

	/**
	* Perform the actual tests here
	*
	* @throws Exception on error
	*/
	public abstract void doTest() throws Exception;

	public static String[] fieldNames = new String[]{"n_ti1", "n_f1", "n_tf1", "n_d1", "n_td1", "n_l1", "n_tl1", "n_dt1", "n_tdt1"};
	public static RandVal[] randVals = new RandVal[]{rint, rfloat, rfloat, rdouble, rdouble, rlong, rlong, rdate, rdate};

	protected String[] getFieldNames() {
	return fieldNames;
	}

	protected RandVal[] getRandValues() {
	return randVals;
	}

	/**
	* Subclasses can override this to change a test's solr home
	* (default is in test-files)
	*/
	public String getSolrHome() {
	return SolrTestCaseJ4.TEST_HOME();
	}

	@Override
	public void setUp() throws Exception {
	SolrTestCaseJ4.resetExceptionIgnores(); // ignore anything with ignore_exception in it
	super.setUp();
	System.setProperty("solr.test.sys.prop1", "propone");
	System.setProperty("solr.test.sys.prop2", "proptwo");
	System.setProperty("solr.solr.home", getSolrHome());
	testDir = new File(TEMP_DIR,
	getClass().getName() + "-" + System.currentTimeMillis());
	testDir.mkdirs();
	}

	@Override
	public void tearDown() throws Exception {
	destroyServers();
	if (!AbstractSolrTestCase.recurseDelete(testDir)) {
	System.err.println("!!!! WARNING: best effort to remove " + testDir.getAbsolutePath() + " FAILED !!!!!");
	}
	super.tearDown();
	}

	private void createServers(int numShards) throws Exception {
	controlJetty = createJetty(testDir, "control");
	controlClient = createNewSolrServer(controlJetty.getLocalPort());

	StringBuilder sb = new StringBuilder();
	for (int i = 1; i <= numShards; i++) {
	if (sb.length() > 0) sb.append(',');
	JettySolrRunner j = createJetty(testDir, "shard" + i);
	jettys.add(j);
	clients.add(createNewSolrServer(j.getLocalPort()));
	sb.append("localhost:").append(j.getLocalPort()).append(context);
	}

	shards = sb.toString();
	}

	protected void destroyServers() throws Exception {
	controlJetty.stop();
	for (JettySolrRunner jetty : jettys) jetty.stop();
	clients.clear();
	jettys.clear();
	}

	public static JettySolrRunner createJetty(File baseDir, String dataDirName) throws Exception {
	File subDir = new File(baseDir, dataDirName);
	subDir.mkdirs();
	System.setProperty("solr.data.dir", subDir.toString());

	JettySolrRunner jetty = new JettySolrRunner("/solr", 0);

	jetty.start();
	return jetty;
	}

	protected SolrServer createNewSolrServer(int port) {
	try {
	// setup the server...
	String url = "http://localhost:" + port + context;
	CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
	s.setConnectionTimeout(100); // 1/10th sec
	s.setDefaultMaxConnectionsPerHost(100);
	s.setMaxTotalConnections(100);
	return s;
	}
	catch (Exception ex) {
	throw new RuntimeException(ex);
	}
	}

	protected void addFields(SolrInputDocument doc, Object... fields) {
	for (int i = 0; i < fields.length; i += 2) {
	doc.addField((String) (fields[i]), fields[i + 1]);
	}
	}// add random fields to the documet before indexing

	protected void indexr(Object... fields) throws Exception {
	SolrInputDocument doc = new SolrInputDocument();
	addFields(doc, fields);
	addFields(doc, "rnd_b", true);
	addFields(doc, getRandFields(getFieldNames(), getRandValues()));
	indexDoc(doc);
	}

	protected void index(Object... fields) throws Exception {
	SolrInputDocument doc = new SolrInputDocument();
	addFields(doc, fields);
	indexDoc(doc);
	}

	protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {
	controlClient.add(doc);

	int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size();
	SolrServer client = clients.get(which);
	client.add(doc);
	}

	protected void index_specific(int serverNumber, Object... fields) throws Exception {
	SolrInputDocument doc = new SolrInputDocument();
	for (int i = 0; i < fields.length; i += 2) {
	doc.addField((String) (fields[i]), fields[i + 1]);
	}
	controlClient.add(doc);

	SolrServer client = clients.get(serverNumber);
	client.add(doc);
	}

	protected void del(String q) throws Exception {
	controlClient.deleteByQuery(q);
	for (SolrServer client : clients) {
	client.deleteByQuery(q);
	}
	}// serial commit...

	protected void commit() throws Exception {
	controlClient.commit();
	for (SolrServer client : clients) client.commit();
	}

	protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
	// query a random server
	int which = r.nextInt(clients.size());
	SolrServer client = clients.get(which);
	QueryResponse rsp = client.query(params);
	return rsp;
	}

	protected void query(Object... q) throws Exception {
	final ModifiableSolrParams params = new ModifiableSolrParams();
	params.add("reqid",Integer.toString(random.nextInt())); // just to help correlate top-level requests w/ sub requests

	for (int i = 0; i < q.length; i += 2) {
	params.add(q[i].toString(), q[i + 1].toString());
	}

	params.add("controlClient","true"); // just to enable easier sorting through log files
	final QueryResponse controlRsp = controlClient.query(params);
	params.remove("controlClient");

	// query a random server
	params.set("shards", shards);
	QueryResponse rsp = queryServer(params);

	compareResponses(rsp, controlRsp);

	if (stress > 0) {
	log.info("starting stress...");
	Thread[] threads = new Thread[nThreads];
	for (int i = 0; i < threads.length; i++) {
	threads[i] = new Thread() {
	@Override
	public void run() {
	for (int j = 0; j < stress; j++) {
	int which = r.nextInt(clients.size());
	SolrServer client = clients.get(which);
	try {
	QueryResponse rsp = client.query(new ModifiableSolrParams(params));
	if (verifyStress) {
	compareResponses(rsp, controlRsp);
	}
	} catch (SolrServerException e) {
	throw new RuntimeException(e);
	}
	}
	}
	};
	threads[i].start();
	}

	for (Thread thread : threads) {
	thread.join();
	}
	}
	}

	public static boolean eq(String a, String b) {
	return a == b \|\| (a != null && a.equals(b));
	}

	public static int flags(Map<String, Integer> handle, Object key) {
	if (handle == null) return 0;
	Integer f = handle.get(key);
	return f == null ? 0 : f;
	}

	public static String compare(NamedList a, NamedList b, int flags, Map<String, Integer> handle) {
	boolean ordered = (flags & UNORDERED) == 0;

	int posa = 0, posb = 0;
	int aSkipped = 0, bSkipped = 0;

	for (; ;) {
	if (posa >= a.size() \|\| posb >= b.size()) {
	break;
	}

	String namea, nameb;
	Object vala, valb = null;

	int flagsa, flagsb;
	for (; ;) {
	namea = a.getName(posa);
	vala = a.getVal(posa);
	posa++;
	flagsa = flags(handle, namea);
	if ((flagsa & SKIP) != 0) {
	aSkipped++;
	continue;
	}
	break;
	}

	if (!ordered) posb = 0; // reset if not ordered

	while (posb < b.size()) {
	nameb = b.getName(posb);
	valb = b.getVal(posb);
	posb++;
	flagsb = flags(handle, nameb);
	if ((flagsb & SKIP) != 0) {
	bSkipped++;
	continue;
	}
	if (eq(namea, nameb)) {
	break;
	}
	if (ordered) {
	return err("." + namea + "!=" + nameb + " (unordered or missing)");
	}
	// if unordered, continue until we find the right field.
	}

	// ok, namea and nameb should be equal here already.
	if ((flagsa & SKIPVAL) != 0) continue; // keys matching is enough

	String cmp = compare(vala, valb, flagsa, handle);
	if (cmp != null) return "." + namea + cmp;
	}


	if (a.size() - aSkipped != b.size() - bSkipped) {
	return err(".size()==" + a.size() + "," + b.size() + "skipped=" + aSkipped + "," + bSkipped);
	}

	return null;
	}

	public static String compare1(Map a, Map b, int flags, Map<String, Integer> handle) {
	String cmp;

	for (Object keya : a.keySet()) {
	Object vala = a.get(keya);
	int flagsa = flags(handle, keya);
	if ((flagsa & SKIP) != 0) continue;
	if (!b.containsKey(keya)) {
	return err("[" + keya + "]==null");
	}
	if ((flagsa & SKIPVAL) != 0) continue;
	Object valb = b.get(keya);
	cmp = compare(vala, valb, flagsa, handle);
	if (cmp != null) return "[" + keya + "]" + cmp;
	}
	return null;
	}

	public static String compare(Map a, Map b, int flags, Map<String, Integer> handle) {
	String cmp;
	cmp = compare1(a, b, flags, handle);
	if (cmp != null) return cmp;
	return compare1(b, a, flags, handle);
	}

	public static String compare(SolrDocument a, SolrDocument b, int flags, Map<String, Integer> handle) {
	return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle);
	}

	public static String compare(SolrDocumentList a, SolrDocumentList b, int flags, Map<String, Integer> handle) {
	boolean ordered = (flags & UNORDERED) == 0;

	String cmp;
	int f = flags(handle, "maxScore");
	if ((f & SKIPVAL) == 0) {
	cmp = compare(a.getMaxScore(), b.getMaxScore(), 0, handle);
	if (cmp != null) return ".maxScore" + cmp;
	} else {
	if (b.getMaxScore() != null) {
	if (a.getMaxScore() == null) {
	return err(".maxScore missing");
	}
	}
	}

	cmp = compare(a.getNumFound(), b.getNumFound(), 0, handle);
	if (cmp != null) return ".numFound" + cmp;

	cmp = compare(a.getStart(), b.getStart(), 0, handle);
	if (cmp != null) return ".start" + cmp;

	cmp = compare(a.size(), b.size(), 0, handle);
	if (cmp != null) return ".size()" + cmp;

	// only for completely ordered results (ties might be in a different order)
	if (ordered) {
	for (int i = 0; i < a.size(); i++) {
	cmp = compare(a.get(i), b.get(i), 0, handle);
	if (cmp != null) return "[" + i + "]" + cmp;
	}
	return null;
	}

	// unordered case
	for (int i = 0; i < a.size(); i++) {
	SolrDocument doc = a.get(i);
	Object key = doc.getFirstValue("id");
	SolrDocument docb = null;
	if (key == null) {
	// no id field to correlate... must compare ordered
	docb = b.get(i);
	} else {
	for (int j = 0; j < b.size(); j++) {
	docb = b.get(j);
	if (key.equals(docb.getFirstValue("id"))) break;
	}
	}
	// if (docb == null) return "[id="+key+"]";
	cmp = compare(doc, docb, 0, handle);
	if (cmp != null) return "[id=" + key + "]" + cmp;
	}
	return null;
	}

	public static String compare(Object[] a, Object[] b, int flags, Map<String, Integer> handle) {
	if (a.length != b.length) {
	return err(".length:" + a.length + "!=" + b.length);
	}
	for (int i = 0; i < a.length; i++) {
	String cmp = compare(a[i], b[i], flags, handle);
	if (cmp != null) return "[" + i + "]" + cmp;
	}
	return null;
	}

	public static String compare(Object a, Object b, int flags, Map<String, Integer> handle) {
	if (a == b) return null;
	if (a == null \|\| b == null) return err(":" + a + "!=" + b);

	if (a instanceof NamedList && b instanceof NamedList) {
	return compare((NamedList) a, (NamedList) b, flags, handle);
	}

	if (a instanceof SolrDocumentList && b instanceof SolrDocumentList) {
	return compare((SolrDocumentList) a, (SolrDocumentList) b, flags, handle);
	}

	if (a instanceof SolrDocument && b instanceof SolrDocument) {
	return compare((SolrDocument) a, (SolrDocument) b, flags, handle);
	}

	if (a instanceof Map && b instanceof Map) {
	return compare((Map) a, (Map) b, flags, handle);
	}

	if (a instanceof Object[] && b instanceof Object[]) {
	return compare((Object[]) a, (Object[]) b, flags, handle);
	}

	if (a instanceof byte[] && b instanceof byte[]) {
	if (!Arrays.equals((byte[]) a, (byte[]) b)) {
	return err(":" + a + "!=" + b);
	}
	return null;
	}

	if (a instanceof List && b instanceof List) {
	return compare(((List) a).toArray(), ((List) b).toArray(), flags, handle);

	}

	if (!(a.equals(b))) {
	return err(":" + a + "!=" + b);
	}

	return null;
	}

	/** This method is called for root level comparison errors and can be helpful to set a
	* breakpoint in to debug comparison failures.
	*/
	public static String err(String msg) {
	return msg;
	}

	protected void compareResponses(QueryResponse a, QueryResponse b) {
	String cmp;
	cmp = compare(a.getResponse(), b.getResponse(), flags, handle);
	if (cmp != null) {
	log.info("Mismatched responses:\n" + a + "\n" + b);
	TestCase.fail(cmp);
	}
	}

	@Test
	public void testDistribSearch() throws Exception {
	if (fixShardCount) {
	createServers(shardCount);
	RandVal.uniqueValues = new HashSet(); //reset random values
	doTest();
	destroyServers();
	} else {
	for (int nServers = 1; nServers < shardCount; nServers++) {
	createServers(nServers);
	RandVal.uniqueValues = new HashSet(); //reset random values
	doTest();
	destroyServers();
	}
	}
	}

	public static Object[] getRandFields(String[] fields, RandVal[] randVals) {
	Object[] o = new Object[fields.length * 2];
	for (int i = 0; i < fields.length; i++) {
	o[i * 2] = fields[i];
	o[i * 2 + 1] = randVals[i].uval();
	}
	return o;
	}

	public static abstract class RandVal {
	public static Random r = random;
	public static Set uniqueValues = new HashSet();

	public abstract Object val();

	public Object uval() {
	for (; ;) {
	Object v = val();
	if (uniqueValues.add(v)) return v;
	}
	}
	}

	public static class RandDate extends RandVal {
	public static TrieDateField df = new TrieDateField();

	@Override
	public Object val() {
	long v = r.nextLong();
	Date d = new Date(v);
	return df.toExternal(d);
	}
	}
	}