| package org.apache.solr; |
| |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.Date; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Random; |
| import java.util.Set; |
| import java.util.SortedMap; |
| import java.util.concurrent.atomic.AtomicInteger; |
| |
| import junit.framework.Assert; |
| |
| import org.apache.commons.io.FileUtils; |
| import org.apache.lucene.search.FieldCache; |
| import org.apache.lucene.util.Constants; |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| import org.apache.solr.client.solrj.SolrResponse; |
| import org.apache.solr.client.solrj.SolrServer; |
| import org.apache.solr.client.solrj.SolrServerException; |
| import org.apache.solr.client.solrj.embedded.JettySolrRunner; |
| import org.apache.solr.client.solrj.impl.HttpSolrServer; |
| import org.apache.solr.client.solrj.request.UpdateRequest; |
| import org.apache.solr.client.solrj.response.QueryResponse; |
| import org.apache.solr.client.solrj.response.UpdateResponse; |
| import org.apache.solr.common.SolrDocument; |
| import org.apache.solr.common.SolrDocumentList; |
| import org.apache.solr.common.SolrInputDocument; |
| import org.apache.solr.common.params.ModifiableSolrParams; |
| import org.apache.solr.common.params.SolrParams; |
| import org.apache.solr.common.util.NamedList; |
| import org.apache.solr.schema.TrieDateField; |
| import org.eclipse.jetty.servlet.ServletHolder; |
| import org.junit.AfterClass; |
| import org.junit.BeforeClass; |
| import org.junit.Test; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| /** |
| * Helper base class for distributed search test cases |
| * |
| * @since solr 1.5 |
| */ |
| public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { |
| // TODO: this shouldn't be static. get the random when you need it to avoid sharing. |
| public static Random r; |
| |
| private AtomicInteger nodeCnt = new AtomicInteger(0); |
| protected boolean useExplicitNodeNames; |
| |
| @BeforeClass |
| public static void initialize() { |
| assumeFalse("SOLR-4147: ibm 64bit has jvm bugs!", Constants.JRE_IS_64BIT && Constants.JAVA_VENDOR.startsWith("IBM")); |
| r = new Random(random().nextLong()); |
| } |
| |
| /** |
| * Set's the value of the "hostContext" system property to a random path |
| * like string (which may or may not contain sub-paths). This is used |
| * in the default constructor for this test to help ensure no code paths have |
| * hardcoded assumptions about the servlet context used to run solr. |
| * <p> |
| * Test configs may use the <code>${hostContext}</code> variable to access |
| * this system property. |
| * </p> |
| * @see #BaseDistributedSearchTestCase() |
| * @see #clearHostContext |
| */ |
| @BeforeClass |
| public static void initHostContext() { |
| // Can't use randomRealisticUnicodeString because unescaped unicode is |
| // not allowed in URL paths |
| // Can't use URLEncoder.encode(randomRealisticUnicodeString) because |
| // Jetty freaks out and returns 404's when the context uses escapes |
| |
| StringBuilder hostContext = new StringBuilder("/"); |
| if (random().nextBoolean()) { |
| // half the time we use the root context, the other half... |
| |
| // Remember: randomSimpleString might be the empty string |
| hostContext.append(TestUtil.randomSimpleString(random(), 2)); |
| if (random().nextBoolean()) { |
| hostContext.append("_"); |
| } |
| hostContext.append(TestUtil.randomSimpleString(random(), 3)); |
| if ( ! "/".equals(hostContext.toString())) { |
| // if our random string is empty, this might add a trailing slash, |
| // but our code should be ok with that |
| hostContext.append("/").append(TestUtil.randomSimpleString(random(), 2)); |
| } else { |
| // we got 'lucky' and still just have the root context, |
| // NOOP: don't try to add a subdir to nothing (ie "//" is bad) |
| } |
| } |
| // paranoia, we *really* don't want to ever get "//" in a path... |
| final String hc = hostContext.toString().replaceAll("\\/+","/"); |
| |
| log.info("Setting hostContext system property: " + hc); |
| System.setProperty("hostContext", hc); |
| } |
| |
| /** |
| * Clears the "hostContext" system property |
| * @see #initHostContext |
| */ |
| @AfterClass |
| public static void clearHostContext() throws Exception { |
| System.clearProperty("hostContext"); |
| } |
| |
| private static String getHostContextSuitableForServletContext() { |
| String ctx = System.getProperty("hostContext","/solr"); |
| if ("".equals(ctx)) ctx = "/solr"; |
| if (ctx.endsWith("/")) ctx = ctx.substring(0,ctx.length()-1);; |
| if (!ctx.startsWith("/")) ctx = "/" + ctx; |
| return ctx; |
| } |
| |
| /** |
| * Constructs a test in which the jetty+solr instances as well as the |
| * solr clients all use the value of the "hostContext" system property. |
| * <p> |
| * If the system property is not set, or is set to the empty string |
| * (neither of which should normally happen unless a subclass explicitly |
| * modifies the property set by {@link #initHostContext} prior to calling |
| * this constructor) a servlet context of "/solr" is used. (this is for |
| * consistency with the default behavior of solr.xml parsing when using |
| * <code>hostContext="${hostContext:}"</code> |
| * </p> |
| * <p> |
| * If the system property is set to a value which does not begin with a |
| * "/" (which should normally happen unless a subclass explicitly |
| * modifies the property set by {@link #initHostContext} prior to calling |
| * this constructor) a leading "/" will be prepended. |
| * </p> |
| * |
| * @see #initHostContext |
| */ |
| protected BaseDistributedSearchTestCase() { |
| this(getHostContextSuitableForServletContext()); |
| } |
| |
| /** |
| * @param context explicit servlet context path to use (eg: "/solr") |
| */ |
| protected BaseDistributedSearchTestCase(final String context) { |
| this.context = context; |
| this.deadServers = new String[] {"[ff01::114]:33332" + context, |
| "[ff01::083]:33332" + context, |
| "[ff01::213]:33332" + context}; |
| } |
| |
| protected int shardCount = 4; // the actual number of solr cores that will be created in the cluster |
| |
| /** |
| * Sub classes can set this flag in their constructor to true if they |
| * want to fix the number of shards to 'shardCount' |
| * |
| * The default is false which means that test will be executed with |
| * 1, 2, 3, ....shardCount number of shards repeatedly |
| */ |
| protected boolean fixShardCount = false; |
| |
| protected JettySolrRunner controlJetty; |
| protected List<SolrServer> clients = new ArrayList<>(); |
| protected List<JettySolrRunner> jettys = new ArrayList<>(); |
| |
| protected String context; |
| protected String[] deadServers; |
| protected String shards; |
| protected String[] shardsArr; |
| protected File testDir; |
| protected SolrServer controlClient; |
| |
| // to stress with higher thread counts and requests, make sure the junit |
| // xml formatter is not being used (all output will be buffered before |
| // transformation to xml and cause an OOM exception). |
| protected int stress = TEST_NIGHTLY ? 2 : 0; |
| protected boolean verifyStress = true; |
| protected int nThreads = 3; |
| |
| |
| public static int ORDERED = 1; |
| public static int SKIP = 2; |
| public static int SKIPVAL = 4; |
| public static int UNORDERED = 8; |
| |
| protected int flags; |
| protected Map<String, Integer> handle = new HashMap<>(); |
| |
| protected String id = "id"; |
| public static Logger log = LoggerFactory.getLogger(BaseDistributedSearchTestCase.class); |
| |
| public static RandVal rint = new RandVal() { |
| @Override |
| public Object val() { |
| return r.nextInt(); |
| } |
| }; |
| |
| public static RandVal rlong = new RandVal() { |
| @Override |
| public Object val() { |
| return r.nextLong(); |
| } |
| }; |
| |
| public static RandVal rfloat = new RandVal() { |
| @Override |
| public Object val() { |
| return r.nextFloat(); |
| } |
| }; |
| |
| public static RandVal rdouble = new RandVal() { |
| @Override |
| public Object val() { |
| return r.nextDouble(); |
| } |
| }; |
| |
| public static RandVal rdate = new RandDate(); |
| |
| /** |
| * Perform the actual tests here |
| * |
| * @throws Exception on error |
| */ |
| public abstract void doTest() throws Exception; |
| |
| public static String[] fieldNames = new String[]{"n_ti1", "n_f1", "n_tf1", "n_d1", "n_td1", "n_l1", "n_tl1", "n_dt1", "n_tdt1"}; |
| public static RandVal[] randVals = new RandVal[]{rint, rfloat, rfloat, rdouble, rdouble, rlong, rlong, rdate, rdate}; |
| |
| protected String[] getFieldNames() { |
| return fieldNames; |
| } |
| |
| protected RandVal[] getRandValues() { |
| return randVals; |
| } |
| |
| /** |
| * Subclasses can override this to change a test's solr home |
| * (default is in test-files) |
| */ |
| public String getSolrHome() { |
| return SolrTestCaseJ4.TEST_HOME(); |
| } |
| |
| @Override |
| public void setUp() throws Exception { |
| SolrTestCaseJ4.resetExceptionIgnores(); // ignore anything with ignore_exception in it |
| super.setUp(); |
| System.setProperty("solr.test.sys.prop1", "propone"); |
| System.setProperty("solr.test.sys.prop2", "proptwo"); |
| testDir = createTempDir(); |
| } |
| |
| @Override |
| public void tearDown() throws Exception { |
| destroyServers(); |
| FieldCache.DEFAULT.purgeAllCaches(); // avoid FC insanity |
| super.tearDown(); |
| } |
| |
| protected void createServers(int numShards) throws Exception { |
| controlJetty = createJetty(new File(getSolrHome()), testDir + "/control/data", null, getSolrConfigFile(), getSchemaFile()); |
| |
| controlClient = createNewSolrServer(controlJetty.getLocalPort()); |
| |
| shardsArr = new String[numShards]; |
| StringBuilder sb = new StringBuilder(); |
| for (int i = 0; i < numShards; i++) { |
| if (sb.length() > 0) sb.append(','); |
| JettySolrRunner j = createJetty(new File(getSolrHome()), |
| testDir + "/shard" + i + "/data", null, getSolrConfigFile(), |
| getSchemaFile()); |
| jettys.add(j); |
| clients.add(createNewSolrServer(j.getLocalPort())); |
| String shardStr = buildUrl(j.getLocalPort()); |
| shardsArr[i] = shardStr; |
| sb.append(shardStr); |
| } |
| |
| shards = sb.toString(); |
| } |
| |
| |
| protected void setDistributedParams(ModifiableSolrParams params) { |
| params.set("shards", getShardsString()); |
| } |
| |
| protected String getShardsString() { |
| if (deadServers == null) return shards; |
| |
| StringBuilder sb = new StringBuilder(); |
| for (String shard : shardsArr) { |
| if (sb.length() > 0) sb.append(','); |
| int nDeadServers = r.nextInt(deadServers.length+1); |
| if (nDeadServers > 0) { |
| List<String> replicas = new ArrayList<>(Arrays.asList(deadServers)); |
| Collections.shuffle(replicas, r); |
| replicas.add(r.nextInt(nDeadServers+1), shard); |
| for (int i=0; i<nDeadServers+1; i++) { |
| if (i!=0) sb.append('|'); |
| sb.append(replicas.get(i)); |
| } |
| } else { |
| sb.append(shard); |
| } |
| } |
| |
| return sb.toString(); |
| } |
| |
| protected void destroyServers() throws Exception { |
| controlJetty.stop(); |
| ((HttpSolrServer) controlClient).shutdown(); |
| for (JettySolrRunner jetty : jettys) jetty.stop(); |
| for (SolrServer client : clients) ((HttpSolrServer) client).shutdown(); |
| clients.clear(); |
| jettys.clear(); |
| } |
| |
| public JettySolrRunner createJetty(File solrHome, String dataDir) throws Exception { |
| return createJetty(solrHome, dataDir, null, null, null); |
| } |
| |
| public JettySolrRunner createJetty(File solrHome, String dataDir, String shardId) throws Exception { |
| return createJetty(solrHome, dataDir, shardId, null, null); |
| } |
| |
| public JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, String solrConfigOverride, String schemaOverride) throws Exception { |
| return createJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, useExplicitNodeNames); |
| } |
| |
| public JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, String solrConfigOverride, String schemaOverride, boolean explicitCoreNodeName) throws Exception { |
| |
| boolean stopAtShutdown = true; |
| JettySolrRunner jetty = new JettySolrRunner |
| (solrHome.getAbsolutePath(), context, 0, solrConfigOverride, schemaOverride, stopAtShutdown, |
| getExtraServlets(), sslConfig, getExtraRequestFilters()); |
| jetty.setShards(shardList); |
| jetty.setDataDir(dataDir); |
| if (explicitCoreNodeName) { |
| jetty.setCoreNodeName(Integer.toString(nodeCnt.incrementAndGet())); |
| } |
| jetty.start(); |
| |
| return jetty; |
| } |
| |
| /** Override this method to insert extra servlets into the JettySolrRunners that are created using createJetty() */ |
| public SortedMap<ServletHolder,String> getExtraServlets() { |
| return null; |
| } |
| |
| /** Override this method to insert extra filters into the JettySolrRunners that are created using createJetty() */ |
| public SortedMap<Class,String> getExtraRequestFilters() { |
| return null; |
| } |
| |
| protected SolrServer createNewSolrServer(int port) { |
| try { |
| // setup the server... |
| HttpSolrServer s = new HttpSolrServer(buildUrl(port)); |
| s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); |
| s.setSoTimeout(90000); |
| s.setDefaultMaxConnectionsPerHost(100); |
| s.setMaxTotalConnections(100); |
| return s; |
| } |
| catch (Exception ex) { |
| throw new RuntimeException(ex); |
| } |
| } |
| |
| protected String buildUrl(int port) { |
| return buildUrl(port, context); |
| } |
| |
| protected void addFields(SolrInputDocument doc, Object... fields) { |
| for (int i = 0; i < fields.length; i += 2) { |
| doc.addField((String) (fields[i]), fields[i + 1]); |
| } |
| }// add random fields to the documet before indexing |
| |
| protected void indexr(Object... fields) throws Exception { |
| SolrInputDocument doc = new SolrInputDocument(); |
| addFields(doc, fields); |
| addFields(doc, "rnd_b", true); |
| addRandFields(doc); |
| indexDoc(doc); |
| } |
| |
| protected SolrInputDocument addRandFields(SolrInputDocument sdoc) { |
| addFields(sdoc, getRandFields(getFieldNames(), getRandValues())); |
| return sdoc; |
| } |
| |
| protected void index(Object... fields) throws Exception { |
| SolrInputDocument doc = new SolrInputDocument(); |
| addFields(doc, fields); |
| indexDoc(doc); |
| } |
| |
| /** |
| * Indexes the document in both the control client, and a randomly selected client |
| */ |
| protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { |
| controlClient.add(doc); |
| |
| int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size(); |
| SolrServer client = clients.get(which); |
| client.add(doc); |
| } |
| |
| /** |
| * Indexes the document in both the control client and the specified client asserting |
| * that the respones are equivilent |
| */ |
| protected UpdateResponse indexDoc(SolrServer server, SolrParams params, SolrInputDocument... sdocs) throws IOException, SolrServerException { |
| UpdateResponse controlRsp = add(controlClient, params, sdocs); |
| UpdateResponse specificRsp = add(server, params, sdocs); |
| compareSolrResponses(specificRsp, controlRsp); |
| return specificRsp; |
| } |
| |
| protected UpdateResponse add(SolrServer server, SolrParams params, SolrInputDocument... sdocs) throws IOException, SolrServerException { |
| UpdateRequest ureq = new UpdateRequest(); |
| ureq.setParams(new ModifiableSolrParams(params)); |
| for (SolrInputDocument sdoc : sdocs) { |
| ureq.add(sdoc); |
| } |
| return ureq.process(server); |
| } |
| |
| protected UpdateResponse del(SolrServer server, SolrParams params, Object... ids) throws IOException, SolrServerException { |
| UpdateRequest ureq = new UpdateRequest(); |
| ureq.setParams(new ModifiableSolrParams(params)); |
| for (Object id: ids) { |
| ureq.deleteById(id.toString()); |
| } |
| return ureq.process(server); |
| } |
| |
| protected UpdateResponse delQ(SolrServer server, SolrParams params, String... queries) throws IOException, SolrServerException { |
| UpdateRequest ureq = new UpdateRequest(); |
| ureq.setParams(new ModifiableSolrParams(params)); |
| for (String q: queries) { |
| ureq.deleteByQuery(q); |
| } |
| return ureq.process(server); |
| } |
| |
| protected void index_specific(int serverNumber, Object... fields) throws Exception { |
| SolrInputDocument doc = new SolrInputDocument(); |
| for (int i = 0; i < fields.length; i += 2) { |
| doc.addField((String) (fields[i]), fields[i + 1]); |
| } |
| controlClient.add(doc); |
| |
| SolrServer client = clients.get(serverNumber); |
| client.add(doc); |
| } |
| |
| protected void del(String q) throws Exception { |
| controlClient.deleteByQuery(q); |
| for (SolrServer client : clients) { |
| client.deleteByQuery(q); |
| } |
| }// serial commit... |
| |
| protected void commit() throws Exception { |
| controlClient.commit(); |
| for (SolrServer client : clients) { |
| client.commit(); |
| } |
| } |
| |
| protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException { |
| // query a random server |
| int which = r.nextInt(clients.size()); |
| SolrServer client = clients.get(which); |
| QueryResponse rsp = client.query(params); |
| return rsp; |
| } |
| |
| /** |
| * Sets distributed params. |
| * Returns the QueryResponse from {@link #queryServer}, |
| */ |
| protected QueryResponse query(Object... q) throws Exception { |
| return query(true, q); |
| } |
| |
| /** |
| * Sets distributed params. |
| * Returns the QueryResponse from {@link #queryServer}, |
| */ |
| protected QueryResponse query(SolrParams params) throws Exception { |
| return query(true, params); |
| } |
| |
| /** |
| * Returns the QueryResponse from {@link #queryServer} |
| */ |
| protected QueryResponse query(boolean setDistribParams, Object[] q) throws Exception { |
| |
| final ModifiableSolrParams params = new ModifiableSolrParams(); |
| |
| for (int i = 0; i < q.length; i += 2) { |
| params.add(q[i].toString(), q[i + 1].toString()); |
| } |
| return query(setDistribParams, params); |
| } |
| |
| /** |
| * Returns the QueryResponse from {@link #queryServer} |
| */ |
| protected QueryResponse query(boolean setDistribParams, SolrParams p) throws Exception { |
| |
| final ModifiableSolrParams params = new ModifiableSolrParams(p); |
| |
| // TODO: look into why passing true causes fails |
| params.set("distrib", "false"); |
| final QueryResponse controlRsp = controlClient.query(params); |
| validateControlData(controlRsp); |
| |
| params.remove("distrib"); |
| if (setDistribParams) setDistributedParams(params); |
| |
| QueryResponse rsp = queryServer(params); |
| |
| compareResponses(rsp, controlRsp); |
| |
| if (stress > 0) { |
| log.info("starting stress..."); |
| Thread[] threads = new Thread[nThreads]; |
| for (int i = 0; i < threads.length; i++) { |
| threads[i] = new Thread() { |
| @Override |
| public void run() { |
| for (int j = 0; j < stress; j++) { |
| int which = r.nextInt(clients.size()); |
| SolrServer client = clients.get(which); |
| try { |
| QueryResponse rsp = client.query(new ModifiableSolrParams(params)); |
| if (verifyStress) { |
| compareResponses(rsp, controlRsp); |
| } |
| } catch (SolrServerException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| } |
| }; |
| threads[i].start(); |
| } |
| |
| for (Thread thread : threads) { |
| thread.join(); |
| } |
| } |
| return rsp; |
| } |
| |
| public QueryResponse queryAndCompare(SolrParams params, SolrServer... servers) throws SolrServerException { |
| return queryAndCompare(params, Arrays.<SolrServer>asList(servers)); |
| } |
| public QueryResponse queryAndCompare(SolrParams params, Iterable<SolrServer> servers) throws SolrServerException { |
| QueryResponse first = null; |
| for (SolrServer server : servers) { |
| QueryResponse rsp = server.query(new ModifiableSolrParams(params)); |
| if (first == null) { |
| first = rsp; |
| } else { |
| compareResponses(first, rsp); |
| } |
| } |
| |
| return first; |
| } |
| |
| public static boolean eq(String a, String b) { |
| return a == b || (a != null && a.equals(b)); |
| } |
| |
| public static int flags(Map<String, Integer> handle, Object key) { |
| if (handle == null) return 0; |
| Integer f = handle.get(key); |
| return f == null ? 0 : f; |
| } |
| |
| public static String compare(NamedList a, NamedList b, int flags, Map<String, Integer> handle) { |
| // System.out.println("resp a:" + a); |
| // System.out.println("resp b:" + b); |
| boolean ordered = (flags & UNORDERED) == 0; |
| |
| if (!ordered) { |
| Map mapA = new HashMap(a.size()); |
| for (int i=0; i<a.size(); i++) { |
| Object prev = mapA.put(a.getName(i), a.getVal(i)); |
| } |
| |
| Map mapB = new HashMap(b.size()); |
| for (int i=0; i<b.size(); i++) { |
| Object prev = mapB.put(b.getName(i), b.getVal(i)); |
| } |
| |
| return compare(mapA, mapB, flags, handle); |
| } |
| |
| int posa = 0, posb = 0; |
| int aSkipped = 0, bSkipped = 0; |
| |
| for (; ;) { |
| if (posa >= a.size() && posb >= b.size()) { |
| break; |
| } |
| |
| String namea = null, nameb = null; |
| Object vala = null, valb = null; |
| |
| int flagsa = 0, flagsb = 0; |
| while (posa < a.size()) { |
| namea = a.getName(posa); |
| vala = a.getVal(posa); |
| posa++; |
| flagsa = flags(handle, namea); |
| if ((flagsa & SKIP) != 0) { |
| namea = null; vala = null; |
| aSkipped++; |
| continue; |
| } |
| break; |
| } |
| |
| while (posb < b.size()) { |
| nameb = b.getName(posb); |
| valb = b.getVal(posb); |
| posb++; |
| flagsb = flags(handle, nameb); |
| if ((flagsb & SKIP) != 0) { |
| nameb = null; valb = null; |
| bSkipped++; |
| continue; |
| } |
| if (eq(namea, nameb)) { |
| break; |
| } |
| return "." + namea + "!=" + nameb + " (unordered or missing)"; |
| // if unordered, continue until we find the right field. |
| } |
| |
| // ok, namea and nameb should be equal here already. |
| if ((flagsa & SKIPVAL) != 0) continue; // keys matching is enough |
| |
| String cmp = compare(vala, valb, flagsa, handle); |
| if (cmp != null) return "." + namea + cmp; |
| } |
| |
| |
| if (a.size() - aSkipped != b.size() - bSkipped) { |
| return ".size()==" + a.size() + "," + b.size() + " skipped=" + aSkipped + "," + bSkipped; |
| } |
| |
| return null; |
| } |
| |
| public static String compare1(Map a, Map b, int flags, Map<String, Integer> handle) { |
| String cmp; |
| |
| for (Object keya : a.keySet()) { |
| Object vala = a.get(keya); |
| int flagsa = flags(handle, keya); |
| if ((flagsa & SKIP) != 0) continue; |
| if (!b.containsKey(keya)) { |
| return "[" + keya + "]==null"; |
| } |
| if ((flagsa & SKIPVAL) != 0) continue; |
| Object valb = b.get(keya); |
| cmp = compare(vala, valb, flagsa, handle); |
| if (cmp != null) return "[" + keya + "]" + cmp; |
| } |
| return null; |
| } |
| |
| public static String compare(Map a, Map b, int flags, Map<String, Integer> handle) { |
| String cmp; |
| cmp = compare1(a, b, flags, handle); |
| if (cmp != null) return cmp; |
| return compare1(b, a, flags, handle); |
| } |
| |
| public static String compare(SolrDocument a, SolrDocument b, int flags, Map<String, Integer> handle) { |
| return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle); |
| } |
| |
| public static String compare(SolrDocumentList a, SolrDocumentList b, int flags, Map<String, Integer> handle) { |
| boolean ordered = (flags & UNORDERED) == 0; |
| |
| String cmp; |
| int f = flags(handle, "maxScore"); |
| if ((f & SKIPVAL) == 0) { |
| cmp = compare(a.getMaxScore(), b.getMaxScore(), 0, handle); |
| if (cmp != null) return ".maxScore" + cmp; |
| } else { |
| if (b.getMaxScore() != null) { |
| if (a.getMaxScore() == null) { |
| return ".maxScore missing"; |
| } |
| } |
| } |
| |
| cmp = compare(a.getNumFound(), b.getNumFound(), 0, handle); |
| if (cmp != null) return ".numFound" + cmp; |
| |
| cmp = compare(a.getStart(), b.getStart(), 0, handle); |
| if (cmp != null) return ".start" + cmp; |
| |
| cmp = compare(a.size(), b.size(), 0, handle); |
| if (cmp != null) return ".size()" + cmp; |
| |
| // only for completely ordered results (ties might be in a different order) |
| if (ordered) { |
| for (int i = 0; i < a.size(); i++) { |
| cmp = compare(a.get(i), b.get(i), 0, handle); |
| if (cmp != null) return "[" + i + "]" + cmp; |
| } |
| return null; |
| } |
| |
| // unordered case |
| for (int i = 0; i < a.size(); i++) { |
| SolrDocument doc = a.get(i); |
| Object key = doc.getFirstValue("id"); |
| SolrDocument docb = null; |
| if (key == null) { |
| // no id field to correlate... must compare ordered |
| docb = b.get(i); |
| } else { |
| for (int j = 0; j < b.size(); j++) { |
| docb = b.get(j); |
| if (key.equals(docb.getFirstValue("id"))) break; |
| } |
| } |
| // if (docb == null) return "[id="+key+"]"; |
| cmp = compare(doc, docb, 0, handle); |
| if (cmp != null) return "[id=" + key + "]" + cmp; |
| } |
| return null; |
| } |
| |
| public static String compare(Object[] a, Object[] b, int flags, Map<String, Integer> handle) { |
| if (a.length != b.length) { |
| return ".length:" + a.length + "!=" + b.length; |
| } |
| for (int i = 0; i < a.length; i++) { |
| String cmp = compare(a[i], b[i], flags, handle); |
| if (cmp != null) return "[" + i + "]" + cmp; |
| } |
| return null; |
| } |
| |
| public static String compare(Object a, Object b, int flags, Map<String, Integer> handle) { |
| if (a == b) return null; |
| if (a == null || b == null) return ":" + a + "!=" + b; |
| |
| if (a instanceof NamedList && b instanceof NamedList) { |
| return compare((NamedList) a, (NamedList) b, flags, handle); |
| } |
| |
| if (a instanceof SolrDocumentList && b instanceof SolrDocumentList) { |
| return compare((SolrDocumentList) a, (SolrDocumentList) b, flags, handle); |
| } |
| |
| if (a instanceof SolrDocument && b instanceof SolrDocument) { |
| return compare((SolrDocument) a, (SolrDocument) b, flags, handle); |
| } |
| |
| if (a instanceof Map && b instanceof Map) { |
| return compare((Map) a, (Map) b, flags, handle); |
| } |
| |
| if (a instanceof Object[] && b instanceof Object[]) { |
| return compare((Object[]) a, (Object[]) b, flags, handle); |
| } |
| |
| if (a instanceof byte[] && b instanceof byte[]) { |
| if (!Arrays.equals((byte[]) a, (byte[]) b)) { |
| return ":" + a + "!=" + b; |
| } |
| return null; |
| } |
| |
| if (a instanceof List && b instanceof List) { |
| return compare(((List) a).toArray(), ((List) b).toArray(), flags, handle); |
| |
| } |
| |
| if (!(a.equals(b))) { |
| return ":" + a + "!=" + b; |
| } |
| |
| return null; |
| } |
| |
| protected void compareSolrResponses(SolrResponse a, SolrResponse b) { |
| String cmp = compare(a.getResponse(), b.getResponse(), flags, handle); |
| if (cmp != null) { |
| log.error("Mismatched responses:\n" + a + "\n" + b); |
| Assert.fail(cmp); |
| } |
| } |
| protected void compareResponses(QueryResponse a, QueryResponse b) { |
| if (System.getProperty("remove.version.field") != null) { |
| // we don't care if one has a version and the other doesnt - |
| // control vs distrib |
| // TODO: this should prob be done by adding an ignore on _version_ rather than mutating the responses? |
| if (a.getResults() != null) { |
| for (SolrDocument doc : a.getResults()) { |
| doc.removeFields("_version_"); |
| } |
| } |
| if (b.getResults() != null) { |
| for (SolrDocument doc : b.getResults()) { |
| doc.removeFields("_version_"); |
| } |
| } |
| } |
| compareSolrResponses(a, b); |
| } |
| |
| @Test |
| public void testDistribSearch() throws Exception { |
| if (fixShardCount) { |
| createServers(shardCount); |
| RandVal.uniqueValues = new HashSet(); //reset random values |
| doTest(); |
| destroyServers(); |
| } else { |
| for (int nServers = 1; nServers < shardCount; nServers++) { |
| createServers(nServers); |
| RandVal.uniqueValues = new HashSet(); //reset random values |
| doTest(); |
| destroyServers(); |
| } |
| } |
| } |
| |
| public static Object[] getRandFields(String[] fields, RandVal[] randVals) { |
| Object[] o = new Object[fields.length * 2]; |
| for (int i = 0; i < fields.length; i++) { |
| o[i * 2] = fields[i]; |
| o[i * 2 + 1] = randVals[i].uval(); |
| } |
| return o; |
| } |
| |
| /** |
| * Implementations can pre-test the control data for basic correctness before using it |
| * as a check for the shard data. This is useful, for instance, if a test bug is introduced |
| * causing a spelling index not to get built: both control & shard data would have no results |
| * but because they match the test would pass. This method gives us a chance to ensure something |
| * exists in the control data. |
| */ |
| public void validateControlData(QueryResponse control) throws Exception { |
| /* no-op */ |
| } |
| |
| public static abstract class RandVal { |
| public static Set uniqueValues = new HashSet(); |
| |
| public abstract Object val(); |
| |
| public Object uval() { |
| for (; ;) { |
| Object v = val(); |
| if (uniqueValues.add(v)) return v; |
| } |
| } |
| } |
| |
| public static class RandDate extends RandVal { |
| public static TrieDateField df = new TrieDateField(); |
| |
| @Override |
| public Object val() { |
| long v = r.nextLong(); |
| Date d = new Date(v); |
| return df.toExternal(d); |
| } |
| } |
| |
| protected String getSolrXml() { |
| return null; |
| } |
| |
| protected void setupJettySolrHome(File jettyHome) throws IOException { |
| FileUtils.copyDirectory(new File(getSolrHome()), jettyHome); |
| String solrxml = getSolrXml(); |
| if (solrxml != null) { |
| FileUtils.copyFile(new File(getSolrHome(), solrxml), new File(jettyHome, "solr.xml")); |
| } |
| } |
| |
| } |