Google Git
Sign in
apache / lucene-solr / refs/heads/branch_4x / . / solr / test-framework / src / java / org / apache / solr / SolrTestCaseJ4.java
blob: 9dcba13fb35a30305e3ac911e0aa4937cae89fe0 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr;
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import org.apache.commons.codec.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.impl.HttpClientConfigurer;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.IpTables;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.XML;
import org.apache.solr.core.ConfigSolr;
import org.apache.solr.core.ConfigSolrXmlOld;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.JsonUpdateRequestHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.servlet.DirectSolrConnection;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.RevertDefaultThreadHandlerRule;
import org.apache.solr.util.SSLTestConfig;
import org.apache.solr.util.TestHarness;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import org.noggit.CharArr;
import org.noggit.JSONUtil;
import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
import javax.xml.xpath.XPathExpressionException;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.logging.ConsoleHandler;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.regex.Pattern;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* A junit4 Solr test harness that extends LuceneTestCaseJ4. To change which core is used when loading the schema and solrconfig.xml, simply
* invoke the {@link #initCore(String, String, String, String)} method.
*
* Unlike {@link AbstractSolrTestCase}, a new core is not created for each test method.
*/
@ThreadLeakFilters(defaultFilters = true, filters = {
SolrIgnoredThreadsFilter.class,
QuickPatchThreadsFilter.class
})
@SuppressSysoutChecks(bugUrl = "Solr dumps tons of logs to console.")
public abstract class SolrTestCaseJ4 extends LuceneTestCase {
private static String coreName = ConfigSolrXmlOld.DEFAULT_DEFAULT_CORE_NAME;
public static int DEFAULT_CONNECTION_TIMEOUT = 60000; // default socket connection timeout in ms
/**
* Annotation for test classes that want to disable SSL
*/
@Documented
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface SuppressSSL {
/** Point to JIRA entry. */
public String bugUrl() default "None";
}
// these are meant to be accessed sequentially, but are volatile just to ensure any test
// thread will read the latest value
protected static volatile SSLTestConfig sslConfig;
@ClassRule
public static TestRule solrClassRules =
RuleChain.outerRule(new SystemPropertiesRestoreRule())
.around(new RevertDefaultThreadHandlerRule());
@Rule
public TestRule solrTestRules =
RuleChain.outerRule(new SystemPropertiesRestoreRule());
@BeforeClass
@SuppressWarnings("unused")
private static void beforeClass() {
initCoreDataDir = createTempDir("init-core-data");
System.err.println("Creating dataDir: " + initCoreDataDir.getAbsolutePath());
System.setProperty("zookeeper.forceSync", "no");
System.setProperty("jetty.testMode", "true");
System.setProperty("enable.update.log", usually() ? "true" : "false");
System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
System.setProperty("solr.clustering.enabled", "false");
setupLogging();
startTrackingSearchers();
startTrackingZkClients();
ignoreException("ignore_exception");
newRandomConfig();
sslConfig = buildSSLConfig();
//will use ssl specific or default depending on sslConfig
HttpClientUtil.setConfigurer(sslConfig.getHttpClientConfigurer());
if(isSSLMode()) {
// SolrCloud tests should usually clear this
System.setProperty("urlScheme", "https");
}
}
@AfterClass
@SuppressWarnings("unused")
private static void afterClass() throws Exception {
try {
deleteCore();
resetExceptionIgnores();
endTrackingSearchers();
endTrackingZkClients();
resetFactory();
coreName = ConfigSolrXmlOld.DEFAULT_DEFAULT_CORE_NAME;
} finally {
initCoreDataDir = null;
System.clearProperty("zookeeper.forceSync");
System.clearProperty("jetty.testMode");
System.clearProperty("tests.shardhandler.randomSeed");
System.clearProperty("enable.update.log");
System.clearProperty("useCompoundFile");
System.clearProperty("urlScheme");
if (isSSLMode()) {
HttpClientUtil.setConfigurer(new HttpClientConfigurer());
}
// clean up static
sslConfig = null;
}
IpTables.unblockAllPorts();
}
protected static boolean isSSLMode() {
return sslConfig != null && sslConfig.isSSLMode();
}
private static boolean changedFactory = false;
private static String savedFactory;
/** Use a different directory factory. Passing "null" sets to an FS-based factory */
public static void useFactory(String factory) throws Exception {
// allow calling more than once so a subclass can override a base class
if (!changedFactory) {
savedFactory = System.getProperty("solr.DirectoryFactory");
}
if (factory == null) {
factory = random().nextInt(100) < 75 ? "solr.NRTCachingDirectoryFactory" : "solr.StandardDirectoryFactory"; // test the default most of the time
}
System.setProperty("solr.directoryFactory", factory);
changedFactory = true;
}
public static void resetFactory() throws Exception {
if (!changedFactory) return;
changedFactory = false;
if (savedFactory != null) {
System.setProperty("solr.directoryFactory", savedFactory);
} else {
System.clearProperty("solr.directoryFactory");
}
}
private static SSLTestConfig buildSSLConfig() {
// test has been disabled
if (RandomizedContext.current().getTargetClass().isAnnotationPresent(SuppressSSL.class)) {
return new SSLTestConfig();
}
final boolean trySsl = random().nextBoolean();
boolean trySslClientAuth = random().nextBoolean();
if (Constants.MAC_OS_X) {
trySslClientAuth = false;
}
log.info("Randomized ssl ({}) and clientAuth ({})", trySsl,
trySslClientAuth);
return new SSLTestConfig(trySsl, trySslClientAuth);
}
protected static String buildUrl(final int port, final String context) {
return (isSSLMode() ? "https" : "http") + "://127.0.0.1:" + port + context;
}
/**
* Call this from @BeforeClass to set up the test harness and update handler with no cores.
*
* @param solrHome The solr home directory.
* @param xmlStr - the text of an XML file to use. If null, use the what's the absolute minimal file.
* @throws Exception Lost of file-type things can go wrong.
*/
public static void setupNoCoreTest(File solrHome, String xmlStr) throws Exception {
File tmpFile = new File(solrHome, ConfigSolr.SOLR_XML_FILE);
if (xmlStr == null) {
xmlStr = "<solr></solr>";
}
FileUtils.write(tmpFile, xmlStr, IOUtils.UTF_8);
SolrResourceLoader loader = new SolrResourceLoader(solrHome.getAbsolutePath());
h = new TestHarness(loader, ConfigSolr.fromFile(loader, new File(solrHome, "solr.xml")));
lrf = h.getRequestFactory("standard", 0, 20, CommonParams.VERSION, "2.2");
}
/** sets system properties based on
* {@link #newIndexWriterConfig(org.apache.lucene.analysis.Analyzer)}
*
* configs can use these system properties to vary the indexwriter settings
*/
public static void newRandomConfig() {
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
System.setProperty("useCompoundFile", String.valueOf(iwc.getUseCompoundFile()));
System.setProperty("solr.tests.maxBufferedDocs", String.valueOf(iwc.getMaxBufferedDocs()));
System.setProperty("solr.tests.ramBufferSizeMB", String.valueOf(iwc.getRAMBufferSizeMB()));
System.setProperty("solr.tests.mergeScheduler", iwc.getMergeScheduler().getClass().getName());
// don't ask iwc.getMaxThreadStates(), sometimes newIWC uses
// RandomDocumentsWriterPerThreadPool and all hell breaks loose
int maxIndexingThreads = rarely(random())
? TestUtil.nextInt(random(), 5, 20) // crazy value
: TestUtil.nextInt(random(), 1, 4); // reasonable value
System.setProperty("solr.tests.maxIndexingThreads", String.valueOf(maxIndexingThreads));
}
public static Throwable getWrappedException(Throwable e) {
while (e != null && e.getCause() != e && e.getCause() != null) {
e = e.getCause();
}
return e;
}
@Override
public void setUp() throws Exception {
super.setUp();
log.info("###Starting " + getTestName()); // returns <unknown>???
}
@Override
public void tearDown() throws Exception {
log.info("###Ending " + getTestName());
super.tearDown();
}
public static SolrLogFormatter formatter;
public static void setupLogging() {
boolean register = false;
Handler[] handlers = java.util.logging.Logger.getLogger("").getHandlers();
ConsoleHandler consoleHandler = null;
for (Handler handler : handlers) {
if (handler instanceof ConsoleHandler) {
consoleHandler = (ConsoleHandler)handler;
break;
}
}
if (consoleHandler == null) {
consoleHandler = new ConsoleHandler();
register = true;
}
consoleHandler.setLevel(Level.ALL);
formatter = new SolrLogFormatter();
consoleHandler.setFormatter(formatter);
if (register) {
java.util.logging.Logger.getLogger("").addHandler(consoleHandler);
}
}
public static void setLoggingLevel(Level level) {
java.util.logging.Logger logger = java.util.logging.Logger.getLogger("");
logger.setLevel(level);
}
/** Call initCore in @BeforeClass to instantiate a solr core in your test class.
* deleteCore will be called for you via SolrTestCaseJ4 @AfterClass */
public static void initCore(String config, String schema) throws Exception {
initCore(config, schema, TEST_HOME());
}
/** Call initCore in @BeforeClass to instantiate a solr core in your test class.
* deleteCore will be called for you via SolrTestCaseJ4 @AfterClass */
public static void initCore(String config, String schema, String solrHome) throws Exception {
assertNotNull(solrHome);
configString = config;
schemaString = schema;
testSolrHome = solrHome;
if (solrHome != null) {
System.setProperty("solr.solr.home", solrHome);
}
initCore();
}
/** Call initCore in @BeforeClass to instantiate a solr core in your test class.
* deleteCore will be called for you via SolrTestCaseJ4 @AfterClass */
public static void initCore(String config, String schema, String solrHome, String pCoreName) throws Exception {
coreName=pCoreName;
initCore(config,schema,solrHome);
}
static long numOpens;
static long numCloses;
public static void startTrackingSearchers() {
numOpens = SolrIndexSearcher.numOpens.getAndSet(0);
numCloses = SolrIndexSearcher.numCloses.getAndSet(0);
if (numOpens != 0 || numCloses != 0) {
// NOTE: some other tests don't use this base class and hence won't reset the counts.
log.warn("startTrackingSearchers: numOpens="+numOpens+" numCloses="+numCloses);
numOpens = numCloses = 0;
}
}
static long zkClientNumOpens;
static long zkClientNumCloses;
public static void startTrackingZkClients() {
zkClientNumOpens = SolrZkClient.numOpens.get();
zkClientNumCloses = SolrZkClient.numCloses.get();
}
public static void endTrackingSearchers() {
long endNumOpens = SolrIndexSearcher.numOpens.get();
long endNumCloses = SolrIndexSearcher.numCloses.get();
// wait a bit in case any ending threads have anything to release
int retries = 0;
while (endNumOpens - numOpens != endNumCloses - numCloses) {
if (retries++ > 120) {
break;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {}
endNumOpens = SolrIndexSearcher.numOpens.get();
endNumCloses = SolrIndexSearcher.numCloses.get();
}
SolrIndexSearcher.numOpens.getAndSet(0);
SolrIndexSearcher.numCloses.getAndSet(0);
if (endNumOpens-numOpens != endNumCloses-numCloses) {
String msg = "ERROR: SolrIndexSearcher opens=" + (endNumOpens-numOpens) + " closes=" + (endNumCloses-numCloses);
log.error(msg);
// if its TestReplicationHandler, ignore it. the test is broken and gets no love
if ("TestReplicationHandler".equals(RandomizedContext.current().getTargetClass().getSimpleName())) {
log.warn("TestReplicationHandler wants to fail!: " + msg);
} else {
fail(msg);
}
}
}
public static void endTrackingZkClients() {
long endNumOpens = SolrZkClient.numOpens.get();
long endNumCloses = SolrZkClient.numCloses.get();
SolrZkClient.numOpens.getAndSet(0);
SolrZkClient.numCloses.getAndSet(0);
if (endNumOpens-zkClientNumOpens != endNumCloses-zkClientNumCloses) {
String msg = "ERROR: SolrZkClient opens=" + (endNumOpens-zkClientNumOpens) + " closes=" + (endNumCloses-zkClientNumCloses);
log.error(msg);
fail(msg);
}
}
/** Causes an exception matching the regex pattern to not be logged. */
public static void ignoreException(String pattern) {
if (SolrException.ignorePatterns == null)
SolrException.ignorePatterns = new HashSet<>();
SolrException.ignorePatterns.add(pattern);
}
public static void unIgnoreException(String pattern) {
if (SolrException.ignorePatterns != null)
SolrException.ignorePatterns.remove(pattern);
}
public static void resetExceptionIgnores() {
SolrException.ignorePatterns = null;
ignoreException("ignore_exception"); // always ignore "ignore_exception"
}
protected static String getClassName() {
return getTestClass().getName();
}
protected static String getSimpleClassName() {
return getTestClass().getSimpleName();
}
protected static String configString;
protected static String schemaString;
protected static String testSolrHome;
protected static SolrConfig solrConfig;
/**
* Harness initialized by initTestHarness.
*
* <p>
* For use in test methods as needed.
* </p>
*/
protected static TestHarness h;
/**
* LocalRequestFactory initialized by initTestHarness using sensible
* defaults.
*
* <p>
* For use in test methods as needed.
* </p>
*/
protected static TestHarness.LocalRequestFactory lrf;
/**
* Subclasses must define this method to return the name of the
* schema.xml they wish to use.
*/
public static String getSchemaFile() {
return schemaString;
}
/**
* Subclasses must define this method to return the name of the
* solrconfig.xml they wish to use.
*/
public static String getSolrConfigFile() {
return configString;
}
/**
* The directory used to story the index managed by the TestHarness
*/
protected static volatile File initCoreDataDir;
// hack due to File dataDir
protected static String hdfsDataDir;
/**
* Initializes things your test might need
*
* <ul>
* <li>Creates a dataDir in the "java.io.tmpdir"</li>
* <li>initializes the TestHarness h using this data directory, and getSchemaPath()</li>
* <li>initializes the LocalRequestFactory lrf using sensible defaults.</li>
* </ul>
*
*/
public static Logger log = LoggerFactory.getLogger(SolrTestCaseJ4.class);
private static String factoryProp;
public static void initCore() throws Exception {
log.info("####initCore");
ignoreException("ignore_exception");
factoryProp = System.getProperty("solr.directoryFactory");
if (factoryProp == null) {
System.setProperty("solr.directoryFactory","solr.RAMDirectoryFactory");
}
// other methods like starting a jetty instance need these too
System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo");
String configFile = getSolrConfigFile();
if (configFile != null) {
createCore();
}
log.info("####initCore end");
}
public static void createCore() {
assertNotNull(testSolrHome);
solrConfig = TestHarness.createConfig(testSolrHome, coreName, getSolrConfigFile());
h = new TestHarness( coreName, hdfsDataDir == null ? initCoreDataDir.getAbsolutePath() : hdfsDataDir,
solrConfig,
getSchemaFile());
lrf = h.getRequestFactory
("standard",0,20,CommonParams.VERSION,"2.2");
}
public static CoreContainer createCoreContainer(String solrHome, String solrXML) {
testSolrHome = checkNotNull(solrHome);
h = new TestHarness(solrHome, solrXML);
lrf = h.getRequestFactory("standard", 0, 20, CommonParams.VERSION, "2.2");
return h.getCoreContainer();
}
public static CoreContainer createDefaultCoreContainer(String solrHome) {
testSolrHome = checkNotNull(solrHome);
h = new TestHarness("collection1", initCoreDataDir.getAbsolutePath(), "solrconfig.xml", "schema.xml");
lrf = h.getRequestFactory("standard", 0, 20, CommonParams.VERSION, "2.2");
return h.getCoreContainer();
}
public static boolean hasInitException(String message) {
for (Map.Entry<String, CoreContainer.CoreLoadFailure> entry : h.getCoreContainer().getCoreInitFailures().entrySet()) {
if (entry.getValue().exception.getMessage().contains(message))
return true;
}
return false;
}
public static boolean hasInitException(Class<? extends Exception> exceptionType) {
for (Map.Entry<String, CoreContainer.CoreLoadFailure> entry : h.getCoreContainer().getCoreInitFailures().entrySet()) {
if (exceptionType.isAssignableFrom(entry.getValue().exception.getClass()))
return true;
}
return false;
}
/** Subclasses that override setUp can optionally call this method
* to log the fact that their setUp process has ended.
*/
public void postSetUp() {
log.info("####POSTSETUP " + getTestName());
}
/** Subclasses that override tearDown can optionally call this method
* to log the fact that the tearDown process has started. This is necessary
* since subclasses will want to call super.tearDown() at the *end* of their
* tearDown method.
*/
public void preTearDown() {
log.info("####PRETEARDOWN " + getTestName());
}
/**
* Shuts down the test harness, and makes the best attempt possible
* to delete dataDir, unless the system property "solr.test.leavedatadir"
* is set.
*/
public static void deleteCore() {
log.info("###deleteCore" );
if (h != null) { h.close(); }
if (factoryProp == null) {
System.clearProperty("solr.directoryFactory");
}
solrConfig = null;
h = null;
lrf = null;
configString = schemaString = null;
}
/** Validates an update XML String is successful
*/
public static void assertU(String update) {
assertU(null, update);
}
/** Validates an update XML String is successful
*/
public static void assertU(String message, String update) {
checkUpdateU(message, update, true);
}
/** Validates an update XML String failed
*/
public static void assertFailedU(String update) {
assertFailedU(null, update);
}
/** Validates an update XML String failed
*/
public static void assertFailedU(String message, String update) {
checkUpdateU(message, update, false);
}
/** Checks the success or failure of an update message
*/
private static void checkUpdateU(String message, String update, boolean shouldSucceed) {
try {
String m = (null == message) ? "" : message + " ";
if (shouldSucceed) {
String res = h.validateUpdate(update);
if (res != null) fail(m + "update was not successful: " + res);
} else {
String res = h.validateErrorUpdate(update);
if (res != null) fail(m + "update succeeded, but should have failed: " + res);
}
} catch (SAXException e) {
throw new RuntimeException("Invalid XML", e);
}
}
/** Validates a query matches some XPath test expressions and closes the query */
public static void assertQ(SolrQueryRequest req, String... tests) {
assertQ(null, req, tests);
}
/** Validates a query matches some XPath test expressions and closes the query */
public static void assertQ(String message, SolrQueryRequest req, String... tests) {
try {
String m = (null == message) ? "" : message + " ";
String response = h.query(req);
if (req.getParams().getBool("facet", false)) {
// add a test to ensure that faceting did not throw an exception
// internally, where it would be added to facet_counts/exception
String[] allTests = new String[tests.length+1];
System.arraycopy(tests,0,allTests,1,tests.length);
allTests[0] = "*[count(//lst[@name='facet_counts']/*[@name='exception'])=0]";
tests = allTests;
}
String results = h.validateXPath(response, tests);
if (null != results) {
String msg = "REQUEST FAILED: xpath=" + results
+ "\n\txml response was: " + response
+ "\n\trequest was:" + req.getParamString();
log.error(msg);
throw new RuntimeException(msg);
}
} catch (XPathExpressionException e1) {
throw new RuntimeException("XPath is invalid", e1);
} catch (Exception e2) {
SolrException.log(log,"REQUEST FAILED: " + req.getParamString(), e2);
throw new RuntimeException("Exception during query", e2);
}
}
/** Makes a query request and returns the JSON string response */
public static String JQ(SolrQueryRequest req) throws Exception {
SolrParams params = req.getParams();
if (!"json".equals(params.get("wt","xml")) || params.get("indent")==null) {
ModifiableSolrParams newParams = new ModifiableSolrParams(params);
newParams.set("wt","json");
if (params.get("indent")==null) newParams.set("indent","true");
req.setParams(newParams);
}
String response;
boolean failed=true;
try {
response = h.query(req);
failed = false;
} finally {
if (failed) {
log.error("REQUEST FAILED: " + req.getParamString());
}
}
return response;
}
/**
* Validates a query matches some JSON test expressions using the default double delta tolerance.
* @see JSONTestUtil#DEFAULT_DELTA
* @see #assertJQ(SolrQueryRequest,double,String...)
* @return The request response as a JSON String if all test patterns pass
*/
public static String assertJQ(SolrQueryRequest req, String... tests) throws Exception {
return assertJQ(req, JSONTestUtil.DEFAULT_DELTA, tests);
}
/**
* Validates a query matches some JSON test expressions and closes the
* query. The text expression is of the form path:JSON. To facilitate
* easy embedding in Java strings, the JSON tests can have double quotes
* replaced with single quotes.
* <p>
* Please use this with care: this makes it easy to match complete
* structures, but doing so can result in fragile tests if you are
* matching more than what you want to test.
* </p>
* @param req Solr request to execute
* @param delta tolerance allowed in comparing float/double values
* @param tests JSON path expression + '==' + expected value
* @return The request response as a JSON String if all test patterns pass
*/
public static String assertJQ(SolrQueryRequest req, double delta, String... tests) throws Exception {
SolrParams params = null;
try {
params = req.getParams();
if (!"json".equals(params.get("wt","xml")) || params.get("indent")==null) {
ModifiableSolrParams newParams = new ModifiableSolrParams(params);
newParams.set("wt","json");
if (params.get("indent")==null) newParams.set("indent","true");
req.setParams(newParams);
}
String response;
boolean failed=true;
try {
response = h.query(req);
failed = false;
} finally {
if (failed) {
log.error("REQUEST FAILED: " + req.getParamString());
}
}
for (String test : tests) {
if (test == null || test.length()==0) continue;
String testJSON = json(test);
try {
failed = true;
String err = JSONTestUtil.match(response, testJSON, delta);
failed = false;
if (err != null) {
log.error("query failed JSON validation. error=" + err +
"\n expected =" + testJSON +
"\n response = " + response +
"\n request = " + req.getParamString()
);
throw new RuntimeException(err);
}
} finally {
if (failed) {
log.error("JSON query validation threw an exception." +
"\n expected =" + testJSON +
"\n response = " + response +
"\n request = " + req.getParamString()
);
}
}
}
return response;
} finally {
// restore the params
if (params != null && params != req.getParams()) req.setParams(params);
}
}
/** Makes sure a query throws a SolrException with the listed response code */
public static void assertQEx(String message, SolrQueryRequest req, int code ) {
try {
ignoreException(".");
h.query(req);
fail( message );
} catch (SolrException sex) {
assertEquals( code, sex.code() );
} catch (Exception e2) {
throw new RuntimeException("Exception during query", e2);
} finally {
unIgnoreException(".");
}
}
public static void assertQEx(String message, SolrQueryRequest req, SolrException.ErrorCode code ) {
try {
ignoreException(".");
h.query(req);
fail( message );
} catch (SolrException e) {
assertEquals( code.code, e.code() );
} catch (Exception e2) {
throw new RuntimeException("Exception during query", e2);
} finally {
unIgnoreException(".");
}
}
/**
* Makes sure a query throws a SolrException with the listed response code and expected message
* @param failMessage The assert message to show when the query doesn't throw the expected exception
* @param exceptionMessage A substring of the message expected in the exception
* @param req Solr request
* @param code expected error code for the query
*/
public static void assertQEx(String failMessage, String exceptionMessage, SolrQueryRequest req, SolrException.ErrorCode code ) {
try {
ignoreException(".");
h.query(req);
fail( failMessage );
} catch (SolrException e) {
assertEquals( code.code, e.code() );
assertTrue("Unexpected error message. Expecting \"" + exceptionMessage +
"\" but got \"" + e.getMessage() + "\"", e.getMessage()!= null && e.getMessage().contains(exceptionMessage));
} catch (Exception e2) {
throw new RuntimeException("Exception during query", e2);
} finally {
unIgnoreException(".");
}
}
/**
* @see TestHarness#optimize
*/
public static String optimize(String... args) {
return TestHarness.optimize(args);
}
/**
* @see TestHarness#commit
*/
public static String commit(String... args) {
return TestHarness.commit(args);
}
/**
* Generates a simple &lt;add&gt;&lt;doc&gt;... XML String with no options
*
* @param fieldsAndValues 0th and Even numbered args are fields names odds are field values.
* @see #add
* @see #doc
*/
public static String adoc(String... fieldsAndValues) {
XmlDoc d = doc(fieldsAndValues);
return add(d);
}
/**
* Generates a simple &lt;add&gt;&lt;doc&gt;... XML String with no options
*/
public static String adoc(SolrInputDocument sdoc) {
StringWriter out = new StringWriter(512);
try {
out.append("<add>");
ClientUtils.writeXML(sdoc, out);
out.append("</add>");
} catch (IOException e) {
throw new RuntimeException("Inexplicable IO error from StringWriter", e);
}
return out.toString();
}
/**
* Generates an &lt;add&gt;&lt;doc&gt;... XML String with options
* on the add.
*
* @param doc the Document to add
* @param args 0th and Even numbered args are param names, Odds are param values.
* @see #add
* @see #doc
*/
public static String add(XmlDoc doc, String... args) {
try {
StringWriter r = new StringWriter();
// this is annoying
if (null == args || 0 == args.length) {
r.write("<add>");
r.write(doc.xml);
r.write("</add>");
} else {
XML.writeUnescapedXML(r, "add", doc.xml, (Object[])args);
}
return r.getBuffer().toString();
} catch (IOException e) {
throw new RuntimeException
("this should never happen with a StringWriter", e);
}
}
/**
* Generates a &lt;delete&gt;... XML string for an ID
*
* @see TestHarness#deleteById
*/
public static String delI(String id) {
return TestHarness.deleteById(id);
}
/**
* Generates a &lt;delete&gt;... XML string for an query
*
* @see TestHarness#deleteByQuery
*/
public static String delQ(String q) {
return TestHarness.deleteByQuery(q);
}
/**
* Generates a simple &lt;doc&gt;... XML String with no options
*
* @param fieldsAndValues 0th and Even numbered args are fields names, Odds are field values.
* @see TestHarness#makeSimpleDoc
*/
public static XmlDoc doc(String... fieldsAndValues) {
XmlDoc d = new XmlDoc();
d.xml = TestHarness.makeSimpleDoc(fieldsAndValues);
return d;
}
public static ModifiableSolrParams params(String... params) {
ModifiableSolrParams msp = new ModifiableSolrParams();
for (int i=0; i<params.length; i+=2) {
msp.add(params[i], params[i+1]);
}
return msp;
}
public static Map map(Object... params) {
LinkedHashMap ret = new LinkedHashMap();
for (int i=0; i<params.length; i+=2) {
Object o = ret.put(params[i], params[i+1]);
// TODO: handle multi-valued map?
}
return ret;
}
/**
* Generates a SolrQueryRequest using the LocalRequestFactory
* @see #lrf
*/
public static SolrQueryRequest req(String... q) {
return lrf.makeRequest(q);
}
/**
* Generates a SolrQueryRequest using the LocalRequestFactory
* @see #lrf
*/
public static SolrQueryRequest req(String[] params, String... moreParams) {
String[] allParams = moreParams;
if (params.length!=0) {
int len = params.length + moreParams.length;
allParams = new String[len];
System.arraycopy(params,0,allParams,0,params.length);
System.arraycopy(moreParams,0,allParams,params.length,moreParams.length);
}
return lrf.makeRequest(allParams);
}
/**
* Generates a SolrQueryRequest
*/
public static SolrQueryRequest req(SolrParams params, String... moreParams) {
ModifiableSolrParams mp = new ModifiableSolrParams(params);
for (int i=0; i<moreParams.length; i+=2) {
mp.add(moreParams[i], moreParams[i+1]);
}
return new LocalSolrQueryRequest(h.getCore(), mp);
}
/** Necessary to make method signatures un-ambiguous */
public static class XmlDoc {
public String xml;
@Override
public String toString() { return xml; }
}
/**
* @see IOUtils#rm(File...)
*/
@Deprecated()
public static boolean recurseDelete(File f) {
try {
IOUtils.rm(f);
return true;
} catch (IOException e) {
System.err.println(e.toString());
return false;
}
}
public void clearIndex() {
assertU(delQ("*:*"));
}
/** Send JSON update commands */
public static String updateJ(String json, SolrParams args) throws Exception {
SolrCore core = h.getCore();
if (args == null) {
args = params("wt","json","indent","true");
} else {
ModifiableSolrParams newArgs = new ModifiableSolrParams(args);
if (newArgs.get("wt") == null) newArgs.set("wt","json");
if (newArgs.get("indent") == null) newArgs.set("indent","true");
args = newArgs;
}
DirectSolrConnection connection = new DirectSolrConnection(core);
SolrRequestHandler handler = core.getRequestHandler("/update/json");
if (handler == null) {
handler = new JsonUpdateRequestHandler();
handler.init(null);
}
return connection.request(handler, args, json);
}
public static SolrInputDocument sdoc(Object... fieldsAndValues) {
SolrInputDocument sd = new SolrInputDocument();
for (int i=0; i<fieldsAndValues.length; i+=2) {
sd.addField((String)fieldsAndValues[i], fieldsAndValues[i+1]);
}
return sd;
}
public static List<SolrInputDocument> sdocs(SolrInputDocument... docs) {
return Arrays.asList(docs);
}
/** Converts "test JSON" and returns standard JSON.
* Currently this only consists of changing unescaped single quotes to double quotes,
* and escaped single quotes to single quotes.
*
* The primary purpose is to be able to easily embed JSON strings in a JAVA string
* with the best readability.
*
* This transformation is automatically applied to JSON test srings (like assertJQ).
*/
public static String json(String testJSON) {
testJSON = nonEscapedSingleQuotePattern.matcher(testJSON).replaceAll("\"");
testJSON = escapedSingleQuotePattern.matcher(testJSON).replaceAll("'");
return testJSON;
}
private static Pattern nonEscapedSingleQuotePattern = Pattern.compile("(?<!\\\\)\'");
private static Pattern escapedSingleQuotePattern = Pattern.compile("\\\\\'");
/** Creates JSON from a SolrInputDocument. Doesn't currently handle boosts.
* @see #json(SolrInputDocument,CharArr)
*/
public static String json(SolrInputDocument doc) {
CharArr out = new CharArr();
json(doc, out);
return out.toString();
}
/**
* Appends to the <code>out</code> array with JSON from the <code>doc</code>.
* Doesn't currently handle boosts, but does recursively handle child documents
*/
public static void json(SolrInputDocument doc, CharArr out) {
try {
out.append('{');
boolean firstField = true;
for (SolrInputField sfield : doc) {
if (firstField) firstField=false;
else out.append(',');
JSONUtil.writeString(sfield.getName(), 0, sfield.getName().length(), out);
out.append(':');
if (sfield.getValueCount() > 1) {
out.append('[');
boolean firstVal = true;
for (Object val : sfield) {
if (firstVal) firstVal=false;
else out.append(',');
out.append(JSONUtil.toJSON(val));
}
out.append(']');
} else {
out.append(JSONUtil.toJSON(sfield.getValue()));
}
}
boolean firstChildDoc = true;
if(doc.hasChildDocuments()) {
out.append(",\"_childDocuments_\": [");
List<SolrInputDocument> childDocuments = doc.getChildDocuments();
for(SolrInputDocument childDocument : childDocuments) {
if (firstChildDoc) firstChildDoc=false;
else out.append(',');
json(childDocument, out);
}
out.append(']');
}
out.append('}');
} catch (IOException e) {
// should never happen
}
}
/** Creates a JSON add command from a SolrInputDocument list. Doesn't currently handle boosts. */
public static String jsonAdd(SolrInputDocument... docs) {
CharArr out = new CharArr();
try {
out.append('[');
boolean firstField = true;
for (SolrInputDocument doc : docs) {
if (firstField) firstField=false;
else out.append(',');
out.append(json(doc));
}
out.append(']');
} catch (IOException e) {
// should never happen
}
return out.toString();
}
/** Creates a JSON delete command from an id list */
public static String jsonDelId(Object... ids) {
CharArr out = new CharArr();
try {
out.append('{');
boolean first = true;
for (Object id : ids) {
if (first) first=false;
else out.append(',');
out.append("\"delete\":{\"id\":");
out.append(JSONUtil.toJSON(id));
out.append('}');
}
out.append('}');
} catch (IOException e) {
// should never happen
}
return out.toString();
}
/** Creates a JSON deleteByQuery command */
public static String jsonDelQ(String... queries) {
CharArr out = new CharArr();
try {
out.append('{');
boolean first = true;
for (Object q : queries) {
if (first) first=false;
else out.append(',');
out.append("\"delete\":{\"query\":");
out.append(JSONUtil.toJSON(q));
out.append('}');
}
out.append('}');
} catch (IOException e) {
// should never happen
}
return out.toString();
}
public static Long addAndGetVersion(SolrInputDocument sdoc, SolrParams params) throws Exception {
if (params==null || params.get("versions") == null) {
ModifiableSolrParams mparams = new ModifiableSolrParams(params);
mparams.set("versions","true");
params = mparams;
}
String response = updateJ(jsonAdd(sdoc), params);
Map rsp = (Map)ObjectBuilder.fromJSON(response);
List lst = (List)rsp.get("adds");
if (lst == null || lst.size() == 0) return null;
return (Long) lst.get(1);
}
public static Long deleteAndGetVersion(String id, SolrParams params) throws Exception {
if (params==null || params.get("versions") == null) {
ModifiableSolrParams mparams = new ModifiableSolrParams(params);
mparams.set("versions","true");
params = mparams;
}
String response = updateJ(jsonDelId(id), params);
Map rsp = (Map)ObjectBuilder.fromJSON(response);
List lst = (List)rsp.get("deletes");
if (lst == null || lst.size() == 0) return null;
return (Long) lst.get(1);
}
public static Long deleteByQueryAndGetVersion(String q, SolrParams params) throws Exception {
if (params==null || params.get("versions") == null) {
ModifiableSolrParams mparams = new ModifiableSolrParams(params);
mparams.set("versions","true");
params = mparams;
}
String response = updateJ(jsonDelQ(q), params);
Map rsp = (Map)ObjectBuilder.fromJSON(response);
List lst = (List)rsp.get("deleteByQuery");
if (lst == null || lst.size() == 0) return null;
return (Long) lst.get(1);
}
/////////////////////////////////////////////////////////////////////////////////////
//////////////////////////// random document / index creation ///////////////////////
/////////////////////////////////////////////////////////////////////////////////////
public abstract static class Vals {
public abstract Comparable get();
public String toJSON(Comparable val) {
return JSONUtil.toJSON(val);
}
protected int between(int min, int max) {
return min != max ? random().nextInt(max-min+1) + min : min;
}
}
public abstract static class IVals extends Vals {
public abstract int getInt();
}
public static class IRange extends IVals {
final int min;
final int max;
public IRange(int min, int max) {
this.min = min;
this.max = max;
}
@Override
public int getInt() {
return between(min,max);
}
@Override
public Comparable get() {
return getInt();
}
}
public static class IValsPercent extends IVals {
final int[] percentAndValue;
public IValsPercent(int... percentAndValue) {
this.percentAndValue = percentAndValue;
}
@Override
public int getInt() {
int r = between(0,99);
int cumulative = 0;
for (int i=0; i<percentAndValue.length; i+=2) {
cumulative += percentAndValue[i];
if (r < cumulative) {
return percentAndValue[i+1];
}
}
return percentAndValue[percentAndValue.length-1];
}
@Override
public Comparable get() {
return getInt();
}
}
public static class FVal extends Vals {
final float min;
final float max;
public FVal(float min, float max) {
this.min = min;
this.max = max;
}
public float getFloat() {
if (min >= max) return min;
return min + random().nextFloat() * (max - min);
}
@Override
public Comparable get() {
return getFloat();
}
}
public static class SVal extends Vals {
char start;
char end;
int minLength;
int maxLength;
public SVal() {
this('a','z',1,10);
}
public SVal(char start, char end, int minLength, int maxLength) {
this.start = start;
this.end = end;
this.minLength = minLength;
this.maxLength = maxLength;
}
@Override
public Comparable get() {
char[] arr = new char[between(minLength,maxLength)];
for (int i=0; i<arr.length; i++) {
arr[i] = (char)between(start, end);
}
return new String(arr);
}
}
public static final IRange ZERO_ONE = new IRange(0,1);
public static final IRange ZERO_TWO = new IRange(0,2);
public static final IRange ONE_ONE = new IRange(1,1);
public static class Doc implements Comparable {
public Comparable id;
public List<Fld> fields;
public int order; // the order this document was added to the index
@Override
public String toString() {
return "Doc("+order+"):"+fields.toString();
}
@Override
public int hashCode() {
return id.hashCode();
}
@Override
public boolean equals(Object o) {
if (!(o instanceof Doc)) return false;
Doc other = (Doc)o;
return this==other || id != null && id.equals(other.id);
}
@Override
public int compareTo(Object o) {
if (!(o instanceof Doc)) return this.getClass().hashCode() - o.getClass().hashCode();
Doc other = (Doc)o;
return this.id.compareTo(other.id);
}
public List<Comparable> getValues(String field) {
for (Fld fld : fields) {
if (fld.ftype.fname.equals(field)) return fld.vals;
}
return null;
}
public Comparable getFirstValue(String field) {
List<Comparable> vals = getValues(field);
return vals==null || vals.size()==0 ? null : vals.get(0);
}
public Map<String,Object> toObject(IndexSchema schema) {
Map<String,Object> result = new HashMap<>();
for (Fld fld : fields) {
SchemaField sf = schema.getField(fld.ftype.fname);
if (!sf.multiValued()) {
result.put(fld.ftype.fname, fld.vals.get(0));
} else {
result.put(fld.ftype.fname, fld.vals);
}
}
return result;
}
}
public static class Fld {
public FldType ftype;
public List<Comparable> vals;
@Override
public String toString() {
return ftype.fname + "=" + (vals.size()==1 ? vals.get(0).toString() : vals.toString());
}
}
protected class FldType {
public String fname;
public IVals numValues;
public Vals vals;
public FldType(String fname, Vals vals) {
this(fname, ZERO_ONE, vals);
}
public FldType(String fname, IVals numValues, Vals vals) {
this.fname = fname;
this.numValues = numValues;
this.vals = vals;
}
public Comparable createValue() {
return vals.get();
}
public List<Comparable> createValues() {
int nVals = numValues.getInt();
if (nVals <= 0) return null;
List<Comparable> vals = new ArrayList<>(nVals);
for (int i=0; i<nVals; i++)
vals.add(createValue());
return vals;
}
public Fld createField() {
List<Comparable> vals = createValues();
if (vals == null) return null;
Fld fld = new Fld();
fld.ftype = this;
fld.vals = vals;
return fld;
}
}
public Map<Comparable,Doc> indexDocs(List<FldType> descriptor, Map<Comparable,Doc> model, int nDocs) throws Exception {
if (model == null) {
model = new LinkedHashMap<>();
}
// commit an average of 10 times for large sets, or 10% of the time for small sets
int commitOneOutOf = Math.max(nDocs/10, 10);
for (int i=0; i<nDocs; i++) {
Doc doc = createDoc(descriptor);
// doc.order = order++;
updateJ(toJSON(doc), null);
model.put(doc.id, doc);
// commit 10% of the time
if (random().nextInt(commitOneOutOf)==0) {
assertU(commit());
}
// duplicate 10% of the docs
if (random().nextInt(10)==0) {
updateJ(toJSON(doc), null);
model.put(doc.id, doc);
}
}
// optimize 10% of the time
if (random().nextInt(10)==0) {
assertU(optimize());
} else {
if (random().nextInt(10) == 0) {
assertU(commit());
} else {
assertU(commit("softCommit","true"));
}
}
// merging segments no longer selects just adjacent segments hence ids (doc.order) can be shuffled.
// we need to look at the index to determine the order.
String responseStr = h.query(req("q","*:*", "fl","id", "sort","_docid_ asc", "rows",Integer.toString(model.size()*2), "wt","json", "indent","true"));
Object response = ObjectBuilder.fromJSON(responseStr);
response = ((Map)response).get("response");
response = ((Map)response).get("docs");
List<Map> docList = (List<Map>)response;
int order = 0;
for (Map doc : docList) {
Object id = doc.get("id");
Doc modelDoc = model.get(id);
if (modelDoc == null) continue; // may be some docs in the index that aren't modeled
modelDoc.order = order++;
}
// make sure we updated the order of all docs in the model
assertEquals(order, model.size());
return model;
}
public static Doc createDoc(List<FldType> descriptor) {
Doc doc = new Doc();
doc.fields = new ArrayList<>();
for (FldType ftype : descriptor) {
Fld fld = ftype.createField();
if (fld != null) {
doc.fields.add(fld);
if ("id".equals(ftype.fname))
doc.id = fld.vals.get(0);
}
}
return doc;
}
public static Comparator<Doc> createSort(IndexSchema schema, List<FldType> fieldTypes, String[] out) {
StringBuilder sortSpec = new StringBuilder();
int nSorts = random().nextInt(4);
List<Comparator<Doc>> comparators = new ArrayList<>();
for (int i=0; i<nSorts; i++) {
if (i>0) sortSpec.append(',');
int which = random().nextInt(fieldTypes.size()+2);
boolean asc = random().nextBoolean();
if (which == fieldTypes.size()) {
// sort by score
sortSpec.append("score").append(asc ? " asc" : " desc");
comparators.add(createComparator("score", asc, false, false, false));
} else if (which == fieldTypes.size() + 1) {
// sort by docid
sortSpec.append("_docid_").append(asc ? " asc" : " desc");
comparators.add(createComparator("_docid_", asc, false, false, false));
} else {
String field = fieldTypes.get(which).fname;
sortSpec.append(field).append(asc ? " asc" : " desc");
SchemaField sf = schema.getField(field);
comparators.add(createComparator(field, asc, sf.sortMissingLast(), sf.sortMissingFirst(), !(sf.sortMissingLast()||sf.sortMissingFirst()) ));
}
}
out[0] = sortSpec.length() > 0 ? sortSpec.toString() : null;
if (comparators.size() == 0) {
// default sort is by score desc
comparators.add(createComparator("score", false, false, false, false));
}
return createComparator(comparators);
}
public static Comparator<Doc> createComparator(final String field, final boolean asc, final boolean sortMissingLast, final boolean sortMissingFirst, final boolean sortMissingAsZero) {
final int mul = asc ? 1 : -1;
if (field.equals("_docid_")) {
return new Comparator<Doc>() {
@Override
public int compare(Doc o1, Doc o2) {
return (o1.order - o2.order) * mul;
}
};
}
if (field.equals("score")) {
return createComparator("score_f", asc, sortMissingLast, sortMissingFirst, sortMissingAsZero);
}
return new Comparator<Doc>() {
private Comparable zeroVal(Comparable template) {
if (template == null) return null;
if (template instanceof String) return null; // fast-path for string
if (template instanceof Integer) return 0;
if (template instanceof Long) return (long)0;
if (template instanceof Float) return (float)0;
if (template instanceof Double) return (double)0;
if (template instanceof Short) return (short)0;
if (template instanceof Byte) return (byte)0;
if (template instanceof Character) return (char)0;
return null;
}
@Override
public int compare(Doc o1, Doc o2) {
Comparable v1 = o1.getFirstValue(field);
Comparable v2 = o2.getFirstValue(field);
v1 = v1 == null ? zeroVal(v2) : v1;
v2 = v2 == null ? zeroVal(v1) : v2;
int c = 0;
if (v1 == v2) {
c = 0;
} else if (v1 == null) {
if (sortMissingLast) c = mul;
else if (sortMissingFirst) c = -mul;
else c = -1;
} else if (v2 == null) {
if (sortMissingLast) c = -mul;
else if (sortMissingFirst) c = mul;
else c = 1;
} else {
c = v1.compareTo(v2);
}
c = c * mul;
return c;
}
};
}
public static Comparator<Doc> createComparator(final List<Comparator<Doc>> comparators) {
return new Comparator<Doc>() {
@Override
public int compare(Doc o1, Doc o2) {
int c = 0;
for (Comparator<Doc> comparator : comparators) {
c = comparator.compare(o1, o2);
if (c!=0) return c;
}
return o1.order - o2.order;
}
};
}
public static String toJSON(Doc doc) {
CharArr out = new CharArr();
try {
out.append("{\"add\":{\"doc\":{");
boolean firstField = true;
for (Fld fld : doc.fields) {
if (firstField) firstField=false;
else out.append(',');
JSONUtil.writeString(fld.ftype.fname, 0, fld.ftype.fname.length(), out);
out.append(':');
if (fld.vals.size() > 1) {
out.append('[');
}
boolean firstVal = true;
for (Comparable val : fld.vals) {
if (firstVal) firstVal=false;
else out.append(',');
out.append(JSONUtil.toJSON(val));
}
if (fld.vals.size() > 1) {
out.append(']');
}
}
out.append("}}}");
} catch (IOException e) {
// should never happen
}
return out.toString();
}
/** Return a Map from field value to a list of document ids */
public Map<Comparable, List<Comparable>> invertField(Map<Comparable, Doc> model, String field) {
Map<Comparable, List<Comparable>> value_to_id = new HashMap<>();
// invert field
for (Comparable key : model.keySet()) {
Doc doc = model.get(key);
List<Comparable> vals = doc.getValues(field);
if (vals == null) continue;
for (Comparable val : vals) {
List<Comparable> ids = value_to_id.get(val);
if (ids == null) {
ids = new ArrayList<>(2);
value_to_id.put(val, ids);
}
ids.add(key);
}
}
return value_to_id;
}
/** Gets a resource from the context classloader as {@link File}. This method should only be used,
* if a real file is needed. To get a stream, code should prefer
* {@link Class#getResourceAsStream} using {@code this.getClass()}.
*/
public static File getFile(String name) {
final URL url = Thread.currentThread().getContextClassLoader().getResource(name.replace(File.separatorChar, '/'));
if (url != null) {
try {
return new File(url.toURI());
} catch (Exception e) {
throw new RuntimeException("Resource was found on classpath, but cannot be resolved to a " +
"normal file (maybe it is part of a JAR file): " + name);
}
}
final File file = new File(name);
if (file.exists()) {
return file;
}
throw new RuntimeException("Cannot find resource in classpath or in file-system (relative to CWD): " + name);
}
public static String TEST_HOME() {
return getFile("solr/collection1").getParent();
}
public static Throwable getRootCause(Throwable t) {
Throwable result = t;
for (Throwable cause = t; null != cause; cause = cause.getCause()) {
result = cause;
}
return result;
}
public static void assertXmlFile(final File file, String... xpath)
throws IOException, SAXException {
try {
String xml = FileUtils.readFileToString(file, "UTF-8");
String results = TestHarness.validateXPath(xml, xpath);
if (null != results) {
String msg = "File XPath failure: file=" + file.getPath() + " xpath="
+ results + "\n\nxml was: " + xml;
fail(msg);
}
} catch (XPathExpressionException e2) {
throw new RuntimeException("XPath is invalid", e2);
}
}
/**
* Fails if the number of documents in the given SolrDocumentList differs
* from the given number of expected values, or if any of the values in the
* given field don't match the expected values in the same order.
*/
public static void assertFieldValues(SolrDocumentList documents, String fieldName, Object... expectedValues) {
if (documents.size() != expectedValues.length) {
fail("Number of documents (" + documents.size()
+ ") is different from number of expected values (" + expectedValues.length);
}
for (int docNum = 1 ; docNum <= documents.size() ; ++docNum) {
SolrDocument doc = documents.get(docNum - 1);
Object expected = expectedValues[docNum - 1];
Object actual = doc.get(fieldName);
if ((null == expected && null != actual) ||
(null != expected && null == actual) ||
(null != expected && null != actual && !expected.equals(actual))) {
fail("Unexpected " + fieldName + " field value in document #" + docNum
+ ": expected=[" + expected + "], actual=[" + actual + "]");
}
}
}
public static void copyMinConf(File dstRoot) throws IOException {
copyMinConf(dstRoot, null);
}
// Creates a minimal conf dir. Optionally adding in a core.properties file from the string passed in
// the string to write to the core.properties file may be null in which case nothing is done with it.
// propertiesContent may be an empty string, which will actually work.
public static void copyMinConf(File dstRoot, String propertiesContent) throws IOException {
File subHome = new File(dstRoot, "conf");
if (! dstRoot.exists()) {
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
}
if (propertiesContent != null) {
FileUtils.writeStringToFile(new File(dstRoot, "core.properties"), propertiesContent, Charsets.UTF_8.toString());
}
String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf";
FileUtils.copyFile(new File(top, "schema-tiny.xml"), new File(subHome, "schema.xml"));
FileUtils.copyFile(new File(top, "solrconfig-minimal.xml"), new File(subHome, "solrconfig.xml"));
FileUtils.copyFile(new File(top, "solrconfig.snippet.randomindexconfig.xml"), new File(subHome, "solrconfig.snippet.randomindexconfig.xml"));
}
// Creates minimal full setup, including the old solr.xml file that used to be hard coded in ConfigSolrXmlOld
// TODO: remove for 5.0
public static void copyMinFullSetup(File dstRoot) throws IOException {
if (! dstRoot.exists()) {
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
}
File xmlF = new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml");
FileUtils.copyFile(xmlF, new File(dstRoot, "solr.xml"));
copyMinConf(dstRoot);
}
// Creates a consistent configuration, _including_ solr.xml at dstRoot. Creates collection1/conf and copies
// the stock files in there. Seems to be indicated for some tests when we remove the default, hard-coded
// solr.xml from being automatically synthesized from SolrConfigXmlOld.DEFAULT_SOLR_XML.
public static void copySolrHomeToTemp(File dstRoot, String collection) throws IOException {
copySolrHomeToTemp(dstRoot, collection, false);
}
public static void copySolrHomeToTemp(File dstRoot, String collection, boolean newStyle) throws IOException {
if (!dstRoot.exists()) {
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
}
if (newStyle) {
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr-no-core.xml"), new File(dstRoot, "solr.xml"));
} else {
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(dstRoot, "solr.xml"));
}
File subHome = new File(dstRoot, collection + File.separator + "conf");
String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf";
FileUtils.copyFile(new File(top, "currency.xml"), new File(subHome, "currency.xml"));
FileUtils.copyFile(new File(top, "mapping-ISOLatin1Accent.txt"), new File(subHome, "mapping-ISOLatin1Accent.txt"));
FileUtils.copyFile(new File(top, "old_synonyms.txt"), new File(subHome, "old_synonyms.txt"));
FileUtils.copyFile(new File(top, "open-exchange-rates.json"), new File(subHome, "open-exchange-rates.json"));
FileUtils.copyFile(new File(top, "protwords.txt"), new File(subHome, "protwords.txt"));
FileUtils.copyFile(new File(top, "schema.xml"), new File(subHome, "schema.xml"));
FileUtils.copyFile(new File(top, "solrconfig.snippet.randomindexconfig.xml"), new File(subHome, "solrconfig.snippet.randomindexconfig.xml"));
FileUtils.copyFile(new File(top, "solrconfig.xml"), new File(subHome, "solrconfig.xml"));
FileUtils.copyFile(new File(top, "stopwords.txt"), new File(subHome, "stopwords.txt"));
FileUtils.copyFile(new File(top, "synonyms.txt"), new File(subHome, "synonyms.txt"));
}
public static CoreDescriptorBuilder buildCoreDescriptor(CoreContainer container, String name, String instancedir) {
return new CoreDescriptorBuilder(container, name, instancedir);
}
public static class CoreDescriptorBuilder {
final String name;
final String instanceDir;
final CoreContainer container;
final Properties properties = new Properties();
public CoreDescriptorBuilder(CoreContainer container, String name, String instancedir) {
this.name = name;
this.instanceDir = instancedir;
this.container = container;
}
public CoreDescriptorBuilder withSchema(String schema) {
properties.setProperty(CoreDescriptor.CORE_SCHEMA, schema);
return this;
}
public CoreDescriptorBuilder withConfig(String config) {
properties.setProperty(CoreDescriptor.CORE_CONFIG, config);
return this;
}
public CoreDescriptorBuilder withDataDir(String datadir) {
properties.setProperty(CoreDescriptor.CORE_DATADIR, datadir);
return this;
}
public CoreDescriptor build() {
return new CoreDescriptor(container, name, instanceDir, properties);
}
public CoreDescriptorBuilder isTransient(boolean isTransient) {
properties.setProperty(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isTransient));
return this;
}
public CoreDescriptorBuilder loadOnStartup(boolean loadOnStartup) {
properties.setProperty(CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(loadOnStartup));
return this;
}
}
public boolean compareSolrDocument(Object expected, Object actual) {
if (!(expected instanceof SolrDocument) || !(actual instanceof SolrDocument)) {
return false;
}
if (expected == actual) {
return true;
}
SolrDocument solrDocument1 = (SolrDocument) expected;
SolrDocument solrDocument2 = (SolrDocument) actual;
if(solrDocument1.getFieldNames().size() != solrDocument1.getFieldNames().size()) {
return false;
}
Iterator<String> iter1 = solrDocument1.getFieldNames().iterator();
Iterator<String> iter2 = solrDocument2.getFieldNames().iterator();
if(iter1.hasNext()) {
String key1 = iter1.next();
String key2 = iter2.next();
Object val1 = solrDocument1.getFieldValues(key1);
Object val2 = solrDocument2.getFieldValues(key2);
if(!key1.equals(key2) || !val1.equals(val2)) {
return false;
}
}
if(solrDocument1.getChildDocuments() == null && solrDocument2.getChildDocuments() == null) {
return true;
}
if(solrDocument1.getChildDocuments() == null || solrDocument2.getChildDocuments() == null) {
return false;
} else if(solrDocument1.getChildDocuments().size() != solrDocument2.getChildDocuments().size()) {
return false;
} else {
Iterator<SolrDocument> childDocsIter1 = solrDocument1.getChildDocuments().iterator();
Iterator<SolrDocument> childDocsIter2 = solrDocument2.getChildDocuments().iterator();
while(childDocsIter1.hasNext()) {
if(!compareSolrDocument(childDocsIter1.next(), childDocsIter2.next())) {
return false;
}
}
return true;
}
}
public boolean compareSolrDocumentList(Object expected, Object actual) {
if (!(expected instanceof SolrDocumentList) || !(actual instanceof SolrDocumentList)) {
return false;
}
if (expected == actual) {
return true;
}
SolrDocumentList list1 = (SolrDocumentList) expected;
SolrDocumentList list2 = (SolrDocumentList) actual;
if(Float.compare(list1.getMaxScore(), list2.getMaxScore()) != 0 || list1.getNumFound() != list2.getNumFound() ||
list1.getStart() != list2.getStart()) {
return false;
}
for(int i=0; i<list1.getNumFound(); i++) {
if(!compareSolrDocument(list1.get(i), list2.get(i))) {
return false;
}
}
return true;
}
public boolean compareSolrInputDocument(Object expected, Object actual) {
if (!(expected instanceof SolrInputDocument) || !(actual instanceof SolrInputDocument)) {
return false;
}
if (expected == actual) {
return true;
}
SolrInputDocument sdoc1 = (SolrInputDocument) expected;
SolrInputDocument sdoc2 = (SolrInputDocument) actual;
if (Float.compare(sdoc1.getDocumentBoost(), sdoc2.getDocumentBoost()) != 0) {
return false;
}
if(sdoc1.getFieldNames().size() != sdoc2.getFieldNames().size()) {
return false;
}
Iterator<String> iter1 = sdoc1.getFieldNames().iterator();
Iterator<String> iter2 = sdoc2.getFieldNames().iterator();
if(iter1.hasNext()) {
String key1 = iter1.next();
String key2 = iter2.next();
Object val1 = sdoc1.getFieldValues(key1);
Object val2 = sdoc2.getFieldValues(key2);
if(!key1.equals(key2) || !val1.equals(val2)) {
return false;
}
}
if(sdoc1.getChildDocuments() == null && sdoc2.getChildDocuments() == null) {
return true;
}
if(sdoc1.getChildDocuments() == null || sdoc2.getChildDocuments() == null) {
return false;
} else if(sdoc1.getChildDocuments().size() != sdoc2.getChildDocuments().size()) {
return false;
} else {
Iterator<SolrInputDocument> childDocsIter1 = sdoc1.getChildDocuments().iterator();
Iterator<SolrInputDocument> childDocsIter2 = sdoc2.getChildDocuments().iterator();
while(childDocsIter1.hasNext()) {
if(!compareSolrInputDocument(childDocsIter1.next(), childDocsIter2.next())) {
return false;
}
}
return true;
}
}
public boolean assertSolrInputFieldEquals(Object expected, Object actual) {
if (!(expected instanceof SolrInputField) || !(actual instanceof SolrInputField)) {
return false;
}
if (expected == actual) {
return true;
}
SolrInputField sif1 = (SolrInputField) expected;
SolrInputField sif2 = (SolrInputField) actual;
if (!sif1.getName().equals(sif2.getName())) {
return false;
}
if (!sif1.getValue().equals(sif2.getValue())) {
return false;
}
if (Float.compare(sif1.getBoost(), sif2.getBoost()) != 0) {
return false;
}
return true;
}
/**
* Returns <code>likely</code> most (1/10) of the time, otherwise <code>unlikely</code>
*/
public static Object skewed(Object likely, Object unlikely) {
return (0 == TestUtil.nextInt(random(), 0, 9)) ? unlikely : likely;
}
/**
* Returns a randomly generated Date in the appropriate Solr external (input) format
* @see #randomSkewedDate
*/
public static String randomDate() {
return TrieDateField.formatExternal(new Date(random().nextLong()));
}
/**
* Returns a Date such that all results from this method always have the same values for
* year+month+day+hour+minute but the seconds are randomized. This can be helpful for
* indexing documents with random date values that are biased for a narrow window
* (one day) to test collisions/overlaps
*
* @see #randomDate
*/
public static String randomSkewedDate() {
return String.format(Locale.ROOT, "2010-10-31T10:31:%02d.000Z",
TestUtil.nextInt(random(), 0, 59));
}
/**
* We want "realistic" unicode strings beyond simple ascii, but because our
* updates use XML we need to ensure we don't get "special" code block.
*/
public static String randomXmlUsableUnicodeString() {
String result = TestUtil.randomRealisticUnicodeString(random());
if (result.matches(".*\\p{InSpecials}.*")) {
result = TestUtil.randomSimpleString(random());
}
return result;
}
}