| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.search.facet; |
| |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| import org.apache.solr.SolrTestCaseJ4; |
| import org.apache.solr.SolrTestUtil; |
| import org.apache.solr.client.solrj.SolrClient; |
| import org.apache.solr.client.solrj.SolrServerException; |
| import org.apache.solr.client.solrj.embedded.JettySolrRunner; |
| import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; |
| import org.apache.solr.client.solrj.impl.Http2SolrClient; |
| import org.apache.solr.client.solrj.request.CollectionAdminRequest; |
| import org.apache.solr.client.solrj.request.QueryRequest; |
| import org.apache.solr.client.solrj.response.QueryResponse; |
| import org.apache.solr.cloud.SolrCloudTestCase; |
| import org.apache.solr.cloud.TestCloudPivotFacet; |
| import org.apache.solr.common.SolrException; |
| import org.apache.solr.common.SolrInputDocument; |
| import org.apache.solr.common.params.ModifiableSolrParams; |
| import org.apache.solr.common.params.SolrParams; |
| import org.apache.solr.common.util.NamedList; |
| import org.junit.AfterClass; |
| import org.junit.BeforeClass; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import java.io.IOException; |
| import java.lang.invoke.MethodHandles; |
| import java.nio.file.Path; |
| import java.nio.file.Paths; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Random; |
| import java.util.concurrent.atomic.AtomicInteger; |
| |
| /** |
| * <p> |
| * Tests randomized JSON Facets, sometimes using query 'join' domain transfers and/or domain 'filter' options |
| * </p> |
| * <p> |
| * The results of each facet constraint count will be compared with a verification query using an equivalent filter |
| * </p> |
| * |
| * @see TestCloudPivotFacet |
| */ |
| public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase { |
| |
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); |
| |
| private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName(); |
| private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection"; |
| |
| private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT; |
| private static final int MAX_FIELD_NUM = 15; |
| private static final int UNIQUE_FIELD_VALS = 20; |
| |
| // NOTE: set to 'true' to see if refinement testing is adequate (should get fails occasionally) |
| private static final boolean FORCE_DISABLE_REFINEMENT = false; |
| |
| /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */ |
| private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" }; |
| /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */ |
| private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" }; |
| |
| /** A basic client for operations at the cloud level, default collection will be set */ |
| private static CloudHttp2SolrClient CLOUD_CLIENT; |
| /** One client per node */ |
| private static final ArrayList<Http2SolrClient> CLIENTS = new ArrayList<>(5); |
| |
| @BeforeClass |
| public static void beforeTestCloudJSONFacetJoinDomain() throws Exception { |
| // sanity check constants |
| assertTrue("bad test constants: some suffixes will never be tested", |
| (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM)); |
| |
| // we need DVs on point fields to compute stats & facets |
| if (Boolean.getBoolean(SolrTestCaseJ4.NUMERIC_POINTS_SYSPROP)) System.setProperty(SolrTestCaseJ4.NUMERIC_DOCVALUES_SYSPROP,"true"); |
| |
| // multi replicas should not matter... |
| final int repFactor; |
| // ... but we definitely want to test multiple shards |
| final int numShards; |
| |
| if (TEST_NIGHTLY) { |
| numShards = TestUtil.nextInt(random(), 1, (LuceneTestCase.usually() ? 2 :3)); |
| repFactor = LuceneTestCase.usually() ? 1 : 2; |
| } else { |
| numShards = 1; |
| repFactor = 1; |
| } |
| |
| final int numNodes = (numShards * repFactor); |
| |
| final String configName = DEBUG_LABEL + "_config-set"; |
| final Path configDir = Paths.get(SolrTestUtil.TEST_HOME(), "collection1", "conf"); |
| |
| configureCluster(numNodes).addConfig(configName, configDir).configure(); |
| |
| Map<String, String> collectionProperties = new LinkedHashMap<>(); |
| collectionProperties.put("config", "solrconfig-tlog.xml"); |
| collectionProperties.put("schema", "schema_latest.xml"); |
| CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor) |
| .setProperties(collectionProperties) |
| .process(cluster.getSolrClient()); |
| |
| CLOUD_CLIENT = cluster.getSolrClient(); |
| CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME); |
| |
| for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { |
| CLIENTS.add(SolrTestCaseJ4.getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/")); |
| } |
| |
| final int numDocs = LuceneTestCase.atLeast(TEST_NIGHTLY ? 100 : 25); |
| for (int id = 0; id < numDocs; id++) { |
| SolrInputDocument doc = SolrTestCaseJ4.sdoc("id", ""+id); |
| for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) { |
| // NOTE: some docs may have no value in a field |
| final int numValsThisDoc = TestUtil.nextInt(random(), 0, (LuceneTestCase.usually() ? 3 : 6)); |
| for (int v = 0; v < numValsThisDoc; v++) { |
| final String fieldValue = randFieldValue(fieldNum); |
| |
| // for each fieldNum, there are actaully two fields: one string, and one integer |
| doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue); |
| doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue); |
| } |
| } |
| CLOUD_CLIENT.add(doc); |
| if (random().nextInt(100) < 1) { |
| CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments |
| } |
| if (random().nextInt(100) < 5) { |
| CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs |
| } |
| } |
| CLOUD_CLIENT.commit(); |
| } |
| |
| @AfterClass |
| public static void afterTestCloudJSONFacetJoinDomain() throws Exception { |
| shutdownCluster(); |
| } |
| |
| /** |
| * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that |
| * uses that number and one of hte specified suffixes in it's name. |
| * |
| * @see #STR_FIELD_SUFFIXES |
| * @see #INT_FIELD_SUFFIXES |
| * @see #MAX_FIELD_NUM |
| * @see #randFieldValue |
| */ |
| private static String field(final String[] suffixes, final int fieldNum) { |
| assert fieldNum < MAX_FIELD_NUM; |
| |
| final String suffix = suffixes[fieldNum % suffixes.length]; |
| return "field_" + fieldNum + suffix; |
| } |
| private static String strfield(final int fieldNum) { |
| return field(STR_FIELD_SUFFIXES, fieldNum); |
| } |
| private static String intfield(final int fieldNum) { |
| return field(INT_FIELD_SUFFIXES, fieldNum); |
| } |
| |
| /** |
| * Given a (random) field number, returns a random (integer based) value for that field. |
| * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS} |
| * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins |
| * will match fewer documents based on how far apart the field numbers are. |
| * |
| * @see #UNIQUE_FIELD_VALS |
| * @see #field |
| */ |
| private static String randFieldValue(final int fieldNum) { |
| return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS)); |
| } |
| |
| |
| @AfterClass |
| private static void afterClass() throws Exception { |
| for (Http2SolrClient client : CLIENTS) { |
| client.close(); |
| } |
| CLIENTS.clear(); |
| CLOUD_CLIENT = null; |
| } |
| |
| /** Sanity check that malformed requests produce errors */ |
| public void testMalformedGivesError() throws Exception { |
| |
| SolrTestCaseJ4.ignoreException(".*'join' domain change.*"); |
| |
| for (String join : Arrays.asList("bogus", |
| "{ }", |
| "{ from:null, to:foo_s }", |
| "{ from:foo_s }", |
| "{ from:foo_s, to:foo_s, bogus:'what what?' }", |
| "{ to:foo_s, bogus:'what what?' }")) { |
| SolrException e = LuceneTestCase.expectThrows(SolrException.class, () -> { |
| final SolrParams req = params("q", "*:*", "json.facet", |
| "{ x : { type:terms, field:x_s, domain: { join:"+join+" } } }"); |
| final NamedList trash = getRandClient(random()).request(new QueryRequest(req)); |
| }); |
| assertEquals(join + " -> " + e, SolrException.ErrorCode.BAD_REQUEST.code, e.code()); |
| assertTrue(join + " -> " + e, e.getMessage().contains("'join' domain change")); |
| } |
| } |
| |
| public void testSanityCheckDomainMethods() throws Exception { |
| { |
| final JoinDomain empty = new JoinDomain(null, null, null); |
| assertEquals(null, empty.toJSONFacetParamValue()); |
| final SolrParams out = empty.applyDomainToQuery("safe_key", params("q","qqq")); |
| assertNotNull(out); |
| assertEquals(null, out.get("safe_key")); |
| assertEquals("qqq", out.get("q")); |
| } |
| { |
| final JoinDomain join = new JoinDomain("xxx", "yyy", null); |
| assertEquals("domain:{join:{from:xxx,to:yyy}}", join.toJSONFacetParamValue().toString()); |
| final SolrParams out = join.applyDomainToQuery("safe_key", params("q","qqq")); |
| assertNotNull(out); |
| assertEquals("qqq", out.get("safe_key")); |
| assertEquals("{!join from=xxx to=yyy v=$safe_key}", out.get("q")); |
| |
| } |
| { |
| final JoinDomain filter = new JoinDomain(null, null, "zzz"); |
| assertEquals("domain:{filter:'zzz'}", filter.toJSONFacetParamValue().toString()); |
| final SolrParams out = filter.applyDomainToQuery("safe_key", params("q","qqq")); |
| assertNotNull(out); |
| assertEquals(null, out.get("safe_key")); |
| assertEquals("zzz AND qqq", out.get("q")); |
| } |
| { |
| final JoinDomain both = new JoinDomain("xxx", "yyy", "zzz"); |
| assertEquals("domain:{join:{from:xxx,to:yyy},filter:'zzz'}", both.toJSONFacetParamValue().toString()); |
| final SolrParams out = both.applyDomainToQuery("safe_key", params("q","qqq")); |
| assertNotNull(out); |
| assertEquals("qqq", out.get("safe_key")); |
| assertEquals("zzz AND {!join from=xxx to=yyy v=$safe_key}", out.get("q")); |
| } |
| } |
| |
| /** |
| * Test some small, hand crafted, but non-trivial queries that are |
| * easier to trace/debug then a pure random monstrosity. |
| * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom) |
| */ |
| @LuceneTestCase.Nightly // needs larger docs counts and such |
| public void testBespoke() throws Exception { |
| |
| { // sanity check our test methods can handle a query matching no docs |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]")); |
| top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null))); |
| facets.put("empty_top", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS); |
| assertFacetCountsAreCorrect(maxBuckets, facets, strfield(7) + ":bogus"); |
| assertEquals("Empty search result shouldn't have found a single bucket", |
| UNIQUE_FIELD_VALS, maxBuckets.get()); |
| } |
| |
| { // sanity check our test methods can handle a query where a facet filter prevents any doc from having terms |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, "-*:*")); |
| top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null))); |
| facets.put("filtered_top", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "*:*"); |
| assertEquals("Empty join filter shouldn't have found a single bucket", |
| UNIQUE_FIELD_VALS, maxBuckets.get()); |
| } |
| |
| { // sanity check our test methods can handle a query where a facet filter prevents any doc from having sub-terms |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(8), strfield(8), null)); |
| top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(null, null, "-*:*"))); |
| facets.put("filtered_top", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "*:*"); |
| assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS); |
| } |
| |
| { // strings |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]")); |
| top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null))); |
| facets.put("facet_4", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)"); |
| assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS); |
| } |
| |
| { // ints |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(intfield(9), new JoinDomain(intfield(5), intfield(9), null)); |
| facets.put("top", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "("+intfield(7)+":16 OR "+intfield(3)+":13)"); |
| // seems to happen sometimes... |
| // assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS); |
| } |
| |
| { // some domains with filter only, no actual join |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, strfield(9)+":[* TO *]")); |
| top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(null, null, strfield(3)+":[* TO 5]"))); |
| facets.put("top", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)"); |
| assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS); |
| |
| } |
| |
| { // low limits, explicit refinement |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), |
| new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"), |
| 5, 0, true); |
| top.subFacets.put("facet_5", new TermFacet(strfield(11), |
| new JoinDomain(strfield(8), strfield(8), null), |
| 10, 0, true)); |
| facets.put("facet_4", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(5 * 10); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)"); |
| assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10); |
| } |
| |
| { // low limit, high overrequest |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), |
| new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"), |
| 5, UNIQUE_FIELD_VALS + 10, false); |
| top.subFacets.put("facet_5", new TermFacet(strfield(11), |
| new JoinDomain(strfield(8), strfield(8), null), |
| 10, UNIQUE_FIELD_VALS + 10, false)); |
| facets.put("facet_4", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(5 * 10); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)"); |
| assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10); |
| } |
| |
| { // low limit, low overrequest, explicit refinement |
| Map<String,TermFacet> facets = new LinkedHashMap<>(); |
| TermFacet top = new TermFacet(strfield(9), |
| new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"), |
| 5, 7, true); |
| top.subFacets.put("facet_5", new TermFacet(strfield(11), |
| new JoinDomain(strfield(8), strfield(8), null), |
| 10, 7, true)); |
| facets.put("facet_4", top); |
| final AtomicInteger maxBuckets = new AtomicInteger(5 * 10); |
| assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)"); |
| assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10); |
| } |
| |
| } |
| |
| public void testTheTestRandomRefineParam() { |
| // sanity check that randomRefineParam never violates isRefinementNeeded |
| // (should be imposisble ... unless someone changes/breaks the randomization logic in the future) |
| final int numIters = LuceneTestCase.atLeast(TEST_NIGHTLY ? 100 : 10); |
| for (int iter = 0; iter < numIters; iter++) { |
| final Integer limit = TermFacet.randomLimitParam(random()); |
| final Integer overrequest = TermFacet.randomOverrequestParam(random()); |
| final Boolean refine = TermFacet.randomRefineParam(random(), limit, overrequest); |
| if (TermFacet.isRefinementNeeded(limit, overrequest)) { |
| assertEquals("limit: " + limit + ", overrequest: " + overrequest + ", refine: " + refine, |
| Boolean.TRUE, refine); |
| } |
| } |
| } |
| |
| public void testTheTestTermFacetShouldFreakOutOnBadRefineOptions() { |
| LuceneTestCase.expectThrows(AssertionError.class, () -> { |
| final TermFacet bogus = new TermFacet("foo", null, 5, 0, false); |
| }); |
| } |
| |
| public void testRandom() throws Exception { |
| |
| // we put a safety valve in place on the maximum number of buckets that we are willing to verify |
| // across *all* the queries that we do. |
| // that way if the randomized queries we build all have relatively small facets, so be it, but if |
| // we get a really big one early on, we can test as much as possible, skip other iterations. |
| // |
| // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them) |
| final int maxBucketsAllowed = LuceneTestCase.atLeast(TEST_NIGHTLY ? 2000 : 200); |
| final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed); |
| |
| final int numIters = LuceneTestCase.atLeast(TEST_NIGHTLY ? 20 : 3); |
| for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) { |
| assertFacetCountsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(), buildRandomQuery()); |
| } |
| assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed); |
| } |
| |
| /** |
| * Generates a random query string across the randomized fields/values in the index |
| * |
| * @see #randFieldValue |
| * @see #field |
| */ |
| private static String buildRandomQuery() { |
| if (0 == TestUtil.nextInt(random(), 0,10)) { |
| return "*:*"; |
| } |
| final int numClauses = TestUtil.nextInt(random(), 3, 10); |
| List<String> clauses = new ArrayList<String>(numClauses); |
| for (int c = 0; c < numClauses; c++) { |
| final int fieldNum = random().nextInt(MAX_FIELD_NUM); |
| // keep queries simple, just use str fields - not point of test |
| clauses.add(strfield(fieldNum) + ":" + randFieldValue(fieldNum)); |
| } |
| return "(" + String.join(" OR ", clauses) + ")"; |
| } |
| |
| /** |
| * Given a set of (potentially nested) term facets, and a base query string, asserts that |
| * the actual counts returned when executing that query with those facets match the expected results |
| * of filtering on the equivalent facet terms+domain |
| */ |
| private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck, |
| Map<String,TermFacet> expected, |
| final String query) throws SolrServerException, IOException { |
| |
| final SolrParams baseParams = params("q", query, "rows","0"); |
| final SolrParams facetParams = params("json.facet", ""+TermFacet.toJSONFacetParamValue(expected)); |
| final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams); |
| |
| log.info("Doing full run: {}", initParams); |
| |
| QueryResponse rsp = null; |
| // JSON Facets not (currently) available from QueryResponse... |
| NamedList topNamedList = null; |
| try { |
| rsp = (new QueryRequest(initParams)).process(getRandClient(random())); |
| assertNotNull(initParams + " is null rsp?", rsp); |
| topNamedList = rsp.getResponse(); |
| assertNotNull(initParams + " is null topNamedList?", topNamedList); |
| } catch (Exception e) { |
| throw new RuntimeException("init query failed: " + initParams + ": " + |
| e.getMessage(), e); |
| } |
| try { |
| final NamedList facetResponse = (NamedList) topNamedList.get("facets"); |
| assertNotNull("null facet results?", facetResponse); |
| assertEquals("numFound mismatch with top count?", |
| rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue()); |
| if (0 == rsp.getResults().getNumFound()) { |
| // when the query matches nothing, we should expect no top level facets |
| expected = Collections.emptyMap(); |
| } |
| assertFacetCountsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse); |
| } catch (AssertionError e) { |
| throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e); |
| } finally { |
| log.info("Ending full run"); |
| } |
| } |
| |
| /** |
| * Recursive Helper method that walks the actual facet response, comparing the counts to the expected output |
| * based on the equivalent filters generated from the original TermFacet. |
| */ |
| private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck, |
| final Map<String,TermFacet> expected, |
| final SolrParams baseParams, |
| final NamedList actualFacetResponse) throws SolrServerException, IOException { |
| |
| for (Map.Entry<String,TermFacet> entry : expected.entrySet()) { |
| final String facetKey = entry.getKey(); |
| final TermFacet facet = entry.getValue(); |
| final NamedList results = (NamedList) actualFacetResponse.get(facetKey); |
| assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results); |
| final List<NamedList> buckets = (List<NamedList>) results.get("buckets"); |
| assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets); |
| |
| if (buckets.isEmpty()) { |
| // should only happen if the baseParams query does not match any docs with our field X |
| final long docsWithField = getRandClient(random()).query |
| (facet.applyValueConstraintAndDomain(baseParams, facetKey, "[* TO *]")).getResults().getNumFound(); |
| assertEquals(facetKey + " has no buckets, but docs in query exist with field: " + facet.field, |
| 0, docsWithField); |
| } |
| |
| for (NamedList bucket : buckets) { |
| final long count = ((Number) bucket.get("count")).longValue(); |
| final String fieldVal = bucket.get("val").toString(); // int or stringified int |
| |
| // change our query to filter on the fieldVal, and wrap in the facet domain (if any) |
| final SolrParams verifyParams = facet.applyValueConstraintAndDomain(baseParams, facetKey, fieldVal); |
| |
| // check the count for this bucket |
| assertEquals(facetKey + ": " + verifyParams, |
| count, getRandClient(random()).query(verifyParams).getResults().getNumFound()); |
| |
| if (maxBucketsToCheck.decrementAndGet() <= 0) { |
| return; |
| } |
| |
| // recursively check subFacets |
| if (! facet.subFacets.isEmpty()) { |
| assertFacetCountsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket); |
| } |
| } |
| } |
| assertTrue("facets have unexpected keys left over: " + actualFacetResponse, |
| // should alwasy be a count, maybe a 'val' if we're a subfacet |
| (actualFacetResponse.size() == expected.size() + 1) || |
| (actualFacetResponse.size() == expected.size() + 2)); |
| } |
| |
| |
| /** |
| * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param. |
| * |
| * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters |
| */ |
| private static final class TermFacet { |
| public final String field; |
| public final Map<String,TermFacet> subFacets = new LinkedHashMap<>(); |
| public final JoinDomain domain; // may be null |
| public final Integer limit; // may be null |
| public final Integer overrequest; // may be null |
| public final Boolean refine; // may be null |
| |
| /** Simplified constructor asks for limit = # unique vals */ |
| public TermFacet(String field, JoinDomain domain) { |
| this(field, domain, UNIQUE_FIELD_VALS, 0, false); |
| } |
| public TermFacet(String field, JoinDomain domain, Integer limit, Integer overrequest, Boolean refine) { |
| assert null != field; |
| this.field = field; |
| this.domain = domain; |
| this.limit = limit; |
| this.overrequest = overrequest; |
| this.refine = refine; |
| if (isRefinementNeeded(limit, overrequest)) { |
| assertEquals("Invalid refine param based on limit & overrequest: " + this.toString(), |
| Boolean.TRUE, refine); |
| } |
| } |
| |
| /** |
| * Returns new SolrParams that: |
| * <ul> |
| * <li>copy the original SolrParams</li> |
| * <li>modify/wrap the original "q" param to capture the domain change for this facet (if any)</li> |
| * <li>add a filter query against this field with the specified value</li> |
| * </ul> |
| * |
| * @see JoinDomain#applyDomainToQuery |
| */ |
| public SolrParams applyValueConstraintAndDomain(SolrParams orig, String facetKey, String facetVal) { |
| // first wrap our original query in the domain if there is one... |
| if (null != domain) { |
| orig = domain.applyDomainToQuery(facetKey + "_q", orig); |
| } |
| // then filter by the facet value we need to test... |
| final ModifiableSolrParams out = new ModifiableSolrParams(orig); |
| out.set("q", field + ":" + facetVal + " AND " + orig.get("q")); |
| |
| return out; |
| } |
| |
| /** |
| * recursively generates the <code>json.facet</code> param value to use for testing this facet |
| */ |
| private CharSequence toJSONFacetParamValue() { |
| final String limitStr = (null == limit) ? "" : (", limit:" + limit); |
| final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest); |
| final String refineStr = (null == refine) ? "" : ", refine:" + refine; |
| final StringBuilder sb = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + refineStr); |
| if (! subFacets.isEmpty()) { |
| sb.append(", facet:"); |
| sb.append(toJSONFacetParamValue(subFacets)); |
| } |
| if (null != domain) { |
| CharSequence ds = domain.toJSONFacetParamValue(); |
| if (null != ds) { |
| sb.append(", ").append(ds); |
| } |
| } |
| sb.append("}"); |
| return sb; |
| } |
| |
| /** |
| * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to |
| * use for testing them against in a solr request. |
| */ |
| public static CharSequence toJSONFacetParamValue(Map<String,TermFacet> facets) { |
| assert null != facets; |
| assert 0 < facets.size(); |
| StringBuilder sb = new StringBuilder("{"); |
| for (String key : facets.keySet()) { |
| sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue()); |
| sb.append(" ,"); |
| } |
| sb.setLength(sb.length() - 1); |
| sb.append("}"); |
| return sb; |
| } |
| |
| /** |
| * Factory method for generating some random (nested) facets. |
| * |
| * For simplicity, each facet will have a unique key name, regardless of it's depth under other facets |
| * |
| * @see JoinDomain |
| */ |
| public static Map<String,TermFacet> buildRandomFacets() { |
| // for simplicity, use a unique facet key regardless of depth - simplifies verification |
| AtomicInteger keyCounter = new AtomicInteger(0); |
| final int maxDepth = TestUtil.nextInt(random(), 0, (LuceneTestCase.usually() ? 2 : 3)); |
| return buildRandomFacets(keyCounter, maxDepth); |
| } |
| |
| /** |
| * picks a random value for the "limit" param, biased in favor of interesting test cases |
| * |
| * @return a number to specify in the request, or null to specify nothing (trigger default behavior) |
| * @see #UNIQUE_FIELD_VALS |
| */ |
| public static Integer randomLimitParam(Random r) { |
| final int limit = 1 + r.nextInt(UNIQUE_FIELD_VALS * 2); |
| if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) { |
| return -1; // unlimited |
| } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) { |
| return null; // sometimes, don't specify limit if it's the default |
| } |
| return limit; |
| } |
| |
| /** |
| * picks a random value for the "overrequest" param, biased in favor of interesting test cases |
| * |
| * @return a number to specify in the request, or null to specify nothing (trigger default behavior) |
| * @see #UNIQUE_FIELD_VALS |
| */ |
| public static Integer randomOverrequestParam(Random r) { |
| switch(r.nextInt(10)) { |
| case 0: |
| case 1: |
| case 2: |
| case 3: |
| return 0; // 40% of the time, no overrequest to better stress refinement |
| case 4: |
| case 5: |
| return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed |
| case 6: |
| return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough |
| default: break; |
| } |
| // else.... either leave param unspecified (or redundently specify the -1 default) |
| return r.nextBoolean() ? null : -1; |
| } |
| |
| /** |
| * picks a random value for the "refine" param, that is garunteed to be suitable for |
| * the specified limit & overrequest params. |
| * |
| * @return a value to specify in the request, or null to specify nothing (trigger default behavior) |
| * @see #randomLimitParam |
| * @see #randomOverrequestParam |
| * @see #UNIQUE_FIELD_VALS |
| */ |
| public static Boolean randomRefineParam(Random r, Integer limitParam, Integer overrequestParam) { |
| if (isRefinementNeeded(limitParam, overrequestParam)) { |
| return true; |
| } |
| |
| // refinement is not required |
| if (0 == r.nextInt(10)) { // once in a while, turn on refinement even if it isn't needed. |
| return true; |
| } |
| // explicitly or implicitly indicate refinement is not needed |
| return r.nextBoolean() ? false : null; |
| } |
| |
| /** |
| * Deterministicly identifies if the specified limit & overrequest params <b>require</b> |
| * a "refine:true" param be used in the the request, in order for the counts to be 100% accurate. |
| * |
| * @see #UNIQUE_FIELD_VALS |
| */ |
| public static boolean isRefinementNeeded(Integer limitParam, Integer overrequestParam) { |
| |
| if (FORCE_DISABLE_REFINEMENT) { |
| return false; |
| } |
| |
| // use the "effective" values if the params are null |
| final int limit = null == limitParam ? DEFAULT_LIMIT : limitParam; |
| final int overrequest = null == overrequestParam ? 0 : overrequestParam; |
| |
| return |
| // don't presume how much overrequest will be done by default, just check the limit |
| (overrequest < 0 && limit < UNIQUE_FIELD_VALS) |
| // if the user specified overrequest is not "enough" to get all unique values |
| || (overrequest >= 0 && (long)limit + overrequest < UNIQUE_FIELD_VALS); |
| } |
| |
| /** |
| * recursive helper method for building random facets |
| * |
| * @param keyCounter used to ensure every generated facet has a unique key name |
| * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level. |
| */ |
| private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) { |
| final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1' |
| Map<String,TermFacet> results = new LinkedHashMap<>(); |
| for (int i = 0; i < numFacets; i++) { |
| final JoinDomain domain = JoinDomain.buildRandomDomain(); |
| assert null != domain; |
| final Integer limit = randomLimitParam(random()); |
| final Integer overrequest = randomOverrequestParam(random()); |
| final TermFacet facet = new TermFacet(field(random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES, |
| random().nextInt(MAX_FIELD_NUM)), |
| domain, limit, overrequest, |
| randomRefineParam(random(), limit, overrequest)); |
| results.put("facet_" + keyCounter.incrementAndGet(), facet); |
| if (0 < maxDepth) { |
| // if we're going wide, don't go deep |
| final int nextMaxDepth = Math.max(0, maxDepth - numFacets); |
| facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth))); |
| } |
| } |
| return results; |
| } |
| } |
| |
| |
| /** |
| * Models a Domain Change which includes either a 'join' or a 'filter' or both |
| */ |
| private static final class JoinDomain { |
| public final String from; |
| public final String to; |
| public final String filter; // not bothering with more then 1 filter, not the point of the test |
| |
| /** |
| * @param from left side of join field name, null if domain involves no joining |
| * @param to right side of join field name, null if domain involves no joining |
| * @param filter filter to apply to domain, null if domain involves no filtering |
| */ |
| public JoinDomain(String from, String to, String filter) { |
| assert ! ((null == from) ^ (null == to)) : "if from is null, to must be null"; |
| this.from = from; |
| this.to = to; |
| this.filter = filter; |
| } |
| |
| /** |
| * @return the JSON string representing this domain for use in a facet param, or null if no domain should be used |
| * */ |
| public CharSequence toJSONFacetParamValue() { |
| if (null == from && null == filter) { |
| return null; |
| } |
| StringBuilder sb = new StringBuilder("domain:{"); |
| if (null != from) { |
| assert null != to; |
| sb. append("join:{from:").append(from).append(",to:").append(to).append("}"); |
| if (null != filter){ |
| sb.append(","); |
| } |
| |
| } |
| if (null != filter) { |
| sb.append("filter:'").append(filter).append("'"); |
| } |
| sb.append("}"); |
| return sb; |
| } |
| |
| /** |
| * Given some original SolrParams, returns new SolrParams where the original "q" param is wrapped |
| * as needed to apply the equivalent transformation to a query as this domain would to a facet |
| */ |
| public SolrParams applyDomainToQuery(String safeKey, SolrParams in) { |
| assert null == in.get(safeKey); // shouldn't be possible if every facet uses a unique key string |
| |
| String q = in.get("q"); |
| final ModifiableSolrParams out = new ModifiableSolrParams(in); |
| if (null != from) { |
| out.set(safeKey, in.get("q")); |
| q = "{!join from="+from+" to="+to+" v=$"+safeKey+"}"; |
| } |
| if (null != filter) { |
| q = filter + " AND " + q; |
| } |
| out.set("q", q); |
| return out; |
| } |
| |
| /** |
| * Factory method for creating a random domain change to use with a facet - may return an 'noop' JoinDomain, |
| * but will never return null. |
| */ |
| public static JoinDomain buildRandomDomain() { |
| |
| // use consistent type on both sides of join |
| final String[] suffixes = random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES; |
| |
| final boolean noJoin = random().nextBoolean(); |
| |
| String from = null; |
| String to = null; |
| for (;;) { |
| if (noJoin) break; |
| from = field(suffixes, random().nextInt(MAX_FIELD_NUM)); |
| to = field(suffixes, random().nextInt(MAX_FIELD_NUM)); |
| // HACK: joined numeric point fields need docValues.. for now just skip _is fields if we are dealing with points. |
| if (Boolean.getBoolean(SolrTestCaseJ4.NUMERIC_POINTS_SYSPROP) && (from.endsWith("_is") || to.endsWith("_is"))) |
| { |
| continue; |
| } |
| break; |
| } |
| |
| // keep it simple, only filter on string fields - not point of test |
| final String filterField = strfield(random().nextInt(MAX_FIELD_NUM)); |
| |
| final String filter = random().nextBoolean() ? null : filterField+":[* TO *]"; |
| return new JoinDomain(from, to, filter); |
| } |
| } |
| |
| /** |
| * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed |
| * at a node in our cluster |
| */ |
| public static SolrClient getRandClient(Random rand) { |
| int numClients = CLIENTS.size(); |
| int idx = TestUtil.nextInt(rand, 0, numClients); |
| |
| return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx); |
| } |
| |
| } |
