| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr; |
| |
| import java.io.IOException; |
| import java.lang.invoke.MethodHandles; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.EnumSet; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.concurrent.Callable; |
| import java.util.concurrent.ExecutorCompletionService; |
| import java.util.concurrent.Future; |
| |
| import org.apache.commons.lang3.StringUtils; |
| import org.apache.lucene.util.LuceneTestCase.Slow; |
| import org.apache.solr.SolrTestCaseJ4.SuppressSSL; |
| import org.apache.solr.client.solrj.SolrClient; |
| import org.apache.solr.client.solrj.SolrQuery; |
| import org.apache.solr.client.solrj.SolrServerException; |
| import org.apache.solr.client.solrj.embedded.JettySolrRunner; |
| import org.apache.solr.client.solrj.impl.HttpSolrClient; |
| import org.apache.solr.client.solrj.response.FacetField; |
| import org.apache.solr.client.solrj.response.FieldStatsInfo; |
| import org.apache.solr.client.solrj.response.QueryResponse; |
| import org.apache.solr.client.solrj.response.RangeFacet; |
| import org.apache.solr.common.EnumFieldValue; |
| import org.apache.solr.common.SolrException; |
| import org.apache.solr.common.SolrException.ErrorCode; |
| import org.apache.solr.common.params.CommonParams; |
| import org.apache.solr.common.params.FacetParams.FacetRangeMethod; |
| import org.apache.solr.common.params.ModifiableSolrParams; |
| import org.apache.solr.common.params.ShardParams; |
| import org.apache.solr.common.params.SolrParams; |
| import org.apache.solr.common.params.StatsParams; |
| import org.apache.solr.common.util.NamedList; |
| import org.apache.solr.handler.component.ShardResponse; |
| import org.apache.solr.handler.component.StatsComponentTest.StatSetCombinations; |
| import org.apache.solr.handler.component.StatsField.Stat; |
| import org.apache.solr.handler.component.TrackingShardHandlerFactory; |
| import org.apache.solr.handler.component.TrackingShardHandlerFactory.RequestTrackingQueue; |
| import org.apache.solr.handler.component.TrackingShardHandlerFactory.ShardRequestAndParams; |
| import org.apache.solr.response.SolrQueryResponse; |
| import org.junit.BeforeClass; |
| import org.junit.Test; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| /** |
| * TODO? perhaps use: |
| * http://docs.codehaus.org/display/JETTY/ServletTester |
| * rather then open a real connection? |
| * |
| * |
| * @since solr 1.3 |
| */ |
| @Slow |
| @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-9061") |
| public class TestDistributedSearch extends BaseDistributedSearchTestCase { |
| |
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); |
| |
| String t1="a_t"; |
| String i1 = pickRandom("a_i1", "a_i_p", "a_i_ni_p"); |
| String nint = pickRandom("n_i", "n_is_p", "n_is_ni_p"); |
| String tint = "n_ti"; |
| String tlong = "other_tl1"; |
| String tdate_a = "a_n_tdt"; |
| String tdate_b = "b_n_tdt"; |
| |
| String oddField="oddField_s"; |
| String s1="a_s"; |
| String missingField="ignore_exception__missing_but_valid_field_t"; |
| String invalidField="ignore_exception__invalid_field_not_in_schema"; |
| |
| @Override |
| protected String getSolrXml() { |
| return "solr-trackingshardhandler.xml"; |
| } |
| |
| @BeforeClass |
| public static void beforeClass() { |
| // we shutdown a jetty and start it and try to use |
| // the same http client pretty fast - this lowered setting makes sure |
| // we validate the connection before use on the restarted |
| // server so that we don't use a bad one |
| System.setProperty("validateAfterInactivity", "200"); |
| |
| System.setProperty("solr.httpclient.retries", "0"); |
| System.setProperty("distribUpdateSoTimeout", "5000"); |
| |
| |
| } |
| |
| public TestDistributedSearch() { |
| // we need DVs on point fields to compute stats & facets |
| if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true"); |
| } |
| |
| @Test |
| @SuppressWarnings({"unchecked"}) |
| public void test() throws Exception { |
| |
| assertEquals(clients.size(), jettys.size()); |
| |
| QueryResponse rsp = null; |
| int backupStress = stress; // make a copy so we can restore |
| |
| del("*:*"); |
| indexr(id,1, i1, 100, tlong, 100,t1,"now is the time for all good men", |
| "foo_sev_enum", "Medium", |
| tdate_a, "2010-04-20T11:00:00Z", |
| tdate_b, "2009-08-20T11:00:00Z", |
| "foo_f", 1.414f, "foo_b", "true", "foo_d", 1.414d, |
| s1, "z${foo}"); |
| indexr(id,2, i1, 50 , tlong, 50,t1,"to come to the aid of their country.", |
| "foo_sev_enum", "Medium", |
| "foo_sev_enum", "High", |
| tdate_a, "2010-05-02T11:00:00Z", |
| tdate_b, "2009-11-02T11:00:00Z", |
| s1, "z${foo}"); |
| indexr(id,3, i1, 2, tlong, 2,t1,"how now brown cow", |
| tdate_a, "2010-05-03T11:00:00Z", |
| s1, "z${foo}"); |
| indexr(id,4, i1, -100 ,tlong, 101, |
| t1,"the quick fox jumped over the lazy dog", |
| tdate_a, "2010-05-03T11:00:00Z", |
| tdate_b, "2010-05-03T11:00:00Z", |
| s1, "a"); |
| indexr(id,5, i1, 500, tlong, 500 , |
| t1,"the quick fox jumped way over the lazy dog", |
| tdate_a, "2010-05-05T11:00:00Z", |
| s1, "b"); |
| indexr(id,6, i1, -600, tlong, 600 ,t1,"humpty dumpy sat on a wall", s1, "c"); |
| indexr(id,7, i1, 123, tlong, 123 ,t1,"humpty dumpy had a great fall", s1, "d"); |
| indexr(id,8, i1, 876, tlong, 876, |
| tdate_b, "2010-01-05T11:00:00Z", |
| "foo_sev_enum", "High", |
| t1,"all the kings horses and all the kings men", s1, "e"); |
| indexr(id,9, i1, 7, tlong, 7,t1,"couldn't put humpty together again", s1, "f"); |
| |
| commit(); // try to ensure there's more than one segment |
| |
| indexr(id,10, i1, 4321, tlong, 4321,t1,"this too shall pass", s1, "g"); |
| indexr(id,11, i1, -987, tlong, 987, |
| "foo_sev_enum", "Medium", |
| t1,"An eye for eye only ends up making the whole world blind.", s1, "h"); |
| indexr(id,12, i1, 379, tlong, 379, |
| t1,"Great works are performed, not by strength, but by perseverance.", s1, "i"); |
| indexr(id,13, i1, 232, tlong, 232, |
| t1,"no eggs on wall, lesson learned", |
| oddField, "odd man out", s1, "j"); |
| |
| indexr(id, "1001", "lowerfilt", "toyota", s1, "k"); // for spellcheck |
| |
| indexr(id, 14, "SubjectTerms_mfacet", new String[] {"mathematical models", "mathematical analysis"}, s1, "l"); |
| indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); |
| indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); |
| String[] vals = new String[100]; |
| for (int i=0; i<100; i++) { |
| vals[i] = "test " + i; |
| } |
| indexr(id, 17, "SubjectTerms_mfacet", vals); |
| |
| |
| |
| for (int i=100; i<150; i++) { |
| indexr(id, i); |
| } |
| |
| commit(); |
| |
| handle.clear(); |
| handle.put("timestamp", SKIPVAL); |
| handle.put("_version_", SKIPVAL); // not a cloud test, but may use updateLog |
| |
| //Test common query parameters. |
| validateCommonQueryParameters(); |
| |
| // random value sort |
| for (String f : fieldNames) { |
| query("q","*:*", "sort",f+" desc"); |
| query("q","*:*", "sort",f+" asc"); |
| } |
| |
| // these queries should be exactly ordered and scores should exactly match |
| query("q","*:*", "sort",i1+" desc"); |
| query("q","*:*", "sort","{!func}testfunc(add("+i1+",5))"+" desc"); |
| query("q",i1 + "[* TO *]", "sort",i1+" asc"); |
| query("q","*:*", "sort",i1+" asc, id desc"); |
| query("q","*:*", "sort",i1+" desc", "fl","*,score"); |
| query("q","*:*", "sort","n_tl1 asc", "fl","*,score"); |
| query("q","*:*", "sort","n_tl1 desc"); |
| |
| handle.put("maxScore", SKIPVAL); |
| testMinExactCount(); |
| |
| query("q","{!func}"+i1);// does not expect maxScore. So if it comes ,ignore it. JavaBinCodec.writeSolrDocumentList() |
| //is agnostic of request params. |
| handle.remove("maxScore"); |
| query("q","{!func}"+i1, "fl","*,score"); // even scores should match exactly here |
| |
| handle.put("highlighting", UNORDERED); |
| handle.put("response", UNORDERED); |
| |
| handle.put("maxScore", SKIPVAL); |
| query("q","quick"); |
| query("q","all","fl","id","start","0"); |
| query("q","all","fl","foofoofoo","start","0"); // no fields in returned docs |
| query("q","all","fl","id","start","100"); |
| |
| handle.put("score", SKIPVAL); |
| query("q","quick","fl","*,score"); |
| query("q","all","fl","*,score","start","1"); |
| query("q","all","fl","*,score","start","100"); |
| |
| query("q","now their fox sat had put","fl","*,score", |
| "hl","true","hl.fl",t1); |
| |
| query("q","now their fox sat had put","fl","foofoofoo", |
| "hl","true","hl.fl",t1); |
| |
| query("q","matchesnothing","fl","*,score"); |
| |
| // test that a single NOW value is propagated to all shards... if that is true |
| // then the primary sort should always be a tie and then the secondary should always decide |
| query("q","{!func}ms(NOW)", "sort","score desc,"+i1+" desc","fl","id"); |
| |
| query("q","*:*", "rows",0, "facet","true", "facet.field",t1, "facet.field",t1); |
| query("q","*:*", "rows",0, "facet","true", "facet.field",t1,"facet.limit",1); |
| query("q","*:*", "rows",0, "facet","true", "facet.query","quick", "facet.query","quick", "facet.query","all", "facet.query","*:*"); |
| query("q","*:*", "rows",0, "facet","true", "facet.field",t1, "facet.mincount",2); |
| |
| // a facet query to test out chars out of the ascii range |
| query("q","*:*", "rows",0, "facet","true", "facet.query","{!term f=foo_s}international\u00ff\u01ff\u2222\u3333"); |
| |
| // simple field facet on date fields |
| rsp = query("q","*:*", "rows", 0, |
| "facet","true", "facet.limit", 1, // TODO: limit shouldn't be needed: SOLR-6386 |
| "facet.field", tdate_a); |
| assertEquals(1, rsp.getFacetFields().size()); |
| rsp = query("q","*:*", "rows", 0, |
| "facet","true", "facet.limit", 1, // TODO: limit shouldn't be needed: SOLR-6386 |
| "facet.field", tdate_b, "facet.field", tdate_a); |
| assertEquals(2, rsp.getFacetFields().size()); |
| |
| String facetQuery = "id_i1:[1 TO 15]"; |
| |
| // simple range facet on one field |
| query("q",facetQuery, "rows",100, "facet","true", |
| "facet.range",tlong, |
| "facet.range",tlong, |
| "facet.range.start",200, |
| "facet.range.gap",100, |
| "facet.range.end",900, |
| "facet.range.method", FacetRangeMethod.FILTER); |
| |
| // simple range facet on one field using dv method |
| query("q",facetQuery, "rows",100, "facet","true", |
| "facet.range",tlong, |
| "facet.range",tlong, |
| "facet.range.start",200, |
| "facet.range.gap",100, |
| "facet.range.end",900, |
| "facet.range.method", FacetRangeMethod.DV); |
| |
| // range facet on multiple fields |
| query("q",facetQuery, "rows",100, "facet","true", |
| "facet.range",tlong, |
| "facet.range",i1, |
| "f."+i1+".facet.range.start",300, |
| "f."+i1+".facet.range.gap",87, |
| "facet.range.end",900, |
| "facet.range.start",200, |
| "facet.range.gap",100, |
| "f."+tlong+".facet.range.end",900, |
| "f."+i1+".facet.range.method", FacetRangeMethod.FILTER, |
| "f."+tlong+".facet.range.method", FacetRangeMethod.DV); |
| |
| // range facet with "other" param |
| QueryResponse response = query("q",facetQuery, "rows",100, "facet","true", |
| "facet.range",tlong, |
| "facet.range.start",200, |
| "facet.range.gap",100, |
| "facet.range.end",900, |
| "facet.range.other","all"); |
| assertEquals(tlong, response.getFacetRanges().get(0).getName()); |
| assertEquals(6, response.getFacetRanges().get(0).getBefore()); |
| assertEquals(5, response.getFacetRanges().get(0).getBetween()); |
| assertEquals(2, response.getFacetRanges().get(0).getAfter()); |
| |
| // Test mincounts. Do NOT want to go through all the stuff where with validateControlData in query() method |
| // Purposely packing a _bunch_ of stuff together here to insure that the proper level of mincount is used for |
| // each |
| ModifiableSolrParams minParams = new ModifiableSolrParams(); |
| minParams.set("q","*:*"); |
| minParams.set("rows", 1); |
| minParams.set("facet", "true"); |
| minParams.set("facet.missing", "true"); |
| minParams.set("facet.field", i1); |
| minParams.set("facet.missing", "true"); |
| minParams.set("facet.mincount", 2); |
| |
| // Return a separate section of ranges over i1. Should respect global range mincount |
| minParams.set("facet.range", i1); |
| minParams.set("f." + i1 + ".facet.range.start", 0); |
| minParams.set("f." + i1 + ".facet.range.gap", 200); |
| minParams.set("f." + i1 + ".facet.range.end", 1200); |
| minParams.set("f." + i1 + ".facet.mincount", 4); |
| |
| |
| // Return a separate section of ranges over tlong Should respect facet.mincount |
| minParams.add("facet.range", tlong); |
| minParams.set("f." + tlong + ".facet.range.start", 0); |
| minParams.set("f." + tlong + ".facet.range.gap", 100); |
| minParams.set("f." + tlong + ".facet.range.end", 1200); |
| // Repeat with a range type of date |
| minParams.add("facet.range", tdate_b); |
| minParams.set("f." + tdate_b + ".facet.range.start", "2009-02-01T00:00:00Z"); |
| minParams.set("f." + tdate_b + ".facet.range.gap", "+1YEAR"); |
| minParams.set("f." + tdate_b + ".facet.range.end", "2011-01-01T00:00:00Z"); |
| minParams.set("f." + tdate_b + ".facet.mincount", 3); |
| |
| // Insure that global mincount is respected for facet queries |
| minParams.set("facet.query", tdate_a + ":[2010-01-01T00:00:00Z TO 2011-01-01T00:00:00Z]"); // Should return some counts |
| //minParams.set("facet.query", tdate_a + ":[* TO *]"); // Should be removed |
| minParams.add("facet.query", tdate_b + ":[2008-01-01T00:00:00Z TO 2009-09-01T00:00:00Z]"); // Should be removed from response |
| |
| |
| setDistributedParams(minParams); |
| QueryResponse minResp = queryServer(minParams); |
| |
| ModifiableSolrParams eParams = new ModifiableSolrParams(); |
| eParams.set("q",tdate_b + ":[* TO *]"); |
| eParams.set("rows", 1000); |
| eParams.set("fl", tdate_b); |
| setDistributedParams(eParams); |
| QueryResponse eResp = queryServer(eParams); |
| |
| // Check that exactly the right numbers of counts came through |
| assertEquals("Should be exactly 2 range facets returned after minCounts taken into account ", 3, minResp.getFacetRanges().size()); |
| assertEquals("Should only be 1 query facets returned after minCounts taken into account ", 1, minResp.getFacetQuery().size()); |
| |
| checkMinCountsField(minResp.getFacetField(i1).getValues(), new Object[]{null, 55L}); // Should just be the null entries for field |
| |
| checkMinCountsRange(minResp.getFacetRanges().get(0).getCounts(), new Object[]{"0", 5L}); // range on i1 |
| checkMinCountsRange(minResp.getFacetRanges().get(1).getCounts(), new Object[]{"0", 3L, "100", 3L}); // range on tlong |
| checkMinCountsRange(minResp.getFacetRanges().get(2).getCounts(), new Object[]{"2009-02-01T00:00:00Z", 3L}); // date (range) on tvh |
| |
| assertTrue("Should have a facet for tdate_a", minResp.getFacetQuery().containsKey("a_n_tdt:[2010-01-01T00:00:00Z TO 2011-01-01T00:00:00Z]")); |
| int qCount = minResp.getFacetQuery().get("a_n_tdt:[2010-01-01T00:00:00Z TO 2011-01-01T00:00:00Z]"); |
| assertEquals("tdate_a should be 5", qCount, 5); |
| |
| // Now let's do some queries, the above is getting too complex |
| minParams = new ModifiableSolrParams(); |
| minParams.set("q","*:*"); |
| minParams.set("rows", 1); |
| minParams.set("facet", "true"); |
| minParams.set("facet.mincount", 3); |
| |
| minParams.set("facet.query", tdate_a + ":[2010-01-01T00:00:00Z TO 2010-05-04T00:00:00Z]"); |
| minParams.add("facet.query", tdate_b + ":[2009-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"); // Should be removed |
| setDistributedParams(minParams); |
| minResp = queryServer(minParams); |
| |
| assertEquals("Should only be 1 query facets returned after minCounts taken into account ", 1, minResp.getFacetQuery().size()); |
| assertTrue("Should be an entry for a_n_tdt", minResp.getFacetQuery().containsKey("a_n_tdt:[2010-01-01T00:00:00Z TO 2010-05-04T00:00:00Z]")); |
| qCount = minResp.getFacetQuery().get("a_n_tdt:[2010-01-01T00:00:00Z TO 2010-05-04T00:00:00Z]"); |
| assertEquals("a_n_tdt should have a count of 4 ", qCount, 4); |
| // variations of fl |
| query("q","*:*", "fl","score","sort",i1 + " desc"); |
| query("q","*:*", "fl",i1 + ",score","sort",i1 + " desc"); |
| query("q","*:*", "fl", i1, "fl","score","sort",i1 + " desc"); |
| query("q","*:*", "fl", "id," + i1,"sort",i1 + " desc"); |
| query("q","*:*", "fl", "id", "fl",i1,"sort",i1 + " desc"); |
| query("q","*:*", "fl",i1, "fl", "id","sort",i1 + " desc"); |
| query("q","*:*", "fl", "id", "fl",nint, "fl",tint,"sort",i1 + " desc"); |
| query("q","*:*", "fl",nint, "fl", "id", "fl",tint,"sort",i1 + " desc"); |
| handle.put("did", SKIPVAL); |
| query("q","*:*", "fl","did:[docid]","sort",i1 + " desc"); |
| handle.remove("did"); |
| query("q","*:*", "fl","log(" + tlong + "),abs(" + tlong + "),score","sort",i1 + " desc"); |
| query("q","*:*", "fl","n_*","sort",i1 + " desc"); |
| |
| // basic spellcheck testing |
| query("q", "toyata", "fl", "id,lowerfilt", "spellcheck", true, "spellcheck.q", "toyata", "qt", "/spellCheckCompRH_Direct", "shards.qt", "/spellCheckCompRH_Direct"); |
| |
| stress=0; // turn off stress... we want to tex max combos in min time |
| for (int i=0; i<25*RANDOM_MULTIPLIER; i++) { |
| String f = fieldNames[random().nextInt(fieldNames.length)]; |
| if (random().nextBoolean()) f = t1; // the text field is a really interesting one to facet on (and it's multi-valued too) |
| |
| // we want a random query and not just *:* so we'll get zero counts in facets also |
| // TODO: do a better random query |
| String q = random().nextBoolean() ? "*:*" : "id:(1 3 5 7 9 11 13) OR id_i1:[100 TO " + random().nextInt(50) + "]"; |
| |
| int nolimit = random().nextBoolean() ? -1 : 10000; // these should be equivalent |
| |
| // if limit==-1, we should always get exact matches |
| query("q",q, "rows",0, "facet","true", "facet.field",f, "facet.limit",nolimit, "facet.sort","count", "facet.mincount",random().nextInt(5), "facet.offset",random().nextInt(10)); |
| query("q",q, "rows",0, "facet","true", "facet.field",f, "facet.limit",nolimit, "facet.sort","index", "facet.mincount",random().nextInt(5), "facet.offset",random().nextInt(10)); |
| // for index sort, we should get exact results for mincount <= 1 |
| query("q",q, "rows",0, "facet","true", "facet.field",f, "facet.sort","index", "facet.mincount",random().nextInt(2), "facet.offset",random().nextInt(10), "facet.limit",random().nextInt(11)-1); |
| } |
| stress = backupStress; // restore stress |
| |
| // test faceting multiple things at once |
| query("q","*:*", "rows",0, "facet","true", "facet.query","quick", "facet.query","all", "facet.query","*:*" |
| ,"facet.field",t1); |
| |
| // test filter tagging, facet exclusion, and naming (multi-select facet support) |
| queryAndCompareUIF("q","*:*", "rows",0, "facet","true", "facet.query","{!key=myquick}quick", "facet.query","{!key=myall ex=a}all", "facet.query","*:*" |
| ,"facet.field","{!key=mykey ex=a}"+t1 |
| ,"facet.field","{!key=other ex=b}"+t1 |
| ,"facet.field","{!key=again ex=a,b}"+t1 |
| ,"facet.field",t1 |
| ,"fq","{!tag=a}id_i1:[1 TO 7]", "fq","{!tag=b}id_i1:[3 TO 9]" |
| ); |
| queryAndCompareUIF("q", "*:*", "facet", "true", "facet.field", "{!ex=t1}SubjectTerms_mfacet", "fq", "{!tag=t1}SubjectTerms_mfacet:(test 1)", "facet.limit", "10", "facet.mincount", "1"); |
| |
| // test field that is valid in schema but missing in all shards |
| query("q","*:*", "rows",100, "facet","true", "facet.field",missingField, "facet.mincount",2); |
| // test field that is valid in schema and missing in some shards |
| query("q","*:*", "rows",100, "facet","true", "facet.field",oddField, "facet.mincount",2); |
| |
| query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", "stats_dt"); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", i1); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", nint); |
| |
| handle.put("stddev", FUZZY); |
| handle.put("sumOfSquares", FUZZY); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", tdate_a); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", tdate_b); |
| handle.remove("stddev"); |
| handle.remove("sumOfSquares"); |
| |
| |
| rsp = query("q", "*:*", "sort", i1 + " desc", "stats", "true", |
| "stats.field", "{!cardinality='true'}" + oddField, |
| "stats.field", "{!cardinality='true'}" + tlong); |
| |
| { // don't leak variabls |
| |
| // long |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(tlong); |
| assertNotNull("missing stats", s); |
| assertEquals("wrong cardinality", Long.valueOf(13), s.getCardinality()); |
| // |
| assertNull("expected null for min", s.getMin()); |
| assertNull("expected null for mean", s.getMean()); |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for stddev", s.getStddev()); |
| assertNull("expected null for sum", s.getSum()); |
| assertNull("expected null for percentiles", s.getSum()); |
| |
| // string |
| s = rsp.getFieldStatsInfo().get(oddField); |
| assertNotNull("missing stats", s); |
| assertEquals("wrong cardinality", Long.valueOf(1), s.getCardinality()); |
| // |
| assertNull("expected null for min", s.getMin()); |
| assertNull("expected null for mean", s.getMean()); |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for stddev", s.getStddev()); |
| assertNull("expected null for sum", s.getSum()); |
| assertNull("expected null for percentiles", s.getSum()); |
| } |
| |
| query("q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", |
| "{!percentiles='1,2,3,4,5'}" + i1); |
| |
| query("q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", |
| "{!percentiles='1,20,30,40,98,99,99.9'}" + i1); |
| |
| rsp = query("q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", |
| "{!percentiles='1.0,99.999,0.001'}" + tlong); |
| { // don't leak variabls |
| Double[] expectedKeys = new Double[] { 1.0D, 99.999D, 0.001D }; |
| Double[] expectedVals = new Double[] { 2.0D, 4320.0D, 2.0D }; |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(tlong); |
| assertNotNull("no stats for " + tlong, s); |
| |
| Map<Double,Double> p = s.getPercentiles(); |
| assertNotNull("no percentils", p); |
| assertEquals("insufficient percentiles", expectedKeys.length, p.size()); |
| Iterator<Double> actualKeys = p.keySet().iterator(); |
| for (int i = 0; i < expectedKeys.length; i++) { |
| Double expectedKey = expectedKeys[i]; |
| assertTrue("Ran out of actual keys as of : "+ i + "->" +expectedKey, |
| actualKeys.hasNext()); |
| assertEquals(expectedKey, actualKeys.next()); |
| assertEquals("percentiles are off: " + p.toString(), |
| expectedVals[i], p.get(expectedKey), 1.0D); |
| } |
| |
| // |
| assertNull("expected null for count", s.getMin()); |
| assertNull("expected null for count", s.getMean()); |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for stddev", s.getStddev()); |
| assertNull("expected null for sum", s.getSum()); |
| } |
| |
| query("q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", |
| "{!percentiles='1,20,50,80,99'}" + tdate_a); |
| |
| query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "fq", "{!tag=nothing}-*:*", |
| "stats.field", "{!key=special_key ex=nothing}stats_dt"); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "f.stats_dt.stats.calcdistinct", "true", |
| "stats.field", "{!key=special_key}stats_dt"); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "f.stats_dt.stats.calcdistinct", "true", |
| "fq", "{!tag=xxx}id_i1:[3 TO 9]", |
| "stats.field", "{!key=special_key}stats_dt", |
| "stats.field", "{!ex=xxx}stats_dt"); |
| |
| handle.put("stddev", FUZZY); |
| handle.put("sumOfSquares", FUZZY); |
| query("q","*:*", "sort",i1+" desc", "stats", "true", |
| // do a really simple query so distributed IDF doesn't cause problems |
| // when comparing with control collection |
| "stats.field", "{!lucene key=q_key}" + i1 + "foo_b:true", |
| "stats.field", "{!func key=f_key}sum(" + tlong +","+i1+")"); |
| |
| query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "stats.field", "stats_dt", |
| "stats.field", i1, |
| "stats.field", tdate_a, |
| "stats.field", tdate_b); |
| |
| // only ask for "min" and "mean", explicitly exclude deps of mean, whitebox check shard responses |
| try { |
| RequestTrackingQueue trackingQueue = new RequestTrackingQueue(); |
| TrackingShardHandlerFactory.setTrackingQueue(jettys, trackingQueue); |
| |
| rsp = query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "stats.field", "{!min=true sum=false mean=true count=false}" + i1); |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1); |
| assertNotNull("no stats for " + i1, s); |
| // |
| assertEquals("wrong min", -987.0D, (Double)s.getMin(), 0.0001D ); |
| assertEquals("wrong mean", 377.153846D, (Double)s.getMean(), 0.0001D ); |
| // |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for stddev", s.getStddev()); |
| assertNull("expected null for sum", s.getSum()); |
| assertNull("expected null for percentiles", s.getPercentiles()); |
| assertNull("expected null for cardinality", s.getCardinality()); |
| |
| // sanity check deps relationship |
| for (Stat dep : EnumSet.of(Stat.sum, Stat.count)) { |
| assertTrue("Purpose of this test is to ensure that asking for some stats works even when the deps " + |
| "of those stats are explicitly excluded -- but the expected dep relationshp is no longer valid. " + |
| "ie: who changed the code and didn't change this test?, expected: " + dep, |
| Stat.mean.getDistribDeps().contains(dep)); |
| } |
| |
| // check our shard requests & responses - ensure we didn't get unneccessary stats from every shard |
| int numStatsShardRequests = 0; |
| EnumSet<Stat> shardStatsExpected = EnumSet.of(Stat.min, Stat.sum, Stat.count); |
| for (List<ShardRequestAndParams> shard : trackingQueue.getAllRequests().values()) { |
| for (ShardRequestAndParams shardReq : shard) { |
| if (shardReq.params.getBool(StatsParams.STATS, false)) { |
| numStatsShardRequests++; |
| for (ShardResponse shardRsp : shardReq.sreq.responses) { |
| NamedList<Object> shardStats = |
| ((NamedList<NamedList<NamedList<Object>>>) |
| shardRsp.getSolrResponse().getResponse().get("stats")).get("stats_fields").get(i1); |
| |
| assertNotNull("no stard stats for " + i1, shardStats); |
| // |
| for (Map.Entry<String,Object> entry : shardStats) { |
| Stat found = Stat.forName(entry.getKey()); |
| assertNotNull("found shardRsp stat key we were not expecting: " + entry, found); |
| assertTrue("found stat we were not expecting: " + entry, shardStatsExpected.contains(found)); |
| |
| } |
| } |
| } |
| } |
| } |
| assertTrue("did't see any stats=true shard requests", 0 < numStatsShardRequests); |
| } finally { |
| TrackingShardHandlerFactory.setTrackingQueue(jettys, null); |
| } |
| |
| // only ask for "min", "mean" and "stddev", |
| rsp = query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "stats.field", "{!min=true mean=true stddev=true}" + i1); |
| { // don't leak variables |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1); |
| assertNotNull("no stats for " + i1, s); |
| // |
| assertEquals("wrong min", -987.0D, (Double)s.getMin(), 0.0001D ); |
| assertEquals("wrong mean", 377.153846D, (Double)s.getMean(), 0.0001D ); |
| assertEquals("wrong stddev", 1271.76215D, s.getStddev(), 0.0001D ); |
| // |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for sum", s.getSum()); |
| assertNull("expected null for percentiles", s.getPercentiles()); |
| assertNull("expected null for cardinality", s.getCardinality()); |
| } |
| |
| // request stats, but disable them all via param refs |
| rsp = query("q","*:*", "sort",i1+" desc", "stats", "true", "doMin", "false", |
| "stats.field", "{!min=$doMin}" + i1); |
| { // don't leak variables |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1); |
| // stats section should exist, even though stats should be null |
| assertNotNull("no stats for " + i1, s); |
| // |
| assertNull("expected null for min", s.getMin() ); |
| assertNull("expected null for mean", s.getMean() ); |
| assertNull("expected null for stddev", s.getStddev() ); |
| // |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for sum", s.getSum()); |
| assertNull("expected null for percentiles", s.getPercentiles()); |
| assertNull("expected null for cardinality", s.getCardinality()); |
| } |
| |
| final String[] stats = new String[] { |
| "min", "max", "sum", "sumOfSquares", "stddev", "mean", "missing", "count" |
| }; |
| |
| // ask for arbitrary pairs of stats |
| for (String stat1 : stats) { |
| for (String stat2 : stats) { |
| // NOTE: stat1 might equal stat2 - good edge case to test for |
| |
| rsp = query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "stats.field", "{!" + stat1 + "=true " + stat2 + "=true}" + i1); |
| |
| final List<String> statsExpected = new ArrayList<String>(2); |
| statsExpected.add(stat1); |
| if ( ! stat1.equals(stat2) ) { |
| statsExpected.add(stat2); |
| } |
| |
| // ignore the FieldStatsInfo convinience class, and look directly at the NamedList |
| // so we don't need any sort of crazy reflection |
| NamedList<Object> svals = |
| ((NamedList<NamedList<NamedList<Object>>>) |
| rsp.getResponse().get("stats")).get("stats_fields").get(i1); |
| |
| assertNotNull("no stats for field " + i1, svals); |
| assertEquals("wrong quantity of stats", statsExpected.size(), svals.size()); |
| |
| |
| for (String s : statsExpected) { |
| assertNotNull("stat shouldn't be null: " + s, svals.get(s)); |
| assertTrue("stat should be a Number: " + s + " -> " + svals.get(s).getClass(), |
| svals.get(s) instanceof Number); |
| // some loose assertions since we're iterating over various stats |
| if (svals.get(s) instanceof Double) { |
| Double val = (Double) svals.get(s); |
| assertFalse("stat shouldn't be NaN: " + s, val.isNaN()); |
| assertFalse("stat shouldn't be Inf: " + s, val.isInfinite()); |
| assertFalse("stat shouldn't be 0: " + s, val.equals(0.0D)); |
| } else { |
| // count or missing |
| assertTrue("stat should be count of missing: " + s, |
| ("count".equals(s) || "missing".equals(s))); |
| assertTrue("stat should be a Long: " + s + " -> " + svals.get(s).getClass(), |
| svals.get(s) instanceof Long); |
| Long val = (Long) svals.get(s); |
| assertFalse("stat shouldn't be 0: " + s, val.equals(0L)); |
| } |
| } |
| } |
| } |
| |
| // all of these diff ways of asking for min & calcdistinct should have the same result |
| for (SolrParams p : new SolrParams[] { |
| params("stats.field", "{!min=true calcdistinct=true}" + i1), |
| params("stats.calcdistinct", "true", |
| "stats.field", "{!min=true}" + i1), |
| params("f."+i1+".stats.calcdistinct", "true", |
| "stats.field", "{!min=true}" + i1), |
| params("stats.calcdistinct", "false", |
| "f."+i1+".stats.calcdistinct", "true", |
| "stats.field", "{!min=true}" + i1), |
| params("stats.calcdistinct", "false", |
| "f."+i1+".stats.calcdistinct", "false", |
| "stats.field", "{!min=true calcdistinct=true}" + i1), |
| params("stats.calcdistinct", "false", |
| "f."+i1+".stats.calcdistinct", "false", |
| "stats.field", "{!min=true countDistinct=true distinctValues=true}" + i1), |
| params("stats.field", "{!min=true countDistinct=true distinctValues=true}" + i1), |
| params("yes", "true", |
| "stats.field", "{!min=$yes countDistinct=$yes distinctValues=$yes}" + i1), |
| }) { |
| |
| rsp = query(SolrParams.wrapDefaults |
| (p, params("q","*:*", "sort",i1+" desc", "stats", "true"))); |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1); |
| assertNotNull(p+" no stats for " + i1, s); |
| // |
| assertEquals(p+" wrong min", -987.0D, (Double)s.getMin(), 0.0001D ); |
| assertEquals(p+" wrong calcDistinct", Long.valueOf(13), s.getCountDistinct()); |
| assertNotNull(p+" expected non-null list for distinct vals", s.getDistinctValues()); |
| assertEquals(p+" expected list for distinct vals", 13, s.getDistinctValues().size()); |
| // |
| assertNull(p+" expected null for mean", s.getMean() ); |
| assertNull(p+" expected null for count", s.getCount()); |
| assertNull(p+" expected null for max", s.getMax()); |
| assertNull(p+" expected null for missing", s.getMissing()); |
| assertNull(p+" expected null for stddev", s.getStddev()); |
| assertNull(p+" expected null for sum", s.getSum()); |
| assertNull(p+" expected null for percentiles", s.getPercentiles()); |
| assertNull(p+" expected null for cardinality", s.getCardinality()); |
| |
| } |
| |
| // all of these diff ways of excluding calcdistinct should have the same result |
| for (SolrParams p : new SolrParams[] { |
| params("stats.field", "{!min=true calcdistinct=false}" + i1), |
| params("stats.calcdistinct", "false", |
| "stats.field", "{!min=true}" + i1), |
| params("f."+i1+".stats.calcdistinct", "false", |
| "stats.field", "{!min=true}" + i1), |
| params("stats.calcdistinct", "true", |
| "f."+i1+".stats.calcdistinct", "false", |
| "stats.field", "{!min=true}" + i1), |
| params("stats.calcdistinct", "true", |
| "f."+i1+".stats.calcdistinct", "true", |
| "stats.field", "{!min=true calcdistinct=false}" + i1), |
| params("stats.calcdistinct", "true", |
| "f."+i1+".stats.calcdistinct", "true", |
| "stats.field", "{!min=true countDistinct=false distinctValues=false}" + i1), |
| }) { |
| |
| rsp = query(SolrParams.wrapDefaults |
| (p, params("q","*:*", "sort",i1+" desc", "stats", "true"))); |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1); |
| assertNotNull(p+" no stats for " + i1, s); |
| // |
| assertEquals(p+" wrong min", -987.0D, (Double)s.getMin(), 0.0001D ); |
| // |
| assertNull(p+" expected null for calcDistinct", s.getCountDistinct()); |
| assertNull(p+" expected null for distinct vals", s.getDistinctValues()); |
| // |
| assertNull(p+" expected null for mean", s.getMean() ); |
| assertNull(p+" expected null for count", s.getCount()); |
| assertNull(p+" expected null for max", s.getMax()); |
| assertNull(p+" expected null for missing", s.getMissing()); |
| assertNull(p+" expected null for stddev", s.getStddev()); |
| assertNull(p+" expected null for sum", s.getSum()); |
| assertNull(p+" expected null for percentiles", s.getPercentiles()); |
| assertNull(p+" expected null for cardinality", s.getCardinality()); |
| } |
| |
| // this field doesn't exist in any doc in the result set. |
| // ensure we get expected values for the stats we ask for, but null for the stats |
| rsp = query("q","*:*", "sort",i1+" desc", "stats", "true", |
| "stats.field", "{!min=true mean=true stddev=true}does_not_exist_i"); |
| { // don't leak variables |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get("does_not_exist_i"); |
| assertNotNull("no stats for bogus field", s); |
| |
| // things we explicit expect because we asked for them |
| // NOTE: min is expected to be null even though requested because of no values |
| assertEquals("wrong min", null, s.getMin()); |
| assertTrue("mean should be NaN", ((Double)s.getMean()).isNaN()); |
| assertEquals("wrong stddev", 0.0D, s.getStddev(), 0.0D ); |
| |
| // things that we didn't ask for, so they better be null |
| assertNull("expected null for count", s.getCount()); |
| assertNull("expected null for calcDistinct", s.getCountDistinct()); |
| assertNull("expected null for distinct vals", s.getDistinctValues()); |
| assertNull("expected null for max", s.getMax()); |
| assertNull("expected null for missing", s.getMissing()); |
| assertNull("expected null for sum", s.getSum()); |
| assertNull("expected null for percentiles", s.getPercentiles()); |
| assertNull("expected null for cardinality", s.getCardinality()); |
| } |
| |
| // look at stats on non numeric fields |
| // |
| // not all stats are supported on every field type, so some of these permutations will |
| // result in no stats being computed but this at least lets us sanity check that for each |
| // of these field+stats(s) combinations we get consistent results between the distribted |
| // request and the single node situation. |
| // |
| // NOTE: percentiles excluded because it doesn't support simple 'true/false' syntax |
| // (and since it doesn't work for non-numerics anyway, we aren't missing any coverage here) |
| EnumSet<Stat> allStats = EnumSet.complementOf(EnumSet.of(Stat.percentiles)); |
| |
| int numTotalStatQueries = 0; |
| // don't go overboard, just do all permutations of 1 or 2 stat params, for each field & query |
| final int numStatParamsAtOnce = 2; |
| for (int numParams = 1; numParams <= numStatParamsAtOnce; numParams++) { |
| for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) { |
| |
| for (String field : new String[] { |
| "foo_f", i1, tlong, tdate_a, oddField, "foo_sev_enum", |
| // fields that no doc has any value in |
| "bogus___s", "bogus___f", "bogus___i", "bogus___tdt", "bogus___sev_enum" |
| }) { |
| |
| for ( String q : new String[] { |
| "*:*", // all docs |
| "bogus___s:bogus", // no docs |
| "id:" + random().nextInt(50 ), // 0 or 1 doc... |
| "id:" + random().nextInt(50 ), |
| "id:" + random().nextInt(100), |
| "id:" + random().nextInt(100), |
| "id:" + random().nextInt(200) |
| }) { |
| |
| // EnumSets use natural ordering, we want to randomize the order of the params |
| List<Stat> combo = new ArrayList<Stat>(set); |
| Collections.shuffle(combo, random()); |
| |
| StringBuilder paras = new StringBuilder("{!key=k "); |
| |
| for (Stat stat : combo) { |
| paras.append(stat + "=true "); |
| } |
| |
| paras.append("}").append(field); |
| numTotalStatQueries++; |
| rsp = query("q", q, "rows", "0", "stats", "true", |
| "stats.field", paras.toString()); |
| // simple assert, mostly relying on comparison with single shard |
| FieldStatsInfo s = rsp.getFieldStatsInfo().get("k"); |
| assertNotNull(s); |
| |
| // TODO: if we had a programatic way to determine what stats are supported |
| // by what field types, we could make more confident asserts here. |
| } |
| } |
| } |
| } |
| handle.remove("stddev"); |
| handle.remove("sumOfSquares"); |
| assertEquals("Sanity check failed: either test broke, or test changed, or you adjusted Stat enum" + |
| " (adjust constant accordingly if intentional)", |
| 5082, numTotalStatQueries); |
| |
| /*** TODO: the failure may come back in "exception" |
| try { |
| // test error produced for field that is invalid for schema |
| query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2); |
| TestCase.fail("SolrServerException expected for invalid field that is not in schema"); |
| } catch (SolrServerException ex) { |
| // expected |
| } |
| ***/ |
| |
| // Try to get better coverage for refinement queries by turning off over requesting. |
| // This makes it much more likely that we may not get the top facet values and hence |
| // we turn of that checking. |
| handle.put("facet_fields", SKIPVAL); |
| query("q","*:*", "rows",0, "facet","true", "facet.field",t1,"facet.limit",5, "facet.shard.limit",5); |
| // check a complex key name |
| query("q","*:*", "rows",0, "facet","true", "facet.field","{!key='$a b/c \\' \\} foo'}"+t1,"facet.limit",5, "facet.shard.limit",5); |
| query("q","*:*", "rows",0, "facet","true", "facet.field","{!key='$a'}"+t1,"facet.limit",5, "facet.shard.limit",5); |
| handle.remove("facet_fields"); |
| // Make sure there is no macro expansion for field values |
| query("q","*:*", "rows",0, "facet","true", "facet.field",s1,"facet.limit",5, "facet.shard.limit",5); |
| query("q","*:*", "rows",0, "facet","true", "facet.field",s1,"facet.limit",5, "facet.shard.limit",5, "expandMacros", "true"); |
| query("q","*:*", "rows",0, "facet","true", "facet.field",s1,"facet.limit",5, "facet.shard.limit",5, "expandMacros", "false"); |
| // Macro expansion should still work for the parameters |
| query("q","*:*", "rows",0, "facet","true", "facet.field","${foo}", "f.${foo}.mincount", 1, "foo", s1); |
| query("q","*:*", "rows",0, "facet","true", "facet.field","${foo}", "f.${foo}.mincount", 1, "foo", s1, "expandMacros", "true"); |
| |
| // index the same document to two servers and make sure things |
| // don't blow up. |
| if (clients.size()>=2) { |
| index(id,100, i1, 107 ,t1,"oh no, a duplicate!"); |
| for (int i=0; i<clients.size(); i++) { |
| index_specific(i, id,100, i1, 107 ,t1,"oh no, a duplicate!"); |
| } |
| commit(); |
| query("q","duplicate", "hl","true", "hl.fl", t1); |
| query("q","fox duplicate horses", "hl","true", "hl.fl", t1); |
| query("q","*:*", "rows",100); |
| } |
| |
| //SOLR 3161 ensure shards.qt=/update fails (anything but search handler really) |
| // Also see TestRemoteStreaming#testQtUpdateFails() |
| |
| //SolrException e = expectThrows(SolrException.class, () -> { |
| // ignoreException("isShard is only acceptable"); |
| // query("q","*:*","shards.qt","/update","stream.body","<delete><query>*:*</query></delete>"); |
| //}); |
| unIgnoreException("isShard is only acceptable"); |
| |
| // test debugging |
| // handle.put("explain", UNORDERED); |
| handle.put("explain", SKIPVAL); // internal docids differ, idf differs w/o global idf |
| handle.put("debug", UNORDERED); |
| handle.put("time", SKIPVAL); |
| handle.put("track", SKIP); //track is not included in single node search |
| query("q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true"); |
| query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); |
| query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); |
| query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); |
| query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); |
| |
| // SOLR-6545, wild card field list |
| indexr(id, "19", "text", "d", "cat_a_sS", "1" ,t1, "2"); |
| commit(); |
| |
| rsp = query("q", "id:19", "fl", "id", "fl", "*a_sS"); |
| assertFieldValues(rsp.getResults(), "id", "19"); |
| |
| rsp = query("q", "id:19", "fl", "id," + t1 + ",cat*"); |
| assertFieldValues(rsp.getResults(), "id", "19"); |
| |
| // Check Info is added to for each shard |
| ModifiableSolrParams q = new ModifiableSolrParams(); |
| q.set("q", "*:*"); |
| q.set(ShardParams.SHARDS_INFO, true); |
| setDistributedParams(q); |
| rsp = queryServer(q); |
| NamedList<?> sinfo = (NamedList<?>) rsp.getResponse().get(ShardParams.SHARDS_INFO); |
| String shards = getShardsString(); |
| int cnt = StringUtils.countMatches(shards, ",")+1; |
| |
| assertNotNull("missing shard info", sinfo); |
| assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, cnt, sinfo.size()); |
| |
| // test shards.tolerant=true |
| |
| List<JettySolrRunner> upJettys = Collections.synchronizedList(new ArrayList<>(jettys)); |
| List<SolrClient> upClients = Collections.synchronizedList(new ArrayList<>(clients)); |
| List<JettySolrRunner> downJettys = Collections.synchronizedList(new ArrayList<>()); |
| List<String> upShards = Collections.synchronizedList(new ArrayList<>(Arrays.asList(shardsArr))); |
| |
| int cap = Math.max(upJettys.size() - 1, 1); |
| |
| int numDownServers = random().nextInt(cap); |
| for (int i = 0; i < numDownServers; i++) { |
| if (upJettys.size() == 1) { |
| continue; |
| } |
| // shut down some of the jettys |
| int indexToRemove = r.nextInt(upJettys.size() - 1); |
| JettySolrRunner downJetty = upJettys.remove(indexToRemove); |
| upClients.remove(indexToRemove); |
| upShards.remove(indexToRemove); |
| downJetty.stop(); |
| downJettys.add(downJetty); |
| } |
| |
| Thread.sleep(100); |
| |
| queryPartialResults(upShards, upClients, |
| "q", "*:*", |
| "facet", "true", |
| "facet.field", t1, |
| "facet.field", t1, |
| "facet.limit", 5, |
| ShardParams.SHARDS_INFO, "true", |
| ShardParams.SHARDS_TOLERANT, "true"); |
| |
| queryPartialResults(upShards, upClients, |
| "q", "*:*", |
| "facet", "true", |
| "facet.query", i1 + ":[1 TO 50]", |
| "facet.query", i1 + ":[1 TO 50]", |
| ShardParams.SHARDS_INFO, "true", |
| ShardParams.SHARDS_TOLERANT, "true"); |
| |
| // test group query |
| queryPartialResults(upShards, upClients, |
| "q", "*:*", |
| "rows", 100, |
| "fl", "id," + i1, |
| "group", "true", |
| "group.query", t1 + ":kings OR " + t1 + ":eggs", |
| "group.limit", 10, |
| "sort", i1 + " asc, id asc", |
| CommonParams.TIME_ALLOWED, 10000, |
| ShardParams.SHARDS_INFO, "true", |
| ShardParams.SHARDS_TOLERANT, "true"); |
| |
| queryPartialResults(upShards, upClients, |
| "q", "*:*", |
| "stats", "true", |
| "stats.field", i1, |
| ShardParams.SHARDS_INFO, "true", |
| ShardParams.SHARDS_TOLERANT, "true"); |
| |
| queryPartialResults(upShards, upClients, |
| "q", "toyata", |
| "spellcheck", "true", |
| "spellcheck.q", "toyata", |
| "qt", "/spellCheckCompRH_Direct", |
| "shards.qt", "/spellCheckCompRH_Direct", |
| ShardParams.SHARDS_INFO, "true", |
| ShardParams.SHARDS_TOLERANT, "true"); |
| |
| // restart the jettys |
| for (JettySolrRunner downJetty : downJettys) { |
| downJetty.start(); |
| } |
| |
| |
| // This index has the same number for every field |
| |
| // TODO: This test currently fails because debug info is obtained only |
| // on shards with matches. |
| // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); |
| |
| // Thread.sleep(10000000000L); |
| |
| del("*:*"); // delete all docs and test stats request |
| commit(); |
| try { |
| query("q", "*:*", "stats", "true", |
| "stats.field", "stats_dt", |
| "stats.field", i1, |
| "stats.field", tdate_a, |
| "stats.field", tdate_b, |
| "stats.calcdistinct", "true"); |
| } catch (HttpSolrClient.RemoteSolrException e) { |
| if (e.getMessage().startsWith("java.lang.NullPointerException")) { |
| fail("NullPointerException with stats request on empty index"); |
| } else { |
| throw e; |
| } |
| } |
| |
| String fieldName = "severity"; |
| indexr("id", "1", fieldName, "Not Available"); |
| indexr("id", "2", fieldName, "Low"); |
| indexr("id", "3", fieldName, "Medium"); |
| indexr("id", "4", fieldName, "High"); |
| indexr("id", "5", fieldName, "Critical"); |
| |
| commit(); |
| |
| rsp = query("q", "*:*", "stats", "true", "stats.field", fieldName); |
| assertEquals(new EnumFieldValue(0, "Not Available"), |
| rsp.getFieldStatsInfo().get(fieldName).getMin()); |
| query("q", "*:*", "stats", "true", "stats.field", fieldName, |
| StatsParams.STATS_CALC_DISTINCT, "true"); |
| assertEquals(new EnumFieldValue(11, "Critical"), |
| rsp.getFieldStatsInfo().get(fieldName).getMax()); |
| |
| handle.put("severity", UNORDERED); // this is stupid, but stats.facet doesn't guarantee order |
| query("q", "*:*", "stats", "true", "stats.field", fieldName, |
| "stats.facet", fieldName); |
| } |
| |
| private void testMinExactCount() throws Exception { |
| assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT, "200", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc"); |
| assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT, "-1", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc"); |
| assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT, "1", CommonParams.ROWS, "200", CommonParams.SORT, "score desc, id asc"); |
| assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", "facet", "true", "facet.field", s1, CommonParams.MIN_EXACT_COUNT,"1", CommonParams.ROWS, "200", CommonParams.SORT, "score desc, id asc"); |
| assertIsExactHitCount("q","{!cache=false}id:1", CommonParams.MIN_EXACT_COUNT,"1", CommonParams.ROWS, "1"); |
| assertApproximatedHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT,"2", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc"); |
| } |
| |
| private void assertIsExactHitCount(Object... requestParams) throws Exception { |
| QueryResponse response = query(requestParams); |
| assertNotNull("Expecting exact hit count in response: " + response.getResults().toString(), |
| response.getResults().getNumFoundExact()); |
| assertTrue("Expecting exact hit count in response: " + response.getResults().toString(), |
| response.getResults().getNumFoundExact()); |
| } |
| |
| private void assertApproximatedHitCount(Object...requestParams) throws Exception { |
| handle.put("numFound", SKIPVAL); |
| QueryResponse response = query(requestParams); |
| assertNotNull("Expecting numFoundExact in response: " + response.getResults().toString(), |
| response.getResults().getNumFoundExact()); |
| assertFalse("Expecting aproximated results in response: " + response.getResults().toString(), |
| response.getResults().getNumFoundExact()); |
| handle.remove("numFound", SKIPVAL); |
| } |
| |
| /** comparing results with facet.method=uif */ |
| private void queryAndCompareUIF(Object ... params) throws Exception { |
| final QueryResponse expect = query(params); |
| |
| final Object[] newParams = Arrays.copyOf(params, params.length+2); |
| newParams[newParams.length-2] = "facet.method"; |
| newParams[newParams.length-1] = "uif"; |
| final QueryResponse uifResult = query(newParams); |
| compareResponses(expect, uifResult); |
| } |
| |
| protected void checkMinCountsField(List<FacetField.Count> counts, Object[] pairs) { |
| assertEquals("There should be exactly " + pairs.length / 2 + " returned counts. There were: " + counts.size(), counts.size(), pairs.length / 2); |
| assertTrue("Variable len param must be an even number, it was: " + pairs.length, (pairs.length % 2) == 0); |
| for (int pairs_idx = 0, counts_idx = 0; pairs_idx < pairs.length; pairs_idx += 2, counts_idx++) { |
| String act_name = counts.get(counts_idx).getName(); |
| long act_count = counts.get(counts_idx).getCount(); |
| String exp_name = (String) pairs[pairs_idx]; |
| long exp_count = (long) pairs[pairs_idx + 1]; |
| assertEquals("Expected ordered entry " + exp_name + " at position " + counts_idx + " got " + act_name, act_name, exp_name); |
| assertEquals("Expected count for entry: " + exp_name + " at position " + counts_idx + " got " + act_count, act_count, exp_count); |
| } |
| } |
| |
| protected void checkMinCountsRange(List<RangeFacet.Count> counts, Object[] pairs) { |
| assertEquals("There should be exactly " + pairs.length / 2 + " returned counts. There were: " + counts.size(), counts.size(), pairs.length / 2); |
| assertTrue("Variable len param must be an even number, it was: " + pairs.length, (pairs.length % 2) == 0); |
| for (int pairs_idx = 0, counts_idx = 0; pairs_idx < pairs.length; pairs_idx += 2, counts_idx++) { |
| String act_name = counts.get(counts_idx).getValue(); |
| long act_count = counts.get(counts_idx).getCount(); |
| String exp_name = (String) pairs[pairs_idx]; |
| long exp_count = (long) pairs[pairs_idx + 1]; |
| assertEquals("Expected ordered entry " + exp_name + " at position " + counts_idx + " got " + act_name, act_name, exp_name); |
| assertEquals("Expected count for entry: " + exp_name + " at position " + counts_idx + " got " + act_count, act_count, exp_count); |
| } |
| } |
| |
| @SuppressWarnings({"unchecked", "rawtypes"}) |
| protected void queryPartialResults(final List<String> upShards, |
| final List<SolrClient> upClients, |
| Object... q) throws Exception { |
| |
| final ModifiableSolrParams params = new ModifiableSolrParams(); |
| |
| for (int i = 0; i < q.length; i += 2) { |
| params.add(q[i].toString(), q[i + 1].toString()); |
| } |
| // TODO: look into why passing true causes fails |
| params.set("distrib", "false"); |
| final QueryResponse controlRsp = controlClient.query(params); |
| // if time.allowed is specified then even a control response can return a partialResults header |
| if (params.get(CommonParams.TIME_ALLOWED) == null) { |
| validateControlData(controlRsp); |
| } |
| |
| params.remove("distrib"); |
| setDistributedParams(params); |
| |
| if (upClients.size() == 0) { |
| return; |
| } |
| QueryResponse rsp = queryRandomUpServer(params, upClients); |
| |
| comparePartialResponses(rsp, controlRsp, upShards); |
| |
| if (stress > 0) { |
| log.info("starting stress..."); |
| Set<Future<Object>> pending = new HashSet<>();; |
| ExecutorCompletionService<Object> cs = new ExecutorCompletionService<>(executor); |
| Callable[] threads = new Callable[nThreads]; |
| for (int i = 0; i < threads.length; i++) { |
| threads[i] = new Callable() { |
| @Override |
| public Object call() { |
| for (int j = 0; j < stress; j++) { |
| int which = r.nextInt(upClients.size()); |
| SolrClient client = upClients.get(which); |
| try { |
| QueryResponse rsp = client.query(new ModifiableSolrParams(params)); |
| if (verifyStress) { |
| comparePartialResponses(rsp, controlRsp, upShards); |
| } |
| } catch (SolrServerException | IOException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| return null; |
| } |
| }; |
| pending.add(cs.submit(threads[i])); |
| } |
| |
| while (pending.size() > 0) { |
| Future<Object> future = cs.take(); |
| pending.remove(future); |
| future.get(); |
| } |
| |
| } |
| } |
| |
| protected QueryResponse queryRandomUpServer(ModifiableSolrParams params, List<SolrClient> upClients) |
| throws SolrServerException, IOException { |
| // query a random "up" server |
| SolrClient client; |
| if (upClients.size() == 1) { |
| client = upClients.get(0); |
| } else { |
| int which = r.nextInt(upClients.size() - 1); |
| client = upClients.get(which); |
| } |
| |
| QueryResponse rsp = client.query(params); |
| return rsp; |
| } |
| |
| protected void comparePartialResponses(QueryResponse rsp, QueryResponse controlRsp, List<String> upShards) |
| { |
| NamedList<?> sinfo = (NamedList<?>) rsp.getResponse().get(ShardParams.SHARDS_INFO); |
| |
| assertNotNull("missing shard info", sinfo); |
| assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, shardsArr.length, sinfo.size()); |
| // identify each one |
| for (Map.Entry<String,?> entry : sinfo) { |
| String shard = entry.getKey(); |
| NamedList<?> info = (NamedList<?>) entry.getValue(); |
| boolean found = false; |
| for(int i=0; i<shardsArr.length; i++) { |
| String s = shardsArr[i]; |
| if (shard.contains(s)) { |
| found = true; |
| // make sure that it responded if it's up and the landing node didn't error before sending the request to the shard |
| if (upShards.contains(s)) { |
| // this is no longer true if there was a query timeout on an up shard |
| // assertTrue("Expected to find numFound in the up shard info",info.get("numFound") != null); |
| boolean timeAllowedError = info.get("error") != null && info.get("error").toString().contains("Time allowed to handle this request"); |
| if (timeAllowedError) { |
| assertEquals("Expected to find the " + SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY + " header set if a shard is down", |
| Boolean.TRUE, rsp.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY)); |
| assertTrue("Expected to find error in the down shard info: " + info.toString(), info.get("error") != null); |
| } else { |
| assertTrue("Expected timeAllowedError or to find shardAddress in the up shard info: " + info.toString(), info.get("shardAddress") != null); |
| } |
| } else { |
| assertEquals("Expected to find the " + SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY + " header set if a shard is down. Response: " + rsp, |
| Boolean.TRUE, rsp.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY)); |
| assertTrue("Expected to find error in the down shard info: " + info.toString(), info.get("error") != null); |
| } |
| } |
| } |
| assertTrue("Couldn't find shard " + shard + " represented in shards info", found); |
| } |
| } |
| |
| @Override |
| public void validateControlData(QueryResponse control) throws Exception { |
| super.validateControlData(control); |
| assertNull("Expected the "+SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" header to be null", |
| control.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY)); |
| } |
| |
| private void validateCommonQueryParameters() throws Exception { |
| ignoreException("parameter cannot be negative"); |
| |
| SolrException e1 = expectThrows(SolrException.class, () -> { |
| SolrQuery query = new SolrQuery(); |
| query.setParam("start", "non_numeric_value").setQuery("*"); |
| QueryResponse resp = query(query); |
| }); |
| assertEquals(ErrorCode.BAD_REQUEST.code, e1.code()); |
| |
| SolrException e2 = expectThrows(SolrException.class, () -> { |
| SolrQuery query = new SolrQuery(); |
| query.setStart(-1).setQuery("*"); |
| QueryResponse resp = query(query); |
| }); |
| assertEquals(ErrorCode.BAD_REQUEST.code, e2.code()); |
| |
| SolrException e3 = expectThrows(SolrException.class, () -> { |
| SolrQuery query = new SolrQuery(); |
| query.setRows(-1).setStart(0).setQuery("*"); |
| QueryResponse resp = query(query); |
| }); |
| assertEquals(ErrorCode.BAD_REQUEST.code, e3.code()); |
| |
| SolrException e4 = expectThrows(SolrException.class, () -> { |
| SolrQuery query = new SolrQuery(); |
| query.setParam("rows", "non_numeric_value").setQuery("*"); |
| QueryResponse resp = query(query); |
| }); |
| assertEquals(ErrorCode.BAD_REQUEST.code, e4.code()); |
| |
| resetExceptionIgnores(); |
| } |
| } |
