| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.cloud; |
| |
| import org.apache.solr.SolrTestCaseJ4; |
| import org.apache.solr.client.solrj.SolrClient; |
| import org.apache.solr.client.solrj.request.UpdateRequest; |
| import org.apache.solr.client.solrj.response.QueryResponse; |
| import org.apache.solr.common.SolrDocument; |
| import org.apache.solr.common.SolrException; |
| import org.apache.solr.common.util.StrUtils; |
| import org.junit.BeforeClass; |
| import org.junit.Ignore; |
| import org.junit.Test; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import java.lang.invoke.MethodHandles; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| |
| @Ignore // MRM TODO: - finish getRandomJettyLeader |
| public class TestDistribDocBasedVersion extends SolrCloudBridgeTestCase { |
| |
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); |
| |
| String bucket1 = "s1"; // shard1: top bits:10 80000000:ffffffff |
| String bucket2 = "s2"; // shard2: top bits:00 00000000:7fffffff |
| |
| private static String vfield = "my_version_l"; |
| |
| |
| @BeforeClass |
| public static void beforeShardHashingTest() throws Exception { |
| useFactory(null); |
| } |
| |
| public TestDistribDocBasedVersion() { |
| schemaString = "schema15.xml"; // we need a string id |
| solrconfigString = "solrconfig-externalversionconstraint.xml"; |
| super.sliceCount = 2; |
| numJettys = 4; |
| |
| |
| /*** |
| hash of a is 3c2569b2 high bits=0 shard=shard3 |
| hash of b is 95de7e03 high bits=2 shard=shard1 |
| hash of c is e132d65f high bits=3 shard=shard2 |
| hash of d is 27191473 high bits=0 shard=shard3 |
| hash of e is 656c4367 high bits=1 shard=shard4 |
| hash of f is 2b64883b high bits=0 shard=shard3 |
| hash of g is f18ae416 high bits=3 shard=shard2 |
| hash of h is d482b2d3 high bits=3 shard=shard2 |
| hash of i is 811a702b high bits=2 shard=shard1 |
| hash of j is ca745a39 high bits=3 shard=shard2 |
| hash of k is cfbda5d1 high bits=3 shard=shard2 |
| hash of l is 1d5d6a2c high bits=0 shard=shard3 |
| hash of m is 5ae4385c high bits=1 shard=shard4 |
| hash of n is c651d8ac high bits=3 shard=shard2 |
| hash of o is 68348473 high bits=1 shard=shard4 |
| hash of p is 986fdf9a high bits=2 shard=shard1 |
| hash of q is ff8209e8 high bits=3 shard=shard2 |
| hash of r is 5c9373f1 high bits=1 shard=shard4 |
| hash of s is ff4acaf1 high bits=3 shard=shard2 |
| hash of t is ca87df4d high bits=3 shard=shard2 |
| hash of u is 62203ae0 high bits=1 shard=shard4 |
| hash of v is bdafcc55 high bits=2 shard=shard1 |
| hash of w is ff439d1f high bits=3 shard=shard2 |
| hash of x is 3e9a9b1b high bits=0 shard=shard3 |
| hash of y is 477d9216 high bits=1 shard=shard4 |
| hash of z is c1f69a17 high bits=3 shard=shard2 |
| ***/ |
| } |
| |
| @Test |
| // @ShardsFixed(num = 4) |
| public void test() throws Exception { |
| boolean testFinished = false; |
| |
| handle.clear(); |
| handle.put("timestamp", SKIPVAL); |
| |
| // MRM TODO: flakey? |
| // doTestDocVersions(); |
| doTestHardFail(); |
| commit(); // work arround SOLR-5628 |
| |
| testFinished = true; |
| |
| } |
| |
| private void doTestHardFail() throws Exception { |
| log.info("### STARTING doTestHardFail"); |
| |
| // use a leader so we test both forwarding and non-forwarding logic |
| cluster.getShardLeaderJetty(COLLECTION, bucket1); |
| |
| // solrClient = cloudClient; CloudSolrServer doesn't currently support propagating error codes |
| |
| doTestHardFail("p!doc1"); |
| doTestHardFail("q!doc1"); |
| doTestHardFail("r!doc1"); |
| doTestHardFail("x!doc1"); |
| } |
| |
| private void doTestHardFail(String id) throws Exception { |
| vdelete(id, 5, "update.chain","external-version-failhard"); |
| vadd(id, 10, "update.chain","external-version-failhard"); |
| vadd(id ,15, "update.chain","external-version-failhard"); |
| vaddFail(id ,11, 409, "update.chain","external-version-failhard"); |
| vdeleteFail(id ,11, 409, "update.chain","external-version-failhard"); |
| vdelete(id, 20, "update.chain","external-version-failhard"); |
| } |
| |
| private void doTestDocVersions() throws Exception { |
| log.info("### STARTING doTestDocVersions"); |
| assertEquals(2, cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION).getSlices().size()); |
| |
| solrClient = cloudClient; |
| |
| vadd("b!doc1", 10); |
| vadd("c!doc2", 11); |
| vadd("d!doc3", 10); |
| vadd("e!doc4", 11); |
| |
| doRTG("b!doc1,c!doc2,d!doc3,e!doc4", "10,11,10,11"); |
| |
| vadd("b!doc1", 5); |
| vadd("c!doc2", 10); |
| vadd("d!doc3", 9); |
| vadd("e!doc4", 8); |
| |
| doRTG("b!doc1,c!doc2,d!doc3,e!doc4", "10,11,10,11"); |
| |
| vadd("b!doc1", 24); |
| vadd("c!doc2", 23); |
| vadd("d!doc3", 22); |
| vadd("e!doc4", 21); |
| |
| doRTG("b!doc1,c!doc2,d!doc3,e!doc4", "24,23,22,21"); |
| |
| vdelete("b!doc1", 20); |
| |
| doRTG("b!doc1,c!doc2,d!doc3,e!doc4", "24,23,22,21"); |
| |
| vdelete("b!doc1", 30); |
| |
| doRTG("b!doc1,c!doc2,d!doc3,e!doc4", "30,23,22,21"); |
| |
| // try delete before add |
| vdelete("b!doc123", 100); |
| vadd("b!doc123", 99); |
| doRTG("b!doc123", "100"); |
| // now add greater |
| vadd("b!doc123", 101); |
| doRTG("b!doc123", "101"); |
| |
| |
| // |
| // now test with a non-smart client |
| // |
| // use a leader so we test both forwarding and non-forwarding logic |
| cluster.getShardLeaderJetty(COLLECTION, bucket1); |
| |
| vadd("b!doc5", 10); |
| vadd("c!doc6", 11); |
| vadd("d!doc7", 10); |
| vadd("e!doc8", 11); |
| |
| doRTG("b!doc5,c!doc6,d!doc7,e!doc8", "10,11,10,11"); |
| |
| vadd("b!doc5", 5); |
| vadd("c!doc6", 10); |
| vadd("d!doc7", 9); |
| vadd("e!doc8", 8); |
| |
| doRTG("b!doc5,c!doc6,d!doc7,e!doc8", "10,11,10,11"); |
| |
| vadd("b!doc5", 24); |
| vadd("c!doc6", 23); |
| vadd("d!doc7", 22); |
| vadd("e!doc8", 21); |
| |
| doRTG("b!doc5,c!doc6,d!doc7,e!doc8", "24,23,22,21"); |
| |
| vdelete("b!doc5", 20); |
| |
| doRTG("b!doc5,c!doc6,d!doc7,e!doc8", "24,23,22,21"); |
| |
| vdelete("b!doc5", 30); |
| |
| doRTG("b!doc5,c!doc6,d!doc7,e!doc8", "30,23,22,21"); |
| |
| // try delete before add |
| vdelete("b!doc1234", 100); |
| vadd("b!doc1234", 99); |
| doRTG("b!doc1234", "100"); |
| // now add greater |
| vadd("b!doc1234", 101); |
| doRTG("b!doc1234", "101"); |
| |
| commit(); |
| |
| // check liveness for all docs |
| doQuery("b!doc123,101,c!doc2,23,d!doc3,22,e!doc4,21,b!doc1234,101,c!doc6,23,d!doc7,22,e!doc8,21", "q","live_b:true"); |
| doQuery("b!doc1,30,b!doc5,30", "q","live_b:false"); |
| |
| // delete by query should just work like normal |
| doDBQ("id:b!doc1 OR id:e*"); |
| commit(); |
| |
| doQuery("b!doc123,101,c!doc2,23,d!doc3,22,b!doc1234,101,c!doc6,23,d!doc7,22", "q","live_b:true"); |
| doQuery("b!doc5,30", "q","live_b:false"); |
| |
| } |
| |
| SolrClient solrClient; |
| |
| void vdelete(String id, long version, String... params) throws Exception { |
| UpdateRequest req = new UpdateRequest(); |
| req.deleteById(id); |
| req.setParam("del_version", Long.toString(version)); |
| for (int i=0; i<params.length; i+=2) { |
| req.setParam( params[i], params[i+1]); |
| } |
| solrClient.request(req); |
| // req.process(cloudClient); |
| } |
| |
| void vadd(String id, long version, String... params) throws Exception { |
| UpdateRequest req = new UpdateRequest(); |
| req.add(SolrTestCaseJ4.sdoc("id", id, vfield, version)); |
| for (int i=0; i<params.length; i+=2) { |
| req.setParam( params[i], params[i+1]); |
| } |
| solrClient.request(req); |
| } |
| |
| void vaddFail(String id, long version, int errCode, String... params) throws Exception { |
| boolean failed = false; |
| try { |
| vadd(id, version, params); |
| } catch (SolrException e) { |
| failed = true; |
| assertEquals(errCode, e.code()); |
| } catch (Exception e) { |
| log.error("ERROR", e); |
| } |
| assertTrue(failed); |
| } |
| |
| void vdeleteFail(String id, long version, int errCode, String... params) throws Exception { |
| boolean failed = false; |
| try { |
| vdelete(id, version, params); |
| } catch (SolrException e) { |
| failed = true; |
| assertEquals(errCode, e.code()); |
| } catch (Exception e) { |
| log.error("ERROR", e); |
| } |
| assertTrue(failed); |
| } |
| |
| |
| void doQuery(String expectedDocs, String... queryParams) throws Exception { |
| |
| List<String> strs = StrUtils.splitSmart(expectedDocs, ",", true); |
| Map<String, Object> expectedIds = new HashMap<>(); |
| for (int i=0; i<strs.size(); i+=2) { |
| String id = strs.get(i); |
| String vS = strs.get(i+1); |
| Long v = Long.valueOf(vS); |
| expectedIds.put(id,v); |
| } |
| |
| QueryResponse rsp = cloudClient.query(params(queryParams)); |
| Map<String, Object> obtainedIds = new HashMap<>(); |
| for (SolrDocument doc : rsp.getResults()) { |
| obtainedIds.put((String) doc.get("id"), doc.get(vfield)); |
| } |
| |
| assertEquals(expectedIds, obtainedIds); |
| } |
| |
| |
| void doRTG(String ids, String versions) throws Exception { |
| Map<String, Object> expectedIds = new HashMap<>(); |
| List<String> strs = StrUtils.splitSmart(ids, ",", true); |
| List<String> verS = StrUtils.splitSmart(versions, ",", true); |
| for (int i=0; i<strs.size(); i++) { |
| expectedIds.put(strs.get(i), Long.valueOf(verS.get(i))); |
| } |
| |
| solrClient.query(params("qt", "/get", "ids", ids)); |
| |
| QueryResponse rsp = cloudClient.query(params("qt","/get", "ids",ids)); |
| Map<String, Object> obtainedIds = new HashMap<>(); |
| for (SolrDocument doc : rsp.getResults()) { |
| obtainedIds.put((String) doc.get("id"), doc.get(vfield)); |
| } |
| |
| assertEquals(expectedIds, obtainedIds); |
| } |
| |
| void doRTG(String ids) throws Exception { |
| solrClient.query(params("qt", "/get", "ids", ids)); |
| |
| Set<String> expectedIds = new HashSet<>( StrUtils.splitSmart(ids, ",", true) ); |
| |
| QueryResponse rsp = cloudClient.query(params("qt","/get", "ids",ids)); |
| Set<String> obtainedIds = new HashSet<>(); |
| for (SolrDocument doc : rsp.getResults()) { |
| obtainedIds.add((String) doc.get("id")); |
| } |
| |
| assertEquals(expectedIds, obtainedIds); |
| } |
| |
| |
| // TODO: refactor some of this stuff into the SolrJ client... it should be easier to use |
| void doDBQ(String q, String... reqParams) throws Exception { |
| UpdateRequest req = new UpdateRequest(); |
| req.deleteByQuery(q); |
| req.setParams(params(reqParams)); |
| req.process(cloudClient); |
| } |
| } |