| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.update.processor; |
| |
| import java.util.ArrayList; |
| import java.util.List; |
| import java.util.concurrent.Callable; |
| import java.util.concurrent.ExecutorService; |
| import java.util.concurrent.Executors; |
| |
| import org.apache.lucene.util.TestUtil; |
| import org.apache.solr.SolrTestCaseJ4; |
| import org.apache.solr.common.SolrException; |
| import org.apache.solr.common.SolrInputDocument; |
| import org.apache.solr.common.util.ExecutorUtil; |
| import org.apache.solr.common.util.NamedList; |
| import org.apache.solr.schema.IndexSchema; |
| import org.apache.solr.schema.SchemaField; |
| import org.apache.solr.common.util.SolrNamedThreadFactory; |
| import org.junit.Before; |
| import org.junit.BeforeClass; |
| |
| import static org.hamcrest.CoreMatchers.equalTo; |
| import static org.hamcrest.CoreMatchers.hasItem; |
| import static org.hamcrest.CoreMatchers.is; |
| import static org.hamcrest.CoreMatchers.not; |
| import static org.hamcrest.CoreMatchers.nullValue; |
| |
| public class TestDocBasedVersionConstraints extends SolrTestCaseJ4 { |
| |
| @BeforeClass |
| public static void beforeClass() throws Exception { |
| initCore("solrconfig-externalversionconstraint.xml", "schema15.xml"); |
| } |
| |
| @Before |
| public void before() throws Exception { |
| assertU(delQ("*:*")); |
| assertU(commit()); |
| } |
| |
| |
| public void testSimpleUpdates() throws Exception { |
| |
| // skip low version against committed data |
| assertU(adoc("id", "aaa", "name", "a1", "my_version_l", "1001")); |
| assertU(commit()); |
| assertU(adoc("id", "aaa", "name", "a2", "my_version_l", "1002")); |
| assertU(commit()); |
| assertU(adoc("id", "aaa", "name", "XX", "my_version_l", "1")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("q","+id:aaa +name:a2"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| |
| // skip low version against uncommitted data from updateLog |
| assertU(adoc("id", "aaa", "name", "a3", "my_version_l", "1003")); |
| assertU(adoc("id", "aaa", "name", "XX", "my_version_l", "7")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a3'}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("q","+id:aaa +name:a3"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a3'}}"); |
| |
| // interleave updates to multiple docs using same versions |
| for (long ver = 1010; ver < 1020; ver++) { |
| for (String id : new String[] {"aaa", "bbb", "ccc", "ddd"}) { |
| assertU(adoc("id", id, "my_version_l", ""+ver)); |
| } |
| } |
| for (String id : new String[] {"aaa", "bbb", "ccc", "ddd"}) { |
| assertU(adoc("id", id, "name", "XX", "my_version_l", "10")); |
| assertJQ(req("qt","/get", "id",id, "fl","my_version_l") |
| , "=={'doc':{'my_version_l':"+1019+"}}"); |
| } |
| assertU(commit()); |
| assertJQ(req("q","name:XX"), "/response/numFound==0"); |
| for (String id : new String[] {"aaa", "bbb", "ccc", "ddd"}) { |
| assertJQ(req("q","+id:"+id), "/response/numFound==1"); |
| assertJQ(req("q","+name:XX +id:"+id), "/response/numFound==0"); |
| assertJQ(req("q","+id:"+id + " +my_version_l:1019"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id",id, "fl","my_version_l") |
| , "=={'doc':{'my_version_l':"+1019+"}}"); |
| } |
| } |
| |
| public void testSimpleDeletes() throws Exception { |
| |
| // skip low version delete against committed doc |
| assertU(adoc("id", "aaa", "name", "a1", "my_version_l", "1001")); |
| assertU(commit()); |
| assertU(adoc("id", "aaa", "name", "a2", "my_version_l", "1002")); |
| assertU(commit()); |
| deleteAndGetVersion("aaa", |
| params("del_version", "7")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:a2"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| |
| // skip low version delete against uncommitted doc from updateLog |
| assertU(adoc("id", "aaa", "name", "a3", "my_version_l", "1003")); |
| deleteAndGetVersion("aaa", |
| params("del_version", "8")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a3'}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:a3"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a3'}}"); |
| |
| // skip low version add against uncommitted "delete" from updateLog |
| deleteAndGetVersion("aaa", params("del_version", "1010")); |
| assertU(adoc("id", "aaa", "name", "XX", "my_version_l", "22")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}"); |
| |
| // skip low version add against committed "delete" |
| // (delete was already done & committed above) |
| assertU(adoc("id", "aaa", "name", "XX", "my_version_l", "23")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}"); |
| } |
| |
| /** |
| * Sanity check that there are no hardcoded assumptions about the |
| * field type used that could byte us in the ass. |
| */ |
| public void testFloatVersionField() throws Exception { |
| |
| // skip low version add & low version delete against committed doc |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "a1", "my_version_f", "10.01")), |
| params("update.chain","external-version-float")); |
| assertU(commit()); |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "XX", "my_version_f", "4.2")), |
| params("update.chain","external-version-float")); |
| assertU(commit()); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a1'}}"); |
| deleteAndGetVersion("aaa", params("del_version", "7", |
| "update.chain","external-version-float")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a1'}}"); |
| assertU(commit()); |
| |
| // skip low version delete against uncommitted doc from updateLog |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "a2", "my_version_f", "10.02")), |
| params("update.chain","external-version-float")); |
| deleteAndGetVersion("aaa", params("del_version", "8", |
| "update.chain","external-version-float")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:a2"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| |
| // skip low version add against uncommitted "delete" from updateLog |
| deleteAndGetVersion("aaa", params("del_version", "10.10", |
| "update.chain","external-version-float")); |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "XX", "my_version_f", "10.05")), |
| params("update.chain","external-version-float")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_f") |
| , "=={'doc':{'my_version_f':10.10}}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_f") |
| , "=={'doc':{'my_version_f':10.10}}"); |
| |
| // skip low version add against committed "delete" |
| // (delete was already done & committed above) |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "XX", "my_version_f", "10.09")), |
| params("update.chain","external-version-float")); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_f") |
| , "=={'doc':{'my_version_f':10.10}}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_f") |
| , "=={'doc':{'my_version_f':10.10}}"); |
| } |
| |
| public void testFailOnOldVersion() throws Exception { |
| |
| // fail low version add & low version delete against committed doc |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "a1", "my_version_l", "1001")), |
| params("update.chain","external-version-failhard")); |
| assertU(commit()); |
| |
| SolrException ex = expectThrows(SolrException.class, () -> { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "XX", "my_version_l", "42")), |
| params("update.chain","external-version-failhard")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| assertU(commit()); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a1'}}"); |
| |
| ex = expectThrows(SolrException.class, () -> { |
| deleteAndGetVersion("aaa", params("del_version", "7", |
| "update.chain","external-version-failhard")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a1'}}"); |
| assertU(commit()); |
| |
| // fail low version delete against uncommitted doc from updateLog |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "a2", "my_version_l", "1002")), |
| params("update.chain","external-version-failhard")); |
| ex = expectThrows(SolrException.class, () -> { |
| deleteAndGetVersion("aaa", params("del_version", "8", |
| "update.chain","external-version-failhard")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:a2"), "/response/numFound==1"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","name") |
| , "=={'doc':{'name':'a2'}}"); |
| |
| // fail low version add against uncommitted "delete" from updateLog |
| deleteAndGetVersion("aaa", params("del_version", "1010", |
| "update.chain","external-version-failhard")); |
| ex = expectThrows(SolrException.class, () -> { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "XX", "my_version_l", "1005")), |
| params("update.chain","external-version-failhard")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}"); |
| |
| // fail low version add against committed "delete" |
| // (delete was already done & committed above) |
| ex = expectThrows(SolrException.class, () -> { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "XX", "my_version_l", "1009")), |
| params("update.chain","external-version-failhard")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}}"); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa"), "/response/numFound==1"); |
| assertJQ(req("q","+id:aaa +name:XX"), "/response/numFound==0"); |
| assertJQ(req("qt","/get", "id","aaa", "fl","my_version_l") |
| , "=={'doc':{'my_version_l':1010}}"); |
| } |
| |
| // Test multiple versions, that it has to be greater than my_version_l and my_version_f |
| public void testMultipleVersions() throws Exception { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "a1", "my_version_l", "1001", "my_version_f", "1.0")), |
| params("update.chain","external-version-failhard-multiple")); |
| assertU(commit()); |
| // All variations of additional versions should fail other than my_version_l greater or my_version_f greater. |
| SolrException ex = expectThrows(SolrException.class, () -> { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "X1", "my_version_l", "1000", "my_version_f", "1.0")), |
| params("update.chain","external-version-failhard-multiple")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| ex = expectThrows(SolrException.class, () -> { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "X2", "my_version_l", "1001", "my_version_f", "0.9")), |
| params("update.chain","external-version-failhard-multiple")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| // Also fails on the exact same version |
| ex = expectThrows(SolrException.class, () -> { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "X3", "my_version_l", "1001", "my_version_f", "1.0")), |
| params("update.chain","external-version-failhard-multiple")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| //Verify we are still unchanged |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa +name:a1"), "/response/numFound==1"); |
| |
| // update version 1 |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "Y1", "my_version_l", "2001", "my_version_f", "1.0")), |
| params("update.chain","external-version-failhard-multiple")); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa +name:Y1"), "/response/numFound==1"); |
| |
| // update version 2 |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "Y2", "my_version_l", "2001", "my_version_f", "2.0")), |
| params("update.chain","external-version-failhard-multiple")); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa +name:Y2"), "/response/numFound==1"); |
| } |
| |
| public void testMultipleVersionDeletes() throws Exception { |
| updateJ(jsonAdd(sdoc("id", "aaa", "name", "a1", "my_version_l", "1001", "my_version_f", "1.0")), |
| params("update.chain","external-version-failhard-multiple")); |
| assertU(commit()); |
| |
| SolrException ex = expectThrows(SolrException.class, () -> { |
| deleteAndGetVersion("aaa", params("del_version", "1000", "del_version_2", "1.0", |
| "update.chain","external-version-failhard-multiple")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| ex = expectThrows(SolrException.class, () -> { |
| deleteAndGetVersion("aaa", params("del_version", "1001", "del_version_2", "0.9", |
| "update.chain","external-version-failhard-multiple")); |
| }); |
| assertEquals(409, ex.code()); |
| |
| // And just verify if we pass version 1, we still error if version 2 isn't found. |
| ignoreException("Delete by ID must specify doc version param"); |
| ex = expectThrows(SolrException.class, () -> { |
| deleteAndGetVersion("aaa", params("del_version", "1001", |
| "update.chain","external-version-failhard-multiple")); |
| }); |
| assertEquals(400, ex.code()); |
| unIgnoreException("Delete by ID must specify doc version param"); |
| |
| //Verify we are still unchanged |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa +name:a1"), "/response/numFound==1"); |
| |
| //And let's verify the actual case. |
| deleteAndGetVersion("aaa", params("del_version", "1001", "del_version_2", "2.0", |
| "update.chain","external-version-failhard-multiple")); |
| assertU(commit()); |
| assertJQ(req("q","+id:aaa +name:a1"), "/response/numFound==0"); //Delete allowed |
| } |
| |
| |
| /** |
| * Proof of concept test demonstrating how to manage and periodically cleanup |
| * the "logically" deleted documents |
| */ |
| public void testManagingDeletes() throws Exception { |
| // add some docs |
| for (long ver = 1010; ver < 1020; ver++) { |
| for (String id : new String[] {"aaa", "bbb", "ccc", "ddd"}) { |
| assertU(adoc("id", id, "name", "name_"+id, "my_version_l", ""+ver)); |
| } |
| } |
| assertU(adoc("id", "aaa", "name", "name_aaa", "my_version_l", "1030")); |
| assertU(commit()); |
| // sample queries |
| assertJQ(req("q","*:*", |
| "fq","live_b:true") |
| ,"/response/numFound==4"); |
| assertJQ(req("q","id:aaa", |
| "fq","live_b:true", |
| "fl","id,my_version_l") |
| ,"/response/numFound==1" |
| ,"/response/docs==[{'id':'aaa','my_version_l':1030}]}"); |
| // logically delete |
| deleteAndGetVersion("aaa", |
| params("del_version", "1031")); |
| assertU(commit()); |
| // sample queries |
| assertJQ(req("q","*:*", |
| "fq","live_b:true") |
| ,"/response/numFound==3"); |
| assertJQ(req("q","id:aaa", |
| "fq","live_b:true") |
| ,"/response/numFound==0"); |
| // placeholder doc is still in the index though |
| assertJQ(req("q","id:aaa", |
| "fq","live_b:false", |
| "fq", "timestamp_tdt:[* TO *]", |
| "fl","id,live_b,my_version_l") |
| ,"/response/numFound==1" |
| ,"/response/docs==[{'id':'aaa','my_version_l':1031,'live_b':false}]}"); |
| // doc can't be re-added with a low version |
| assertU(adoc("id", "aaa", "name", "XX", "my_version_l", "1025")); |
| assertU(commit()); |
| assertJQ(req("q","id:aaa", |
| "fq","live_b:true") |
| ,"/response/numFound==0"); |
| |
| // "dead" placeholder docs can be periodically cleaned up |
| // ie: assertU(delQ("+live_b:false +timestamp_tdt:[* TO NOW/MINUTE-5MINUTE]")); |
| // but to prevent the test from ebing time sensitive we'll just purge them all |
| assertU(delQ("+live_b:false")); |
| assertU(commit()); |
| // now doc can be re-added w/any version, no matter how low |
| assertU(adoc("id", "aaa", "name", "aaa", "my_version_l", "7")); |
| assertU(commit()); |
| assertJQ(req("q","id:aaa", |
| "fq","live_b:true", |
| "fl","id,live_b,my_version_l") |
| ,"/response/numFound==1" |
| ,"/response/docs==[{'id':'aaa','my_version_l':7,'live_b':true}]}"); |
| |
| } |
| |
| /** |
| * Constantly hammer the same doc with multiple concurrent threads and diff versions, |
| * confirm that the highest version wins. |
| */ |
| public void testConcurrentAdds() throws Exception { |
| final int NUM_DOCS = atLeast(50); |
| final int MAX_CONCURENT = atLeast(10); |
| ExecutorService runner = ExecutorUtil.newMDCAwareFixedThreadPool(MAX_CONCURENT, new SolrNamedThreadFactory("TestDocBasedVersionConstraints")); |
| // runner = Executors.newFixedThreadPool(1); // to test single threaded |
| try { |
| for (int id = 0; id < NUM_DOCS; id++) { |
| final int numAdds = TestUtil.nextInt(random(), 3, MAX_CONCURENT); |
| final int winner = TestUtil.nextInt(random(), 0, numAdds - 1); |
| final int winnerVersion = atLeast(100); |
| final boolean winnerIsDeleted = (0 == TestUtil.nextInt(random(), 0, 4)); |
| List<Callable<Object>> tasks = new ArrayList<>(numAdds); |
| for (int variant = 0; variant < numAdds; variant++) { |
| final boolean iShouldWin = (variant==winner); |
| final long version = (iShouldWin ? winnerVersion |
| : TestUtil.nextInt(random(), 1, winnerVersion - 1)); |
| if ((iShouldWin && winnerIsDeleted) |
| || (!iShouldWin && 0 == TestUtil.nextInt(random(), 0, 4))) { |
| tasks.add(delayedDelete(""+id, ""+version)); |
| } else { |
| tasks.add(delayedAdd("id",""+id,"name","name"+id+"_"+variant, |
| "my_version_l", ""+ version)); |
| } |
| } |
| runner.invokeAll(tasks); |
| final String expectedDoc = "{'id':'"+id+"','my_version_l':"+winnerVersion + |
| ( ! winnerIsDeleted ? ",'name':'name"+id+"_"+winner+"'}" : "}"); |
| |
| assertJQ(req("qt","/get", "id",""+id, "fl","id,name,my_version_l") |
| , "=={'doc':" + expectedDoc + "}"); |
| assertU(commit()); |
| assertJQ(req("q","id:"+id, |
| "fl","id,name,my_version_l") |
| ,"/response/numFound==1" |
| ,"/response/docs==["+expectedDoc+"]"); |
| } |
| } finally { |
| ExecutorUtil.shutdownAndAwaitTermination(runner); |
| } |
| } |
| |
| public void testMissingVersionOnOldDocs() throws Exception { |
| String version = "2"; |
| |
| // Write one doc with version, one doc without version using the "no version" chain |
| updateJ(json("[{\"id\": \"a\", \"name\": \"a1\", \"my_version_l\": " + version + "}]"), |
| params("update.chain", "no-external-version")); |
| updateJ(json("[{\"id\": \"b\", \"name\": \"b1\"}]"), params("update.chain", "no-external-version")); |
| assertU(commit()); |
| assertJQ(req("q","*:*"), "/response/numFound==2"); |
| assertJQ(req("q","id:a"), "/response/numFound==1"); |
| assertJQ(req("q","id:b"), "/response/numFound==1"); |
| |
| // Try updating both with a new version and using the enforced version chain, expect id=b to fail bc old |
| // doc is missing the version field |
| String newVersion = "3"; |
| updateJ(json("[{\"id\": \"a\", \"name\": \"a1\", \"my_version_l\": " + newVersion + "}]"), |
| params("update.chain", "external-version-constraint")); |
| |
| ignoreException("Doc exists in index, but has null versionField: my_version_l"); |
| SolrException ex = expectThrows(SolrException.class, () -> { |
| updateJ(json("[{\"id\": \"b\", \"name\": \"b1\", \"my_version_l\": " + newVersion + "}]"), |
| params("update.chain", "external-version-constraint")); |
| }); |
| assertEquals("Doc exists in index, but has null versionField: my_version_l", ex.getMessage()); |
| unIgnoreException("Doc exists in index, but has null versionField: my_version_l"); |
| |
| assertU(commit()); |
| assertJQ(req("q","*:*"), "/response/numFound==2"); |
| assertJQ(req("qt","/get", "id", "a", "fl", "id,my_version_l"), "=={'doc':{'id':'a', 'my_version_l':3}}"); // version changed to 3 |
| assertJQ(req("qt","/get", "id", "b", "fl", "id,my_version_l"), "=={'doc':{'id':'b'}}"); // no version, because update failed |
| |
| // Try to update again using the external version enforcement, but allowing old docs to not have the version |
| // field. Expect id=a to fail because version is lower, expect id=b to succeed. |
| version = "1"; |
| updateJ(json("[{\"id\": \"a\", \"name\": \"a1\", \"my_version_l\": " + version + "}]"), |
| params("update.chain", "external-version-support-missing")); |
| System.out.println("send b"); |
| updateJ(json("[{\"id\": \"b\", \"name\": \"b1\", \"my_version_l\": " + version + "}]"), |
| params("update.chain", "external-version-support-missing")); |
| assertU(commit()); |
| assertJQ(req("q","*:*"), "/response/numFound==2"); |
| assertJQ(req("qt","/get", "id", "a", "fl", "id,my_version_l"), "=={'doc':{'id':'a', 'my_version_l':3}}"); |
| assertJQ(req("qt","/get", "id", "b", "fl", "id,my_version_l"), "=={'doc':{'id':'b', 'my_version_l':1}}"); |
| } |
| |
| public void testTombstoneConfig() throws Exception { |
| assertJQ(req("q","*:*"),"/response/numFound==0"); |
| updateWithChain("tombstone-config", |
| "id", "b!doc1", |
| "my_version_l", "1"); |
| assertU(commit()); |
| assertJQ(req("q","*:*"),"/response/numFound==1"); |
| assertJQ(req("q","foo_b:true"),"/response/numFound==0"); |
| assertJQ(req("q","foo_i:1"),"/response/numFound==0"); |
| assertJQ(req("q","foo_l:1"),"/response/numFound==0"); |
| assertJQ(req("q","foo_f:1.5"),"/response/numFound==0"); |
| assertJQ(req("q","foo_s:bar"),"/response/numFound==0"); |
| assertJQ(req("q","foo_ss:bar1"),"/response/numFound==0"); |
| assertJQ(req("q","foo_ss:bar2"),"/response/numFound==0"); |
| |
| deleteAndGetVersion("b!doc1", |
| params("del_version", "2", "update.chain", |
| "tombstone-config")); |
| assertU(commit()); |
| |
| assertJQ(req("q","foo_b:true"),"/response/numFound==1"); |
| assertJQ(req("q","foo_i:1"),"/response/numFound==1"); |
| assertJQ(req("q","foo_l:1"),"/response/numFound==1"); |
| assertJQ(req("q","foo_f:1.5"),"/response/numFound==1"); |
| assertJQ(req("q","foo_s:bar"),"/response/numFound==1"); |
| assertJQ(req("q","foo_ss:bar1"),"/response/numFound==1"); |
| assertJQ(req("q","foo_ss:bar2"),"/response/numFound==1"); |
| } |
| |
| public void testCanCreateTombstonesBasic() { |
| DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory(); |
| NamedList<Object> config = new NamedList<>(); |
| config.add("versionField", "_version_"); |
| factory.init(config); |
| IndexSchema schema = h.getCore().getLatestSchema(); |
| assertThat(factory.canCreateTombstoneDocument(schema), is(true)); |
| } |
| |
| public void testCanCreateTombstonesMissingRequiredField() { |
| DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory(); |
| NamedList<Object> config = new NamedList<>(); |
| config.add("versionField", "_version_"); |
| factory.init(config); |
| IndexSchema schema = h.getCore().getLatestSchema(); |
| SchemaField sf = schema.getField("sku1"); |
| assertThat(sf, is(not(nullValue()))); |
| assertThat(schema.getRequiredFields(), not(hasItem(sf))); |
| try { |
| schema.getRequiredFields().add(sf); |
| assertThat(factory.canCreateTombstoneDocument(schema), is(false)); |
| } finally { |
| schema.getRequiredFields().remove(sf); |
| } |
| } |
| |
| public void testCanCreateTombstonesRequiredFieldWithDefault() { |
| DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory(); |
| NamedList<Object> config = new NamedList<>(); |
| config.add("versionField", "_version_"); |
| factory.init(config); |
| IndexSchema schema = h.getCore().getLatestSchema(); |
| SchemaField sf = schema.getField("sku1"); |
| SchemaField sf2 = new SchemaField("sku1_with_default", sf.getType(), sf.getProperties(), "foo"); |
| try { |
| schema.getRequiredFields().add(sf2); |
| assertThat(factory.canCreateTombstoneDocument(schema), is(true)); |
| } finally { |
| schema.getRequiredFields().remove(sf2); |
| } |
| } |
| |
| public void testCanCreateTombstonesRequiredFieldInTombstoneConfig() { |
| DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory(); |
| NamedList<Object> config = new NamedList<>(); |
| config.add("versionField", "_version_"); |
| NamedList<Object> tombstoneConfig = new NamedList<>(); |
| config.add("tombstoneConfig", tombstoneConfig); |
| tombstoneConfig.add("sku1", "foo"); |
| factory.init(config); |
| IndexSchema schema = h.getCore().getLatestSchema(); |
| SchemaField sf = schema.getField("sku1"); |
| assertThat(sf, is(not(nullValue()))); |
| assertThat(schema.getRequiredFields(), not(hasItem(sf))); |
| try { |
| schema.getRequiredFields().add(sf); |
| assertThat(factory.canCreateTombstoneDocument(schema), is(true)); |
| } finally { |
| schema.getRequiredFields().remove(sf); |
| } |
| } |
| |
| public void testCanCreateTombstonesVersionFieldRequired() { |
| DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory(); |
| NamedList<Object> config = new NamedList<>(); |
| config.add("versionField", "_version_"); |
| factory.init(config); |
| IndexSchema schema = h.getCore().getLatestSchema(); |
| SchemaField versionField = schema.getField("_version_"); |
| assertThat(versionField, is(not(nullValue()))); |
| assertThat(schema.getRequiredFields(), not(hasItem(versionField))); |
| try { |
| schema.getRequiredFields().add(versionField); |
| assertThat(factory.canCreateTombstoneDocument(schema), is(true)); |
| } finally { |
| schema.getRequiredFields().remove(versionField); |
| } |
| } |
| |
| public void testCanCreateTombstonesUniqueKeyFieldRequired() { |
| DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory(); |
| NamedList<Object> config = new NamedList<>(); |
| config.add("versionField", "_version_"); |
| factory.init(config); |
| IndexSchema schema = h.getCore().getLatestSchema(); |
| SchemaField uniqueKeyField = schema.getField("id"); |
| assertThat(uniqueKeyField, is(not(nullValue()))); |
| assertThat(uniqueKeyField, equalTo(schema.getUniqueKeyField())); |
| assertThat(schema.getRequiredFields(), hasItem(schema.getUniqueKeyField())); |
| assertThat(factory.canCreateTombstoneDocument(schema), is(true)); |
| } |
| |
| private void updateWithChain(String chain, String...fields) throws Exception { |
| assert fields.length % 2 == 0; |
| SolrInputDocument doc = new SolrInputDocument(fields); |
| updateJ(jsonAdd(doc), params("update.chain", chain)); |
| } |
| |
| private Callable<Object> delayedAdd(final String... fields) { |
| return Executors.callable(() -> { |
| // log.info("ADDING DOC: " + adoc(fields)); |
| assertU(adoc(fields)); |
| }); |
| } |
| private Callable<Object> delayedDelete(final String id, final String externalVersion) { |
| return Executors.callable(() -> { |
| try { |
| // Why does this throw "Exception" ??? |
| // log.info("DELETING DOC: " + id + " v="+externalVersion); |
| deleteAndGetVersion(id, params("del_version", externalVersion)); |
| } catch (Exception e) { |
| throw new RuntimeException(e); |
| } |
| }); |
| } |
| |
| } |
