blob: 9833e908913e4d7dadcbe5f5e0172ca16aca2b46 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.Map;
import java.util.TimeZone;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.CompositeIdRouter;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.util.DateMathParser;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.apache.solr.client.solrj.RoutedAliasTypes.TIME;
/**
* Direct http tests of the CreateRoutedAlias functionality.
*/
@SolrTestCaseJ4.SuppressSSL
public class CreateRoutedAliasTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(2).configure();
// final Properties properties = new Properties();
// properties.setProperty("immutable", "true"); // we won't modify it in this test
// new ConfigSetAdminRequest.Create()
// .setConfigSetName(configName)
// .setBaseConfigSetName("_default")
// .setNewConfigSetProperties(properties)
// .process(cluster.getSolrClient());
}
private CloudSolrClient solrClient;
@Before
public void doBefore() throws Exception {
solrClient = getCloudSolrClient(cluster);
}
@After
public void doAfter() throws Exception {
cluster.deleteAllCollections(); // deletes aliases too
if (null != solrClient) {
solrClient.close();
solrClient = null;
}
}
// This is a fairly complete test where we set many options and see that it both affected the created
// collection and that the alias metadata was saved accordingly
@Test
public void testV2() throws Exception {
// note we don't use TZ in this test, thus it's UTC
final String aliasName = getSaferTestName();
String createNode = cluster.getRandomJetty(random()).getNodeName();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
//TODO fix Solr test infra so that this /____v2/ becomes /api/
HttpPost post = new HttpPost(baseUrl + "/____v2/c");
post.setEntity(new StringEntity("{\n" +
" \"create-alias\" : {\n" +
" \"name\": \"" + aliasName + "\",\n" +
" \"router\" : {\n" +
" \"name\": \"time\",\n" +
" \"field\": \"evt_dt\",\n" +
" \"start\":\"NOW/DAY\",\n" + // small window for test failure once a day.
" \"interval\":\"+2HOUR\",\n" +
" \"maxFutureMs\":\"14400000\"\n" +
" },\n" +
//TODO should we use "NOW=" param? Won't work with v2 and is kinda a hack any way since intended for distrib
" \"create-collection\" : {\n" +
" \"router\": {\n" +
" \"name\":\"implicit\",\n" +
" \"field\":\"foo_s\"\n" +
" },\n" +
" \"shards\":\"foo,bar\",\n" +
" \"config\":\"_default\",\n" +
" \"tlogReplicas\":1,\n" +
" \"pullReplicas\":1,\n" +
" \"maxShardsPerNode\":4,\n" + // note: we also expect the 'policy' to work fine
" \"nodeSet\": '" + createNode + "',\n" +
" \"properties\" : {\n" +
" \"foobar\":\"bazbam\",\n" +
" \"foobar2\":\"bazbam2\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}", ContentType.APPLICATION_JSON));
assertSuccess(post);
Date startDate = DateMathParser.parseMath(new Date(), "NOW/DAY");
String initialCollectionName = TimeRoutedAlias.formatCollectionNameFromInstant(aliasName, startDate.toInstant());
// small chance could fail due to "NOW"; see above
assertCollectionExists(initialCollectionName);
Thread.sleep(1000);
// Test created collection:
final DocCollection coll = solrClient.getClusterStateProvider().getState(initialCollectionName).get();
//System.err.println(coll);
//TODO how do we assert the configSet ?
assertEquals(ImplicitDocRouter.class, coll.getRouter().getClass());
assertEquals("foo_s", ((Map)coll.get("router")).get("field"));
assertEquals(2, coll.getSlices().size()); // numShards
assertEquals(4, coll.getSlices().stream()
.mapToInt(s -> s.getReplicas().size()).sum()); // num replicas
// we didn't ask for any NRT replicas
assertEquals(0, coll.getSlices().stream()
.mapToInt(s -> s.getReplicas(r -> r.getType() == Replica.Type.NRT).size()).sum());
//assertEquals(1, coll.getNumNrtReplicas().intValue()); // TODO seems to be erroneous; I figured 'null'
assertEquals(1, coll.getNumTlogReplicas().intValue()); // per-shard
assertEquals(1, coll.getNumPullReplicas().intValue()); // per-shard
assertEquals(4, coll.getMaxShardsPerNode());
//TODO SOLR-11877 assertEquals(2, coll.getStateFormat());
assertTrue("nodeSet didn't work?",
coll.getSlices().stream().flatMap(s -> s.getReplicas().stream())
.map(Replica::getNodeName).allMatch(createNode::equals));
// Test Alias metadata:
Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases();
Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
assertEquals(initialCollectionName, collectionAliasMap.get(aliasName));
Map<String, String> meta = aliases.getCollectionAliasProperties(aliasName);
//System.err.println(new TreeMap(meta));
assertEquals("evt_dt",meta.get("router.field"));
assertEquals("_default",meta.get("create-collection.collection.configName"));
assertEquals("foo_s",meta.get("create-collection.router.field"));
assertEquals("bazbam",meta.get("create-collection.property.foobar"));
assertEquals("bazbam2",meta.get("create-collection.property.foobar2"));
assertEquals(createNode,meta.get("create-collection.createNodeSet"));
}
@Test
public void testV1() throws Exception {
final String aliasName = getSaferTestName();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
Instant start = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=xml" +
"&name=" + aliasName +
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=" + start +
"&router.interval=%2B30MINUTE" +
"&create-collection.collection.configName=_default" +
"&create-collection.router.field=foo_s" +
"&create-collection.numShards=1" +
"&create-collection.replicationFactor=2");
assertSuccess(get);
String initialCollectionName = TimeRoutedAlias.formatCollectionNameFromInstant(aliasName, start);
assertCollectionExists(initialCollectionName);
// Test created collection:
final DocCollection coll = solrClient.getClusterStateProvider().getState(initialCollectionName).get();
//TODO how do we assert the configSet ?
assertEquals(CompositeIdRouter.class, coll.getRouter().getClass());
assertEquals("foo_s", ((Map)coll.get("router")).get("field"));
assertEquals(1, coll.getSlices().size()); // numShards
assertEquals(2, coll.getReplicationFactor().intValue()); // num replicas
//TODO SOLR-11877 assertEquals(2, coll.getStateFormat());
// Test Alias metadata
Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases();
Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
String alias = collectionAliasMap.get(aliasName);
assertNotNull(alias);
Map<String, String> meta = aliases.getCollectionAliasProperties(aliasName);
assertNotNull(meta);
assertEquals("evt_dt",meta.get("router.field"));
assertEquals("_default",meta.get("create-collection.collection.configName"));
assertEquals(null,meta.get("start"));
}
// TZ should not affect the first collection name if absolute date given for start
@Test
public void testTimezoneAbsoluteDate() throws Exception {
final String aliasName = getSaferTestName();
try (SolrClient client = getCloudSolrClient(cluster)) {
CollectionAdminRequest.createTimeRoutedAlias(
aliasName,
"2018-01-15T00:00:00Z",
"+30MINUTE",
"evt_dt",
CollectionAdminRequest.createCollection("_ignored_", "_default", 1, 1)
)
.setTimeZone(TimeZone.getTimeZone("GMT-10"))
.process(client);
}
assertCollectionExists(aliasName + TIME.getSeparatorPrefix() +"2018-01-15");
}
@Test
public void testCollectionNamesMustBeAbsent() throws Exception {
CollectionAdminRequest.createCollection("collection1meta", "_default", 2, 1).process(cluster.getSolrClient());
CollectionAdminRequest.createCollection("collection2meta", "_default", 1, 1).process(cluster.getSolrClient());
cluster.waitForActiveCollection("collection1meta", 2, 2);
cluster.waitForActiveCollection("collection2meta", 1, 1);
waitForState("Expected collection1 to be created with 2 shards and 1 replica", "collection1meta", clusterShape(2, 2));
waitForState("Expected collection2 to be created with 1 shard and 1 replica", "collection2meta", clusterShape(1, 1));
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
zkStateReader.createClusterStateWatchersAndUpdate();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=" + getTestName() +
"&collections=collection1meta,collection2meta" +
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=2018-01-15T00:00:00Z" +
"&router.interval=%2B30MINUTE" +
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, "Collections cannot be specified");
}
@Test
public void testAliasNameMustBeValid() throws Exception {
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=735741!45" + // ! not allowed
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=2018-01-15T00:00:00Z" +
"&router.interval=%2B30MINUTE" +
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, "Invalid alias");
}
@Test
public void testRandomRouterNameFails() throws Exception {
final String aliasName = getSaferTestName();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=" + aliasName +
"&router.field=evt_dt" +
"&router.name=tiafasme" + //bad
"&router.start=2018-01-15T00:00:00Z" +
"&router.interval=%2B30MINUTE" +
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, " is not in supported types, ");
}
@Test
public void testTimeStampWithMsFails() throws Exception {
final String aliasName = getSaferTestName();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=" + aliasName +
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=2018-01-15T00:00:00.001Z" + // bad: no milliseconds permitted
"&router.interval=%2B30MINUTE" +
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, "Date or date math for start time includes milliseconds");
}
@Test
public void testBadDateMathIntervalFails() throws Exception {
final String aliasName = getSaferTestName();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=" + aliasName +
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=2018-01-15T00:00:00Z" +
"&router.interval=%2B30MINUTEx" + // bad; trailing 'x'
"&router.maxFutureMs=60000" +
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, "Unit not recognized");
}
@Test
public void testNegativeFutureFails() throws Exception {
final String aliasName = getSaferTestName();
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=" + aliasName +
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=2018-01-15T00:00:00Z" +
"&router.interval=%2B30MINUTE" +
"&router.maxFutureMs=-60000" + // bad: negative
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, "must be >= 0");
}
@Test
public void testUnParseableFutureFails() throws Exception {
final String aliasName = "testAlias";
final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEALIAS" +
"&wt=json" +
"&name=" + aliasName +
"&router.field=evt_dt" +
"&router.name=time" +
"&router.start=2018-01-15T00:00:00Z" +
"&router.interval=%2B30MINUTE" +
"&router.maxFutureMs=SixtyThousandMilliseconds" + // bad
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
assertFailure(get, "SixtyThousandMilliseconds"); //TODO improve SolrParams.getLong
}
private void assertSuccess(HttpUriRequest msg) throws IOException {
CloseableHttpClient httpClient = (CloseableHttpClient) solrClient.getHttpClient();
try (CloseableHttpResponse response = httpClient.execute(msg)) {
if (200 != response.getStatusLine().getStatusCode()) {
System.err.println(EntityUtils.toString(response.getEntity()));
fail("Unexpected status: " + response.getStatusLine());
}
}
}
private void assertFailure(HttpUriRequest msg, String expectedErrorSubstring) throws IOException {
CloseableHttpClient httpClient = (CloseableHttpClient) solrClient.getHttpClient();
try (CloseableHttpResponse response = httpClient.execute(msg)) {
assertEquals(400, response.getStatusLine().getStatusCode());
String entity = EntityUtils.toString(response.getEntity());
assertTrue("Didn't find expected error string within response: " + entity,
entity.contains(expectedErrorSubstring));
}
}
private void assertCollectionExists(String name) throws IOException, SolrServerException {
solrClient.getClusterStateProvider().connect(); // TODO get rid of this
// https://issues.apache.org/jira/browse/SOLR-9784?focusedCommentId=16332729
assertNotNull(name + " not found", solrClient.getClusterStateProvider().getState(name));
// note: could also do:
//List collections = CollectionAdminRequest.listCollections(solrClient);
}
// not testing collection parameters, those should inherit error checking from the collection creation code.
}