| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.solr.cloud.api.collections; |
| |
| |
| import java.lang.invoke.MethodHandles; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.NoSuchElementException; |
| import java.util.Set; |
| import java.util.concurrent.atomic.AtomicInteger; |
| import java.util.concurrent.atomic.AtomicReference; |
| |
| import org.apache.solr.client.solrj.cloud.DistribStateManager; |
| import org.apache.solr.client.solrj.cloud.NodeStateProvider; |
| import org.apache.solr.client.solrj.cloud.SolrCloudManager; |
| import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; |
| import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo; |
| import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; |
| import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; |
| import org.apache.solr.client.solrj.request.CoreAdminRequest; |
| import org.apache.solr.cloud.Overseer; |
| import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker; |
| import org.apache.solr.cloud.overseer.OverseerAction; |
| import org.apache.solr.common.SolrException; |
| import org.apache.solr.common.SolrException.ErrorCode; |
| import org.apache.solr.common.cloud.ClusterState; |
| import org.apache.solr.common.cloud.CompositeIdRouter; |
| import org.apache.solr.common.cloud.DocCollection; |
| import org.apache.solr.common.cloud.DocRouter; |
| import org.apache.solr.common.cloud.PlainIdRouter; |
| import org.apache.solr.common.cloud.Replica; |
| import org.apache.solr.common.cloud.ReplicaPosition; |
| import org.apache.solr.common.cloud.Slice; |
| import org.apache.solr.common.cloud.ZkNodeProps; |
| import org.apache.solr.common.cloud.ZkStateReader; |
| import org.apache.solr.common.cloud.rule.ImplicitSnitch; |
| import org.apache.solr.common.params.CommonAdminParams; |
| import org.apache.solr.common.params.CommonParams; |
| import org.apache.solr.common.params.CoreAdminParams; |
| import org.apache.solr.common.params.ModifiableSolrParams; |
| import org.apache.solr.common.util.NamedList; |
| import org.apache.solr.common.util.SimpleOrderedMap; |
| import org.apache.solr.common.util.Utils; |
| import org.apache.solr.handler.component.ShardHandler; |
| import org.apache.solr.update.SolrIndexSplitter; |
| import org.apache.solr.util.RTimerTree; |
| import org.apache.solr.util.TestInjection; |
| import org.apache.zookeeper.CreateMode; |
| import org.apache.zookeeper.KeeperException; |
| import org.apache.zookeeper.data.Stat; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; |
| import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; |
| import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; |
| import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; |
| import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; |
| import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; |
| import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; |
| import static org.apache.solr.common.params.CommonAdminParams.ASYNC; |
| import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS; |
| |
| |
| public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { |
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); |
| private static final int MIN_NUM_SUB_SHARDS = 2; |
| // This is an arbitrary number that seems reasonable at this time. |
| private static final int MAX_NUM_SUB_SHARDS = 8; |
| private static final int DEFAULT_NUM_SUB_SHARDS = 2; |
| |
| private final OverseerCollectionMessageHandler ocmh; |
| |
| public SplitShardCmd(OverseerCollectionMessageHandler ocmh) { |
| this.ocmh = ocmh; |
| } |
| |
| @SuppressWarnings("unchecked") |
| @Override |
| public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception { |
| split(state, message,(NamedList<Object>) results); |
| } |
| |
| @SuppressWarnings({"rawtypes"}) |
| public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList<Object> results) throws Exception { |
| final String asyncId = message.getStr(ASYNC); |
| |
| boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); |
| String methodStr = message.getStr(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower()); |
| SolrIndexSplitter.SplitMethod splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr); |
| if (splitMethod == null) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown value '" + CommonAdminParams.SPLIT_METHOD + |
| ": " + methodStr); |
| } |
| boolean withTiming = message.getBool(CommonParams.TIMING, false); |
| |
| String extCollectionName = message.getStr(CoreAdminParams.COLLECTION); |
| |
| boolean followAliases = message.getBool(FOLLOW_ALIASES, false); |
| String collectionName; |
| if (followAliases) { |
| collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName); |
| } else { |
| collectionName = extCollectionName; |
| } |
| |
| log.debug("Split shard invoked: {}", message); |
| ZkStateReader zkStateReader = ocmh.zkStateReader; |
| zkStateReader.forceUpdateCollection(collectionName); |
| AtomicReference<String> slice = new AtomicReference<>(); |
| slice.set(message.getStr(ZkStateReader.SHARD_ID_PROP)); |
| Set<String> offlineSlices = new HashSet<>(); |
| RTimerTree timings = new RTimerTree(); |
| |
| String splitKey = message.getStr("split.key"); |
| DocCollection collection = clusterState.getCollection(collectionName); |
| |
| PolicyHelper.SessionWrapper sessionWrapper = null; |
| |
| Slice parentSlice = getParentSlice(clusterState, collectionName, slice, splitKey); |
| if (parentSlice.getState() != Slice.State.ACTIVE) { |
| throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Parent slice is not active: " + |
| collectionName + "/ " + parentSlice.getName() + ", state=" + parentSlice.getState()); |
| } |
| |
| // find the leader for the shard |
| Replica parentShardLeader; |
| try { |
| parentShardLeader = zkStateReader.getLeaderRetry(collectionName, slice.get(), 10000); |
| } catch (InterruptedException e) { |
| Thread.currentThread().interrupt(); |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted."); |
| } |
| |
| RTimerTree t; |
| if (ocmh.overseer.getCoreContainer().getNodeConfig().getMetricsConfig().isEnabled()) { |
| t = timings.sub("checkDiskSpace"); |
| checkDiskSpace(collectionName, slice.get(), parentShardLeader, splitMethod, ocmh.cloudManager); |
| t.stop(); |
| } |
| |
| // let's record the ephemeralOwner of the parent leader node |
| Stat leaderZnodeStat = zkStateReader.getZkClient().exists(ZkStateReader.LIVE_NODES_ZKNODE + "/" + parentShardLeader.getNodeName(), null, true); |
| if (leaderZnodeStat == null) { |
| // we just got to know the leader but its live node is gone already! |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The shard leader node: " + parentShardLeader.getNodeName() + " is not live anymore!"); |
| } |
| |
| List<DocRouter.Range> subRanges = new ArrayList<>(); |
| List<String> subSlices = new ArrayList<>(); |
| List<String> subShardNames = new ArrayList<>(); |
| |
| // reproduce the currently existing number of replicas per type |
| AtomicInteger numNrt = new AtomicInteger(); |
| AtomicInteger numTlog = new AtomicInteger(); |
| AtomicInteger numPull = new AtomicInteger(); |
| parentSlice.getReplicas().forEach(r -> { |
| switch (r.getType()) { |
| case NRT: |
| numNrt.incrementAndGet(); |
| break; |
| case TLOG: |
| numTlog.incrementAndGet(); |
| break; |
| case PULL: |
| numPull.incrementAndGet(); |
| } |
| }); |
| int repFactor = numNrt.get() + numTlog.get() + numPull.get(); |
| |
| boolean success = false; |
| try { |
| // type of the first subreplica will be the same as leader |
| boolean firstNrtReplica = parentShardLeader.getType() == Replica.Type.NRT; |
| // verify that we indeed have the right number of correct replica types |
| if ((firstNrtReplica && numNrt.get() < 1) || (!firstNrtReplica && numTlog.get() < 1)) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "aborting split - inconsistent replica types in collection " + collectionName + |
| ": nrt=" + numNrt.get() + ", tlog=" + numTlog.get() + ", pull=" + numPull.get() + ", shard leader type is " + |
| parentShardLeader.getType()); |
| } |
| |
| // check for the lock |
| if (!lockForSplit(ocmh.cloudManager, collectionName, parentSlice.getName())) { |
| // mark as success to avoid clearing the lock in the "finally" block |
| success = true; |
| throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Can't lock parent slice for splitting (another split operation running?): " + |
| collectionName + "/" + parentSlice.getName()); |
| } |
| |
| List<Map<String, Object>> replicas = new ArrayList<>((repFactor - 1) * 2); |
| |
| @SuppressWarnings("deprecation") |
| ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(ocmh.overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient()); |
| |
| |
| if (message.getBool(CommonAdminParams.SPLIT_BY_PREFIX, false)) { |
| t = timings.sub("getRanges"); |
| |
| ModifiableSolrParams params = new ModifiableSolrParams(); |
| params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString()); |
| params.set(CoreAdminParams.GET_RANGES, "true"); |
| params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower()); |
| params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core")); |
| // Only 2 is currently supported |
| // int numSubShards = message.getInt(NUM_SUB_SHARDS, DEFAULT_NUM_SUB_SHARDS); |
| // params.set(NUM_SUB_SHARDS, Integer.toString(numSubShards)); |
| |
| { |
| final ShardRequestTracker shardRequestTracker = ocmh.syncRequestTracker(); |
| shardRequestTracker.sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler); |
| SimpleOrderedMap<Object> getRangesResults = new SimpleOrderedMap<>(); |
| String msgOnError = "SPLITSHARD failed to invoke SPLIT.getRanges core admin command"; |
| shardRequestTracker.processResponses(getRangesResults, shardHandler, true, msgOnError); |
| |
| // Extract the recommended splits from the shard response (if it exists) |
| // example response: getRangesResults={success={127.0.0.1:62086_solr={responseHeader={status=0,QTime=1},ranges=10-20,3a-3f}}} |
| NamedList successes = (NamedList)getRangesResults.get("success"); |
| if (successes != null && successes.size() > 0) { |
| NamedList shardRsp = (NamedList)successes.getVal(0); |
| String splits = (String)shardRsp.get(CoreAdminParams.RANGES); |
| if (splits != null) { |
| log.info("Resulting split ranges to be used: {} slice={} leader={}", splits, slice, parentShardLeader); |
| // change the message to use the recommended split ranges |
| message = message.plus(CoreAdminParams.RANGES, splits); |
| } |
| } |
| } |
| |
| t.stop(); |
| } |
| |
| |
| t = timings.sub("fillRanges"); |
| |
| String rangesStr = fillRanges(ocmh.cloudManager, message, collection, parentSlice, subRanges, subSlices, subShardNames, firstNrtReplica); |
| t.stop(); |
| |
| boolean oldShardsDeleted = false; |
| for (String subSlice : subSlices) { |
| Slice oSlice = collection.getSlice(subSlice); |
| if (oSlice != null) { |
| final Slice.State state = oSlice.getState(); |
| if (state == Slice.State.ACTIVE) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, |
| "Sub-shard: " + subSlice + " exists in active state. Aborting split shard."); |
| } else { |
| // delete the shards |
| log.info("Sub-shard: {} already exists therefore requesting its deletion", subSlice); |
| Map<String, Object> propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, "deleteshard"); |
| propMap.put(COLLECTION_PROP, collectionName); |
| propMap.put(SHARD_ID_PROP, subSlice); |
| ZkNodeProps m = new ZkNodeProps(propMap); |
| try { |
| ocmh.commandMap.get(DELETESHARD).call(clusterState, m, new NamedList()); |
| } catch (Exception e) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to delete already existing sub shard: " + subSlice, |
| e); |
| } |
| |
| oldShardsDeleted = true; |
| } |
| } |
| } |
| |
| if (oldShardsDeleted) { |
| // refresh the locally cached cluster state |
| // we know we have the latest because otherwise deleteshard would have failed |
| clusterState = zkStateReader.getClusterState(); |
| collection = clusterState.getCollection(collectionName); |
| } |
| |
| String nodeName = parentShardLeader.getNodeName(); |
| |
| t = timings.sub("createSubSlicesAndLeadersInState"); |
| for (int i = 0; i < subRanges.size(); i++) { |
| String subSlice = subSlices.get(i); |
| String subShardName = subShardNames.get(i); |
| DocRouter.Range subRange = subRanges.get(i); |
| |
| log.debug("Creating slice {} of collection {} on {}", subSlice, collectionName, nodeName); |
| |
| Map<String, Object> propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD.toLower()); |
| propMap.put(ZkStateReader.SHARD_ID_PROP, subSlice); |
| propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); |
| propMap.put(ZkStateReader.SHARD_RANGE_PROP, subRange.toString()); |
| propMap.put(ZkStateReader.SHARD_STATE_PROP, Slice.State.CONSTRUCTION.toString()); |
| propMap.put(ZkStateReader.SHARD_PARENT_PROP, parentSlice.getName()); |
| propMap.put("shard_parent_node", nodeName); |
| propMap.put("shard_parent_zk_session", leaderZnodeStat.getEphemeralOwner()); |
| |
| ocmh.overseer.offerStateUpdate(Utils.toJSON(new ZkNodeProps(propMap))); |
| |
| // wait until we are able to see the new shard in cluster state and refresh the local view of the cluster state |
| clusterState = ocmh.waitForNewShard(collectionName, subSlice); |
| |
| log.debug("Adding first replica {} as part of slice {} of collection {} on {}" |
| , subShardName, subSlice, collectionName, nodeName); |
| propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); |
| propMap.put(COLLECTION_PROP, collectionName); |
| propMap.put(SHARD_ID_PROP, subSlice); |
| propMap.put(REPLICA_TYPE, firstNrtReplica ? Replica.Type.NRT.toString() : Replica.Type.TLOG.toString()); |
| propMap.put("node", nodeName); |
| propMap.put(CoreAdminParams.NAME, subShardName); |
| propMap.put(CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.toString(waitForFinalState)); |
| // copy over property params: |
| for (String key : message.keySet()) { |
| if (key.startsWith(OverseerCollectionMessageHandler.COLL_PROP_PREFIX)) { |
| propMap.put(key, message.getStr(key)); |
| } |
| } |
| // add async param |
| if (asyncId != null) { |
| propMap.put(ASYNC, asyncId); |
| } |
| ocmh.addReplica(clusterState, new ZkNodeProps(propMap), results, null); |
| } |
| |
| |
| { |
| final ShardRequestTracker syncRequestTracker = ocmh.syncRequestTracker(); |
| String msgOnError = "SPLITSHARD failed to create subshard leaders"; |
| syncRequestTracker.processResponses(results, shardHandler, true, msgOnError); |
| handleFailureOnAsyncRequest(results, msgOnError); |
| } |
| t.stop(); |
| t = timings.sub("waitForSubSliceLeadersAlive"); |
| { |
| final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); |
| for (String subShardName : subShardNames) { |
| // wait for parent leader to acknowledge the sub-shard core |
| log.debug("Asking parent leader to wait for: {} to be alive on: {}", subShardName, nodeName); |
| String coreNodeName = ocmh.waitForCoreNodeName(collectionName, nodeName, subShardName); |
| CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); |
| cmd.setCoreName(subShardName); |
| cmd.setNodeName(nodeName); |
| cmd.setCoreNodeName(coreNodeName); |
| cmd.setState(Replica.State.ACTIVE); |
| cmd.setCheckLive(true); |
| cmd.setOnlyIfLeader(true); |
| |
| ModifiableSolrParams p = new ModifiableSolrParams(cmd.getParams()); |
| shardRequestTracker.sendShardRequest(nodeName, p, shardHandler); |
| } |
| |
| String msgOnError = "SPLITSHARD timed out waiting for subshard leaders to come up"; |
| shardRequestTracker.processResponses(results, shardHandler, true, msgOnError); |
| handleFailureOnAsyncRequest(results, msgOnError); |
| } |
| t.stop(); |
| |
| log.debug("Successfully created all sub-shards for collection {} parent shard: {} on: {}" |
| , collectionName, slice, parentShardLeader); |
| |
| if (log.isInfoEnabled()) { |
| log.info("Splitting shard {} as part of slice {} of collection {} on {}" |
| , parentShardLeader.getName(), slice, collectionName, parentShardLeader); |
| } |
| |
| ModifiableSolrParams params = new ModifiableSolrParams(); |
| params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString()); |
| params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower()); |
| params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core")); |
| for (int i = 0; i < subShardNames.size(); i++) { |
| String subShardName = subShardNames.get(i); |
| params.add(CoreAdminParams.TARGET_CORE, subShardName); |
| } |
| params.set(CoreAdminParams.RANGES, rangesStr); |
| |
| t = timings.sub("splitParentCore"); |
| { |
| final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); |
| shardRequestTracker.sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler); |
| |
| String msgOnError = "SPLITSHARD failed to invoke SPLIT core admin command"; |
| shardRequestTracker.processResponses(results, shardHandler, true, msgOnError); |
| handleFailureOnAsyncRequest(results, msgOnError); |
| } |
| t.stop(); |
| |
| if (log.isDebugEnabled()) { |
| log.debug("Index on shard: {} split into {} successfully", nodeName, subShardNames.size()); |
| } |
| |
| t = timings.sub("applyBufferedUpdates"); |
| // apply buffered updates on sub-shards |
| { |
| final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); |
| |
| for (int i = 0; i < subShardNames.size(); i++) { |
| String subShardName = subShardNames.get(i); |
| |
| log.debug("Applying buffered updates on : {}", subShardName); |
| |
| params = new ModifiableSolrParams(); |
| params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString()); |
| params.set(CoreAdminParams.NAME, subShardName); |
| |
| shardRequestTracker.sendShardRequest(nodeName, params, shardHandler); |
| } |
| |
| String msgOnError = "SPLITSHARD failed while asking sub shard leaders to apply buffered updates"; |
| shardRequestTracker.processResponses(results, shardHandler, true, msgOnError); |
| handleFailureOnAsyncRequest(results, msgOnError); |
| } |
| t.stop(); |
| |
| log.debug("Successfully applied buffered updates on : {}", subShardNames); |
| |
| // Replica creation for the new Slices |
| // replica placement is controlled by the autoscaling policy framework |
| |
| Set<String> nodes = clusterState.getLiveNodes(); |
| List<String> nodeList = new ArrayList<>(nodes.size()); |
| nodeList.addAll(nodes); |
| |
| // TODO: Have maxShardsPerNode param for this operation? |
| |
| // Remove the node that hosts the parent shard for replica creation. |
| nodeList.remove(nodeName); |
| |
| // TODO: change this to handle sharding a slice into > 2 sub-shards. |
| |
| // we have already created one subReplica for each subShard on the parent node. |
| // identify locations for the remaining replicas |
| if (firstNrtReplica) { |
| numNrt.decrementAndGet(); |
| } else { |
| numTlog.decrementAndGet(); |
| } |
| |
| t = timings.sub("identifyNodesForReplicas"); |
| Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() |
| .forCollection(collectionName) |
| .forShard(subSlices) |
| .assignNrtReplicas(numNrt.get()) |
| .assignTlogReplicas(numTlog.get()) |
| .assignPullReplicas(numPull.get()) |
| .onNodes(new ArrayList<>(clusterState.getLiveNodes())) |
| .build(); |
| Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager); |
| Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, collection); |
| List<ReplicaPosition> replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest); |
| sessionWrapper = PolicyHelper.getLastSessionWrapper(true); |
| t.stop(); |
| |
| t = timings.sub("createReplicaPlaceholders"); |
| for (ReplicaPosition replicaPosition : replicaPositions) { |
| String sliceName = replicaPosition.shard; |
| String subShardNodeName = replicaPosition.node; |
| String solrCoreName = Assign.buildSolrCoreName(collectionName, sliceName, replicaPosition.type, replicaPosition.index); |
| |
| log.debug("Creating replica shard {} as part of slice {} of collection {} on {}" |
| , solrCoreName, sliceName, collectionName, subShardNodeName); |
| |
| // we first create all replicas in DOWN state without actually creating their cores in order to |
| // avoid a race condition where Overseer may prematurely activate the new sub-slices (and deactivate |
| // the parent slice) before all new replicas are added. This situation may lead to a loss of performance |
| // because the new shards will be activated with possibly many fewer replicas. |
| ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), |
| ZkStateReader.COLLECTION_PROP, collectionName, |
| ZkStateReader.SHARD_ID_PROP, sliceName, |
| ZkStateReader.CORE_NAME_PROP, solrCoreName, |
| ZkStateReader.REPLICA_TYPE, replicaPosition.type.name(), |
| ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), |
| ZkStateReader.BASE_URL_PROP, zkStateReader.getBaseUrlForNodeName(subShardNodeName), |
| ZkStateReader.NODE_NAME_PROP, subShardNodeName, |
| CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.toString(waitForFinalState)); |
| ocmh.overseer.offerStateUpdate(Utils.toJSON(props)); |
| |
| HashMap<String, Object> propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); |
| propMap.put(COLLECTION_PROP, collectionName); |
| propMap.put(SHARD_ID_PROP, sliceName); |
| propMap.put(REPLICA_TYPE, replicaPosition.type.name()); |
| propMap.put("node", subShardNodeName); |
| propMap.put(CoreAdminParams.NAME, solrCoreName); |
| // copy over property params: |
| for (String key : message.keySet()) { |
| if (key.startsWith(OverseerCollectionMessageHandler.COLL_PROP_PREFIX)) { |
| propMap.put(key, message.getStr(key)); |
| } |
| } |
| // add async param |
| if (asyncId != null) { |
| propMap.put(ASYNC, asyncId); |
| } |
| // special flag param to instruct addReplica not to create the replica in cluster state again |
| propMap.put(OverseerCollectionMessageHandler.SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, "true"); |
| |
| propMap.put(CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.toString(waitForFinalState)); |
| |
| replicas.add(propMap); |
| } |
| t.stop(); |
| assert TestInjection.injectSplitFailureBeforeReplicaCreation(); |
| |
| long ephemeralOwner = leaderZnodeStat.getEphemeralOwner(); |
| // compare against the ephemeralOwner of the parent leader node |
| leaderZnodeStat = zkStateReader.getZkClient().exists(ZkStateReader.LIVE_NODES_ZKNODE + "/" + parentShardLeader.getNodeName(), null, true); |
| if (leaderZnodeStat == null || ephemeralOwner != leaderZnodeStat.getEphemeralOwner()) { |
| // put sub-shards in recovery_failed state |
| |
| Map<String, Object> propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); |
| for (String subSlice : subSlices) { |
| propMap.put(subSlice, Slice.State.RECOVERY_FAILED.toString()); |
| } |
| propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); |
| ZkNodeProps m = new ZkNodeProps(propMap); |
| ocmh.overseer.offerStateUpdate(Utils.toJSON(m)); |
| |
| if (leaderZnodeStat == null) { |
| // the leader is not live anymore, fail the split! |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The shard leader node: " + parentShardLeader.getNodeName() + " is not live anymore!"); |
| } else if (ephemeralOwner != leaderZnodeStat.getEphemeralOwner()) { |
| // there's a new leader, fail the split! |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, |
| "The zk session id for the shard leader node: " + parentShardLeader.getNodeName() + " has changed from " |
| + ephemeralOwner + " to " + leaderZnodeStat.getEphemeralOwner() + ". This can cause data loss so we must abort the split"); |
| } |
| } |
| |
| // we must set the slice state into recovery before actually creating the replica cores |
| // this ensures that the logic inside ReplicaMutator to update sub-shard state to 'active' |
| // always gets a chance to execute. See SOLR-7673 |
| |
| if (repFactor == 1) { |
| // A commit is needed so that documents are visible when the sub-shard replicas come up |
| // (Note: This commit used to be after the state switch, but was brought here before the state switch |
| // as per SOLR-13945 so that sub shards don't come up empty, momentarily, after being marked active) |
| t = timings.sub("finalCommit"); |
| ocmh.commit(results, slice.get(), parentShardLeader); |
| t.stop(); |
| // switch sub shard states to 'active' |
| log.info("Replication factor is 1 so switching shard states"); |
| Map<String, Object> propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); |
| propMap.put(slice.get(), Slice.State.INACTIVE.toString()); |
| for (String subSlice : subSlices) { |
| propMap.put(subSlice, Slice.State.ACTIVE.toString()); |
| } |
| propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); |
| ZkNodeProps m = new ZkNodeProps(propMap); |
| ocmh.overseer.offerStateUpdate(Utils.toJSON(m)); |
| } else { |
| log.info("Requesting shard state be set to 'recovery'"); |
| Map<String, Object> propMap = new HashMap<>(); |
| propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); |
| for (String subSlice : subSlices) { |
| propMap.put(subSlice, Slice.State.RECOVERY.toString()); |
| } |
| propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); |
| ZkNodeProps m = new ZkNodeProps(propMap); |
| ocmh.overseer.offerStateUpdate(Utils.toJSON(m)); |
| } |
| |
| t = timings.sub("createCoresForReplicas"); |
| // now actually create replica cores on sub shard nodes |
| for (Map<String, Object> replica : replicas) { |
| ocmh.addReplica(clusterState, new ZkNodeProps(replica), results, null); |
| } |
| |
| assert TestInjection.injectSplitFailureAfterReplicaCreation(); |
| |
| { |
| final ShardRequestTracker syncRequestTracker = ocmh.syncRequestTracker(); |
| String msgOnError = "SPLITSHARD failed to create subshard replicas"; |
| syncRequestTracker.processResponses(results, shardHandler, true, msgOnError); |
| handleFailureOnAsyncRequest(results, msgOnError); |
| } |
| t.stop(); |
| |
| log.info("Successfully created all replica shards for all sub-slices {}", subSlices); |
| |
| // The final commit was added in SOLR-4997 so that documents are visible |
| // when the sub-shard replicas come up |
| if (repFactor > 1) { |
| t = timings.sub("finalCommit"); |
| ocmh.commit(results, slice.get(), parentShardLeader); |
| t.stop(); |
| } |
| |
| if (withTiming) { |
| results.add(CommonParams.TIMING, timings.asNamedList()); |
| } |
| success = true; |
| // don't unlock the shard yet - only do this if the final switch-over in |
| // ReplicaMutator succeeds (or fails) |
| return true; |
| } catch (SolrException e) { |
| throw e; |
| } catch (Exception e) { |
| log.error("Error executing split operation for collection: {} parent shard: {}", collectionName, slice, e); |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e); |
| } finally { |
| if (sessionWrapper != null) sessionWrapper.release(); |
| if (!success) { |
| cleanupAfterFailure(zkStateReader, collectionName, parentSlice.getName(), subSlices, offlineSlices); |
| unlockForSplit(ocmh.cloudManager, collectionName, parentSlice.getName()); |
| } |
| } |
| } |
| |
| /** |
| * In case of async requests, the ShardRequestTracker's processResponses() does not |
| * abort on failure (as it should). Handling this here temporarily for now. |
| */ |
| private void handleFailureOnAsyncRequest(@SuppressWarnings({"rawtypes"})NamedList results, String msgOnError) { |
| Object splitResultFailure = results.get("failure"); |
| if (splitResultFailure != null) { |
| throw new SolrException(ErrorCode.SERVER_ERROR, msgOnError); |
| } |
| } |
| |
| // public and static to facilitate reuse in the simulation framework and in tests |
| public static void checkDiskSpace(String collection, String shard, Replica parentShardLeader, SolrIndexSplitter.SplitMethod method, SolrCloudManager cloudManager) throws SolrException { |
| // check that enough disk space is available on the parent leader node |
| // otherwise the actual index splitting will always fail |
| NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider(); |
| Map<String, Object> nodeValues = nodeStateProvider.getNodeValues(parentShardLeader.getNodeName(), |
| Collections.singletonList(ImplicitSnitch.DISK)); |
| Map<String, Map<String, List<ReplicaInfo>>> infos = nodeStateProvider.getReplicaInfo(parentShardLeader.getNodeName(), |
| Collections.singletonList(Type.CORE_IDX.metricsAttribute)); |
| if (infos.get(collection) == null || infos.get(collection).get(shard) == null) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing replica information for parent shard leader"); |
| } |
| // find the leader |
| List<ReplicaInfo> lst = infos.get(collection).get(shard); |
| Double indexSize = null; |
| for (ReplicaInfo info : lst) { |
| if (info.getCore().equals(parentShardLeader.getCoreName())) { |
| Number size = (Number)info.getVariable(Type.CORE_IDX.metricsAttribute); |
| if (size == null) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing index size information for parent shard leader"); |
| } |
| indexSize = (Double) Type.CORE_IDX.convertVal(size); |
| break; |
| } |
| } |
| if (indexSize == null) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing replica information for parent shard leader"); |
| } |
| Number freeSize = (Number)nodeValues.get(ImplicitSnitch.DISK); |
| if (freeSize == null) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing node disk space information for parent shard leader"); |
| } |
| // 100% more for REWRITE, 5% more for LINK |
| double neededSpace = method == SolrIndexSplitter.SplitMethod.REWRITE ? 2.0 * indexSize : 1.05 * indexSize; |
| if (freeSize.doubleValue() < neededSpace) { |
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "not enough free disk space to perform index split on node " + |
| parentShardLeader.getNodeName() + ", required: " + neededSpace + ", available: " + freeSize); |
| } |
| } |
| |
| private void cleanupAfterFailure(ZkStateReader zkStateReader, String collectionName, String parentShard, |
| List<String> subSlices, Set<String> offlineSlices) { |
| log.info("Cleaning up after a failed split of {}/{}", collectionName, parentShard); |
| // get the latest state |
| try { |
| zkStateReader.forceUpdateCollection(collectionName); |
| } catch (KeeperException | InterruptedException e) { |
| log.warn("Cleanup failed after failed split of {}/{} : (force update collection)", collectionName, parentShard, e); |
| return; |
| } |
| ClusterState clusterState = zkStateReader.getClusterState(); |
| DocCollection coll = clusterState.getCollectionOrNull(collectionName); |
| |
| if (coll == null) { // may have been deleted |
| return; |
| } |
| |
| // If parent is inactive and all sub shards are active, then rolling back |
| // to make the parent active again will cause data loss. |
| if (coll.getSlice(parentShard).getState() == Slice.State.INACTIVE) { |
| boolean allSubSlicesActive = true; |
| for (String sub: subSlices) { |
| if (coll.getSlice(sub).getState() != Slice.State.ACTIVE) { |
| allSubSlicesActive = false; |
| break; |
| } |
| } |
| if (allSubSlicesActive) { |
| return; |
| } |
| } |
| |
| // set already created sub shards states to CONSTRUCTION - this prevents them |
| // from entering into RECOVERY or ACTIVE (SOLR-9455) |
| final Map<String, Object> propMap = new HashMap<>(); |
| boolean sendUpdateState = false; |
| propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); |
| propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); |
| for (Slice s : coll.getSlices()) { |
| if (!subSlices.contains(s.getName())) { |
| continue; |
| } |
| propMap.put(s.getName(), Slice.State.CONSTRUCTION.toString()); |
| sendUpdateState = true; |
| } |
| |
| // if parent is inactive activate it again |
| Slice parentSlice = coll.getSlice(parentShard); |
| if (parentSlice.getState() == Slice.State.INACTIVE) { |
| sendUpdateState = true; |
| propMap.put(parentShard, Slice.State.ACTIVE.toString()); |
| } |
| // plus any other previously deactivated slices |
| for (String sliceName : offlineSlices) { |
| propMap.put(sliceName, Slice.State.ACTIVE.toString()); |
| sendUpdateState = true; |
| } |
| |
| if (sendUpdateState) { |
| try { |
| ZkNodeProps m = new ZkNodeProps(propMap); |
| ocmh.overseer.offerStateUpdate(Utils.toJSON(m)); |
| } catch (Exception e) { |
| // don't give up yet - just log the error, we may still be able to clean up |
| log.warn("Cleanup failed after failed split of {}/{}: (slice state changes)", collectionName, parentShard, e); |
| } |
| } |
| |
| // delete existing subShards |
| for (String subSlice : subSlices) { |
| Slice s = coll.getSlice(subSlice); |
| if (s == null) { |
| continue; |
| } |
| log.debug("- sub-shard: {} exists therefore requesting its deletion", subSlice); |
| HashMap<String, Object> props = new HashMap<>(); |
| props.put(Overseer.QUEUE_OPERATION, "deleteshard"); |
| props.put(COLLECTION_PROP, collectionName); |
| props.put(SHARD_ID_PROP, subSlice); |
| ZkNodeProps m = new ZkNodeProps(props); |
| try { |
| ocmh.commandMap.get(DELETESHARD).call(clusterState, m, new NamedList<Object>()); |
| } catch (Exception e) { |
| log.warn("Cleanup failed after failed split of {}/{} : (deleting existing sub shard{})", collectionName, parentShard, subSlice, e); |
| } |
| } |
| } |
| |
| public static Slice getParentSlice(ClusterState clusterState, String collectionName, AtomicReference<String> slice, String splitKey) { |
| DocCollection collection = clusterState.getCollection(collectionName); |
| DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; |
| |
| Slice parentSlice; |
| |
| if (slice.get() == null) { |
| if (router instanceof CompositeIdRouter) { |
| Collection<Slice> searchSlices = router.getSearchSlicesSingle(splitKey, new ModifiableSolrParams(), collection); |
| if (searchSlices.isEmpty()) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to find an active shard for split.key: " + splitKey); |
| } |
| if (searchSlices.size() > 1) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, |
| "Splitting a split.key: " + splitKey + " which spans multiple shards is not supported"); |
| } |
| parentSlice = searchSlices.iterator().next(); |
| slice.set(parentSlice.getName()); |
| log.info("Split by route.key: {}, parent shard is: {} ", splitKey, slice); |
| } else { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, |
| "Split by route key can only be used with CompositeIdRouter or subclass. Found router: " |
| + router.getClass().getName()); |
| } |
| } else { |
| parentSlice = collection.getSlice(slice.get()); |
| } |
| |
| if (parentSlice == null) { |
| // no chance of the collection being null because ClusterState#getCollection(String) would have thrown |
| // an exception already |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shard with the specified name exists: " + slice); |
| } |
| return parentSlice; |
| } |
| |
| public static String fillRanges(SolrCloudManager cloudManager, ZkNodeProps message, DocCollection collection, Slice parentSlice, |
| List<DocRouter.Range> subRanges, List<String> subSlices, List<String> subShardNames, |
| boolean firstReplicaNrt) { |
| String splitKey = message.getStr("split.key"); |
| String rangesStr = message.getStr(CoreAdminParams.RANGES); |
| String fuzzStr = message.getStr(CommonAdminParams.SPLIT_FUZZ, "0"); |
| float fuzz = 0.0f; |
| try { |
| fuzz = Float.parseFloat(fuzzStr); |
| } catch (Exception e) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid numeric value of 'fuzz': " + fuzzStr); |
| } |
| |
| DocRouter.Range range = parentSlice.getRange(); |
| if (range == null) { |
| range = new PlainIdRouter().fullRange(); |
| } |
| DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; |
| if (rangesStr != null) { |
| String[] ranges = rangesStr.split(","); |
| if (ranges.length == 0 || ranges.length == 1) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard"); |
| } else { |
| for (int i = 0; i < ranges.length; i++) { |
| String r = ranges[i]; |
| try { |
| subRanges.add(DocRouter.DEFAULT.fromString(r)); |
| } catch (Exception e) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e); |
| } |
| if (!subRanges.get(i).isSubsetOf(range)) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, |
| "Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString()); |
| } |
| } |
| List<DocRouter.Range> temp = new ArrayList<>(subRanges); // copy to preserve original order |
| Collections.sort(temp); |
| if (!range.equals(new DocRouter.Range(temp.get(0).min, temp.get(temp.size() - 1).max))) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, |
| "Specified hash ranges: " + rangesStr + " do not cover the entire range of parent shard: " + range); |
| } |
| for (int i = 1; i < temp.size(); i++) { |
| if (temp.get(i - 1).max + 1 != temp.get(i).min) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Specified hash ranges: " + rangesStr |
| + " either overlap with each other or " + "do not cover the entire range of parent shard: " + range); |
| } |
| } |
| } |
| } else if (splitKey != null) { |
| if (router instanceof CompositeIdRouter) { |
| CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router; |
| List<DocRouter.Range> tmpSubRanges = compositeIdRouter.partitionRangeByKey(splitKey, range); |
| if (tmpSubRanges.size() == 1) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The split.key: " + splitKey |
| + " has a hash range that is exactly equal to hash range of shard: " + parentSlice.getName()); |
| } |
| for (DocRouter.Range subRange : tmpSubRanges) { |
| if (subRange.min == subRange.max) { |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The split.key: " + splitKey + " must be a compositeId"); |
| } |
| } |
| subRanges.addAll(tmpSubRanges); |
| if (log.isInfoEnabled()) { |
| log.info("Partitioning parent shard {} range: {} yields: {}", parentSlice.getName(), parentSlice.getRange(), subRanges); |
| } |
| rangesStr = ""; |
| for (int i = 0; i < subRanges.size(); i++) { |
| DocRouter.Range subRange = subRanges.get(i); |
| rangesStr += subRange.toString(); |
| if (i < subRanges.size() - 1) rangesStr += ','; |
| } |
| } |
| } else { |
| int numSubShards = message.getInt(NUM_SUB_SHARDS, DEFAULT_NUM_SUB_SHARDS); |
| log.info("{} set at: {}", NUM_SUB_SHARDS, numSubShards); |
| |
| if(numSubShards < MIN_NUM_SUB_SHARDS || numSubShards > MAX_NUM_SUB_SHARDS) |
| throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, |
| "A shard can only be split into "+MIN_NUM_SUB_SHARDS+" to " + MAX_NUM_SUB_SHARDS |
| + " subshards in one split request. Provided "+NUM_SUB_SHARDS+"=" + numSubShards); |
| subRanges.addAll(router.partitionRange(numSubShards, range, fuzz)); |
| } |
| |
| for (int i = 0; i < subRanges.size(); i++) { |
| String subSlice = parentSlice.getName() + "_" + i; |
| subSlices.add(subSlice); |
| String subShardName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), collection, subSlice, |
| firstReplicaNrt ? Replica.Type.NRT : Replica.Type.TLOG); |
| subShardNames.add(subShardName); |
| } |
| return rangesStr; |
| } |
| |
| public static boolean lockForSplit(SolrCloudManager cloudManager, String collection, String shard) throws Exception { |
| String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/" + shard + "-splitting"; |
| final DistribStateManager stateManager = cloudManager.getDistribStateManager(); |
| synchronized (stateManager) { |
| if (stateManager.hasData(path)) { |
| VersionedData vd = stateManager.getData(path); |
| return false; |
| } |
| Map<String, Object> map = new HashMap<>(); |
| map.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs())); |
| byte[] data = Utils.toJSON(map); |
| try { |
| cloudManager.getDistribStateManager().makePath(path, data, CreateMode.EPHEMERAL, true); |
| } catch (Exception e) { |
| throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Can't lock parent slice for splitting (another split operation running?): " + |
| collection + "/" + shard, e); |
| } |
| return true; |
| } |
| } |
| |
| public static void unlockForSplit(SolrCloudManager cloudManager, String collection, String shard) throws Exception { |
| if (shard != null) { |
| String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/" + shard + "-splitting"; |
| cloudManager.getDistribStateManager().removeRecursively(path, true, true); |
| } else { |
| String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection; |
| try { |
| List<String> names = cloudManager.getDistribStateManager().listData(path); |
| for (String name : cloudManager.getDistribStateManager().listData(path)) { |
| if (name.endsWith("-splitting")) { |
| try { |
| cloudManager.getDistribStateManager().removeData(path + "/" + name, -1); |
| } catch (NoSuchElementException nse) { |
| // ignore |
| } |
| } |
| } |
| } catch (NoSuchElementException nse) { |
| // ignore |
| } |
| } |
| } |
| } |
