| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.handler; |
| |
| import com.fasterxml.jackson.annotation.JsonProperty; |
| import io.swagger.v3.oas.annotations.media.Schema; |
| import java.io.IOException; |
| import java.lang.invoke.MethodHandles; |
| import java.net.URI; |
| import java.nio.file.Paths; |
| import java.text.SimpleDateFormat; |
| import java.time.Instant; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.Date; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Objects; |
| import java.util.Optional; |
| import java.util.function.Consumer; |
| import org.apache.lucene.index.IndexCommit; |
| import org.apache.lucene.store.Directory; |
| import org.apache.solr.client.api.model.SolrJerseyResponse; |
| import org.apache.solr.common.SolrException; |
| import org.apache.solr.common.SolrException.ErrorCode; |
| import org.apache.solr.common.util.NamedList; |
| import org.apache.solr.common.util.SimpleOrderedMap; |
| import org.apache.solr.core.DirectoryFactory.DirContext; |
| import org.apache.solr.core.IndexDeletionPolicyWrapper; |
| import org.apache.solr.core.SolrCore; |
| import org.apache.solr.core.backup.repository.BackupRepository; |
| import org.apache.solr.core.backup.repository.BackupRepository.PathType; |
| import org.apache.solr.core.backup.repository.LocalFileSystemRepository; |
| import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; |
| import org.apache.solr.handler.api.V2ApiUtils; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| /** |
| * Provides functionality equivalent to the snapshooter script This is no longer used in standard |
| * replication. |
| * |
| * @since solr 1.4 |
| */ |
| public class SnapShooter { |
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); |
| private SolrCore solrCore; |
| private String snapshotName = null; |
| private String directoryName = null; |
| private URI baseSnapDirPath = null; |
| private URI snapshotDirPath = null; |
| private BackupRepository backupRepo = null; |
| private String commitName; // can be null |
| |
| @Deprecated |
| public SnapShooter(SolrCore core, String location, String snapshotName) { |
| String snapDirStr = null; |
| // Note - This logic is only applicable to the usecase where a shared file-system is exposed via |
| // local file-system interface (primarily for backwards compatibility). For other use-cases, |
| // users will be required to specify "location" where the backup should be stored. |
| if (location == null) { |
| snapDirStr = core.getDataDir(); |
| } else { |
| snapDirStr = |
| core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); |
| } |
| initialize( |
| new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null); |
| } |
| |
| public SnapShooter( |
| BackupRepository backupRepo, |
| SolrCore core, |
| URI location, |
| String snapshotName, |
| String commitName) { |
| initialize(backupRepo, core, location, snapshotName, commitName); |
| } |
| |
| private void initialize( |
| BackupRepository backupRepo, |
| SolrCore core, |
| URI location, |
| String snapshotName, |
| String commitName) { |
| this.solrCore = Objects.requireNonNull(core); |
| this.backupRepo = Objects.requireNonNull(backupRepo); |
| this.baseSnapDirPath = location; |
| this.snapshotName = snapshotName; |
| if ("file".equals(location.getScheme())) { |
| solrCore.getCoreContainer().assertPathAllowed(Paths.get(location)); |
| } |
| if (snapshotName != null) { |
| directoryName = "snapshot." + snapshotName; |
| } else { |
| SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT); |
| directoryName = "snapshot." + fmt.format(new Date()); |
| } |
| this.snapshotDirPath = backupRepo.resolveDirectory(location, directoryName); |
| this.commitName = commitName; |
| } |
| |
| public BackupRepository getBackupRepository() { |
| return backupRepo; |
| } |
| |
| /** |
| * Gets the parent directory of the snapshots. This is the {@code location} given in the |
| * constructor. |
| */ |
| public URI getLocation() { |
| return this.baseSnapDirPath; |
| } |
| |
| public void validateDeleteSnapshot() { |
| Objects.requireNonNull(this.snapshotName); |
| |
| boolean dirFound = false; |
| String[] paths; |
| try { |
| paths = backupRepo.listAll(baseSnapDirPath); |
| for (String path : paths) { |
| if (path.equals(this.directoryName) |
| && backupRepo.getPathType(backupRepo.resolveDirectory(baseSnapDirPath, path)) |
| == PathType.DIRECTORY) { |
| dirFound = true; |
| break; |
| } |
| } |
| if (dirFound == false) { |
| throw new SolrException( |
| SolrException.ErrorCode.BAD_REQUEST, |
| "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath); |
| } |
| } catch (IOException e) { |
| throw new SolrException( |
| SolrException.ErrorCode.SERVER_ERROR, |
| "Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, |
| e); |
| } |
| } |
| |
| protected void deleteSnapAsync(final ReplicationHandler replicationHandler) { |
| new Thread(() -> deleteNamedSnapshot(replicationHandler), "DeleteNamedSnapshot").start(); |
| } |
| |
| public void validateCreateSnapshot() throws IOException { |
| // Note - Removed the current behavior of creating the directory hierarchy. |
| // Do we really need to provide this support? |
| if (!backupRepo.exists(baseSnapDirPath)) { |
| throw new SolrException( |
| SolrException.ErrorCode.BAD_REQUEST, " Directory does not exist: " + snapshotDirPath); |
| } |
| |
| if (backupRepo.exists(snapshotDirPath)) { |
| throw new SolrException( |
| SolrException.ErrorCode.BAD_REQUEST, |
| "Snapshot directory already exists: " + snapshotDirPath); |
| } |
| } |
| |
| public CoreSnapshotResponse createSnapshot() throws Exception { |
| final IndexCommit indexCommit = getAndSaveIndexCommit(); |
| try { |
| return createSnapshot(indexCommit); |
| } finally { |
| solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration()); |
| } |
| } |
| |
| /** |
| * If {@link #commitName} is non-null, then fetches the generation from the {@link |
| * SolrSnapshotMetaDataManager} and then returns {@link |
| * IndexDeletionPolicyWrapper#getAndSaveCommitPoint}, otherwise it returns {@link |
| * IndexDeletionPolicyWrapper#getAndSaveLatestCommit}. |
| * |
| * <p>Either way: |
| * |
| * <ul> |
| * <li>This method does error handling for all cases where the commit can't be found and wraps |
| * them in {@link SolrException} |
| * <li>If this method returns, the result will be non null, and the caller <em>MUST</em> call |
| * {@link IndexDeletionPolicyWrapper#releaseCommitPoint} when finished |
| * </ul> |
| */ |
| private IndexCommit getAndSaveIndexCommit() throws IOException { |
| final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy(); |
| if (null != commitName) { |
| return getAndSaveNamedIndexCommit(solrCore, commitName); |
| } |
| // else: not a named commit... |
| final IndexCommit commit = delPolicy.getAndSaveLatestCommit(); |
| if (null == commit) { |
| throw new SolrException( |
| ErrorCode.BAD_REQUEST, |
| "Index does not yet have any commits for core " + solrCore.getName()); |
| } |
| if (log.isDebugEnabled()) { |
| log.debug("Using latest commit: generation={}", commit.getGeneration()); |
| } |
| return commit; |
| } |
| |
| public static IndexCommit getAndSaveNamedIndexCommit(SolrCore solrCore, String commitName) |
| throws IOException { |
| final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy(); |
| final SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager(); |
| // We're going to tell the delPolicy to "save" this commit -- even though it's a named snapshot |
| // that will already be protected -- just in case another thread deletes the name. |
| // Because of this, we want to sync on the delPolicy to ensure there is no window of time after |
| // snapshotMgr confirms commitName exists, but before we have a chance to 'save' it, when |
| // the commitName might be deleted *and* the IndexWriter might call onCommit() |
| synchronized (delPolicy) { |
| final Optional<IndexCommit> namedCommit = snapshotMgr.getIndexCommitByName(commitName); |
| if (namedCommit.isPresent()) { |
| final IndexCommit commit = namedCommit.get(); |
| if (log.isDebugEnabled()) { |
| log.debug( |
| "Using named commit: name={}, generation={}", commitName, commit.getGeneration()); |
| } |
| delPolicy.saveCommitPoint(commit.getGeneration()); |
| return commit; |
| } |
| } // else... |
| throw new SolrException( |
| ErrorCode.BAD_REQUEST, |
| "Unable to find an index commit with name " |
| + commitName |
| + " for core " |
| + solrCore.getName()); |
| } |
| |
| public void createSnapAsync(final int numberToKeep, Consumer<NamedList<?>> result) |
| throws IOException { |
| // TODO should use Solr's ExecutorUtil |
| new Thread( |
| () -> { |
| NamedList<Object> snapShootDetails = new SimpleOrderedMap<>(); |
| try { |
| V2ApiUtils.squashIntoNamedListWithoutHeader(snapShootDetails, createSnapshot()); |
| } catch (Exception e) { |
| log.error("Exception while creating snapshot", e); |
| snapShootDetails = new SimpleOrderedMap<>(); |
| snapShootDetails.add("exception", e.getMessage()); |
| } |
| if (snapshotName == null) { |
| try { |
| deleteOldBackups(numberToKeep); |
| } catch (IOException e) { |
| log.warn("Unable to delete old snapshots ", e); |
| } |
| } |
| if (null != snapShootDetails) result.accept(snapShootDetails); |
| }, |
| "CreateSnapshot") |
| .start(); |
| } |
| |
| /** |
| * Handles the logic of creating a snapshot |
| * |
| * <p><b>NOTE:</b> The caller <em>MUST</em> ensure that the {@link IndexCommit} is saved prior to |
| * calling this method, and released after calling this method, or there is no no garuntee that |
| * the method will function correctly. |
| * |
| * @see IndexDeletionPolicyWrapper#saveCommitPoint |
| * @see IndexDeletionPolicyWrapper#releaseCommitPoint |
| */ |
| protected CoreSnapshotResponse createSnapshot(final IndexCommit indexCommit) throws Exception { |
| assert indexCommit != null; |
| if (log.isInfoEnabled()) { |
| log.info( |
| "Creating backup snapshot {} at {}", |
| (snapshotName == null ? "<not named>" : snapshotName), |
| baseSnapDirPath); |
| } |
| boolean success = false; |
| try { |
| CoreSnapshotResponse details = new CoreSnapshotResponse(); |
| details.startTime = Instant.now().toString(); |
| |
| Collection<String> files = indexCommit.getFileNames(); |
| Directory dir = |
| solrCore |
| .getDirectoryFactory() |
| .get( |
| solrCore.getIndexDir(), |
| DirContext.DEFAULT, |
| solrCore.getSolrConfig().indexConfig.lockType); |
| try { |
| for (String fileName : files) { |
| log.debug( |
| "Copying fileName={} from dir={} to snapshot={}", fileName, dir, snapshotDirPath); |
| backupRepo.copyFileFrom(dir, fileName, snapshotDirPath); |
| } |
| } finally { |
| solrCore.getDirectoryFactory().release(dir); |
| } |
| |
| String endTime = Instant.now().toString(); |
| |
| // DEPRECATED: fileCount for removal, replaced with indexFileCount |
| details.fileCount = files.size(); |
| details.indexFileCount = files.size(); |
| details.status = "success"; |
| details.snapshotCompletedAt = endTime; // DEPRECATED: for removal, replaced with endTime |
| details.endTime = endTime; |
| details.snapshotName = snapshotName; |
| details.directoryName = directoryName; |
| if (log.isInfoEnabled()) { |
| log.info( |
| "Done creating backup snapshot: {} into {}", |
| (snapshotName == null ? "<not named>" : snapshotName), |
| snapshotDirPath); |
| } |
| success = true; |
| return details; |
| } finally { |
| if (!success) { |
| try { |
| backupRepo.deleteDirectory(snapshotDirPath); |
| } catch (Exception excDuringDelete) { |
| log.warn( |
| "Failed to delete {} after snapshot creation failed due to: {}", |
| snapshotDirPath, |
| excDuringDelete); |
| } |
| } |
| } |
| } |
| |
| private void deleteOldBackups(int numberToKeep) throws IOException { |
| String[] paths = backupRepo.listAll(baseSnapDirPath); |
| List<OldBackupDirectory> dirs = new ArrayList<>(); |
| for (String f : paths) { |
| if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY) { |
| OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f); |
| if (obd.getTimestamp().isPresent()) { |
| dirs.add(obd); |
| } |
| } |
| } |
| if (numberToKeep > dirs.size() - 1) { |
| return; |
| } |
| Collections.sort(dirs); |
| int i = 1; |
| for (OldBackupDirectory dir : dirs) { |
| if (i++ > numberToKeep) { |
| backupRepo.deleteDirectory(dir.getPath()); |
| } |
| } |
| } |
| |
| protected void deleteNamedSnapshot(ReplicationHandler replicationHandler) { |
| log.info("Deleting snapshot: {}", snapshotName); |
| |
| NamedList<Object> details = new NamedList<>(); |
| details.add("startTime", Instant.now().toString()); |
| details.add("snapshotName", snapshotName); |
| |
| try { |
| URI path = baseSnapDirPath.resolve("snapshot." + snapshotName); |
| backupRepo.deleteDirectory(path); |
| |
| String endTime = Instant.now().toString(); |
| |
| details.add("status", "success"); |
| details.add("snapshotDeletedAt", endTime); // DEPRECATED: for removal, replaced with endTime |
| details.add("endTime", endTime); |
| |
| } catch (IOException e) { |
| details.add("status", "Unable to delete snapshot: " + snapshotName); |
| log.warn("Unable to delete snapshot: {}", snapshotName, e); |
| } |
| |
| replicationHandler.snapShootDetails = details; |
| } |
| |
| public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; |
| |
| public static class CoreSnapshotResponse extends SolrJerseyResponse { |
| @Schema(description = "The time at which snapshot started at.") |
| @JsonProperty("startTime") |
| public String startTime; |
| |
| @Schema(description = "The number of files in the snapshot.") |
| @JsonProperty("fileCount") |
| public int fileCount; |
| |
| @Schema(description = "The number of index files in the snapshot.") |
| @JsonProperty("indexFileCount") |
| public int indexFileCount; |
| |
| @Schema(description = "The status of the snapshot") |
| @JsonProperty("status") |
| public String status; |
| |
| @Schema(description = "The time at which snapshot completed at.") |
| @JsonProperty("snapshotCompletedAt") |
| public String snapshotCompletedAt; |
| |
| @Schema(description = "The time at which snapshot completed at.") |
| @JsonProperty("endTime") |
| public String endTime; |
| |
| @Schema(description = "The name of the snapshot") |
| @JsonProperty("snapshotName") |
| public String snapshotName; |
| |
| @Schema(description = "The name of the directory where snapshot created.") |
| @JsonProperty("directoryName") |
| public String directoryName; |
| } |
| } |
