lucene/replicator/src/java/org/apache/lucene/replicator/nrt/CopyOneFile.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.replicator.nrt;

import java.io.Closeable;
import java.io.IOException;
import java.util.Locale;

import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;

/** Copies one file from an incoming DataInput to a dest filename in a local Directory */

public class CopyOneFile implements Closeable {
  private final DataInput in;
  private final IndexOutput out;
  private final ReplicaNode dest;
  public final String name;
  public final String tmpName;
  public final FileMetaData metaData;
  public final long bytesToCopy;
  private final long copyStartNS;
  private final byte[] buffer;

  private long bytesCopied;

  public CopyOneFile(DataInput in, ReplicaNode dest, String name, FileMetaData metaData, byte[] buffer) throws IOException {
    this.in = in;
    this.name = name;
    this.dest = dest;
    this.buffer = buffer;
    // TODO: pass correct IOCtx, e.g. seg total size
    out = dest.createTempOutput(name, "copy", IOContext.DEFAULT);
    tmpName = out.getName();

    // last 8 bytes are checksum, which we write ourselves after copying all bytes and confirming checksum:
    bytesToCopy = metaData.length - Long.BYTES;

    if (Node.VERBOSE_FILES) {
      dest.message("file " + name + ": start copying to tmp file " + tmpName + " length=" + (8+bytesToCopy));
    }

    copyStartNS = System.nanoTime();
    this.metaData = metaData;
    dest.startCopyFile(name);
  }

  /** Transfers this file copy to another input, continuing where the first one left off */
  public CopyOneFile(CopyOneFile other, DataInput in) {
    this.in = in;
    this.dest = other.dest;
    this.name = other.name;
    this.out = other.out;
    this.tmpName = other.tmpName;
    this.metaData = other.metaData;
    this.bytesCopied = other.bytesCopied;
    this.bytesToCopy = other.bytesToCopy;
    this.copyStartNS = other.copyStartNS;
    this.buffer = other.buffer;
  }

  public void close() throws IOException {
    out.close();
    dest.finishCopyFile(name);
  }

  /** Copy another chunk of bytes, returning true once the copy is done */
  public boolean visit() throws IOException {
    // Copy up to 640 KB per visit:
    for(int i=0;i<10;i++) {
      long bytesLeft = bytesToCopy - bytesCopied;
      if (bytesLeft == 0) {
        long checksum = out.getChecksum();
        if (checksum != metaData.checksum) {
          // Bits flipped during copy!
          dest.message("file " + tmpName + ": checksum mismatch after copy (bits flipped during network copy?) after-copy checksum=" + checksum + " vs expected=" + metaData.checksum + "; cancel job");
          throw new IOException("file " + name + ": checksum mismatch after file copy");
        }

        // Paranoia: make sure the primary node is not smoking crack, by somehow sending us an already corrupted file whose checksum (in its
        // footer) disagrees with reality:
        long actualChecksumIn = in.readLong();
        if (actualChecksumIn != checksum) {
          dest.message("file " + tmpName + ": checksum claimed by primary disagrees with the file's footer: claimed checksum=" + checksum + " vs actual=" + actualChecksumIn);
          throw new IOException("file " + name + ": checksum mismatch after file copy");
        }
        out.writeLong(checksum);
        bytesCopied += Long.BYTES;
        close();

        if (Node.VERBOSE_FILES) {
          dest.message(String.format(Locale.ROOT, "file %s: done copying [%s, %.3fms]",
                                     name,
                                     Node.bytesToString(metaData.length),
                                     (System.nanoTime() - copyStartNS)/1000000.0));
        }

        return true;
      }

      int toCopy = (int) Math.min(bytesLeft, buffer.length);
      in.readBytes(buffer, 0, toCopy);
      out.writeBytes(buffer, 0, toCopy);

      // TODO: rsync will fsync a range of the file; maybe we should do that here for large files in case we crash/killed
      bytesCopied += toCopy;
    }

    return false;
  }

  public long getBytesCopied() {
    return bytesCopied;
  }
}