core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.testing.core.continuous;

import static java.nio.charset.StandardCharsets.UTF_8;

import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.TimeUnit;

import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.io.Text;

import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;

public class ContinuousScanner {

  public static void main(String[] args) throws Exception {

    Properties props = TestProps.loadFromFile(args[0]);
    ContinuousEnv env = new ContinuousEnv(props);

    Random r = new Random();

    long distance = 1000000000000l;

    Connector conn = env.getAccumuloConnector();
    Authorizations auths = env.getRandomAuthorizations();
    Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), auths);
    scanner.setBatchSize(env.getScannerBatchSize());

    int numToScan = Integer.parseInt(props.getProperty(TestProps.CI_SCANNER_ENTRIES));
    int scannerSleepMs = Integer.parseInt(props.getProperty(TestProps.CI_SCANNER_SLEEP_MS));

    double delta = Math.min(.05, .05 / (numToScan / 1000.0));

    while (true) {
      long startRow = ContinuousIngest.genLong(env.getRowMin(), env.getRowMax() - distance, r);
      byte[] scanStart = ContinuousIngest.genRow(startRow);
      byte[] scanStop = ContinuousIngest.genRow(startRow + distance);

      scanner.setRange(new Range(new Text(scanStart), new Text(scanStop)));

      int count = 0;
      Iterator<Entry<Key,Value>> iter = scanner.iterator();

      long t1 = System.currentTimeMillis();

      while (iter.hasNext()) {
        Entry<Key,Value> entry = iter.next();
        ContinuousWalk.validate(entry.getKey(), entry.getValue());
        count++;
      }

      long t2 = System.currentTimeMillis();

      // System.out.println("P1 " +count +" "+((1-delta) * numToScan)+" "+((1+delta) * numToScan)+" "+numToScan);

      if (count < (1 - delta) * numToScan || count > (1 + delta) * numToScan) {
        if (count == 0) {
          distance = distance * 10;
          if (distance < 0)
            distance = 1000000000000l;
        } else {
          double ratio = (double) numToScan / count;
          // move ratio closer to 1 to make change slower
          ratio = ratio - (ratio - 1.0) * (2.0 / 3.0);
          distance = (long) (ratio * distance);
        }

        // System.out.println("P2 "+delta +" "+numToScan+" "+distance+"  "+((double)numToScan/count ));
      }

      System.out.printf("SCN %d %s %d %d%n", t1, new String(scanStart, UTF_8), (t2 - t1), count);

      if (scannerSleepMs > 0) {
        sleepUninterruptibly(scannerSleepMs, TimeUnit.MILLISECONDS);
      }
    }

  }
}