src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.chukwa.datacollection.writer.hbase;

import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
import org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter;
import org.apache.hadoop.chukwa.datacollection.writer.PipelineableWriter;
import org.apache.hadoop.chukwa.datacollection.writer.WriterException;
import org.apache.hadoop.chukwa.extraction.hbase.AbstractProcessor;
import org.apache.hadoop.chukwa.extraction.hbase.ProcessorFactory;
import org.apache.hadoop.chukwa.extraction.hbase.UnknownRecordTypeException;
import org.apache.hadoop.chukwa.util.ExceptionUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.log4j.Logger;

public class HBaseWriter extends PipelineableWriter {
  static Logger log = Logger.getLogger(HBaseWriter.class);
  private static final String CHUKWA_TABLE = "chukwa";
  private static final String CHUKWA_META_TABLE = "chukwa_meta";
  boolean reportStats;
  volatile long dataSize = 0;
  final Timer statTimer;
  private ArrayList<Put> output;
  private Reporter reporter;
  private ChukwaConfiguration conf;
  private Configuration hconf;
  String defaultProcessor;
  private static Connection connection;
  
  private class StatReportingTask extends TimerTask {
    private long lastTs = System.currentTimeMillis();
    private long lastDataSize = 0;

    public void run() {
      long time = System.currentTimeMillis();
      long interval = time - lastTs;
      lastTs = time;

      long ds = dataSize;
      long dataRate = 1000 * (ds - lastDataSize) / interval; // bytes/sec
      // refers only to data field, not including http or chukwa headers
      lastDataSize = ds;

      log.info("stat=HBaseWriter|dataRate="
          + dataRate);
    }
  };

  public HBaseWriter() throws IOException {
    this(true);
  }

  public HBaseWriter(boolean reportStats) throws IOException {
    /* HBase Version >= 0.89.x */
    this(reportStats, new ChukwaConfiguration(), HBaseConfiguration.create());
  }

  public HBaseWriter(ChukwaConfiguration conf, Configuration hconf) throws IOException {
    this(true, conf, hconf);
  }

  private HBaseWriter(boolean reportStats, ChukwaConfiguration conf, Configuration hconf) throws IOException {
    this.reportStats = reportStats;
    this.conf = conf;
    this.hconf = hconf;
    this.statTimer = new Timer();
    this.defaultProcessor = conf.get(
      "chukwa.demux.mapper.default.processor",
      "org.apache.hadoop.chukwa.extraction.hbase.DefaultProcessor");
    log.info("hbase.zookeeper.quorum: " + hconf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + hconf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
    if (reportStats) {
      statTimer.schedule(new StatReportingTask(), 1000, 10 * 1000);
    }
    output = new ArrayList<Put>();
    try {
      reporter = new Reporter();
    } catch (NoSuchAlgorithmException e) {
      throw new IOException("Can not register hashing algorithm.");
    }
    if (connection == null || connection.isClosed()) {
      connection = ConnectionFactory.createConnection(hconf);
    }
  }

  public void close() {
    if (reportStats) {
      statTimer.cancel();
    }
  }

  public void init(Configuration conf) throws WriterException {
    if (connection == null || connection.isClosed()) {
      try {
        connection = ConnectionFactory.createConnection(hconf);
      } catch (IOException e) {
        throw new WriterException("HBase is offline, retry later...");
      }
    }
  }

  @Override
  public CommitStatus add(List<Chunk> chunks) throws WriterException {
    CommitStatus rv = ChukwaWriter.COMMIT_OK;
    Table hbase;
    Table meta;
    try {
      if (connection == null || connection.isClosed()) {
        try {
          connection = ConnectionFactory.createConnection(hconf);
        } catch (IOException e) {
          throw new WriterException("HBase is offline, retry later...");
        }
      }
      hbase = connection.getTable(TableName.valueOf(CHUKWA_TABLE));
      meta = connection.getTable(TableName.valueOf(CHUKWA_META_TABLE));
      for(Chunk chunk : chunks) {
        synchronized (this) {
          try {
            AbstractProcessor processor = getProcessor(chunk.getDataType());
            processor.process(chunk, output, reporter);
            hbase.put(output);
            meta.put(reporter.getInfo());
          } catch (Throwable e) {
            log.warn("Unable to process data:");
            log.warn(new String(chunk.getData()));
            log.warn(ExceptionUtil.getStackTrace(e));
          }
          dataSize += chunk.getData().length;
          output.clear();
          reporter.clear();
        }
      }
      hbase.close();
      meta.close();
    } catch (Exception e) {
      log.error(ExceptionUtil.getStackTrace(e));
      if(connection != null) {
        try {
          connection.close();
        } catch(IOException e2) {
          connection = null;
          throw new WriterException("HBase connection maybe leaking.");
        }
      }
    } 
    if (next != null) {
      rv = next.add(chunks); //pass data through
    }
    return rv;
  }

  private AbstractProcessor getProcessor(String dataType) throws UnknownRecordTypeException {
    String processorClass = findProcessor(conf.get(dataType, defaultProcessor), defaultProcessor);
    return ProcessorFactory.getProcessor(processorClass);
  }

  /**
   * Look for mapper parser class in the demux configuration.
   * Demux configuration has been changed since CHUKWA-581 to
   * support mapping of both mapper and reducer, and this utility
   * class is to detect the mapper class and return the mapper
   * class only.
   *
   */
  private String findProcessor(String processors, String defaultProcessor) {
    if(processors.startsWith(",")) {
      // No mapper class defined.
      return defaultProcessor;
    } else if(processors.contains(",")) {
      // Both mapper and reducer defined.
      String[] parsers = processors.split(",");
      return parsers[0];
    }
    // No reducer defined.
    return processors;
  }
}