| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.storm.hbase.trident.state; |
| |
| import backtype.storm.topology.FailedException; |
| import backtype.storm.tuple.Values; |
| import com.google.common.collect.Lists; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.hbase.HBaseConfiguration; |
| import org.apache.hadoop.hbase.client.*; |
| import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria; |
| import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper; |
| import org.apache.storm.hbase.common.ColumnList; |
| import org.apache.storm.hbase.common.HBaseClient; |
| import org.apache.storm.hbase.trident.mapper.TridentHBaseMapper; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| import storm.trident.operation.TridentCollector; |
| import storm.trident.state.State; |
| import storm.trident.tuple.TridentTuple; |
| |
| import java.io.Serializable; |
| import java.util.List; |
| import java.util.Map; |
| |
| public class HBaseState implements State { |
| |
| private static final Logger LOG = LoggerFactory.getLogger(HBaseState.class); |
| |
| private Options options; |
| private HBaseClient hBaseClient; |
| private Map map; |
| private int numPartitions; |
| private int partitionIndex; |
| |
| protected HBaseState(Map map, int partitionIndex, int numPartitions, Options options) { |
| this.options = options; |
| this.map = map; |
| this.partitionIndex = partitionIndex; |
| this.numPartitions = numPartitions; |
| } |
| |
| public static class Options implements Serializable { |
| private TridentHBaseMapper mapper; |
| private Durability durability = Durability.SKIP_WAL; |
| private HBaseProjectionCriteria projectionCriteria; |
| private HBaseValueMapper rowToStormValueMapper; |
| private String configKey; |
| private String tableName; |
| |
| public Options withDurability(Durability durability) { |
| this.durability = durability; |
| return this; |
| } |
| |
| public Options withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) { |
| this.projectionCriteria = projectionCriteria; |
| return this; |
| } |
| |
| public Options withConfigKey(String configKey) { |
| this.configKey = configKey; |
| return this; |
| } |
| |
| public Options withTableName(String tableName) { |
| this.tableName = tableName; |
| return this; |
| } |
| |
| public Options withRowToStormValueMapper(HBaseValueMapper rowToStormValueMapper) { |
| this.rowToStormValueMapper = rowToStormValueMapper; |
| return this; |
| } |
| |
| public Options withMapper(TridentHBaseMapper mapper) { |
| this.mapper = mapper; |
| return this; |
| } |
| } |
| |
| protected void prepare() { |
| final Configuration hbConfig = HBaseConfiguration.create(); |
| Map<String, Object> conf = (Map<String, Object>) map.get(options.configKey); |
| if(conf == null){ |
| LOG.info("HBase configuration not found using key '" + options.configKey + "'"); |
| LOG.info("Using HBase config from first hbase-site.xml found on classpath."); |
| } else { |
| if (conf.get("hbase.rootdir") == null) { |
| LOG.warn("No 'hbase.rootdir' value found in configuration! Using HBase defaults."); |
| } |
| for (String key : conf.keySet()) { |
| hbConfig.set(key, String.valueOf(map.get(key))); |
| } |
| } |
| |
| this.hBaseClient = new HBaseClient(conf, hbConfig, options.tableName); |
| } |
| |
| @Override |
| public void beginCommit(Long aLong) { |
| LOG.debug("beginCommit is noop."); |
| } |
| |
| @Override |
| public void commit(Long aLong) { |
| LOG.debug("commit is noop."); |
| } |
| |
| public void updateState(List<TridentTuple> tuples, TridentCollector collector) { |
| List<Mutation> mutations = Lists.newArrayList(); |
| |
| for (TridentTuple tuple : tuples) { |
| byte[] rowKey = options.mapper.rowKey(tuple); |
| ColumnList cols = options.mapper.columns(tuple); |
| mutations.addAll(hBaseClient.constructMutationReq(rowKey, cols, options.durability)); |
| } |
| |
| try { |
| hBaseClient.batchMutate(mutations); |
| } catch (Exception e) { |
| LOG.warn("Batch write failed but some requests might have succeeded. Triggering replay.", e); |
| throw new FailedException(e); |
| } |
| } |
| |
| public List<List<Values>> batchRetrieve(List<TridentTuple> tridentTuples) { |
| List<List<Values>> batchRetrieveResult = Lists.newArrayList(); |
| List<Get> gets = Lists.newArrayList(); |
| for (TridentTuple tuple : tridentTuples) { |
| byte[] rowKey = options.mapper.rowKey(tuple); |
| gets.add(hBaseClient.constructGetRequests(rowKey, options.projectionCriteria)); |
| } |
| |
| try { |
| Result[] results = hBaseClient.batchGet(gets); |
| for(int i = 0; i < results.length; i++) { |
| Result result = results[i]; |
| TridentTuple tuple = tridentTuples.get(i); |
| List<Values> values = options.rowToStormValueMapper.toValues(tuple, result); |
| batchRetrieveResult.add(values); |
| } |
| } catch (Exception e) { |
| LOG.warn("Batch get operation failed. Triggering replay.", e); |
| throw new FailedException(e); |
| } |
| return batchRetrieveResult; |
| } |
| } |