external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicBrokersReader.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.storm.kafka;

import org.apache.storm.Config;
import org.apache.storm.utils.Utils;
import com.google.common.base.Preconditions;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.storm.kafka.trident.GlobalPartitionInformation;

import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;


public class DynamicBrokersReader {

    private static final Logger LOG = LoggerFactory.getLogger(DynamicBrokersReader.class);

    private CuratorFramework _curator;
    private String _zkPath;
    private String _topic;
    private Boolean _isWildcardTopic;

    public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) {
        // Check required parameters
        Preconditions.checkNotNull(conf, "conf cannot be null");

        validateConfig(conf);

        Preconditions.checkNotNull(zkStr,"zkString cannot be null");
        Preconditions.checkNotNull(zkPath, "zkPath cannot be null");
        Preconditions.checkNotNull(topic, "topic cannot be null");

        _zkPath = zkPath;
        _topic = topic;
        _isWildcardTopic = Utils.getBoolean(conf.get("kafka.topic.wildcard.match"), false);
        try {
            _curator = CuratorFrameworkFactory.newClient(
                    zkStr,
                    Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
                    Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)),
                    new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
                            Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
            _curator.start();
        } catch (Exception ex) {
            LOG.error("Couldn't connect to zookeeper", ex);
            throw new RuntimeException(ex);
        }
    }

    /**
     * Get all partitions with their current leaders
     */
    public List<GlobalPartitionInformation> getBrokerInfo() throws SocketTimeoutException {
      List<String> topics =  getTopics();
      List<GlobalPartitionInformation> partitions =  new ArrayList<GlobalPartitionInformation>();

      for (String topic : topics) {
          GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(topic, this._isWildcardTopic);
          try {
              int numPartitionsForTopic = getNumPartitions(topic);
              String brokerInfoPath = brokerPath();
              for (int partition = 0; partition < numPartitionsForTopic; partition++) {
                  int leader = getLeaderFor(topic,partition);
                  String path = brokerInfoPath + "/" + leader;
                  try {
                      byte[] brokerData = _curator.getData().forPath(path);
                      Broker hp = getBrokerHost(brokerData);
                      globalPartitionInformation.addPartition(partition, hp);
                  } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
                      LOG.error("Node {} does not exist ", path);
                  }
              }
          } catch (SocketTimeoutException e) {
              throw e;
          } catch (Exception e) {
              throw new RuntimeException(e);
          }
          LOG.info("Read partition info from zookeeper: " + globalPartitionInformation);
          partitions.add(globalPartitionInformation);
      }
        return partitions;
    }

    private int getNumPartitions(String topic) {
        try {
            String topicBrokersPath = partitionPath(topic);
            List<String> children = _curator.getChildren().forPath(topicBrokersPath);
            return children.size();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    private List<String> getTopics() {
        List<String> topics = new ArrayList<String>();
        if (!_isWildcardTopic) {
            topics.add(_topic);
            return topics;
        } else {
            try {
                List<String> children = _curator.getChildren().forPath(topicsPath());
                for (String t : children) {
                    if (t.matches(_topic)) {
                        LOG.info(String.format("Found matching topic %s", t));
                        topics.add(t);
                    }
                }
                return topics;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    }

    public String topicsPath () {
        return _zkPath + "/topics";
    }
    public String partitionPath(String topic) {
        return topicsPath() + "/" + topic + "/partitions";
    }

    public String brokerPath() {
        return _zkPath + "/ids";
    }



    /**
     * get /brokers/topics/distributedTopic/partitions/1/state
     * { "controller_epoch":4, "isr":[ 1, 0 ], "leader":1, "leader_epoch":1, "version":1 }
     * @param topic
     * @param partition
     * @return
     */
    private int getLeaderFor(String topic, long partition) {
        try {
            String topicBrokersPath = partitionPath(topic);
            byte[] hostPortData = _curator.getData().forPath(topicBrokersPath + "/" + partition + "/state");
            Map<Object, Object> value = (Map<Object, Object>) JSONValue.parseWithException(new String(hostPortData, "UTF-8"));
            Integer leader = ((Number) value.get("leader")).intValue();
            if (leader == -1) {
                throw new RuntimeException("No leader found for partition " + partition);
            }
            return leader;
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public void close() {
        _curator.close();
    }

    /**
     * [zk: localhost:2181(CONNECTED) 56] get /brokers/ids/0
     * { "host":"localhost", "jmx_port":9999, "port":9092, "version":1 }
     *
     * @param contents
     * @return
     */
    private Broker getBrokerHost(byte[] contents) {
        try {
            Map<Object, Object> value = (Map<Object, Object>) JSONValue.parseWithException(new String(contents, "UTF-8"));
            String host = (String) value.get("host");
            Integer port = ((Long) value.get("port")).intValue();
            return new Broker(host, port);
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * Validate required parameters in the input configuration Map
     * @param conf
     */
    private void validateConfig(final Map conf) {
        Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT),
                "%s cannot be null", Config.STORM_ZOOKEEPER_SESSION_TIMEOUT);
        Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT),
                "%s cannot be null", Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT);
        Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES),
                "%s cannot be null", Config.STORM_ZOOKEEPER_RETRY_TIMES);
        Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL),
                "%s cannot be null", Config.STORM_ZOOKEEPER_RETRY_INTERVAL);
    }

}