| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.yarn.server.timelineservice.storage.application; |
| |
| import java.io.IOException; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.hbase.HColumnDescriptor; |
| import org.apache.hadoop.hbase.HTableDescriptor; |
| import org.apache.hadoop.hbase.TableName; |
| import org.apache.hadoop.hbase.client.Admin; |
| import org.apache.hadoop.hbase.regionserver.BloomType; |
| import org.apache.hadoop.yarn.conf.YarnConfiguration; |
| import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; |
| import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants; |
| |
| /** |
| * The application table as column families info, config and metrics. Info |
| * stores information about a YARN application entity, config stores |
| * configuration data of a YARN application, metrics stores the metrics of a |
| * YARN application. This table is entirely analogous to the entity table but |
| * created for better performance. |
| * |
| * Example application table record: |
| * |
| * <pre> |
| * |-------------------------------------------------------------------------| |
| * | Row | Column Family | Column Family| Column Family| |
| * | key | info | metrics | config | |
| * |-------------------------------------------------------------------------| |
| * | clusterId! | id:appId | metricId1: | configKey1: | |
| * | userName! | | metricValue1 | configValue1 | |
| * | flowName! | created_time: | @timestamp1 | | |
| * | flowRunId! | 1392993084018 | | configKey2: | |
| * | AppId | | metriciD1: | configValue2 | |
| * | | i!infoKey: | metricValue2 | | |
| * | | infoValue | @timestamp2 | | |
| * | | | | | |
| * | | r!relatesToKey: | metricId2: | | |
| * | | id3=id4=id5 | metricValue1 | | |
| * | | | @timestamp2 | | |
| * | | s!isRelatedToKey: | | | |
| * | | id7=id9=id6 | | | |
| * | | | | | |
| * | | e!eventId=timestamp=infoKey: | | | |
| * | | eventInfoValue | | | |
| * | | | | | |
| * | | flowVersion: | | | |
| * | | versionValue | | | |
| * |-------------------------------------------------------------------------| |
| * </pre> |
| */ |
| public class ApplicationTable extends BaseTable<ApplicationTable> { |
| /** application prefix. */ |
| private static final String PREFIX = |
| YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".application"; |
| |
| /** config param name that specifies the application table name. */ |
| public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name"; |
| |
| /** |
| * config param name that specifies the TTL for metrics column family in |
| * application table. |
| */ |
| private static final String METRICS_TTL_CONF_NAME = PREFIX |
| + ".table.metrics.ttl"; |
| |
| /** default value for application table name. */ |
| private static final String DEFAULT_TABLE_NAME = |
| "timelineservice.application"; |
| |
| /** default TTL is 30 days for metrics timeseries. */ |
| private static final int DEFAULT_METRICS_TTL = 2592000; |
| |
| /** default max number of versions. */ |
| private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000; |
| |
| private static final Log LOG = LogFactory.getLog(ApplicationTable.class); |
| |
| public ApplicationTable() { |
| super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME); |
| } |
| |
| /* |
| * (non-Javadoc) |
| * |
| * @see |
| * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable |
| * (org.apache.hadoop.hbase.client.Admin, |
| * org.apache.hadoop.conf.Configuration) |
| */ |
| public void createTable(Admin admin, Configuration hbaseConf) |
| throws IOException { |
| |
| TableName table = getTableName(hbaseConf); |
| if (admin.tableExists(table)) { |
| // do not disable / delete existing table |
| // similar to the approach taken by map-reduce jobs when |
| // output directory exists |
| throw new IOException("Table " + table.getNameAsString() |
| + " already exists."); |
| } |
| |
| HTableDescriptor applicationTableDescp = new HTableDescriptor(table); |
| HColumnDescriptor infoCF = |
| new HColumnDescriptor(ApplicationColumnFamily.INFO.getBytes()); |
| infoCF.setBloomFilterType(BloomType.ROWCOL); |
| applicationTableDescp.addFamily(infoCF); |
| |
| HColumnDescriptor configCF = |
| new HColumnDescriptor(ApplicationColumnFamily.CONFIGS.getBytes()); |
| configCF.setBloomFilterType(BloomType.ROWCOL); |
| configCF.setBlockCacheEnabled(true); |
| applicationTableDescp.addFamily(configCF); |
| |
| HColumnDescriptor metricsCF = |
| new HColumnDescriptor(ApplicationColumnFamily.METRICS.getBytes()); |
| applicationTableDescp.addFamily(metricsCF); |
| metricsCF.setBlockCacheEnabled(true); |
| // always keep 1 version (the latest) |
| metricsCF.setMinVersions(1); |
| metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS); |
| metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME, |
| DEFAULT_METRICS_TTL)); |
| applicationTableDescp.setRegionSplitPolicyClassName( |
| "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy"); |
| applicationTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length", |
| TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH); |
| admin.createTable(applicationTableDescp, |
| TimelineHBaseSchemaConstants.getUsernameSplits()); |
| LOG.info("Status of table creation for " + table.getNameAsString() + "=" |
| + admin.tableExists(table)); |
| } |
| |
| /** |
| * @param metricsTTL time to live parameter for the metrics in this table. |
| * @param hbaseConf configuration in which to set the metrics TTL config |
| * variable. |
| */ |
| public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) { |
| hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL); |
| } |
| |
| } |