| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hcatalog.common; |
| |
| import org.apache.hadoop.hive.conf.HiveConf; |
| import org.apache.hadoop.mapred.SequenceFileInputFormat; |
| import org.apache.hadoop.mapred.SequenceFileOutputFormat; |
| |
| public final class HCatConstants { |
| |
| public static final String HIVE_RCFILE_IF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileInputFormat"; |
| public static final String HIVE_RCFILE_OF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileOutputFormat"; |
| |
| public static final String SEQUENCEFILE_INPUT = SequenceFileInputFormat.class.getName(); |
| public static final String SEQUENCEFILE_OUTPUT = SequenceFileOutputFormat.class.getName(); |
| |
| public static final String HCAT_PIG_STORAGE_CLASS = "org.apache.pig.builtin.PigStorage"; |
| public static final String HCAT_PIG_LOADER = "hcat.pig.loader"; |
| public static final String HCAT_PIG_LOADER_LOCATION_SET = HCAT_PIG_LOADER + ".location.set" ; |
| public static final String HCAT_PIG_LOADER_ARGS = "hcat.pig.loader.args"; |
| public static final String HCAT_PIG_STORER = "hcat.pig.storer"; |
| public static final String HCAT_PIG_STORER_ARGS = "hcat.pig.storer.args"; |
| public static final String HCAT_PIG_ARGS_DELIMIT = "hcat.pig.args.delimiter"; |
| public static final String HCAT_PIG_ARGS_DELIMIT_DEFAULT = ","; |
| public static final String HCAT_PIG_STORER_LOCATION_SET = HCAT_PIG_STORER + ".location.set" ; |
| public static final String HCAT_PIG_INNER_TUPLE_NAME = "hcat.pig.inner.tuple.name"; |
| public static final String HCAT_PIG_INNER_TUPLE_NAME_DEFAULT = "innertuple"; |
| public static final String HCAT_PIG_INNER_FIELD_NAME = "hcat.pig.inner.field.name"; |
| public static final String HCAT_PIG_INNER_FIELD_NAME_DEFAULT = "innerfield"; |
| |
| //The keys used to store info into the job Configuration |
| public static final String HCAT_KEY_BASE = "mapreduce.lib.hcat"; |
| |
| public static final String HCAT_KEY_OUTPUT_SCHEMA = HCAT_KEY_BASE + ".output.schema"; |
| |
| public static final String HCAT_KEY_JOB_INFO = HCAT_KEY_BASE + ".job.info"; |
| |
| // hcatalog specific configurations, that can be put in hive-site.xml |
| public static final String HCAT_HIVE_CLIENT_EXPIRY_TIME = "hcatalog.hive.client.cache.expiry.time"; |
| |
| private HCatConstants() { // restrict instantiation |
| } |
| |
| public static final String HCAT_TABLE_SCHEMA = "hcat.table.schema"; |
| |
| public static final String HCAT_METASTORE_URI = HiveConf.ConfVars.METASTOREURIS.varname; |
| |
| public static final String HCAT_PERMS = "hcat.perms"; |
| |
| public static final String HCAT_GROUP = "hcat.group"; |
| |
| public static final String HCAT_CREATE_TBL_NAME = "hcat.create.tbl.name"; |
| |
| public static final String HCAT_CREATE_DB_NAME = "hcat.create.db.name"; |
| |
| public static final String HCAT_METASTORE_PRINCIPAL |
| = HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname; |
| |
| // IMPORTANT IMPORTANT IMPORTANT!!!!! |
| //The keys used to store info into the job Configuration. |
| //If any new keys are added, the HCatStorer needs to be updated. The HCatStorer |
| //updates the job configuration in the backend to insert these keys to avoid |
| //having to call setOutput from the backend (which would cause a metastore call |
| //from the map jobs) |
| public static final String HCAT_KEY_OUTPUT_BASE = "mapreduce.lib.hcatoutput"; |
| public static final String HCAT_KEY_OUTPUT_INFO = HCAT_KEY_OUTPUT_BASE + ".info"; |
| public static final String HCAT_KEY_HIVE_CONF = HCAT_KEY_OUTPUT_BASE + ".hive.conf"; |
| public static final String HCAT_KEY_TOKEN_SIGNATURE = HCAT_KEY_OUTPUT_BASE + ".token.sig"; |
| |
| public static final String[] OUTPUT_CONFS_TO_SAVE = { |
| HCAT_KEY_OUTPUT_INFO, |
| HCAT_KEY_HIVE_CONF, |
| HCAT_KEY_TOKEN_SIGNATURE |
| }; |
| |
| |
| public static final String HCAT_MSG_CLEAN_FREQ = "hcat.msg.clean.freq"; |
| public static final String HCAT_MSG_EXPIRY_DURATION = "hcat.msg.expiry.duration"; |
| |
| public static final String HCAT_MSGBUS_TOPIC_NAME = "hcat.msgbus.topic.name"; |
| public static final String HCAT_MSGBUS_TOPIC_NAMING_POLICY = "hcat.msgbus.topic.naming.policy"; |
| public static final String HCAT_MSGBUS_TOPIC_PREFIX = "hcat.msgbus.topic.prefix"; |
| |
| public static final String HCAT_DYNAMIC_PTN_JOBID = HCAT_KEY_OUTPUT_BASE + "dynamic.jobid"; |
| public static final boolean HCAT_IS_DYNAMIC_MAX_PTN_CHECK_ENABLED = false; |
| |
| // Message Bus related properties. |
| public static final String HCAT_DEFAULT_TOPIC_PREFIX = "hcat"; |
| public static final String HCAT_EVENT = "HCAT_EVENT"; |
| public static final String HCAT_ADD_PARTITION_EVENT = "HCAT_ADD_PARTITION"; |
| public static final String HCAT_DROP_PARTITION_EVENT = "HCAT_DROP_PARTITION"; |
| public static final String HCAT_PARTITION_DONE_EVENT = "HCAT_PARTITION_DONE"; |
| public static final String HCAT_ADD_TABLE_EVENT = "HCAT_ADD_TABLE"; |
| public static final String HCAT_DROP_TABLE_EVENT = "HCAT_DROP_TABLE"; |
| public static final String HCAT_ADD_DATABASE_EVENT = "HCAT_ADD_DATABASE"; |
| public static final String HCAT_DROP_DATABASE_EVENT = "HCAT_DROP_DATABASE"; |
| |
| // System environment variables |
| public static final String SYSENV_HADOOP_TOKEN_FILE_LOCATION = "HADOOP_TOKEN_FILE_LOCATION"; |
| |
| // Hadoop Conf Var Names |
| public static final String CONF_MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; |
| |
| //*************************************************************************** |
| // Data-related configuration properties. |
| //*************************************************************************** |
| |
| /** |
| * {@value} (default: {@value #HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT}). |
| * Pig < 0.10.0 does not have boolean support, and scripts written for pre-boolean Pig versions |
| * will not expect boolean values when upgrading Pig. For integration the option is offered to |
| * convert boolean fields to integers by setting this Hadoop configuration key. |
| */ |
| public static final String HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER = |
| "hcat.data.convert.boolean.to.integer"; |
| public static final boolean HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT = false; |
| |
| /** |
| * {@value} (default: {@value #HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT}). |
| * Hive tables support tinyint and smallint columns, while not all processing frameworks support |
| * these types (Pig only has integer for example). Enable this property to promote tinyint and |
| * smallint columns to integer at runtime. Note that writes to tinyint and smallint columns |
| * enforce bounds checking and jobs will fail if attempting to write values outside the column |
| * bounds. |
| */ |
| public static final String HCAT_DATA_TINY_SMALL_INT_PROMOTION = |
| "hcat.data.tiny.small.int.promotion"; |
| public static final boolean HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT = false; |
| |
| /** |
| * {@value} (default: {@value #HCAT_INPUT_BAD_RECORD_THRESHOLD_DEFAULT}). |
| * Threshold for the ratio of bad records that will be silently skipped without causing a task |
| * failure. This is useful when processing large data sets with corrupt records, when its |
| * acceptable to skip some bad records. |
| */ |
| public static final String HCAT_INPUT_BAD_RECORD_THRESHOLD_KEY = "hcat.input.bad.record.threshold"; |
| public static final float HCAT_INPUT_BAD_RECORD_THRESHOLD_DEFAULT = 0.0001f; |
| |
| /** |
| * {@value} (default: {@value #HCAT_INPUT_BAD_RECORD_MIN_DEFAULT}). |
| * Number of bad records that will be accepted before applying |
| * {@value #HCAT_INPUT_BAD_RECORD_THRESHOLD_KEY}. This is necessary to prevent an initial bad |
| * record from causing a task failure. |
| */ |
| public static final String HCAT_INPUT_BAD_RECORD_MIN_KEY = "hcat.input.bad.record.min"; |
| public static final int HCAT_INPUT_BAD_RECORD_MIN_DEFAULT = 2; |
| } |