conf/accumulo-testing.properties - accumulo-testing - Git at Google

 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #   https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
 #

 ###################
 # Common properties
 ###################

 # HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
 test.common.hdfs.root=hdfs://localhost:8020
 # YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml
 test.common.yarn.resource.manager=localhost
 # Memory (in MB) given to each container (if running in YARN)
 test.common.yarn.container.memory.mb=1024
 # Number of cores given to each container (if running in YARN)
 test.common.yarn.container.cores=1

 ###################################
 # Continuous ingest test properties
 ###################################

 # Common
 # ------
 # Accumulo table used by continuous tests
 test.ci.common.accumulo.table=ci
 # Number of tablets that should exist in Accumulo table when created
 test.ci.common.accumulo.num.tablets=20
 # Optional authorizations that if specified will be randomly selected by scanners and walkers
 # Format: a,b|a,b,c|c
 test.ci.common.auths=
 # Accumulo tserver properties to set when creating a table
 test.ci.common.accumulo.server.props=\
 compaction.service.cs1.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \
 compaction.service.cs1.planner.opts.executors=\
 [{"name":"small","type":"internal","maxSize":"16M","numThreads":8},\
 {"name":"medium","type":"internal","maxSize":"128M","numThreads":4},\
 {"name":"large","type":"internal","numThreads":2}]

 # Accumulo table properties to set when creating table
 test.ci.common.accumulo.table.props=\
 table.compaction.dispatcher=org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher \
 table.compaction.dispatcher.opts.service=cs1

 # Ingest
 # ------
 # Number of entries each ingest client should write
 test.ci.ingest.client.entries=9223372036854775807
 # Flush batch writer after this many entries.
 test.ci.ingest.entries.flush=1000000
 # Minimum random row to generate
 test.ci.ingest.row.min=0
 # Maximum random row to generate
 test.ci.ingest.row.max=9223372036854775807
 # Maximum number of random column families to generate
 test.ci.ingest.max.cf=32767
 # Maximum number of random column qualifiers to generate
 test.ci.ingest.max.cq=32767
 # Optional visibilities (in CSV format) that if specified will be randomly selected by ingesters for
 # each linked list
 test.ci.ingest.visibilities=
 # Checksums will be generated during ingest if set to true
 test.ci.ingest.checksum=true
 # Enables periodic pausing of ingest. Pause checks are only done after a flush. To write small
 # amounts of data and then pause, set pause.wait.max and entries.flush small.
 test.ci.ingest.pause.enabled=false
 # Minimum wait between ingest pauses (in seconds)
 test.ci.ingest.pause.wait.min=120
 # Maximum wait between ingest pauses (in seconds)
 test.ci.ingest.pause.wait.max=180
 # Minimum pause duration (in seconds)
 test.ci.ingest.pause.duration.min=60
 # Maximum pause duration (in seconds)
 test.ci.ingest.pause.duration.max=120
 # The probability (between 0.0 and 1.0) that a set of entries will be deleted during continuous ingest
 # To disable deletes, set probability to 0.0
 test.ci.ingest.delete.probability=0.1

 # Batch walker
 # ------------
 # Sleep time between batch scans (in ms)
 test.ci.batch.walker.sleep.ms=10000
 # Scan batch size
 test.ci.batch.walker.batch.size=10000
 # Consistency Level (immediate or eventual)
 test.ci.batch.walker.consistency.level=immediate

 # Walker
 # ------
 # Sleep time between scans (in ms)
 test.ci.walker.sleep.ms=10000
 # Consistency Level (immediate or eventual)
 test.ci.walker.consistency.level=immediate

 # Scanner
 # -------
 # Sleep time between scans (in ms)
 test.ci.scanner.sleep.ms=10000
 # Scanner entries
 test.ci.scanner.entries=5000
 # Consistency Level (immediate or eventual)
 test.ci.scanner.consistency.level=immediate

 # Verify
 # -----
 # Maximum number of mapreduce mappers
 test.ci.verify.max.maps=4096
 # Number of mapreduce reducers
 test.ci.verify.reducers=64
 # Consistency Level (immediate or eventual)
 test.ci.verify.scan.consistency.level=immediate
 # Perform the verification directly on the files while the table is offline
 test.ci.verify.scan.offline=false
 # Comma separated list of auths to use for verify
 test.ci.verify.auths=
 # Location in HDFS to store output. Must not exist.
 test.ci.verify.output.dir=/tmp/ci-verify

 # Bulk Ingest
 # -----------
 # The number of map task to run.
 test.ci.bulk.map.task=10
 # The number of nodes to generate per map task.
 test.ci.bulk.map.nodes=1000000
 # The number of reducers will be the minimum of this prop and table splits+1.  Each reducer will
 # produce a bulk import file.
 test.ci.bulk.reducers.max=1024

 # Splits Scaling
 # -----------
 # The number of tables to create
 test.ci.split.table.count=3
 # Minimum random row to generate
 test.ci.split.ingest.row.min=0
 # Maximum random row to generate
 test.ci.split.ingest.row.max=9223372036854775807
 # Maximum number of random column families to generate
 test.ci.split.ingest.max.cf=32767
 # Maximum number of random column qualifiers to generate
 test.ci.split.ingest.max.cq=32767
 # The number of tablets to create on each table on table creation
 test.ci.split.initial.tablets=1
 # The amount of data to write to each table
 test.ci.split.write.size=10000000
 # The split threshold to set for each table on creation
 test.ci.split.threshold=1G
 # The factor to reduce the split threshold by for each iteration of the test
 test.ci.split.threshold.reduction.factor=10
 # Number of rounds to run the test
 test.ci.split.test.rounds=3

 ###############################
 # Garbage Collection Simulation
 ###############################

 # Name of Accumulo table to use for test
 test.gcs.table=gcs
 # Max number of buckets for references.  Buckets correspond to tablets in the Accumulo GC.
 test.gcs.maxBuckets=100000
 # Split each data section with this many tablets when creating table.
 test.gcs.tablets=10
 # Total number of work list the generator should create before it exits.
 test.gcs.maxWork=100000000
 # Max number of work list a generator should be concurrently working on.
 test.gcs.maxActiveWork=10000
 # Number of entries collector and verifier will read into memory.
 test.gcs.batchSize=100000

 #################
 # MapReduce Tests
 #################

 # RowHash test
 # ------------
 # Table containing input data
 test.rowhash.input.table = terasort
 # Table where data will be output to
 test.rowhash.output.table = rowhash
 # Column that is fetched in input table
 test.rowhash.column = c

 # TeraSort ingest
 # ---------------
 # Table to ingest into
 test.terasort.table = terasort
 # Number of rows to ingest
 test.terasort.num.rows = 10000
 # Minimum key size
 test.terasort.min.keysize = 10
 # Maximum key size
 test.terasort.max.keysize = 10
 # Minimum value size
 test.terasort.min.valuesize = 78
 # Maximum value size
 test.terasort.max.valuesize = 78
 # Number of table splits
 test.terasort.num.splits = 4
	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#

	###################
	# Common properties
	###################

	# HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
	test.common.hdfs.root=hdfs://localhost:8020
	# YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml
	test.common.yarn.resource.manager=localhost
	# Memory (in MB) given to each container (if running in YARN)
	test.common.yarn.container.memory.mb=1024
	# Number of cores given to each container (if running in YARN)
	test.common.yarn.container.cores=1

	###################################
	# Continuous ingest test properties
	###################################

	# Common
	# ------
	# Accumulo table used by continuous tests
	test.ci.common.accumulo.table=ci
	# Number of tablets that should exist in Accumulo table when created
	test.ci.common.accumulo.num.tablets=20
	# Optional authorizations that if specified will be randomly selected by scanners and walkers
	# Format: a,b\|a,b,c\|c
	test.ci.common.auths=
	# Accumulo tserver properties to set when creating a table
	test.ci.common.accumulo.server.props=\
	compaction.service.cs1.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \
	compaction.service.cs1.planner.opts.executors=\
	[{"name":"small","type":"internal","maxSize":"16M","numThreads":8},\
	{"name":"medium","type":"internal","maxSize":"128M","numThreads":4},\
	{"name":"large","type":"internal","numThreads":2}]

	# Accumulo table properties to set when creating table
	test.ci.common.accumulo.table.props=\
	table.compaction.dispatcher=org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher \
	table.compaction.dispatcher.opts.service=cs1

	# Ingest
	# ------
	# Number of entries each ingest client should write
	test.ci.ingest.client.entries=9223372036854775807
	# Flush batch writer after this many entries.
	test.ci.ingest.entries.flush=1000000
	# Minimum random row to generate
	test.ci.ingest.row.min=0
	# Maximum random row to generate
	test.ci.ingest.row.max=9223372036854775807
	# Maximum number of random column families to generate
	test.ci.ingest.max.cf=32767
	# Maximum number of random column qualifiers to generate
	test.ci.ingest.max.cq=32767
	# Optional visibilities (in CSV format) that if specified will be randomly selected by ingesters for
	# each linked list
	test.ci.ingest.visibilities=
	# Checksums will be generated during ingest if set to true
	test.ci.ingest.checksum=true
	# Enables periodic pausing of ingest. Pause checks are only done after a flush. To write small
	# amounts of data and then pause, set pause.wait.max and entries.flush small.
	test.ci.ingest.pause.enabled=false
	# Minimum wait between ingest pauses (in seconds)
	test.ci.ingest.pause.wait.min=120
	# Maximum wait between ingest pauses (in seconds)
	test.ci.ingest.pause.wait.max=180
	# Minimum pause duration (in seconds)
	test.ci.ingest.pause.duration.min=60
	# Maximum pause duration (in seconds)
	test.ci.ingest.pause.duration.max=120
	# The probability (between 0.0 and 1.0) that a set of entries will be deleted during continuous ingest
	# To disable deletes, set probability to 0.0
	test.ci.ingest.delete.probability=0.1

	# Batch walker
	# ------------
	# Sleep time between batch scans (in ms)
	test.ci.batch.walker.sleep.ms=10000
	# Scan batch size
	test.ci.batch.walker.batch.size=10000
	# Consistency Level (immediate or eventual)
	test.ci.batch.walker.consistency.level=immediate

	# Walker
	# ------
	# Sleep time between scans (in ms)
	test.ci.walker.sleep.ms=10000
	# Consistency Level (immediate or eventual)
	test.ci.walker.consistency.level=immediate

	# Scanner
	# -------
	# Sleep time between scans (in ms)
	test.ci.scanner.sleep.ms=10000
	# Scanner entries
	test.ci.scanner.entries=5000
	# Consistency Level (immediate or eventual)
	test.ci.scanner.consistency.level=immediate

	# Verify
	# -----
	# Maximum number of mapreduce mappers
	test.ci.verify.max.maps=4096
	# Number of mapreduce reducers
	test.ci.verify.reducers=64
	# Consistency Level (immediate or eventual)
	test.ci.verify.scan.consistency.level=immediate
	# Perform the verification directly on the files while the table is offline
	test.ci.verify.scan.offline=false
	# Comma separated list of auths to use for verify
	test.ci.verify.auths=
	# Location in HDFS to store output. Must not exist.
	test.ci.verify.output.dir=/tmp/ci-verify

	# Bulk Ingest
	# -----------
	# The number of map task to run.
	test.ci.bulk.map.task=10
	# The number of nodes to generate per map task.
	test.ci.bulk.map.nodes=1000000
	# The number of reducers will be the minimum of this prop and table splits+1. Each reducer will
	# produce a bulk import file.
	test.ci.bulk.reducers.max=1024

	# Splits Scaling
	# -----------
	# The number of tables to create
	test.ci.split.table.count=3
	# Minimum random row to generate
	test.ci.split.ingest.row.min=0
	# Maximum random row to generate
	test.ci.split.ingest.row.max=9223372036854775807
	# Maximum number of random column families to generate
	test.ci.split.ingest.max.cf=32767
	# Maximum number of random column qualifiers to generate
	test.ci.split.ingest.max.cq=32767
	# The number of tablets to create on each table on table creation
	test.ci.split.initial.tablets=1
	# The amount of data to write to each table
	test.ci.split.write.size=10000000
	# The split threshold to set for each table on creation
	test.ci.split.threshold=1G
	# The factor to reduce the split threshold by for each iteration of the test
	test.ci.split.threshold.reduction.factor=10
	# Number of rounds to run the test
	test.ci.split.test.rounds=3

	###############################
	# Garbage Collection Simulation
	###############################

	# Name of Accumulo table to use for test
	test.gcs.table=gcs
	# Max number of buckets for references. Buckets correspond to tablets in the Accumulo GC.
	test.gcs.maxBuckets=100000
	# Split each data section with this many tablets when creating table.
	test.gcs.tablets=10
	# Total number of work list the generator should create before it exits.
	test.gcs.maxWork=100000000
	# Max number of work list a generator should be concurrently working on.
	test.gcs.maxActiveWork=10000
	# Number of entries collector and verifier will read into memory.
	test.gcs.batchSize=100000

	#################
	# MapReduce Tests
	#################

	# RowHash test
	# ------------
	# Table containing input data
	test.rowhash.input.table = terasort
	# Table where data will be output to
	test.rowhash.output.table = rowhash
	# Column that is fetched in input table
	test.rowhash.column = c

	# TeraSort ingest
	# ---------------
	# Table to ingest into
	test.terasort.table = terasort
	# Number of rows to ingest
	test.terasort.num.rows = 10000
	# Minimum key size
	test.terasort.min.keysize = 10
	# Maximum key size
	test.terasort.max.keysize = 10
	# Minimum value size
	test.terasort.min.valuesize = 78
	# Maximum value size
	test.terasort.max.valuesize = 78
	# Number of table splits
	test.terasort.num.splits = 4