HBase 0.94.1RC0 release
git-svn-id: https://svn.apache.org/repos/asf/hbase/tags/0.94.1RC0@1362338 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index ffd18fc..4bf00b2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
HBase Change Log
-Release 0.94.1 - 7/12/2012
+Release 0.94.1 - 7/16/2012
Sub-task
[HBASE-5342] - Grant/Revoke global permissions
@@ -98,7 +98,14 @@
[HBASE-6328] - FSHDFSUtils#recoverFileLease tries to rethrow InterruptedException but actually shallows it
[HBASE-6329] - Stopping META regionserver when splitting region could cause daughter region to be assigned twice
[HBASE-6337] - [MTTR] Remove renaming tmp log file in SplitLogManager
+ [HBASE-6357] - Failed distributed log splitting stuck on master web UI
[HBASE-6369] - HTable is not closed in AggregationClient
+ [HBASE-6375] - Master may be using a stale list of region servers for creating assignment plan during startup
+ [HBASE-6377] - HBASE-5533 metrics miss all operations submitted via MultiAction
+ [HBASE-6380] - bulkload should update the store.storeSize
+ [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments
+ [HBASE-6394] - verifyrep MR job map tasks throws NullPointerException
+ [HBASE-6397] - [hbck] print out bulk load commands for sidelined regions if necessary
Improvement
@@ -131,6 +138,8 @@
[HBASE-6332] - Improve POM for better integration with downstream ivy projects
[HBASE-6334] - TestImprovement for TestHRegion.testWritesWhileGetting
[HBASE-6341] - Publicly expose HConnectionKey
+ [HBASE-6382] - Upgrade Jersey to 1.8 to match Hadoop 1 and 2
+ [HBASE-6384] - hbck should group together those sidelined regions need to be bulk loaded later
New Feature
diff --git a/pom.xml b/pom.xml
index 9fe2b3c..ac0d75c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<packaging>jar</packaging>
- <version>0.94.1-SNAPSHOT</version>
+ <version>0.94.1</version>
<name>HBase</name>
<description>
HBase is the &lt;a href="http://hadoop.apache.org"&rt;Hadoop</a&rt; database. Use it when you need
@@ -984,7 +984,7 @@
<jaxb-api.version>2.1</jaxb-api.version>
<jetty.version>6.1.26</jetty.version>
<jetty.jspapi.version>6.1.14</jetty.jspapi.version>
- <jersey.version>1.4</jersey.version>
+ <jersey.version>1.8</jersey.version>
<jruby.version>1.6.5</jruby.version>
<junit.version>4.10-HBASE-1</junit.version>
<slf4j.version>1.4.3</slf4j.version>
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index c37a4fb..b7d540b 100644
--- a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -35,7 +35,6 @@
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
@@ -137,7 +136,10 @@
}
protected void cleanup(Context context) {
- replicatedScanner.close();
+ if (replicatedScanner != null) {
+ replicatedScanner.close();
+ replicatedScanner = null;
+ }
}
}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 0b5a657..287ec8d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2247,15 +2247,6 @@
* @throws IOException
*/
public void assignAllUserRegions() throws IOException, InterruptedException {
- // Get all available servers
- List<ServerName> servers = serverManager.getOnlineServersList();
-
- // Remove the deadNotExpired servers from the server list.
- removeDeadNotExpiredServers(servers);
-
- // If there are no servers we need not proceed with region assignment.
- if(servers.isEmpty()) return;
-
// Skip assignment for regions of tables in DISABLING state also because
// during clean cluster startup no RS is alive and regions map also doesn't
// have any information about the regions. See HBASE-6281.
@@ -2266,6 +2257,15 @@
disablingAndDisabledTables, true);
if (allRegions == null || allRegions.isEmpty()) return;
+ // Get all available servers
+ List<ServerName> servers = serverManager.getOnlineServersList();
+
+ // Remove the deadNotExpired servers from the server list.
+ removeDeadNotExpiredServers(servers);
+
+ // If there are no servers we need not proceed with region assignment.
+ if(servers.isEmpty()) return;
+
// Determine what type of assignment to do on startup
boolean retainAssignment = master.getConfiguration().
getBoolean("hbase.master.startup.retainassign", true);
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 0ed3200..28e8682 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -557,12 +557,12 @@
* Wait for the region servers to report in.
* We will wait until one of this condition is met:
* - the master is stopped
- * - the 'hbase.master.wait.on.regionservers.timeout' is reached
* - the 'hbase.master.wait.on.regionservers.maxtostart' number of
* region servers is reached
* - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
* there have been no new region server in for
- * 'hbase.master.wait.on.regionservers.interval' time
+ * 'hbase.master.wait.on.regionservers.interval' time AND
+ * the 'hbase.master.wait.on.regionservers.timeout' is reached
*
* @throws InterruptedException
*/
@@ -571,11 +571,18 @@
final long interval = this.master.getConfiguration().
getLong("hbase.master.wait.on.regionservers.interval", 1500);
final long timeout = this.master.getConfiguration().
- getLong("hbase.master.wait.on.regionservers.timeout", 4500);
+ getLong("hbase.master.wait.on.regionservers.timeout", 4500);
final int minToStart = this.master.getConfiguration().
- getInt("hbase.master.wait.on.regionservers.mintostart", 1);
- final int maxToStart = this.master.getConfiguration().
- getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
+ getInt("hbase.master.wait.on.regionservers.mintostart", 1);
+ int maxToStart = this.master.getConfiguration().
+ getInt("hbase.master.wait.on.regionservers.maxtostart", Integer.MAX_VALUE);
+ if (maxToStart < minToStart) {
+ LOG.warn(String.format(
+ "The value of 'hbase.master.wait.on.regionservers.maxtostart' (%d)" +
+ " is set less than 'hbase.master.wait.on.regionservers.mintostart'" +
+ " (%d), ignoring.", maxToStart, minToStart));
+ maxToStart = Integer.MAX_VALUE;
+ }
long now = System.currentTimeMillis();
final long startTime = now;
@@ -586,9 +593,8 @@
int oldCount = 0;
while (
!this.master.isStopped() &&
- slept < timeout &&
count < maxToStart &&
- (lastCountChange+interval > now || count < minToStart)
+ (lastCountChange+interval > now || timeout > slept || count < minToStart)
){
// Log some info at every interval time or if there is a change
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b1e7233..230d6b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1952,15 +1952,12 @@
/** {@inheritDoc} */
public Result get(byte[] regionName, Get get) throws IOException {
checkOpen();
- final long startTime = System.nanoTime();
requestCount.incrementAndGet();
try {
HRegion region = getRegion(regionName);
return region.get(get, getLockFromId(get.getLockId()));
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
- } finally {
- this.metrics.getLatencies.update(System.nanoTime() - startTime);
}
}
@@ -1992,7 +1989,6 @@
throw new IllegalArgumentException("update has null row");
}
- final long startTime = System.nanoTime();
checkOpen();
this.requestCount.incrementAndGet();
HRegion region = getRegion(regionName);
@@ -2004,8 +2000,6 @@
region.put(put, getLockFromId(put.getLockId()), writeToWAL);
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
- } finally {
- this.metrics.putLatencies.update(System.nanoTime() - startTime);
}
}
@@ -2015,7 +2009,6 @@
HRegion region = null;
int i = 0;
- final long startTime = System.nanoTime();
try {
region = getRegion(regionName);
if (!region.getRegionInfo().isMetaTable()) {
@@ -2040,14 +2033,6 @@
return -1;
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
- } finally {
- // going to count this as puts.size() PUTs for latency calculations
- final long totalTime = System.nanoTime() - startTime;
- final long putCount = i;
- final long perPutTime = totalTime / putCount;
- for (int request = 0; request < putCount; request++) {
- this.metrics.putLatencies.update(perPutTime);
- }
}
}
@@ -2520,7 +2505,6 @@
public void delete(final byte[] regionName, final Delete delete)
throws IOException {
checkOpen();
- final long startTime = System.nanoTime();
try {
boolean writeToWAL = delete.getWriteToWAL();
this.requestCount.incrementAndGet();
@@ -2532,8 +2516,6 @@
region.delete(delete, lid, writeToWAL);
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t));
- } finally {
- this.metrics.deleteLatencies.update(System.nanoTime() - startTime);
}
}
@@ -2551,12 +2533,10 @@
int size = deletes.size();
Integer[] locks = new Integer[size];
for (Delete delete : deletes) {
- final long startTime = System.nanoTime();
this.requestCount.incrementAndGet();
locks[i] = getLockFromId(delete.getLockId());
region.delete(delete, locks[i], delete.getWriteToWAL());
i++;
- this.metrics.deleteLatencies.update(System.nanoTime() - startTime);
}
} catch (WrongRegionException ex) {
LOG.debug("Batch deletes: " + i, ex);
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index f7491fd..68219b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -554,8 +554,10 @@
this.family.getBloomFilterType(), this.dataBlockEncoder);
passSchemaMetricsTo(sf);
- sf.createReader();
-
+ StoreFile.Reader r = sf.createReader();
+ this.storeSize += r.length();
+ this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+
LOG.info("Moved hfile " + srcPath + " into store directory " +
homedir + " - updating store file list.");
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
index b00e4b8..9ce40b1 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
@@ -130,25 +130,6 @@
/** Block hit caching ratio for past N periods */
public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
- /**
- * a latency histogram on 'get' requests
- */
- public final MetricsHistogram getLatencies =
- new MetricsHistogram("getRequestLatency", registry);
-
- /**
- * a latency histogram on 'delete' requests
- */
- public final MetricsHistogram deleteLatencies =
- new MetricsHistogram("deleteRequestLatency", registry);
-
- /**
- * a latency histogram on 'put' requests
- */
- public final MetricsHistogram putLatencies =
- new MetricsHistogram("putRequestLatency", registry);
-
-
/*
* Count of requests to the regionservers since last call to metrics update
*/
@@ -395,10 +376,6 @@
this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
- this.putLatencies.pushMetric(this.metricsRecord);
- this.deleteLatencies.pushMetric(this.metricsRecord);
- this.getLatencies.pushMetric(this.metricsRecord);
-
// Mix in HFile and HLog metrics
// Be careful. Here is code for MTVR from up in hadoop:
// public synchronized void inc(final int numOps, final long time) {
@@ -586,9 +563,6 @@
Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
Long.valueOf(this.slowHLogAppendCount.get()));
- sb = appendHistogram(sb, this.deleteLatencies);
- sb = appendHistogram(sb, this.getLatencies);
- sb = appendHistogram(sb, this.putLatencies);
sb = appendHistogram(sb, this.fsReadLatencyHistogram);
sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
sb = appendHistogram(sb, this.fsWriteLatencyHistogram);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 98fca20..707c947 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -144,6 +144,7 @@
private static boolean rsSupportsOffline = true;
private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
private static final int DEFAULT_MAX_MERGE = 5;
+ private static final String TO_BE_LOADED = "to_be_loaded";
/**********************
* Internal resources
@@ -873,8 +874,20 @@
/**
* Sideline a region dir (instead of deleting it)
*/
- Path sidelineRegionDir(FileSystem fs, HbckInfo hi)
- throws IOException {
+ Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
+ return sidelineRegionDir(fs, null, hi);
+ }
+
+ /**
+ * Sideline a region dir (instead of deleting it)
+ *
+ * @param parentDir if specified, the region will be sidelined to
+ * folder like .../parentDir/<table name>/<region name>. The purpose
+ * is to group together similar regions sidelined, for example, those
+ * regions should be bulk loaded back later on. If null, it is ignored.
+ */
+ Path sidelineRegionDir(FileSystem fs,
+ String parentDir, HbckInfo hi) throws IOException {
String tableName = Bytes.toString(hi.getTableName());
Path regionDir = hi.getHdfsRegionDir();
@@ -883,7 +896,11 @@
return null;
}
- Path sidelineTableDir= new Path(getSidelineDir(), tableName);
+ Path rootDir = getSidelineDir();
+ if (parentDir != null) {
+ rootDir = new Path(rootDir, parentDir);
+ }
+ Path sidelineTableDir= new Path(rootDir, tableName);
Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
fs.mkdirs(sidelineRegionDir);
boolean success = false;
@@ -1957,7 +1974,7 @@
offline(regionToSideline.getRegionName());
LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
- Path sidelineRegionDir = sidelineRegionDir(fs, regionToSideline);
+ Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
if (sidelineRegionDir != null) {
sidelinedRegions.put(sidelineRegionDir, regionToSideline);
LOG.info("After sidelined big overlapped region: "
@@ -2117,9 +2134,14 @@
}
public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
- for (Path k : regions.keySet()) {
- System.out.println("To be bulk loaded sidelined region dir: "
- + k.toString());
+ for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
+ String tableName = Bytes.toStringBinary(entry.getValue().getTableName());
+ Path path = entry.getKey();
+ System.out.println("This sidelined region dir should be bulk loaded: "
+ + path.toString());
+ System.out.println("Bulk load command looks like: "
+ + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
+ + path.toUri().getPath() + " "+ tableName);
}
}
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 19fca58..4b8eafe 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -624,8 +624,9 @@
// These settings will make the server waits until this exact number of
// regions servers are connected.
- conf.setInt("hbase.master.wait.on.regionservers.mintostart", numSlaves);
- conf.setInt("hbase.master.wait.on.regionservers.maxtostart", numSlaves);
+ String count = String.valueOf(numSlaves);
+ conf.setIfUnset("hbase.master.wait.on.regionservers.mintostart", count);
+ conf.setIfUnset("hbase.master.wait.on.regionservers.maxtostart", count);
Configuration c = new Configuration(this.conf);
this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
index a0ed0bd..431420c 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
@@ -66,8 +66,11 @@
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Set it so that this test runs with my custom master
- TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL,
- TestingMaster.class, HMaster.class);
+ Configuration conf = TESTUTIL.getConfiguration();
+ conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
+ conf.setInt("hbase.master.wait.on.regionservers.mintostart", 3);
+ conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 4);
+
// Start up the cluster.
TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
}
