Merged trunk into this branch.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_btree_updates_next@830 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
index feedd36..cbfbfa3 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
@@ -147,7 +147,7 @@
@Override
public void start() throws Exception {
- LOGGER.log(Level.INFO, "Starting ClusterControllerService");
+ LOGGER.log(Level.INFO, "Starting ClusterControllerService: " + this);
Registry registry = LocateRegistry.createRegistry(ccConfig.port);
registry.rebind(IHyracksClientInterface.class.getName(), ccci);
registry.rebind(IClusterController.class.getName(), this);
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
index 21fac2f..0b1fea0 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
@@ -26,6 +26,8 @@
import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema.GarbageCollectorInfo;
public class NodeControllerState {
private static final int RRD_SIZE = 720;
@@ -46,6 +48,8 @@
private final int nProcessors;
+ private final HeartbeatSchema hbSchema;
+
private final long[] hbTime;
private final long[] heapInitSize;
@@ -70,6 +74,12 @@
private final double[] systemLoadAverage;
+ private final String[] gcNames;
+
+ private final long[][] gcCollectionCounts;
+
+ private final long[][] gcCollectionTimes;
+
private int rrdPtr;
private int lastHeartbeatDuration;
@@ -84,6 +94,7 @@
arch = reg.getArch();
osVersion = reg.getOSVersion();
nProcessors = reg.getNProcessors();
+ hbSchema = reg.getHeartbeatSchema();
hbTime = new long[RRD_SIZE];
heapInitSize = new long[RRD_SIZE];
@@ -97,6 +108,14 @@
threadCount = new int[RRD_SIZE];
peakThreadCount = new int[RRD_SIZE];
systemLoadAverage = new double[RRD_SIZE];
+ GarbageCollectorInfo[] gcInfos = hbSchema.getGarbageCollectorInfos();
+ int gcN = gcInfos.length;
+ gcNames = new String[gcN];
+ for (int i = 0; i < gcN; ++i) {
+ gcNames[i] = gcInfos[i].getName();
+ }
+ gcCollectionCounts = new long[gcN][RRD_SIZE];
+ gcCollectionTimes = new long[gcN][RRD_SIZE];
rrdPtr = 0;
}
@@ -115,6 +134,11 @@
threadCount[rrdPtr] = hbData.threadCount;
peakThreadCount[rrdPtr] = hbData.peakThreadCount;
systemLoadAverage[rrdPtr] = hbData.systemLoadAverage;
+ int gcN = hbSchema.getGarbageCollectorInfos().length;
+ for (int i = 0; i < gcN; ++i) {
+ gcCollectionCounts[i][rrdPtr] = hbData.gcCollectionCounts[i];
+ gcCollectionTimes[i][rrdPtr] = hbData.gcCollectionTimes[i];
+ }
rrdPtr = (rrdPtr + 1) % RRD_SIZE;
}
@@ -172,6 +196,9 @@
o.put("thread-counts", threadCount);
o.put("peak-thread-counts", peakThreadCount);
o.put("system-load-averages", systemLoadAverage);
+ o.put("gc-names", gcNames);
+ o.put("gc-collection-counts", gcCollectionCounts);
+ o.put("gc-collection-times", gcCollectionTimes);
return o;
}
diff --git a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html
index c1d78ba..c79ae20 100644
--- a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html
+++ b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html
@@ -1,4 +1,14 @@
<wicket:extend xmlns:wicket>
+ <div id='system-load-chart' class="jqplot-target time-chart">
+ </div>
+ <div id='gc-charts'>
+ </div>
+ <div id='heap-usage-chart' class="jqplot-target time-chart">
+ </div>
+ <div id='nonheap-usage-chart' class="jqplot-target time-chart">
+ </div>
+ <div id='thread-chart' class="jqplot-target time-chart">
+ </div>
<script type="text/javascript">
$(document).ready(function() {
$.timer(function() {
@@ -17,6 +27,8 @@
var nonheapUsageMaxSizes = result['nonheap-max-sizes'];
var threadCounts = result['thread-counts'];
var peakThreadCounts = result['peak-thread-counts'];
+ var gcCollectionCounts = result['gc-collection-counts'];
+ var gcCollectionTimes = result['gc-collection-times'];
var sysLoadArray = [];
var heapUsageInitSizesArray = [];
var heapUsageUsedSizesArray = [];
@@ -28,6 +40,20 @@
var nonheapUsageMaxSizesArray = [];
var threadCountsArray = [];
var peakThreadCountsArray = [];
+ var gcCollectionCountsArray = [];
+ var gcCollectionTimesArray = [];
+ var gcChartsDiv = document.getElementById('gc-charts');
+ for(var i = 0; i < gcCollectionCounts.length; ++i) {
+ gcCollectionCountsArray.push([]);
+ gcCollectionTimesArray.push([]);
+ var gcCollectionCountDiv = document.createElement('div');
+ gcCollectionCountDiv.id = 'gc-collection-count-' + i;
+ gcCollectionCountDiv.style = "jqplot-target time-chart";
+ gcChartsDiv.appendChild(gcCollectionCountDiv);
+ var gcCollectionTimeDiv = document.createElement('div');
+ gcCollectionTimeDiv.id = 'gc-collection-time-' + i;
+ gcChartsDiv.appendChild(gcCollectionTimeDiv);
+ }
var rrdPtr = result['rrd-ptr'];
for(var i = 0; i < sysLoad.length; ++i) {
sysLoadArray.push([i, sysLoad[rrdPtr]]);
@@ -41,6 +67,12 @@
nonheapUsageMaxSizesArray.push([i, nonheapUsageMaxSizes[rrdPtr] / (1024 * 1024)]);
threadCountsArray.push([i, threadCounts[rrdPtr]]);
peakThreadCountsArray.push([i, peakThreadCounts[rrdPtr]]);
+ for(var j = 0; j < gcCollectionCounts.length; ++j) {
+ gcCollectionCountsArray[j].push([i, gcCollectionCounts[j][rrdPtr]]);
+ }
+ for(var j = 0; j < gcCollectionTimes.length; ++j) {
+ gcCollectionTimesArray[j].push([i, gcCollectionTimes[j][rrdPtr]]);
+ }
rrdPtr = (rrdPtr + 1) % sysLoad.length;
}
$('#system-load-chart').empty();
@@ -68,6 +100,59 @@
show: false
}
});
+ var gcNames = result['gc-names'];
+ for(var i = 0; i < gcCollectionCounts.length; ++i) {
+ $('#gc-collection-count-' + i).addClass("jqplot-target time-chart");
+ $.jqplot('gc-collection-count-' + i, [gcCollectionCountsArray[i]], {
+ title: 'GC Collection Counts(' + gcNames[i] + ')',
+ axes: {
+ xaxis: {
+ label: 'Time',
+ tickOptions: {
+ formatString:'%d'
+ }
+ },
+ yaxis: {
+ label: 'Collection Counts',
+ tickOptions:{
+ formatString:'%d'
+ }
+ }
+ },
+ highlighter: {
+ show: true,
+ sizeAdjust: 7.5
+ },
+ cursor: {
+ show: false
+ }
+ });
+ $('#gc-collection-time-' + i).addClass("jqplot-target time-chart");
+ $.jqplot('gc-collection-time-' + i, [gcCollectionTimesArray[i]], {
+ title: 'GC Collection Times(' + gcNames[i] + ')',
+ axes: {
+ xaxis: {
+ label: 'Time',
+ tickOptions: {
+ formatString:'%d'
+ }
+ },
+ yaxis: {
+ label: 'Collection Times',
+ tickOptions:{
+ formatString:'%d'
+ }
+ }
+ },
+ highlighter: {
+ show: true,
+ sizeAdjust: 7.5
+ },
+ cursor: {
+ show: false
+ }
+ });
+ }
$('#heap-usage-chart').empty();
$.jqplot('heap-usage-chart', [heapUsageInitSizesArray, heapUsageUsedSizesArray, heapUsageCommittedSizesArray, heapUsageMaxSizesArray], {
title: 'Heap Usage',
@@ -188,12 +273,4 @@
});
});
</script>
- <div id='system-load-chart' class="jqplot-target time-chart">
- </div>
- <div id='heap-usage-chart' class="jqplot-target time-chart">
- </div>
- <div id='nonheap-usage-chart' class="jqplot-target time-chart">
- </div>
- <div id='thread-chart' class="jqplot-target time-chart">
- </div>
</wicket:extend>
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
index 9b10d27..ff92c2d 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
public final class NodeRegistration implements Serializable {
private static final long serialVersionUID = 1L;
@@ -38,8 +39,10 @@
private final int nProcessors;
+ private final HeartbeatSchema hbSchema;
+
public NodeRegistration(INodeController nc, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
- String osName, String arch, String osVersion, int nProcessors) {
+ String osName, String arch, String osVersion, int nProcessors, HeartbeatSchema hbSchema) {
this.nc = nc;
this.nodeId = nodeId;
this.ncConfig = ncConfig;
@@ -48,6 +51,7 @@
this.arch = arch;
this.osVersion = osVersion;
this.nProcessors = nProcessors;
+ this.hbSchema = hbSchema;
}
public INodeController getNodeController() {
@@ -81,4 +85,8 @@
public int getNProcessors() {
return nProcessors;
}
+
+ public HeartbeatSchema getHeartbeatSchema() {
+ return hbSchema;
+ }
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
index 1816cd5..cd9086f 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -31,4 +31,6 @@
public int peakThreadCount;
public long totalStartedThreadCount;
public double systemLoadAverage;
+ public long[] gcCollectionCounts;
+ public long[] gcCollectionTimes;
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatSchema.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatSchema.java
new file mode 100644
index 0000000..e53e9a8
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatSchema.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.heartbeat;
+
+import java.io.Serializable;
+
+public class HeartbeatSchema implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final GarbageCollectorInfo[] gcInfos;
+
+ public HeartbeatSchema(GarbageCollectorInfo[] gcInfos) {
+ this.gcInfos = gcInfos;
+ }
+
+ public GarbageCollectorInfo[] getGarbageCollectorInfos() {
+ return gcInfos;
+ }
+
+ public static class GarbageCollectorInfo implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String name;
+
+ public GarbageCollectorInfo(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
index a59b17e..3cd3c2f 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.control.nc;
import java.io.File;
+import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
@@ -55,6 +56,7 @@
import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
import edu.uci.ics.hyracks.control.common.work.FutureValue;
@@ -107,6 +109,8 @@
private final MemoryMXBean memoryMXBean;
+ private final List<GarbageCollectorMXBean> gcMXBeans;
+
private final ThreadMXBean threadMXBean;
private final RuntimeMXBean runtimeMXBean;
@@ -132,6 +136,7 @@
NodeControllerService.class.getName()), id));
applications = new Hashtable<String, NCApplicationContext>();
memoryMXBean = ManagementFactory.getMemoryMXBean();
+ gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
threadMXBean = ManagementFactory.getThreadMXBean();
runtimeMXBean = ManagementFactory.getRuntimeMXBean();
osMXBean = ManagementFactory.getOperatingSystemMXBean();
@@ -157,9 +162,14 @@
connectionManager.start();
Registry registry = LocateRegistry.getRegistry(ncConfig.ccHost, ncConfig.ccPort);
IClusterController cc = (IClusterController) registry.lookup(IClusterController.class.getName());
+ HeartbeatSchema.GarbageCollectorInfo[] gcInfos = new HeartbeatSchema.GarbageCollectorInfo[gcMXBeans.size()];
+ for (int i = 0; i < gcInfos.length; ++i) {
+ gcInfos[i] = new HeartbeatSchema.GarbageCollectorInfo(gcMXBeans.get(i).getName());
+ }
+ HeartbeatSchema hbSchema = new HeartbeatSchema(gcInfos);
this.nodeParameters = cc.registerNode(new NodeRegistration(this, id, ncConfig, connectionManager
.getNetworkAddress(), osMXBean.getName(), osMXBean.getArch(), osMXBean.getVersion(), osMXBean
- .getAvailableProcessors()));
+ .getAvailableProcessors(), hbSchema));
queue.start();
heartbeatTask = new HeartbeatTask(cc);
@@ -296,6 +306,8 @@
public HeartbeatTask(IClusterController cc) {
this.cc = cc;
hbData = new HeartbeatData();
+ hbData.gcCollectionCounts = new long[gcMXBeans.size()];
+ hbData.gcCollectionTimes = new long[gcMXBeans.size()];
}
@Override
@@ -314,6 +326,12 @@
hbData.peakThreadCount = threadMXBean.getPeakThreadCount();
hbData.totalStartedThreadCount = threadMXBean.getTotalStartedThreadCount();
hbData.systemLoadAverage = osMXBean.getSystemLoadAverage();
+ int gcN = gcMXBeans.size();
+ for (int i = 0; i < gcN; ++i) {
+ GarbageCollectorMXBean gcMXBean = gcMXBeans.get(i);
+ hbData.gcCollectionCounts[i] = gcMXBean.getCollectionCount();
+ hbData.gcCollectionTimes[i] = gcMXBean.getCollectionTime();
+ }
try {
cc.nodeHeartbeat(id, hbData);
} catch (Exception e) {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
index 2a2141a..38e0b91 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
@@ -218,16 +218,15 @@
int entry = hpc.partition(accessor0, i, numPartitions);
ByteBuffer outbuf = outbufs[entry];
- appender.reset(outbuf, true);
- while (true) {
- if (appender.append(accessor0, i)) {
- break;
- } else {
- // buffer is full, ie. we cannot fit the tuple
- // into the buffer -- write it to disk
- write(entry, outbuf);
- outbuf.clear();
- appender.reset(outbuf, true);
+ appender.reset(outbuf, false);
+ if (!appender.append(accessor0, i)) {
+ // buffer is full, ie. we cannot fit the tuple
+ // into the buffer -- write it to disk
+ write(entry, outbuf);
+ outbuf.clear();
+ appender.reset(outbuf, true);
+ if (!appender.append(accessor0, i)) {
+ throw new HyracksDataException("Item too big to fit in frame");
}
}
}
diff --git a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
index 3cbe50a..8ed7bb7 100644
--- a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
+++ b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
@@ -51,7 +51,7 @@
HyracksCCProcess ccp = new HyracksCCProcess(ccConfig);
ccp.start();
- Thread.sleep(2000);
+ Thread.sleep(5000);
HyracksNCProcess ncps[] = new HyracksNCProcess[options.n];
for (int i = 0; i < options.n; ++i) {
diff --git a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
index 0531665..6a0e45f 100644
--- a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
+++ b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
@@ -63,6 +63,7 @@
List<String> cList = new ArrayList<String>();
cList.add(getJavaCommand());
cList.add("-Dbasedir=" + System.getProperty("basedir"));
+ cList.add("-Djava.rmi.server.hostname=127.0.0.1");
cList.add("-classpath");
cList.add(getClasspath());
cList.add(getMainClassName());
