script/pig/ClusterSummary.pig - chukwa - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 %default START '1234567890';
 SystemMetrics = load 'hbase://SystemMetrics' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('tags:cluster cpu:combined cpu:idle cpu:sys cpu:user disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage.1 memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets swap:Total swap:Used swap:Free swap:PageIn swap:PageOut','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut);
 CleanseBuffer = foreach SystemMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut;
 ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut;
 TimeSeries = GROUP ConcatBuffer BY rowId;
 ComputeBuffer = FOREACH TimeSeries GENERATE group, AVG(ConcatBuffer.cpuCombined), AVG(ConcatBuffer.cpuIdle), AVG(ConcatBuffer.cpuSys), AVG(ConcatBuffer.cpuUser), AVG(ConcatBuffer.diskReadBytes), AVG(ConcatBuffer.diskReads), AVG(ConcatBuffer.diskWriteBytes), AVG(ConcatBuffer.diskWrites), AVG(ConcatBuffer.LoadAverage), AVG(ConcatBuffer.memoryFreePercent), AVG(ConcatBuffer.memoryUsedPercent), AVG(ConcatBuffer.networkRxBytes), AVG(ConcatBuffer.networkRxDropped), AVG(ConcatBuffer.networkRxErrors), AVG(ConcatBuffer.networkRxPackets), AVG(ConcatBuffer.networkTxBytes), AVG(ConcatBuffer.networkTxCollisions), AVG(ConcatBuffer.networkTxErrors), AVG(ConcatBuffer.networkTxPackets), AVG(ConcatBuffer.swapTotal), AVG(ConcatBuffer.swapUsed), AVG(ConcatBuffer.swapFree), AVG(ConcatBuffer.swapPageIn), AVG(ConcatBuffer.swapPageOut);
 STORE ComputeBuffer INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('cpu:Combined cpu:Idle cpu:Sys cpu:User disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets swap:Total swap:Used swap:Free swap:PageIn swap:PageOut');
 HDFSMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('dfs_FSNamesystem:cluster dfs_FSNamesystem:CapacityTotalGB dfs_FSNamesystem:CapacityUsedGB dfs_FSNamesystem:CapacityRemainingGB dfs_FSNamesystem:BlockCapacity dfs_FSNamesystem:BlocksTotal dfs_FSNamesystem:MissingBlocks dfs_FSNamesystem:CorruptBlocks dfs_FSNamesystem:UnderReplicatedBlocks dfs_FSNamesystem:FilesTotal','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal);
 CleanseBuffer = foreach HDFSMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal;
 ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal;
 STORE ConcatBuffer INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('hdfs:CapacityTotalGB hdfs:CapacityUsedGB hdfs:CapacityRemainingGB hdfs:BlockCapacity hdfs:BlocksTotal hdfs:MissingBlocks hdfs:CorruptBlocks hdfs:UnderReplicatedBlocks hdfs:FilesTotal');
 MapReduceMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapred_tasktracker:cluster mapred_tasktracker:mapTaskSlots mapred_tasktracker:maps_running mapred_tasktracker:reduceTaskSlots mapred_tasktracker:reduces_running mapred_tasktracker:tasks_completed mapred_tasktracker:tasks_failed_ping mapred_tasktracker:tasks_failed_timeout mapred_jobtracker:occupied_map_slots','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, mapTaskSlots, mapsRunning, reduceTaskSlots, reduceRunning, tasksCompleted, tasksFailedPing, tasksFailedTimeout);
 CleanseBuffer = foreach MapReduceMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, mapTaskSlots, mapsRunning, reduceTaskSlots, reduceRunning, tasksCompleted, tasksFailedPing, tasksFailedTimeout;
 GroupBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, mapTaskSlots, mapsRunning, reduceTaskSlots, reduceRunning, tasksCompleted, tasksFailedPing, tasksFailedTimeout;
 TimeSeries = GROUP GroupBuffer BY rowId;
 MapReduceSummary = FOREACH TimeSeries GENERATE group, SUM(GroupBuffer.mapTaskSlots), SUM(GroupBuffer.mapsRunning), SUM(GroupBuffer.reduceTaskSlots), SUM(GroupBuffer.reduceRunning), SUM(GroupBuffer.tasksCompleted), SUM(GroupBuffer.tasksFailedPing), SUM(GroupBuffer.tasksFailedTimeout);
 STORE MapReduceSummary INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapreduce:mapTaskSlots mapreduce:mapsRunning mapreduce:reduceTaskSlots mapreduce:reduceRunning mapreduce:tasksCompleted mapreduce:tasksFailedPing mapreduce:tasksFailedTimeout');

 MapReduceMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapred_jobtracker:cluster mapred_jobtracker:occupied_map_slots mapred_jobtracker:occupied_reduce_slots','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, occupiedMapSlots, occupiedReduceSlots);
 CleanseBuffer = foreach MapReduceMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, occupiedMapSlots, occupiedReduceSlots;
 GroupBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, occupiedMapSlots, occupiedReduceSlots;
 TimeSeries = GROUP GroupBuffer BY rowId;
 MapReduceSummary = FOREACH TimeSeries GENERATE group, SUM(GroupBuffer.occupiedMapSlots), SUM(GroupBuffer.occupiedReduceSlots);
 STORE MapReduceSummary INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapreduce:occupiedMapSlots mapreduce:occupiedReduceSlots');
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	%default START '1234567890';
	SystemMetrics = load 'hbase://SystemMetrics' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('tags:cluster cpu:combined cpu:idle cpu:sys cpu:user disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage.1 memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets swap:Total swap:Used swap:Free swap:PageIn swap:PageOut','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut);
	CleanseBuffer = foreach SystemMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut;
	ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut;
	TimeSeries = GROUP ConcatBuffer BY rowId;
	ComputeBuffer = FOREACH TimeSeries GENERATE group, AVG(ConcatBuffer.cpuCombined), AVG(ConcatBuffer.cpuIdle), AVG(ConcatBuffer.cpuSys), AVG(ConcatBuffer.cpuUser), AVG(ConcatBuffer.diskReadBytes), AVG(ConcatBuffer.diskReads), AVG(ConcatBuffer.diskWriteBytes), AVG(ConcatBuffer.diskWrites), AVG(ConcatBuffer.LoadAverage), AVG(ConcatBuffer.memoryFreePercent), AVG(ConcatBuffer.memoryUsedPercent), AVG(ConcatBuffer.networkRxBytes), AVG(ConcatBuffer.networkRxDropped), AVG(ConcatBuffer.networkRxErrors), AVG(ConcatBuffer.networkRxPackets), AVG(ConcatBuffer.networkTxBytes), AVG(ConcatBuffer.networkTxCollisions), AVG(ConcatBuffer.networkTxErrors), AVG(ConcatBuffer.networkTxPackets), AVG(ConcatBuffer.swapTotal), AVG(ConcatBuffer.swapUsed), AVG(ConcatBuffer.swapFree), AVG(ConcatBuffer.swapPageIn), AVG(ConcatBuffer.swapPageOut);
	STORE ComputeBuffer INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('cpu:Combined cpu:Idle cpu:Sys cpu:User disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets swap:Total swap:Used swap:Free swap:PageIn swap:PageOut');
	HDFSMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('dfs_FSNamesystem:cluster dfs_FSNamesystem:CapacityTotalGB dfs_FSNamesystem:CapacityUsedGB dfs_FSNamesystem:CapacityRemainingGB dfs_FSNamesystem:BlockCapacity dfs_FSNamesystem:BlocksTotal dfs_FSNamesystem:MissingBlocks dfs_FSNamesystem:CorruptBlocks dfs_FSNamesystem:UnderReplicatedBlocks dfs_FSNamesystem:FilesTotal','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal);
	CleanseBuffer = foreach HDFSMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal;
	ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal;
	STORE ConcatBuffer INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('hdfs:CapacityTotalGB hdfs:CapacityUsedGB hdfs:CapacityRemainingGB hdfs:BlockCapacity hdfs:BlocksTotal hdfs:MissingBlocks hdfs:CorruptBlocks hdfs:UnderReplicatedBlocks hdfs:FilesTotal');
	MapReduceMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapred_tasktracker:cluster mapred_tasktracker:mapTaskSlots mapred_tasktracker:maps_running mapred_tasktracker:reduceTaskSlots mapred_tasktracker:reduces_running mapred_tasktracker:tasks_completed mapred_tasktracker:tasks_failed_ping mapred_tasktracker:tasks_failed_timeout mapred_jobtracker:occupied_map_slots','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, mapTaskSlots, mapsRunning, reduceTaskSlots, reduceRunning, tasksCompleted, tasksFailedPing, tasksFailedTimeout);
	CleanseBuffer = foreach MapReduceMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, mapTaskSlots, mapsRunning, reduceTaskSlots, reduceRunning, tasksCompleted, tasksFailedPing, tasksFailedTimeout;
	GroupBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, mapTaskSlots, mapsRunning, reduceTaskSlots, reduceRunning, tasksCompleted, tasksFailedPing, tasksFailedTimeout;
	TimeSeries = GROUP GroupBuffer BY rowId;
	MapReduceSummary = FOREACH TimeSeries GENERATE group, SUM(GroupBuffer.mapTaskSlots), SUM(GroupBuffer.mapsRunning), SUM(GroupBuffer.reduceTaskSlots), SUM(GroupBuffer.reduceRunning), SUM(GroupBuffer.tasksCompleted), SUM(GroupBuffer.tasksFailedPing), SUM(GroupBuffer.tasksFailedTimeout);
	STORE MapReduceSummary INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapreduce:mapTaskSlots mapreduce:mapsRunning mapreduce:reduceTaskSlots mapreduce:reduceRunning mapreduce:tasksCompleted mapreduce:tasksFailedPing mapreduce:tasksFailedTimeout');

	MapReduceMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapred_jobtracker:cluster mapred_jobtracker:occupied_map_slots mapred_jobtracker:occupied_reduce_slots','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, occupiedMapSlots, occupiedReduceSlots);
	CleanseBuffer = foreach MapReduceMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, occupiedMapSlots, occupiedReduceSlots;
	GroupBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, occupiedMapSlots, occupiedReduceSlots;
	TimeSeries = GROUP GroupBuffer BY rowId;
	MapReduceSummary = FOREACH TimeSeries GENERATE group, SUM(GroupBuffer.occupiedMapSlots), SUM(GroupBuffer.occupiedReduceSlots);
	STORE MapReduceSummary INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('mapreduce:occupiedMapSlots mapreduce:occupiedReduceSlots');