src/main/java/org/apache/hadoop/chukwa/database/DatabaseConfig.java - chukwa - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package org.apache.hadoop.chukwa.database;


 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import java.util.*;
 import java.io.File;
 import java.io.FilenameFilter;

 public class DatabaseConfig {
   private Configuration config = null;
   public final static long CENTURY = 36500 * 24 * 60 * 60 * 1000L;
   public final static long DECADE = 3650 * 24 * 60 * 60 * 1000L;
   public final static long YEAR = 365 * 24 * 60 * 60 * 1000L;
   public final static long QUARTER = 91250 * 24 * 60 * 60L;
   public final static long MONTH = 30 * 24 * 60 * 60 * 1000L;
   public final static long WEEK = 7 * 24 * 60 * 60 * 1000L;
   public final static long DAY = 24 * 60 * 60 * 1000L;
   public final static String MDL_XML = "mdl.xml";

   public DatabaseConfig(String path) {
     Path fileResource = new Path(path);
     config = new Configuration();
     config.addResource(fileResource);
   }

   public DatabaseConfig() {
     String dataConfig = System.getenv("CHUKWA_CONF_DIR");
     if (dataConfig == null) {
       dataConfig = MDL_XML;
     } else {
       dataConfig += File.separator + MDL_XML;
     }
     Path fileResource = new Path(dataConfig);
     config = new Configuration();
     config.addResource(fileResource);

     if (System.getenv("CHUKWA_CONF_DIR") != null) {
       // Allow site-specific MDL files to be included in the
       // configuration so as to keep the "main" mdl.xml pure.
       File confDir = new File(System.getenv("CHUKWA_CONF_DIR"));
       File[] confFiles = confDir.listFiles(new FilenameFilter() {

         @Override
         public boolean accept(File dir, String name) {
           // Implements a naming convention of ending with "mdl.xml"
           // but is careful not to pick up mdl.xml itself again.
           return name.endsWith(MDL_XML) && !name.equals(MDL_XML);
         }

       });

       if (confFiles != null) {
         for (File confFile : confFiles)
           config.addResource(new Path(confFile.getAbsolutePath()));
       }
     }
   }

   public String get(String key) {
     return config.get(key);
   }

   public void put(String key, String value) {
     this.config.set(key, value);
   }

   public Iterator<?> iterator() {
     return this.config.iterator();
   }

   public HashMap<String, String> startWith(String key) {
     HashMap<String, String> transformer = new HashMap<String, String>();
     Iterator<?> entries = config.iterator();
     while (entries.hasNext()) {
       String entry = entries.next().toString();
       if (entry.startsWith(key)) {
         String[] metrics = entry.split("=");
         transformer.put(metrics[0], metrics[1]);
       }
     }
     return transformer;
   }

   public String[] findTableName(String tableName, long start, long end) {
     String[] tableNames = null;
     String tableType = "_week";
     long now = (new Date()).getTime();
     long timeWindow = end - start;
     long partitionSize = WEEK;
     boolean fallback = true;

     if (config.get("consolidator.table." + tableName) == null) {
       tableNames = new String[1];
       tableNames[0] = tableName;
       return tableNames;
     }
     if (timeWindow <= 0) {
       timeWindow = 1;
     }
     if (timeWindow > DECADE) {
       tableType = "_century";
       partitionSize = CENTURY;
     } else if (timeWindow > YEAR) {
       tableType = "_decade";
       partitionSize = DECADE;
     } else if (timeWindow > QUARTER) {
       tableType = "_year";
       partitionSize = YEAR;
     } else if (timeWindow > MONTH) {
       tableType = "_quarter";
       partitionSize = QUARTER;
     } else if (timeWindow > WEEK) {
       tableType = "_month";
       partitionSize = MONTH;
     } else {
       tableType = "_week";
       partitionSize = WEEK;
     }

     long currentPartition = now / partitionSize;
     long startPartition = start / partitionSize;
     long endPartition = end / partitionSize;
     while (fallback && partitionSize != CENTURY * 100) {
       // Check if the starting date is in the far distance from current time. If
       // it is, use down sampled data.
       if (startPartition + 2 < currentPartition) {
         fallback = true;
         if (partitionSize == DAY) {
           tableType = "_week";
           partitionSize = WEEK;
         } else if (partitionSize == WEEK) {
           tableType = "_month";
           partitionSize = MONTH;
         } else if (partitionSize == MONTH) {
           tableType = "_year";
           partitionSize = YEAR;
         } else if (partitionSize == YEAR) {
           tableType = "_decade";
           partitionSize = DECADE;
         } else if (partitionSize == DECADE) {
           tableType = "_century";
           partitionSize = CENTURY;
         } else {
           partitionSize = 100 * CENTURY;
         }
         currentPartition = now / partitionSize;
         startPartition = start / partitionSize;
         endPartition = end / partitionSize;
       } else {
         fallback = false;
       }
     }

     if (startPartition != endPartition) {
       int delta = (int) (endPartition - startPartition);
       tableNames = new String[delta + 1];
       for (int i = 0; i <= delta; i++) {
         long partition = startPartition + (long) i;
         tableNames[i] = tableName + "_" + partition + tableType;
       }
     } else {
       tableNames = new String[1];
       tableNames[0] = tableName + "_" + startPartition + tableType;
     }
     return tableNames;
   }

   public String[] findTableNameForCharts(String tableName, long start, long end) {
     String[] tableNames = null;
     String tableType = "_week";
     long now = (new Date()).getTime();
     long timeWindow = end - start;
     if (timeWindow > 60 * 60 * 1000) {
       timeWindow = timeWindow + 1;
     }
     long partitionSize = WEEK;
     boolean fallback = true;

     if (config.get("consolidator.table." + tableName) == null) {
       tableNames = new String[1];
       tableNames[0] = tableName;
       return tableNames;
     }

     if (timeWindow <= 0) {
       timeWindow = 1;
     }
     if (timeWindow > DECADE) {
       tableType = "_decade";
       partitionSize = CENTURY;
     } else if (timeWindow > YEAR) {
       tableType = "_decade";
       partitionSize = CENTURY;
     } else if (timeWindow > QUARTER) {
       tableType = "_decade";
       partitionSize = DECADE;
     } else if (timeWindow > MONTH) {
       tableType = "_year";
       partitionSize = YEAR;
     } else if (timeWindow > WEEK) {
       tableType = "_quarter";
       partitionSize = QUARTER;
     } else if (timeWindow > DAY) {
       tableType = "_month";
       partitionSize = MONTH;
     } else {
       tableType = "_week";
       partitionSize = WEEK;
     }


     long currentPartition = now / partitionSize;
     long startPartition = start / partitionSize;
     long endPartition = end / partitionSize;
     while (fallback && partitionSize != DECADE * 100) {
       // Check if the starting date is in the far distance from current time. If
       // it is, use down sampled data.
       if (startPartition + 2 < currentPartition) {
         fallback = true;
         if (partitionSize == DAY) {
           tableType = "_month";
           partitionSize = MONTH;
         } else if (partitionSize == WEEK) {
           tableType = "_quarter";
           partitionSize = QUARTER;
         } else if (partitionSize == MONTH) {
           tableType = "_year";
           partitionSize = YEAR;
         } else if (partitionSize == YEAR) {
           tableType = "_decade";
           partitionSize = DECADE;
         } else {
           partitionSize = CENTURY;
         }
         currentPartition = now / partitionSize;
         startPartition = start / partitionSize;
         endPartition = end / partitionSize;
       } else {
         fallback = false;
       }
     }

     if (startPartition != endPartition) {
       int delta = (int) (endPartition - startPartition);
       tableNames = new String[delta + 1];
       for (int i = 0; i <= delta; i++) {
         long partition = startPartition + (long) i;
         tableNames[i] = tableName + "_" + partition + tableType;
       }
     } else {
       tableNames = new String[1];
       tableNames[0] = tableName + "_" + startPartition + tableType;
     }

     return tableNames;
   }

   public static void main(String[] args) {
     DatabaseConfig dbc = new DatabaseConfig();
     String[] names = dbc.findTableName("system_metrics", 1216140020000L,
         1218645620000L);
     for (String n : names) {
       System.out.println("name:" + n);
     }
   }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package org.apache.hadoop.chukwa.database;


	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.Path;
	import java.util.*;
	import java.io.File;
	import java.io.FilenameFilter;

	public class DatabaseConfig {
	private Configuration config = null;
	public final static long CENTURY = 36500 * 24 * 60 * 60 * 1000L;
	public final static long DECADE = 3650 * 24 * 60 * 60 * 1000L;
	public final static long YEAR = 365 * 24 * 60 * 60 * 1000L;
	public final static long QUARTER = 91250 * 24 * 60 * 60L;
	public final static long MONTH = 30 * 24 * 60 * 60 * 1000L;
	public final static long WEEK = 7 * 24 * 60 * 60 * 1000L;
	public final static long DAY = 24 * 60 * 60 * 1000L;
	public final static String MDL_XML = "mdl.xml";

	public DatabaseConfig(String path) {
	Path fileResource = new Path(path);
	config = new Configuration();
	config.addResource(fileResource);
	}

	public DatabaseConfig() {
	String dataConfig = System.getenv("CHUKWA_CONF_DIR");
	if (dataConfig == null) {
	dataConfig = MDL_XML;
	} else {
	dataConfig += File.separator + MDL_XML;
	}
	Path fileResource = new Path(dataConfig);
	config = new Configuration();
	config.addResource(fileResource);

	if (System.getenv("CHUKWA_CONF_DIR") != null) {
	// Allow site-specific MDL files to be included in the
	// configuration so as to keep the "main" mdl.xml pure.
	File confDir = new File(System.getenv("CHUKWA_CONF_DIR"));
	File[] confFiles = confDir.listFiles(new FilenameFilter() {

	@Override
	public boolean accept(File dir, String name) {
	// Implements a naming convention of ending with "mdl.xml"
	// but is careful not to pick up mdl.xml itself again.
	return name.endsWith(MDL_XML) && !name.equals(MDL_XML);
	}

	});

	if (confFiles != null) {
	for (File confFile : confFiles)
	config.addResource(new Path(confFile.getAbsolutePath()));
	}
	}
	}

	public String get(String key) {
	return config.get(key);
	}

	public void put(String key, String value) {
	this.config.set(key, value);
	}

	public Iterator<?> iterator() {
	return this.config.iterator();
	}

	public HashMap<String, String> startWith(String key) {
	HashMap<String, String> transformer = new HashMap<String, String>();
	Iterator<?> entries = config.iterator();
	while (entries.hasNext()) {
	String entry = entries.next().toString();
	if (entry.startsWith(key)) {
	String[] metrics = entry.split("=");
	transformer.put(metrics[0], metrics[1]);
	}
	}
	return transformer;
	}

	public String[] findTableName(String tableName, long start, long end) {
	String[] tableNames = null;
	String tableType = "_week";
	long now = (new Date()).getTime();
	long timeWindow = end - start;
	long partitionSize = WEEK;
	boolean fallback = true;

	if (config.get("consolidator.table." + tableName) == null) {
	tableNames = new String[1];
	tableNames[0] = tableName;
	return tableNames;
	}
	if (timeWindow <= 0) {
	timeWindow = 1;
	}
	if (timeWindow > DECADE) {
	tableType = "_century";
	partitionSize = CENTURY;
	} else if (timeWindow > YEAR) {
	tableType = "_decade";
	partitionSize = DECADE;
	} else if (timeWindow > QUARTER) {
	tableType = "_year";
	partitionSize = YEAR;
	} else if (timeWindow > MONTH) {
	tableType = "_quarter";
	partitionSize = QUARTER;
	} else if (timeWindow > WEEK) {
	tableType = "_month";
	partitionSize = MONTH;
	} else {
	tableType = "_week";
	partitionSize = WEEK;
	}

	long currentPartition = now / partitionSize;
	long startPartition = start / partitionSize;
	long endPartition = end / partitionSize;
	while (fallback && partitionSize != CENTURY * 100) {
	// Check if the starting date is in the far distance from current time. If
	// it is, use down sampled data.
	if (startPartition + 2 < currentPartition) {
	fallback = true;
	if (partitionSize == DAY) {
	tableType = "_week";
	partitionSize = WEEK;
	} else if (partitionSize == WEEK) {
	tableType = "_month";
	partitionSize = MONTH;
	} else if (partitionSize == MONTH) {
	tableType = "_year";
	partitionSize = YEAR;
	} else if (partitionSize == YEAR) {
	tableType = "_decade";
	partitionSize = DECADE;
	} else if (partitionSize == DECADE) {
	tableType = "_century";
	partitionSize = CENTURY;
	} else {
	partitionSize = 100 * CENTURY;
	}
	currentPartition = now / partitionSize;
	startPartition = start / partitionSize;
	endPartition = end / partitionSize;
	} else {
	fallback = false;
	}
	}

	if (startPartition != endPartition) {
	int delta = (int) (endPartition - startPartition);
	tableNames = new String[delta + 1];
	for (int i = 0; i <= delta; i++) {
	long partition = startPartition + (long) i;
	tableNames[i] = tableName + "_" + partition + tableType;
	}
	} else {
	tableNames = new String[1];
	tableNames[0] = tableName + "_" + startPartition + tableType;
	}
	return tableNames;
	}

	public String[] findTableNameForCharts(String tableName, long start, long end) {
	String[] tableNames = null;
	String tableType = "_week";
	long now = (new Date()).getTime();
	long timeWindow = end - start;
	if (timeWindow > 60 * 60 * 1000) {
	timeWindow = timeWindow + 1;
	}
	long partitionSize = WEEK;
	boolean fallback = true;

	if (config.get("consolidator.table." + tableName) == null) {
	tableNames = new String[1];
	tableNames[0] = tableName;
	return tableNames;
	}

	if (timeWindow <= 0) {
	timeWindow = 1;
	}
	if (timeWindow > DECADE) {
	tableType = "_decade";
	partitionSize = CENTURY;
	} else if (timeWindow > YEAR) {
	tableType = "_decade";
	partitionSize = CENTURY;
	} else if (timeWindow > QUARTER) {
	tableType = "_decade";
	partitionSize = DECADE;
	} else if (timeWindow > MONTH) {
	tableType = "_year";
	partitionSize = YEAR;
	} else if (timeWindow > WEEK) {
	tableType = "_quarter";
	partitionSize = QUARTER;
	} else if (timeWindow > DAY) {
	tableType = "_month";
	partitionSize = MONTH;
	} else {
	tableType = "_week";
	partitionSize = WEEK;
	}


	long currentPartition = now / partitionSize;
	long startPartition = start / partitionSize;
	long endPartition = end / partitionSize;
	while (fallback && partitionSize != DECADE * 100) {
	// Check if the starting date is in the far distance from current time. If
	// it is, use down sampled data.
	if (startPartition + 2 < currentPartition) {
	fallback = true;
	if (partitionSize == DAY) {
	tableType = "_month";
	partitionSize = MONTH;
	} else if (partitionSize == WEEK) {
	tableType = "_quarter";
	partitionSize = QUARTER;
	} else if (partitionSize == MONTH) {
	tableType = "_year";
	partitionSize = YEAR;
	} else if (partitionSize == YEAR) {
	tableType = "_decade";
	partitionSize = DECADE;
	} else {
	partitionSize = CENTURY;
	}
	currentPartition = now / partitionSize;
	startPartition = start / partitionSize;
	endPartition = end / partitionSize;
	} else {
	fallback = false;
	}
	}

	if (startPartition != endPartition) {
	int delta = (int) (endPartition - startPartition);
	tableNames = new String[delta + 1];
	for (int i = 0; i <= delta; i++) {
	long partition = startPartition + (long) i;
	tableNames[i] = tableName + "_" + partition + tableType;
	}
	} else {
	tableNames = new String[1];
	tableNames[0] = tableName + "_" + startPartition + tableType;
	}

	return tableNames;
	}

	public static void main(String[] args) {
	DatabaseConfig dbc = new DatabaseConfig();
	String[] names = dbc.findTableName("system_metrics", 1216140020000L,
	1218645620000L);
	for (String n : names) {
	System.out.println("name:" + n);
	}
	}
	}