changed backup/restore functionality to account for artifacts created by log manager
git-svn-id: https://asterixdb.googlecode.com/svn/branches/asterix_lsm_stabilization_installer@1448 eaa15691-b419-025a-1212-ee371bd00084
diff --git a/asterix-events/src/main/resources/events/backup/backup.sh b/asterix-events/src/main/resources/events/backup/backup.sh
index e10a4c9..601b81d 100755
--- a/asterix-events/src/main/resources/events/backup/backup.sh
+++ b/asterix-events/src/main/resources/events/backup/backup.sh
@@ -1,35 +1,50 @@
+echo $@ >> ~/backup.log
WORKING_DIR=$1
ASTERIX_INSTANCE_NAME=$2
-ASTERIX_DATA_DIR=$3
-BACKUP_ID=$4
-BACKUP_DIR=$5
-BACKUP_TYPE=$6
-NODE_ID=$7
+ASTERIX_IODEVICES=$3
+NODE_STORE=$4
+ASTERIX_ROOT_METADATA_DIR=$5
+BACKUP_ID=$6
+BACKUP_DIR=$7
+BACKUP_TYPE=$8
+NODE_ID=$9
-nodeStores=$(echo $ASTERIX_DATA_DIR | tr "," "\n")
+nodeIODevices=$(echo $ASTERIX_IODEVICES | tr "," "\n")
if [ $BACKUP_TYPE == "hdfs" ];
then
- HDFS_URL=$8
- HADOOP_VERSION=$9
+ HDFS_URL=${10}
+ HADOOP_VERSION=${11}
export HADOOP_HOME=$WORKING_DIR/hadoop-$HADOOP_VERSION
- for nodeStore in $nodeStores
+ index=1
+ for nodeIODevice in $nodeIODevices
do
- MANGLED_DIR_NAME=`echo $nodeStores | tr / _`
+ STORE_DIR=$nodeIODevice/$NODE_STORE
+ MANGLED_DIR_NAME=`echo $STORE_DIR | tr / _`
NODE_BACKUP_DIR=$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/$MANGLED_DIR_NAME
- echo "$HADOOP_HOME/bin/hadoop fs -copyFromLocal $nodeStore/$NODE_ID/$ASTERIX_INSTANCE_NAME/ $HDFS_URL/$NODE_BACKUP_DIR/" >> ~/backup.log
- $HADOOP_HOME/bin/hadoop fs -copyFromLocal $nodeStore/$NODE_ID/$ASTERIX_INSTANCE_NAME/ $HDFS_URL/$NODE_BACKUP_DIR/
+ $HADOOP_HOME/bin/hadoop fs -copyFromLocal $STORE_DIR/ $HDFS_URL/$NODE_BACKUP_DIR/
+ if [ $index -eq 1 ];
+ then
+ $HADOOP_HOME/bin/hadoop fs -copyFromLocal $nodeIODevice/$ASTERIX_ROOT_METADATA_DIR $HDFS_URL/$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/
+ fi
+ index=`expr $index + 1`
done
else
- for nodeStore in $nodeStores
+ index=1
+ for nodeIODevice in $nodeIODevices
do
- MANGLED_DIR_NAME=`echo $nodeStores | tr / _`
+ STORE_DIR=$nodeIODevice/$NODE_STORE
+ MANGLED_DIR_NAME=`echo $STORE_DIR | tr / _`
NODE_BACKUP_DIR=$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/$MANGLED_DIR_NAME
if [ ! -d $NODE_BACKUP_DIR ];
then
mkdir -p $NODE_BACKUP_DIR
fi
- echo "cp -r $nodeStore/$NODE_ID/$ASTERIX_INSTANCE_NAME/* $NODE_BACKUP_DIR/" >> ~/backup.log
- cp -r $nodeStore/$NODE_ID/$ASTERIX_INSTANCE_NAME/* $NODE_BACKUP_DIR/
+ cp -r $STORE_DIR/* $NODE_BACKUP_DIR/
+ if [ $index -eq 1 ];
+ then
+ cp -r $nodeIODevice/$ASTERIX_ROOT_METADATA_DIR $BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/
+ fi
+ index=`expr $index + 1`
done
fi
diff --git a/asterix-events/src/main/resources/events/restore/restore.sh b/asterix-events/src/main/resources/events/restore/restore.sh
index 515efc1..85e3eb3 100755
--- a/asterix-events/src/main/resources/events/restore/restore.sh
+++ b/asterix-events/src/main/resources/events/restore/restore.sh
@@ -1,32 +1,53 @@
WORKING_DIR=$1
ASTERIX_INSTANCE_NAME=$2
-ASTERIX_DATA_DIR=$3
-BACKUP_ID=$4
-BACKUP_DIR=$5
-BACKUP_TYPE=$6
-NODE_ID=$7
-HDFS_URL=$8
-HADOOP_VERSION=$9
+ASTERIX_IODEVICES=$3
+NODE_STORE=$4
+ASTERIX_ROOT_METADATA_DIR=$5
+BACKUP_ID=$6
+BACKUP_DIR=$7
+BACKUP_TYPE=$8
+NODE_ID=$9
+HDFS_URL=${10}
+HADOOP_VERSION=${11}
export HADOOP_HOME=$WORKING_DIR/hadoop-$HADOOP_VERSION
-nodeStores=$(echo $ASTERIX_DATA_DIR | tr "," "\n")
+iodevices=$(echo $ASTERIX_IODEVICES | tr "," "\n")
-for nodeStore in $nodeStores
+index=1
+for iodevice in $iodevices
do
- MANGLED_BACKUP_DIR=`echo $nodeStore | tr / _`
+ MANGLED_BACKUP_DIR=`echo $iodevice | tr / _`
NODE_BACKUP_DIR=$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/$MANGLED_BACKUP_DIR
- DEST_DIR=$nodeStore/$NODE_ID/$ASTERIX_INSTANCE_NAME
+ DEST_DIR=$iodevice/$NODE_STORE/
if [ ! -d $DEST_DIR ]
then
mkdir -p $DEST_DIR
else
- rm -rf $DEST_DIR/*
+ rm -rf $DEST_DIR
fi
-
+
+ if [ ! -d $iodevice/$ASTERIX_ROOT_METADATA_DIR ]
+ then
+ mkdir -p $iodevice/$ASTERIX_ROOT_METADATA_DIR
+ else
+ rm -rf $iodevice/$ASTERIX_ROOT_METADATA_DIR
+ fi
+
if [ $BACKUP_TYPE == "hdfs" ];
then
$HADOOP_HOME/bin/hadoop fs -copyToLocal $HDFS_URL/$NODE_BACKUP_DIR/* $DEST_DIR/
+ if [ $index -eq 1 ];
+ then
+ $HADOOP_HOME/bin/hadoop fs -copyToLocal $HDFS_URL/$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/$ASTERIX_ROOT_METADATA_DIR $iodevice/
+ fi
else
+ echo "cp -r $NODE_BACKUP_DIR/* $DEST_DIR/" >> ~/restore.log
cp -r $NODE_BACKUP_DIR/* $DEST_DIR/
+ if [ $index -eq 1 ];
+ then
+ echo "cp -r $BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/$ASTERIX_ROOT_METADATA_DIR $iodevice/" >> ~/restore.log
+ cp -r $BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/$ASTERIX_ROOT_METADATA_DIR $iodevice/
+ fi
fi
+ index=`expr $index + 1`
done
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java
index 603862c..297a168 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java
@@ -31,6 +31,8 @@
public class BackupCommand extends AbstractCommand {
+ public static final String ASTERIX_ROOT_METADATA_DIR = "asterix_root_metadata";
+
@Override
protected void execCommand() throws Exception {
InstallerDriver.initConfig();
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java
index af472da..56cd345 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java
@@ -55,9 +55,6 @@
JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
Unmarshaller unmarshaller = ctx.createUnmarshaller();
cluster = (Cluster) unmarshaller.unmarshal(new File(createConfig.clusterPath));
- cluster.setWorkingDir(new WorkingDir(cluster.getWorkingDir().getDir() + File.separator + asterixInstanceName,
- cluster.getWorkingDir().isNFS()));
- cluster.setLogdir(cluster.getLogdir() + File.separator + asterixInstanceName);
AsterixInstance asterixInstance = InstallerUtil.createAsterixInstance(asterixInstanceName, cluster);
InstallerUtil.evaluateConflictWithOtherInstances(asterixInstance);
InstallerUtil.createAsterixZip(asterixInstance, true);
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java
index 321fd1a..3164bd7 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java
@@ -131,6 +131,8 @@
buffer.append("configure" + ":" + " Auto-generate configuration for local psedu-distributed Asterix instance"
+ "\n");
buffer.append("shutdown " + ":" + " Shutdown the installer service" + "\n");
+ buffer.append("help " + ":" + " Provides usage description of a command" + "\n");
+
LOGGER.info(buffer.toString());
}
}
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
index a688abf..2f118c4 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
@@ -68,16 +68,40 @@
return instance;
}
- public static void createAsterixZip(AsterixInstance asterixInstance, boolean newDeployment) throws IOException {
+ public static void createAsterixZip(AsterixInstance asterixInstance, boolean newDeployment) throws IOException,
+ InterruptedException {
+
+ String modifiedZipPath = injectAsterixPropertyFile(InstallerDriver.getAsterixZip(), asterixInstance,
+ newDeployment);
+ injectAsterixLogPropertyFile(modifiedZipPath, asterixInstance);
+ }
+
+ private static String injectAsterixPropertyFile(String origZipFile, AsterixInstance asterixInstance,
+ boolean newDeployment) throws IOException {
writeAsterixConfigurationFile(asterixInstance, newDeployment);
String asterixInstanceDir = InstallerDriver.getAsterixDir() + File.separator + asterixInstance.getName();
- unzip(InstallerDriver.getAsterixZip(), asterixInstanceDir);
+ unzip(origZipFile, asterixInstanceDir);
File sourceJar = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+ asterixInstance.getAsterixVersion() + ".jar");
+ String asterixPropertyFile = "test.properties";
+ File replacementFile = new File(asterixInstanceDir + File.separator + "test.properties");
+ replaceInJar(sourceJar, asterixPropertyFile, replacementFile);
+ new File(asterixInstanceDir + File.separator + "test.properties").delete();
+ String asterixZipName = InstallerDriver.getAsterixZip().substring(
+ InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
+ zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
+ return asterixInstanceDir + File.separator + asterixZipName;
+ }
+ private static String injectAsterixLogPropertyFile(String origZipFile, AsterixInstance asterixInstance)
+ throws IOException {
+ String asterixInstanceDir = InstallerDriver.getAsterixDir() + File.separator + asterixInstance.getName();
+ unzip(origZipFile, asterixInstanceDir);
+ File sourceJar1 = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+ + asterixInstance.getAsterixVersion() + ".jar");
String txnLogPropertyFile = "log.properties";
Properties txnLogProperties = new Properties();
- URLClassLoader urlClassLoader = new URLClassLoader(new URL[] { sourceJar.toURI().toURL() });
+ URLClassLoader urlClassLoader = new URLClassLoader(new URL[] { sourceJar1.toURI().toURL() });
InputStream in = urlClassLoader.getResourceAsStream(txnLogPropertyFile);
if (in != null) {
txnLogProperties.load(in);
@@ -85,20 +109,16 @@
writeAsterixLogConfigurationFile(asterixInstance.getName(), asterixInstance.getCluster(), txnLogProperties);
- String asterixPropertyFile = "test.properties";
- File replacementFile = new File(asterixInstanceDir + File.separator + "test.properties");
- replaceInJar(sourceJar, asterixPropertyFile, replacementFile);
+ File sourceJar2 = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+ + asterixInstance.getAsterixVersion() + ".jar");
+ File replacementFile = new File(asterixInstanceDir + File.separator + "log.properties");
+ replaceInJar(sourceJar2, txnLogPropertyFile, replacementFile);
- replacementFile = new File(asterixInstanceDir + File.separator + "log.properties");
- replaceInJar(sourceJar, txnLogPropertyFile, replacementFile);
-
- new File(asterixInstanceDir + File.separator + "test.properties").delete();
new File(asterixInstanceDir + File.separator + "log.properties").delete();
-
String asterixZipName = InstallerDriver.getAsterixZip().substring(
InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
-
+ return asterixInstanceDir + File.separator + asterixZipName;
}
public static void addLibraryToAsterixZip(AsterixInstance asterixInstance, String dataverseName,
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
index 887ebda..61f0d27 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
@@ -29,6 +29,7 @@
import edu.uci.ics.asterix.event.schema.pattern.Pattern;
import edu.uci.ics.asterix.event.schema.pattern.Patterns;
import edu.uci.ics.asterix.event.schema.pattern.Value;
+import edu.uci.ics.asterix.installer.command.BackupCommand;
import edu.uci.ics.asterix.installer.command.StopCommand;
import edu.uci.ics.asterix.installer.driver.InstallerDriver;
import edu.uci.ics.asterix.installer.error.VerificationUtil;
@@ -133,21 +134,23 @@
private Patterns getHDFSBackUpAsterixPattern(AsterixInstance instance, Backup backupConf) throws Exception {
Cluster cluster = instance.getCluster();
- String clusterStore = instance.getCluster().getStore();
String hdfsUrl = backupConf.getHdfs().getUrl();
String hadoopVersion = backupConf.getHdfs().getVersion();
String hdfsBackupDir = backupConf.getBackupDir();
VerificationUtil.verifyBackupRestoreConfiguration(hdfsUrl, hadoopVersion, hdfsBackupDir);
String workingDir = cluster.getWorkingDir().getDir();
String backupId = "" + instance.getBackupInfo().size();
- String nodeStore;
+ String store;
String pargs;
+ String iodevices;
List<Pattern> patternList = new ArrayList<Pattern>();
for (Node node : cluster.getNode()) {
Nodeid nodeid = new Nodeid(new Value(null, node.getId()));
- nodeStore = node.getStore() == null ? clusterStore : node.getStore();
- pargs = workingDir + " " + instance.getName() + " " + nodeStore + " " + backupId + " " + hdfsBackupDir
- + " " + "hdfs" + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion;
+ iodevices = node.getIodevices() == null ? instance.getCluster().getIodevices() : node.getIodevices();
+ store = node.getStore() == null ? cluster.getStore() : node.getStore();
+ pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " "
+ + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + backupId + " " + hdfsBackupDir + " " + "hdfs"
+ + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion;
Event event = new Event("backup", nodeid, pargs);
patternList.add(new Pattern(null, 1, null, event));
}
@@ -156,18 +159,20 @@
private Patterns getLocalBackUpAsterixPattern(AsterixInstance instance, Backup backupConf) throws Exception {
Cluster cluster = instance.getCluster();
- String clusterStore = instance.getCluster().getStore();
String backupDir = backupConf.getBackupDir();
String workingDir = cluster.getWorkingDir().getDir();
String backupId = "" + instance.getBackupInfo().size();
- String nodeStore;
+ String iodevices;
+ String store;
String pargs;
List<Pattern> patternList = new ArrayList<Pattern>();
for (Node node : cluster.getNode()) {
Nodeid nodeid = new Nodeid(new Value(null, node.getId()));
- nodeStore = node.getStore() == null ? clusterStore : node.getStore();
- pargs = workingDir + " " + instance.getName() + " " + nodeStore + " " + backupId + " " + backupDir + " "
- + "local" + " " + node.getId();
+ iodevices = node.getIodevices() == null ? instance.getCluster().getIodevices() : node.getIodevices();
+ store = node.getStore() == null ? cluster.getStore() : node.getStore();
+ pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " "
+ + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + backupId + " " + backupDir + " " + "local" + " "
+ + node.getId();
Event event = new Event("backup", nodeid, pargs);
patternList.add(new Pattern(null, 1, null, event));
}
@@ -188,9 +193,11 @@
List<Pattern> patternList = new ArrayList<Pattern>();
for (Node node : cluster.getNode()) {
Nodeid nodeid = new Nodeid(new Value(null, node.getId()));
+ String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
nodeStore = node.getStore() == null ? clusterStore : node.getStore();
- pargs = workingDir + " " + instance.getName() + " " + nodeStore + " " + backupId + " " + hdfsBackupDir
- + " " + "hdfs" + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion;
+ pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + nodeStore + " "
+ + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + backupId + " " + " " + hdfsBackupDir + " "
+ + "hdfs" + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion;
Event event = new Event("restore", nodeid, pargs);
patternList.add(new Pattern(null, 1, null, event));
}
@@ -208,9 +215,11 @@
List<Pattern> patternList = new ArrayList<Pattern>();
for (Node node : cluster.getNode()) {
Nodeid nodeid = new Nodeid(new Value(null, node.getId()));
+ String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
nodeStore = node.getStore() == null ? clusterStore : node.getStore();
- pargs = workingDir + " " + instance.getName() + " " + nodeStore + " " + backupId + " " + backupDir + " "
- + "local" + " " + node.getId();
+ pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + nodeStore + " "
+ + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + backupId + " " + backupDir + " " + "local" + " "
+ + node.getId();
Event event = new Event("restore", nodeid, pargs);
patternList.add(new Pattern(null, 1, null, event));
}
