TAJO-2016: Fix storage tests hangs.
diff --git a/CHANGES b/CHANGES
index d55e4b8..feb723e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,8 @@
BUG FIXES
+ TAJO-2016: Fix storage tests hangs. (jinho)
+
TAJO-2013: FilterPushDownRule fails due to the unsupported index. (jinho)
TAJO-2014: TestRpcClientManager fails occasionally. (jinho)
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
index b6f65df..8472095 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
@@ -85,7 +85,7 @@
fs.close();
}
- @Test
+ @Test(timeout = 60000)
public void testReaderWithDFS() throws Exception {
final Configuration conf = new HdfsConfiguration();
String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString();
@@ -94,8 +94,7 @@
conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(2).build();
- cluster.waitClusterUp();
+ .numDataNodes(2).waitSafeMode(true).build();
TajoConf tajoConf = new TajoConf(conf);
tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo");
@@ -122,7 +121,7 @@
lineReader.close();
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index 9c30202..96e58eb 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -68,9 +69,9 @@
@Parameterized.Parameters
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][]{
- {"TEXT"},
- {"RCFILE"},
- {"SEQUENCEFILE"}
+ {BuiltinStorages.TEXT},
+ {BuiltinStorages.RCFILE},
+ {BuiltinStorages.SEQUENCE_FILE}
});
}
@@ -81,11 +82,11 @@
@Test
public void testGzipCodecCompressionData() throws IOException {
- if (dataFormat.equalsIgnoreCase("RCFILE")) {
+ if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(dataFormat, GzipCodec.class);
}
- } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
+ } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(dataFormat, GzipCodec.class);
}
@@ -131,7 +132,7 @@
extension = ((DelimitedTextFile.DelimitedTextFileAppender) appender).getExtension();
}
- int tupleNum = 100000;
+ int tupleNum = 1000;
VTuple vTuple;
for (int i = 0; i < tupleNum; i++) {
@@ -154,7 +155,7 @@
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, tablets[0], schema);
scanner.init();
- if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
+ if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
index b11aa5a..c99e0dc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
@@ -18,13 +18,13 @@
package org.apache.tajo.storage;
-import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import net.minidev.json.JSONObject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.*;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -70,7 +70,7 @@
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
VTuple[] tuples = new VTuple[4];
for(int i=0; i < tuples.length; i++) {
@@ -82,7 +82,7 @@
Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv");
localFs.mkdirs(path.getParent());
- FileTablespace fileStorageManager = (FileTablespace) TablespaceManager.getLocalFs();
+ FileTablespace fileStorageManager = TablespaceManager.getLocalFs();
assertEquals(localFs.getUri(), fileStorageManager.getFileSystem().getUri());
Appender appender = fileStorageManager.getAppender(meta, schema, path);
@@ -99,6 +99,7 @@
i++;
}
assertEquals(4,i);
+ localFs.delete(path, true);
}
@Test(timeout = 60000)
@@ -137,7 +138,7 @@
schema.addColumn("id", Type.INT4);
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
List<Fragment> splits = Lists.newArrayList();
// Get FileFragments in partition batch
@@ -154,7 +155,7 @@
assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]);
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
@@ -203,7 +204,7 @@
assertEquals(0, splits.size());
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
@@ -254,7 +255,7 @@
assertNotEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]);
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
@@ -271,7 +272,6 @@
new MiniDFSCluster.Builder(hdfsConf).numDataNodes(1).build();
URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo");
- Optional<Tablespace> existingTs = Optional.absent();
try {
/* Local FileSystem */
FileTablespace space = TablespaceManager.getLocalFs();
@@ -279,16 +279,17 @@
FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null);
distTablespace.init(conf);
- existingTs = TablespaceManager.addTableSpaceForTest(distTablespace);
+
+ TablespaceManager.addTableSpaceForTest(distTablespace);
/* Distributed FileSystem */
- space = (FileTablespace) TablespaceManager.get(uri);
+ space = TablespaceManager.get(uri);
assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri());
- space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace");
+ space = TablespaceManager.getByName("testGetFileTablespace");
assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri());
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
index 179c9eb..78e3390 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
@@ -117,7 +117,7 @@
@AfterClass
public static void tearDownClass() throws InterruptedException {
- cluster.shutdown(true);
+ cluster.shutdown();
}
public Path getTestDir(FileSystem fs, String dir) throws IOException {
