blob: a78b82af61aa0b98bb1f91d3812580b6324d8031 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.test;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Set;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.pig.LoadFunc;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestLoadFunc {
private static Path curHdfsDir;
private static String curHdfsRoot =
"hdfs://localhost.localdomain:12345";
@BeforeClass
public static void setUpBeforeClass() throws Exception {
curHdfsDir = new Path(curHdfsRoot + "/user/pig/");
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@Test
public void testGetAbsolutePath() throws IOException {
// test case: simple absolute path
Assert.assertEquals("/hadoop/test/passwd",
LoadFunc.getAbsolutePath("/hadoop/test/passwd", curHdfsDir));
}
@Test
public void testGetAbsolutePath2() throws IOException {
// test case: simple relative path
Assert.assertEquals(curHdfsRoot + "/user/pig/data/passwd",
LoadFunc.getAbsolutePath("data/passwd", curHdfsDir));
}
@Test
public void testGetAbsolutePath3() throws IOException {
// test case: remote hdfs path
String absPath = "hdfs://myhost.mydomain:37765/data/passwd";
Assert.assertEquals(absPath,
LoadFunc.getAbsolutePath(absPath, curHdfsDir));
}
@Test
public void testGetAbsolutePath4() throws IOException {
// test case: non dfs scheme
Assert.assertEquals("http://myhost:12345/data/passwd",
LoadFunc.getAbsolutePath("http://myhost:12345/data/passwd",
curHdfsDir));
}
@Test
public void testCommaSeparatedString() throws Exception {
// test case: comma separated absolute paths
Assert.assertEquals("/usr/pig/a,/usr/pig/b,/usr/pig/c",
LoadFunc.getAbsolutePath("/usr/pig/a,/usr/pig/b,/usr/pig/c",
curHdfsDir));
}
@Test
public void testCommaSeparatedString2() throws Exception {
// test case: comma separated relative paths
Assert.assertEquals(curHdfsRoot + "/user/pig/t?s*," +
curHdfsRoot + "/user/pig/test",
LoadFunc.getAbsolutePath("t?s*,test", curHdfsDir));
}
@Test
public void testCommaSeparatedString4() throws Exception {
// test case: comma separated paths with hadoop glob
Assert.assertEquals(curHdfsRoot + "/user/pig/test/{a,c}," +
curHdfsRoot + "/user/pig/test/b",
LoadFunc.getAbsolutePath("test/{a,c},test/b", curHdfsDir));
}
@Test
public void testCommaSeparatedString5() throws Exception {
// test case: comma separated paths with hadoop glob
Assert.assertEquals("/test/data/{a,c}," +
curHdfsRoot + "/user/pig/test/b",
LoadFunc.getAbsolutePath("/test/data/{a,c},test/b",
curHdfsDir));
}
@Test
public void testCommaSeparatedString6() throws Exception {
// test case: comma separated paths with hasoop glob
Assert.assertEquals(curHdfsRoot + "/user/pig/test/{a,c},/test/data/b",
LoadFunc.getAbsolutePath("test/{a,c},/test/data/b",
curHdfsDir));
}
@Test
public void testCommaSeparatedString7() throws Exception {
// test case: comma separated paths with white spaces
Assert.assertEquals(curHdfsRoot + "/user/pig/test/{a,c},/test/data/b",
LoadFunc.getAbsolutePath("test/{a,c}, /test/data/b",
curHdfsDir));
}
@Test(expected=IllegalArgumentException.class)
public void testCommaSeparatedString8() throws Exception {
// test case: comma separated paths with empty string
Assert.assertEquals(curHdfsRoot + "/user/pig/," +
curHdfsRoot + "/test/data/b",
LoadFunc.getAbsolutePath(", /test/data/b",
curHdfsDir));
}
@Test
public void testHarUrl() throws Exception {
// test case: input location is a har:// url
Assert.assertEquals("har:///user/pig/harfile",
LoadFunc.getAbsolutePath("har:///user/pig/harfile",
curHdfsDir));
}
@Test
public void testGlobPaths() throws IOException {
final String basedir = "file://" + System.getProperty("user.dir");
final String tempdir = Long.toString(System.currentTimeMillis());
final String nonexistentpath = basedir + "/" + tempdir + "/this_path_does_not_exist";
String locationStr = null;
Set<Path> paths;
Configuration conf = new Configuration();
// existent path
locationStr = basedir;
paths = LoadFunc.getGlobPaths(locationStr, conf, true);
assertFalse(paths.isEmpty());
// non-existent path
locationStr = nonexistentpath;
try {
paths = LoadFunc.getGlobPaths(locationStr, conf, true);
fail("Paths with pattern are not readable");
}
catch (IOException e) {
assertTrue(e.getMessage().contains("matches 0 files"));
}
// empty glob pattern
locationStr = basedir + "/{}";
try {
paths = LoadFunc.getGlobPaths(locationStr, conf, true);
fail();
}
catch (IOException e) {
assertTrue(e.getMessage().contains("matches 0 files"));
}
paths = LoadFunc.getGlobPaths(locationStr, conf, false);
assertTrue(paths.isEmpty());
// bad glob pattern
locationStr = basedir + "/{1,";
try {
LoadFunc.getGlobPaths(locationStr, conf, true);
Assert.fail("Negative test to test illegal file pattern. Should not be succeeding!");
}
catch (IOException e) {
// The message of the exception for illegal file pattern is rather
// long, so we simply confirm if it contains 'illegal file pattern'.
assertTrue(e.getMessage().contains("Illegal file pattern"));
}
}
}