PIG-4939: QueryParserUtils.setHdfsServers(QueryParserUtils.java:104) should not be called for non-dfs methods
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1768807 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 587aaf3..3e6c55c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -24,6 +24,9 @@
IMPROVEMENTS
+PIG-4939: QueryParserUtils.setHdfsServers(QueryParserUtils.java:104) should not be called for non-dfs
+ methods (szita via daijy)
+
PIG-5034: Remove org.apache.hadoop.hive.serde2.objectinspector.primitive package (nkollar via daijy)
PIG-5036: Remove biggish from e2e input dataset (daijy)
diff --git a/src/docs/src/documentation/content/xdocs/udf.xml b/src/docs/src/documentation/content/xdocs/udf.xml
index 5f2b77e..fb7f836 100644
--- a/src/docs/src/documentation/content/xdocs/udf.xml
+++ b/src/docs/src/documentation/content/xdocs/udf.xml
@@ -1038,6 +1038,9 @@
<li id="loadpredicatepushdown"><a href="http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/LoadPredicatePushdown.java?view=markup">LoadPredicatePushdown</a>
has the methods to push predicates to the loader. It is different than LoadMetadata.setPartitionFilter in that loader may load records which does not satisfy the predicates. In other words, predicates is only a hint. Note this interface is still in development and might change in next version. Currently only OrcStorage implements this interface.</li>
+
+<li id="nonfsloadfunc"><a href="http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/NonFSLoadFunc.java?view=markup">NonFSLoadFunc</a>
+ is a marker interface to indicate that a LoadFunc implementation is not a filesystem loader. This is useful for LoadFunc classes that for example supply queries instead of filesystem pathes to the LOAD operator.</li>
</ul>
<p>The LoadFunc abstract class is the main class to extend for implementing a loader. The methods which need to be overridden are explained below:</p>
diff --git a/src/org/apache/pig/NonFSLoadFunc.java b/src/org/apache/pig/NonFSLoadFunc.java
new file mode 100644
index 0000000..605b693
--- /dev/null
+++ b/src/org/apache/pig/NonFSLoadFunc.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+/**
+ * Marker interface to distinguish LoadFunc implementations that don't use file system sources.
+ */
+public interface NonFSLoadFunc {
+
+}
diff --git a/src/org/apache/pig/parser/LogicalPlanBuilder.java b/src/org/apache/pig/parser/LogicalPlanBuilder.java
index df4a92a..4cc2e18 100644
--- a/src/org/apache/pig/parser/LogicalPlanBuilder.java
+++ b/src/org/apache/pig/parser/LogicalPlanBuilder.java
@@ -34,6 +34,7 @@
import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
+import org.apache.pig.NonFSLoadFunc;
import org.apache.pig.PigConfiguration;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
@@ -888,7 +889,7 @@
if (absolutePath == null) {
absolutePath = loFunc.relativeToAbsolutePath( filename, QueryParserUtils.getCurrentDir( pigContext ) );
- if (absolutePath!=null) {
+ if (absolutePath!=null && !(loFunc instanceof NonFSLoadFunc)) {
QueryParserUtils.setHdfsServers( absolutePath, pigContext );
}
fileNameMap.put( fileNameKey, absolutePath );
diff --git a/test/org/apache/pig/parser/TestQueryParserUtils.java b/test/org/apache/pig/parser/TestQueryParserUtils.java
index abfc768..a6fb391 100644
--- a/test/org/apache/pig/parser/TestQueryParserUtils.java
+++ b/test/org/apache/pig/parser/TestQueryParserUtils.java
@@ -19,10 +19,20 @@
import static org.junit.Assert.assertEquals;
+import java.io.IOException;
import java.util.Properties;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.pig.ExecType;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.NonFSLoadFunc;
+import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.test.Util;
import org.junit.Test;
@@ -107,8 +117,44 @@
}
-
-
}
+
+ @Test
+ public void testNonFSLoadFunc() throws Exception {
+ PigServer pigServer = new PigServer(Util.getLocalTestMode(), new Properties());
+ pigServer.registerQuery("A = load 'hbase://query/SELECT ID, NAME, DATE FROM HIRES WHERE DATE > TO_DATE(\"1990-12-21 05:55:00.000\")' using org.apache.pig.parser.TestQueryParserUtils$DummyNonFSLoader();");
+ pigServer.shutdown();
+ }
+
+ /**
+ * Test class for testNonFSLoadFuncNoSetHdfsServersCall test case
+ */
+ public static class DummyNonFSLoader extends LoadFunc implements NonFSLoadFunc {
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public InputFormat getInputFormat() throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public Tuple getNext() throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public String relativeToAbsolutePath(String location, Path curDir) throws IOException {
+ return location;
+ }
+ }
}
