HDFS-11394. Support for getting erasure coding policy through WebHDFS#FileStatus.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 095b9ac..83fbc6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -134,6 +134,9 @@
}
if (status.isErasureCoded()) {
m.put("ecBit", true);
+ if (status.getErasureCodingPolicy() != null) {
+ m.put("ecPolicy", status.getErasureCodingPolicy().getName());
+ }
}
if (status.isSnapshotEnabled()) {
m.put("snapshotEnabled", status.isSnapshotEnabled());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index c94122e..8571d82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -44,6 +44,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Random;
@@ -81,6 +82,7 @@
import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@@ -109,6 +111,7 @@
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DataChecksum;
import org.apache.log4j.Level;
+import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
@@ -1578,6 +1581,114 @@
}
}
+ /**
+ * Tests that the LISTSTATUS ang GETFILESTATUS WebHDFS calls return the
+ * ecPolicy for EC files.
+ */
+ @Test(timeout=300000)
+ public void testECPolicyInFileStatus() throws Exception {
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
+ final String ecPolicyName = ecPolicy.getName();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(5)
+ .build();
+ cluster.waitActive();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+
+ // Create an EC dir and write a test file in it
+ final Path ecDir = new Path("/ec");
+ Path ecFile = new Path(ecDir, "ec_file.txt");
+ Path nonEcFile = new Path(ecDir, "non_ec_file.txt");
+ fs.mkdirs(ecDir);
+
+ // Create a non-EC file before enabling ec policy
+ DFSTestUtil.createFile(fs, nonEcFile, 1024, (short) 1, 0);
+
+ fs.enableErasureCodingPolicy(ecPolicyName);
+ fs.setErasureCodingPolicy(ecDir, ecPolicyName);
+
+ // Create a EC file
+ DFSTestUtil.createFile(fs, ecFile, 1024, (short) 1, 0);
+
+ // Query webhdfs REST API to list statuses of files/directories in ecDir
+ InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
+ URL listStatusUrl = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + ecDir.toString() + "?op=LISTSTATUS");
+
+ HttpURLConnection conn = (HttpURLConnection) listStatusUrl.openConnection();
+ conn.setRequestMethod("GET");
+ conn.setInstanceFollowRedirects(false);
+ String listStatusResponse = IOUtils.toString(conn.getInputStream(),
+ StandardCharsets.UTF_8);
+ Assert.assertEquals("Response wasn't " + HttpURLConnection.HTTP_OK,
+ HttpURLConnection.HTTP_OK, conn.getResponseCode());
+
+ // Verify that ecPolicy is set in the ListStatus response for ec file
+ String ecpolicyForECfile = getECPolicyFromFileStatusJson(
+ getFileStatusJson(listStatusResponse, ecFile.getName()));
+ assertEquals("EC policy for ecFile should match the set EC policy",
+ ecpolicyForECfile, ecPolicyName);
+
+ // Verify that ecPolicy is not set in the ListStatus response for non-ec
+ // file
+ String ecPolicyForNonECfile = getECPolicyFromFileStatusJson(
+ getFileStatusJson(listStatusResponse, nonEcFile.getName()));
+ assertEquals("EC policy for nonEcFile should be null (not set)",
+ ecPolicyForNonECfile, null);
+
+ // Query webhdfs REST API to get fileStatus for ecFile
+ URL getFileStatusUrl = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + ecFile.toString() +
+ "?op=GETFILESTATUS");
+
+ conn = (HttpURLConnection) getFileStatusUrl.openConnection();
+ conn.setRequestMethod("GET");
+ conn.setInstanceFollowRedirects(false);
+ String getFileStatusResponse = IOUtils.toString(conn.getInputStream(),
+ StandardCharsets.UTF_8);
+ Assert.assertEquals("Response wasn't " + HttpURLConnection.HTTP_OK,
+ HttpURLConnection.HTTP_OK, conn.getResponseCode());
+
+ // Verify that ecPolicy is set in getFileStatus response for ecFile
+ JSONObject fileStatusObject = new JSONObject(getFileStatusResponse)
+ .getJSONObject("FileStatus");
+ ecpolicyForECfile = getECPolicyFromFileStatusJson(fileStatusObject);
+ assertEquals("EC policy for ecFile should match the set EC policy",
+ ecpolicyForECfile, ecPolicyName);
+ }
+
+ /**
+ * Get FileStatus JSONObject from ListStatus response.
+ */
+ private JSONObject getFileStatusJson(String response, String fileName)
+ throws JSONException {
+ JSONObject listStatusResponseJson = new JSONObject(response);
+ JSONArray fileStatusArray = listStatusResponseJson
+ .getJSONObject("FileStatuses")
+ .getJSONArray("FileStatus");
+ for (int i = 0; i < fileStatusArray.length(); i++) {
+ JSONObject fileStatusJsonObject = fileStatusArray.getJSONObject(i);
+ if (fileName.equals(fileStatusJsonObject.get("pathSuffix"))) {
+ return fileStatusJsonObject;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get ECPolicy name from FileStatus JSONObject.
+ */
+ private String getECPolicyFromFileStatusJson(JSONObject fileStatusJsonObject)
+ throws JSONException {
+ if (fileStatusJsonObject.has("ecPolicy")) {
+ return fileStatusJsonObject.getString("ecPolicy");
+ } else {
+ return null;
+ }
+ }
+
final static class DummyThrowable extends Throwable {
}
}
