blob: dfb8c33c72d4be7f026693af179007014a780480 [file] [log] [blame]
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.federation.router;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.store.protocol.*;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Collections;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* This is a test through the Router move data to the Trash.
*/
public class TestRouterTrash {
public static final Logger LOG =
LoggerFactory.getLogger(TestRouterTrash.class);
private static StateStoreDFSCluster cluster;
private static MiniRouterDFSCluster.RouterContext routerContext;
private static MountTableResolver mountTable;
private static FileSystem routerFs;
private static FileSystem nnFs;
private static final String TEST_USER = "test-trash";
private static MiniRouterDFSCluster.NamenodeContext nnContext;
private static String ns0;
private static String ns1;
private static final String MOUNT_POINT = "/home/data";
private static final String FILE = MOUNT_POINT + "/file1";
private static final String TRASH_ROOT = "/user/" + TEST_USER + "/.Trash";
private static final String CURRENT = "/Current";
@BeforeClass
public static void globalSetUp() throws Exception {
// Build and start a federated cluster
cluster = new StateStoreDFSCluster(false, 2);
Configuration conf = new RouterConfigBuilder()
.stateStore()
.admin()
.rpc()
.http()
.build();
conf.set(FS_TRASH_INTERVAL_KEY, "100");
cluster.addRouterOverrides(conf);
cluster.startCluster();
cluster.startRouters();
cluster.waitClusterUp();
ns0 = cluster.getNameservices().get(0);
ns1 = cluster.getNameservices().get(1);
routerContext = cluster.getRandomRouter();
routerFs = routerContext.getFileSystem();
nnContext = cluster.getNamenode(ns0, null);
nnFs = nnContext.getFileSystem();
Router router = routerContext.getRouter();
mountTable = (MountTableResolver) router.getSubclusterResolver();
}
@AfterClass
public static void tearDown() {
if (cluster != null) {
cluster.stopRouter(routerContext);
cluster.shutdown();
cluster = null;
}
}
@After
public void clearMountTable() throws IOException {
RouterClient client = routerContext.getAdminClient();
MountTableManager mountTableManager = client.getMountTableManager();
GetMountTableEntriesRequest req1 =
GetMountTableEntriesRequest.newInstance("/");
GetMountTableEntriesResponse response =
mountTableManager.getMountTableEntries(req1);
for (MountTable entry : response.getEntries()) {
RemoveMountTableEntryRequest req2 =
RemoveMountTableEntryRequest.newInstance(entry.getSourcePath());
mountTableManager.removeMountTableEntry(req2);
}
}
@After
public void clearFile() throws IOException {
FileStatus[] fileStatuses = nnFs.listStatus(new Path("/"));
for (FileStatus file : fileStatuses) {
nnFs.delete(file.getPath(), true);
}
}
private boolean addMountTable(final MountTable entry) throws IOException {
RouterClient client = routerContext.getAdminClient();
MountTableManager mountTableManager = client.getMountTableManager();
AddMountTableEntryRequest addRequest =
AddMountTableEntryRequest.newInstance(entry);
AddMountTableEntryResponse addResponse =
mountTableManager.addMountTableEntry(addRequest);
// Reload the Router cache
mountTable.loadCache(true);
return addResponse.getStatus();
}
@Test
public void testMoveToTrashNoMountPoint() throws IOException,
URISyntaxException, InterruptedException {
MountTable addEntry = MountTable.newInstance(MOUNT_POINT,
Collections.singletonMap(ns0, MOUNT_POINT));
assertTrue(addMountTable(addEntry));
// current user client
DFSClient client = nnContext.getClient();
client.setOwner("/", TEST_USER, TEST_USER);
UserGroupInformation ugi = UserGroupInformation.
createRemoteUser(TEST_USER);
// test user client
client = nnContext.getClient(ugi);
client.mkdirs(MOUNT_POINT, new FsPermission("777"), true);
assertTrue(client.exists(MOUNT_POINT));
// create test file
client.create(FILE, true);
Path filePath = new Path(FILE);
FileStatus[] fileStatuses = routerFs.listStatus(filePath);
assertEquals(1, fileStatuses.length);
assertEquals(TEST_USER, fileStatuses[0].getOwner());
// move to Trash
Configuration routerConf = routerContext.getConf();
FileSystem fs =
DFSTestUtil.getFileSystemAs(ugi, routerConf);
Trash trash = new Trash(fs, routerConf);
assertTrue(trash.moveToTrash(filePath));
fileStatuses = nnFs.listStatus(
new Path(TRASH_ROOT + CURRENT + MOUNT_POINT));
assertEquals(1, fileStatuses.length);
assertTrue(nnFs.exists(new Path(TRASH_ROOT + CURRENT + FILE)));
assertTrue(nnFs.exists(new Path("/user/" +
TEST_USER + "/.Trash/Current" + FILE)));
// When the target path in Trash already exists.
client.create(FILE, true);
filePath = new Path(FILE);
fileStatuses = routerFs.listStatus(filePath);
assertEquals(1, fileStatuses.length);
assertTrue(trash.moveToTrash(filePath));
fileStatuses = routerFs.listStatus(
new Path(TRASH_ROOT + CURRENT + MOUNT_POINT));
assertEquals(2, fileStatuses.length);
}
@Test
public void testMoveToTrashWithKerberosUser() throws IOException,
URISyntaxException, InterruptedException {
//Constructs the structure of the KerBoers user name
String kerberosUser = "randomUser/dev@HADOOP.COM";
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(kerberosUser);
MountTable addEntry = MountTable.newInstance(MOUNT_POINT,
Collections.singletonMap(ns1, MOUNT_POINT));
assertTrue(addMountTable(addEntry));
// current user client
MiniRouterDFSCluster.NamenodeContext nn1Context = cluster.getNamenode(ns1, null);
DFSClient currentUserClientNs0 = nnContext.getClient();
DFSClient currentUserClientNs1 = nn1Context.getClient();
currentUserClientNs0.setOwner("/", ugi.getShortUserName(), ugi.getShortUserName());
currentUserClientNs1.setOwner("/", ugi.getShortUserName(), ugi.getShortUserName());
// test user client
DFSClient testUserClientNs1 = nn1Context.getClient(ugi);
testUserClientNs1.mkdirs(MOUNT_POINT, new FsPermission("777"), true);
assertTrue(testUserClientNs1.exists(MOUNT_POINT));
// create test file
testUserClientNs1.create(FILE, true);
Path filePath = new Path(FILE);
FileStatus[] fileStatuses = routerFs.listStatus(filePath);
assertEquals(1, fileStatuses.length);
assertEquals(ugi.getShortUserName(), fileStatuses[0].getOwner());
// move to Trash
Configuration routerConf = routerContext.getConf();
FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, routerConf);
Trash trash = new Trash(fs, routerConf);
assertTrue(trash.moveToTrash(filePath));
fileStatuses = fs.listStatus(
new Path("/user/" + ugi.getShortUserName() + "/.Trash/Current" + MOUNT_POINT));
assertEquals(1, fileStatuses.length);
}
@Test
public void testDeleteToTrashExistMountPoint() throws IOException,
URISyntaxException, InterruptedException {
MountTable addEntry = MountTable.newInstance(MOUNT_POINT,
Collections.singletonMap(ns0, MOUNT_POINT));
assertTrue(addMountTable(addEntry));
// add Trash mount point
addEntry = MountTable.newInstance(TRASH_ROOT,
Collections.singletonMap(ns1, TRASH_ROOT));
assertTrue(addMountTable(addEntry));
// current user client
DFSClient client = nnContext.getClient();
client.setOwner("/", TEST_USER, TEST_USER);
UserGroupInformation ugi = UserGroupInformation.
createRemoteUser(TEST_USER);
// test user client
client = nnContext.getClient(ugi);
client.mkdirs(MOUNT_POINT, new FsPermission("777"), true);
assertTrue(client.exists(MOUNT_POINT));
// create test file
client.create(FILE, true);
Path filePath = new Path(FILE);
FileStatus[] fileStatuses = routerFs.listStatus(filePath);
assertEquals(1, fileStatuses.length);
assertEquals(TEST_USER, fileStatuses[0].getOwner());
// move to Trash
Configuration routerConf = routerContext.getConf();
FileSystem fs =
DFSTestUtil.getFileSystemAs(ugi, routerConf);
Trash trash = new Trash(fs, routerConf);
assertTrue(trash.moveToTrash(filePath));
fileStatuses = nnFs.listStatus(
new Path(TRASH_ROOT + CURRENT + MOUNT_POINT));
assertEquals(1, fileStatuses.length);
assertTrue(nnFs.exists(new Path(TRASH_ROOT + CURRENT + FILE)));
// When the target path in Trash already exists.
client.create(FILE, true);
filePath = new Path(FILE);
fileStatuses = nnFs.listStatus(filePath);
assertEquals(1, fileStatuses.length);
assertTrue(trash.moveToTrash(filePath));
fileStatuses = nnFs.listStatus(
new Path(TRASH_ROOT + CURRENT + MOUNT_POINT));
assertEquals(2, fileStatuses.length);
}
@Test
public void testIsTrashPath() throws IOException {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
assertNotNull(ugi);
assertTrue(MountTableResolver.isTrashPath(
"/user/" + ugi.getUserName() + "/.Trash/Current" + MOUNT_POINT));
assertTrue(MountTableResolver.isTrashPath(
"/user/" + ugi.getUserName() +
"/.Trash/" + Time.now() + MOUNT_POINT));
assertFalse(MountTableResolver.isTrashPath(MOUNT_POINT));
// Contains TrashCurrent but does not begin with TrashCurrent.
assertFalse(MountTableResolver.isTrashPath("/home/user/" +
ugi.getUserName() + "/.Trash/Current" + MOUNT_POINT));
assertFalse(MountTableResolver.isTrashPath("/home/user/" +
ugi.getUserName() + "/.Trash/" + Time.now() + MOUNT_POINT));
// Special cases.
assertFalse(MountTableResolver.isTrashPath(""));
assertFalse(MountTableResolver.isTrashPath(
"/home/user/empty.Trash/Current"));
assertFalse(MountTableResolver.isTrashPath(
"/home/user/.Trash"));
assertFalse(MountTableResolver.isTrashPath(
"/.Trash/Current"));
}
@Test
public void testSubtractTrashCurrentPath() throws IOException {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
assertNotNull(ugi);
assertEquals(MOUNT_POINT, MountTableResolver.subtractTrashCurrentPath(
"/user/" + ugi.getUserName() + "/.Trash/Current" + MOUNT_POINT));
assertEquals(MOUNT_POINT, MountTableResolver.subtractTrashCurrentPath(
"/user/" + ugi.getUserName() +
"/.Trash/" + Time.now() + MOUNT_POINT));
// Contains TrashCurrent but does not begin with TrashCurrent.
assertEquals("/home/user/" + ugi.getUserName() +
"/.Trash/Current" + MOUNT_POINT, MountTableResolver.
subtractTrashCurrentPath("/home/user/" +
ugi.getUserName() + "/.Trash/Current" + MOUNT_POINT));
long time = Time.now();
assertEquals("/home/user/" + ugi.getUserName() +
"/.Trash/" + time + MOUNT_POINT, MountTableResolver.
subtractTrashCurrentPath("/home/user/" + ugi.getUserName() +
"/.Trash/" + time + MOUNT_POINT));
// Special cases.
assertEquals("", MountTableResolver.subtractTrashCurrentPath(""));
assertEquals("/home/user/empty.Trash/Current", MountTableResolver.
subtractTrashCurrentPath("/home/user/empty.Trash/Current"));
assertEquals("/home/user/.Trash", MountTableResolver.
subtractTrashCurrentPath("/home/user/.Trash"));
assertEquals("/.Trash/Current", MountTableResolver.
subtractTrashCurrentPath("/.Trash/Current"));
}
}