| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.ozone.om; |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.nio.file.Path; |
| import java.nio.file.Paths; |
| import java.util.ArrayList; |
| import java.util.BitSet; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.TreeSet; |
| import java.util.UUID; |
| |
| import org.apache.commons.io.FileUtils; |
| import org.apache.commons.lang.RandomStringUtils; |
| import org.apache.hadoop.conf.StorageUnit; |
| import org.apache.hadoop.hdds.HddsConfigKeys; |
| import org.apache.hadoop.hdds.client.BlockID; |
| import org.apache.hadoop.hdds.conf.OzoneConfiguration; |
| import org.apache.hadoop.hdds.protocol.DatanodeDetails; |
| import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; |
| import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; |
| import org.apache.hadoop.hdds.scm.TestUtils; |
| import org.apache.hadoop.hdds.scm.container.MockNodeManager; |
| import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; |
| import org.apache.hadoop.hdds.scm.exceptions.SCMException; |
| import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; |
| import org.apache.hadoop.hdds.scm.net.NetworkTopology; |
| import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; |
| import org.apache.hadoop.hdds.scm.net.NodeSchema; |
| import org.apache.hadoop.hdds.scm.net.NodeSchemaManager; |
| import org.apache.hadoop.hdds.scm.node.NodeManager; |
| import org.apache.hadoop.hdds.scm.pipeline.Pipeline; |
| import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; |
| import org.apache.hadoop.hdds.scm.server.SCMConfigurator; |
| import org.apache.hadoop.hdds.scm.server.StorageContainerManager; |
| import org.apache.hadoop.ozone.OzoneAcl; |
| import org.apache.hadoop.ozone.OzoneConfigKeys; |
| import org.apache.hadoop.ozone.OzoneTestUtils; |
| import org.apache.hadoop.ozone.om.exceptions.OMException; |
| import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; |
| import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; |
| import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; |
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; |
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; |
| import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; |
| import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; |
| import org.apache.hadoop.ozone.om.helpers.OpenKeySession; |
| import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; |
| import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; |
| import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; |
| import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; |
| import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; |
| import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; |
| import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; |
| import org.apache.hadoop.ozone.security.acl.OzoneObj; |
| import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; |
| import org.apache.hadoop.ozone.security.acl.RequestContext; |
| import org.apache.hadoop.security.UserGroupInformation; |
| import org.apache.hadoop.test.GenericTestUtils; |
| import org.apache.hadoop.test.LambdaTestUtils; |
| |
| import org.apache.hadoop.util.Time; |
| import org.junit.After; |
| import org.junit.AfterClass; |
| import org.junit.Assert; |
| import org.junit.Assume; |
| import org.junit.BeforeClass; |
| import org.junit.Rule; |
| import org.junit.Test; |
| import org.junit.rules.ExpectedException; |
| import org.mockito.Mockito; |
| |
| import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; |
| import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX; |
| import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; |
| import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; |
| |
| import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA; |
| import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA; |
| import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA; |
| |
| import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL; |
| |
| /** |
| * Test class for @{@link KeyManagerImpl}. |
| */ |
| public class TestKeyManagerImpl { |
| |
| private static PrefixManager prefixManager; |
| private static KeyManagerImpl keyManager; |
| private static NodeManager nodeManager; |
| private static StorageContainerManager scm; |
| private static ScmBlockLocationProtocol mockScmBlockLocationProtocol; |
| private static OzoneConfiguration conf; |
| private static OMMetadataManager metadataManager; |
| private static File dir; |
| private static long scmBlockSize; |
| private static final String KEY_NAME = "key1"; |
| private static final String BUCKET_NAME = "bucket1"; |
| private static final String VOLUME_NAME = "vol1"; |
| |
| @Rule |
| public ExpectedException exception = ExpectedException.none(); |
| |
| @BeforeClass |
| public static void setUp() throws Exception { |
| conf = new OzoneConfiguration(); |
| dir = GenericTestUtils.getRandomizedTestDir(); |
| conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString()); |
| conf.set(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, "true"); |
| mockScmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class); |
| metadataManager = new OmMetadataManagerImpl(conf); |
| nodeManager = new MockNodeManager(true, 10); |
| NodeSchema[] schemas = new NodeSchema[] |
| {ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA}; |
| NodeSchemaManager schemaManager = NodeSchemaManager.getInstance(); |
| schemaManager.init(schemas, false); |
| NetworkTopology clusterMap = new NetworkTopologyImpl(schemaManager); |
| nodeManager.getAllNodes().stream().forEach(node -> { |
| node.setNetworkName(node.getUuidString()); |
| clusterMap.add(node); |
| }); |
| ((MockNodeManager)nodeManager).setNetworkTopology(clusterMap); |
| SCMConfigurator configurator = new SCMConfigurator(); |
| configurator.setScmNodeManager(nodeManager); |
| configurator.setNetworkTopology(clusterMap); |
| scm = TestUtils.getScm(conf, configurator); |
| scm.start(); |
| scm.exitSafeMode(); |
| scmBlockSize = (long) conf |
| .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, |
| StorageUnit.BYTES); |
| conf.setLong(OZONE_KEY_PREALLOCATION_BLOCKS_MAX, 10); |
| |
| keyManager = |
| new KeyManagerImpl(scm.getBlockProtocolServer(), metadataManager, conf, |
| "om1", null); |
| prefixManager = new PrefixManagerImpl(metadataManager, false); |
| |
| Mockito.when(mockScmBlockLocationProtocol |
| .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), |
| Mockito.any(ReplicationType.class), |
| Mockito.any(ReplicationFactor.class), Mockito.anyString(), |
| Mockito.any(ExcludeList.class))).thenThrow( |
| new SCMException("SafeModePrecheck failed for allocateBlock", |
| ResultCodes.SAFE_MODE_EXCEPTION)); |
| createVolume(VOLUME_NAME); |
| createBucket(VOLUME_NAME, BUCKET_NAME); |
| } |
| |
| @AfterClass |
| public static void cleanup() throws Exception { |
| scm.stop(); |
| scm.join(); |
| metadataManager.stop(); |
| keyManager.stop(); |
| FileUtils.deleteDirectory(dir); |
| } |
| |
| @After |
| public void cleanupTest() throws IOException { |
| List<OzoneFileStatus> fileStatuses = keyManager |
| .listStatus(createBuilder().setKeyName("").build(), true, "", 100000); |
| for (OzoneFileStatus fileStatus : fileStatuses) { |
| if (fileStatus.isFile()) { |
| keyManager.deleteKey( |
| createKeyArgs(fileStatus.getPath().toString().substring(1))); |
| } else { |
| keyManager.deleteKey(createKeyArgs(OzoneFSUtils |
| .addTrailingSlashIfNeeded( |
| fileStatus.getPath().toString().substring(1)))); |
| } |
| } |
| } |
| |
| private static void createBucket(String volumeName, String bucketName) |
| throws IOException { |
| OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .build(); |
| |
| TestOMRequestUtils.addBucketToOM(metadataManager, bucketInfo); |
| } |
| |
| private static void createVolume(String volumeName) throws IOException { |
| OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder() |
| .setVolume(volumeName) |
| .setAdminName("bilbo") |
| .setOwnerName("bilbo") |
| .build(); |
| TestOMRequestUtils.addVolumeToOM(metadataManager, volumeArgs); |
| } |
| |
| @Test |
| public void allocateBlockFailureInSafeMode() throws Exception { |
| KeyManager keyManager1 = new KeyManagerImpl(mockScmBlockLocationProtocol, |
| metadataManager, conf, "om1", null); |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(KEY_NAME) |
| .build(); |
| |
| // As now openKey will allocate at least one block, even if the size |
| // passed is 0. So adding an entry to openKeyTable manually to test |
| // allocateBlock failure. |
| OmKeyInfo omKeyInfo = new OmKeyInfo.Builder() |
| .setVolumeName(keyArgs.getVolumeName()) |
| .setBucketName(keyArgs.getBucketName()) |
| .setKeyName(keyArgs.getKeyName()) |
| .setOmKeyLocationInfos(Collections.singletonList( |
| new OmKeyLocationInfoGroup(0, new ArrayList<>()))) |
| .setCreationTime(Time.now()) |
| .setModificationTime(Time.now()) |
| .setDataSize(0) |
| .setReplicationType(keyArgs.getType()) |
| .setReplicationFactor(keyArgs.getFactor()) |
| .setFileEncryptionInfo(null).build(); |
| metadataManager.getOpenKeyTable().put( |
| metadataManager.getOpenKey(VOLUME_NAME, BUCKET_NAME, KEY_NAME, 1L), |
| omKeyInfo); |
| LambdaTestUtils.intercept(OMException.class, |
| "SafeModePrecheck failed for allocateBlock", () -> { |
| keyManager1 |
| .allocateBlock(keyArgs, 1L, new ExcludeList()); |
| }); |
| } |
| |
| @Test |
| public void openKeyFailureInSafeMode() throws Exception { |
| UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); |
| KeyManager keyManager1 = new KeyManagerImpl(mockScmBlockLocationProtocol, |
| metadataManager, conf, "om1", null); |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(KEY_NAME) |
| .setDataSize(1000) |
| .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(), |
| ALL, ALL)) |
| .build(); |
| LambdaTestUtils.intercept(OMException.class, |
| "SafeModePrecheck failed for allocateBlock", () -> { |
| keyManager1.openKey(keyArgs); |
| }); |
| } |
| |
| @Test |
| public void openKeyWithMultipleBlocks() throws IOException { |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(UUID.randomUUID().toString()) |
| .setDataSize(scmBlockSize * 10) |
| .build(); |
| OpenKeySession keySession = keyManager.openKey(keyArgs); |
| OmKeyInfo keyInfo = keySession.getKeyInfo(); |
| Assert.assertEquals(10, |
| keyInfo.getLatestVersionLocations().getLocationList().size()); |
| } |
| |
| @Test |
| public void testCreateDirectory() throws IOException { |
| // Create directory where the parent directory does not exist |
| String keyName = RandomStringUtils.randomAlphabetic(5); |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| for (int i =0; i< 5; i++) { |
| keyName += "/" + RandomStringUtils.randomAlphabetic(5); |
| } |
| keyManager.createDirectory(keyArgs); |
| Path path = Paths.get(keyName); |
| while (path != null) { |
| // verify parent directories are created |
| Assert.assertTrue(keyManager.getFileStatus(keyArgs).isDirectory()); |
| path = path.getParent(); |
| } |
| |
| // make sure create directory fails where parent is a file |
| keyName = RandomStringUtils.randomAlphabetic(5); |
| keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| OpenKeySession keySession = keyManager.openKey(keyArgs); |
| keyArgs.setLocationInfoList( |
| keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| for (int i =0; i< 5; i++) { |
| keyName += "/" + RandomStringUtils.randomAlphabetic(5); |
| } |
| try { |
| keyManager.createDirectory(keyArgs); |
| Assert.fail("Creation should fail for directory."); |
| } catch (OMException e) { |
| Assert.assertEquals(e.getResult(), |
| OMException.ResultCodes.FILE_ALREADY_EXISTS); |
| } |
| |
| // create directory for root directory |
| keyName = ""; |
| keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| keyManager.createDirectory(keyArgs); |
| Assert.assertTrue(keyManager.getFileStatus(keyArgs).isDirectory()); |
| |
| // create directory where parent is root |
| keyName = RandomStringUtils.randomAlphabetic(5); |
| keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| keyManager.createDirectory(keyArgs); |
| Assert.assertTrue(keyManager.getFileStatus(keyArgs).isDirectory()); |
| } |
| |
| @Test |
| public void testOpenFile() throws IOException { |
| // create key |
| String keyName = RandomStringUtils.randomAlphabetic(5); |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| OpenKeySession keySession = keyManager.createFile(keyArgs, false, false); |
| keyArgs.setLocationInfoList( |
| keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| |
| // try to open created key with overWrite flag set to false |
| try { |
| keyManager.createFile(keyArgs, false, false); |
| Assert.fail("Open key should fail for non overwrite create"); |
| } catch (OMException ex) { |
| if (ex.getResult() != OMException.ResultCodes.FILE_ALREADY_EXISTS) { |
| throw ex; |
| } |
| } |
| |
| // create file should pass with overwrite flag set to true |
| keyManager.createFile(keyArgs, true, false); |
| |
| // try to create a file where parent directories do not exist and |
| // recursive flag is set to false |
| keyName = RandomStringUtils.randomAlphabetic(5); |
| for (int i =0; i< 5; i++) { |
| keyName += "/" + RandomStringUtils.randomAlphabetic(5); |
| } |
| keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| try { |
| keyManager.createFile(keyArgs, false, false); |
| Assert.fail("Open file should fail for non recursive write"); |
| } catch (OMException ex) { |
| if (ex.getResult() != OMException.ResultCodes.DIRECTORY_NOT_FOUND) { |
| throw ex; |
| } |
| } |
| |
| // file create should pass when recursive flag is set to true |
| keySession = keyManager.createFile(keyArgs, false, true); |
| keyArgs.setLocationInfoList( |
| keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| Assert.assertTrue(keyManager |
| .getFileStatus(keyArgs).isFile()); |
| |
| // try creating a file over a directory |
| keyArgs = createBuilder() |
| .setKeyName("") |
| .build(); |
| try { |
| keyManager.createFile(keyArgs, true, true); |
| Assert.fail("Open file should fail for non recursive write"); |
| } catch (OMException ex) { |
| if (ex.getResult() != OMException.ResultCodes.NOT_A_FILE) { |
| throw ex; |
| } |
| } |
| } |
| |
| @Test |
| public void testCheckAccessForFileKey() throws Exception { |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName("testdir/deep/NOTICE.txt") |
| .build(); |
| OpenKeySession keySession = keyManager.createFile(keyArgs, false, true); |
| keyArgs.setLocationInfoList( |
| keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| |
| OzoneObj fileKey = OzoneObjInfo.Builder.fromKeyArgs(keyArgs) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| RequestContext context = currentUserReads(); |
| Assert.assertTrue(keyManager.checkAccess(fileKey, context)); |
| |
| OzoneObj parentDirKey = OzoneObjInfo.Builder.fromKeyArgs(keyArgs) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .setKeyName("testdir") |
| .build(); |
| Assert.assertTrue(keyManager.checkAccess(parentDirKey, context)); |
| } |
| |
| @Test |
| public void testCheckAccessForNonExistentKey() throws Exception { |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName("testdir/deep/NO_SUCH_FILE.txt") |
| .build(); |
| OzoneObj nonExistentKey = OzoneObjInfo.Builder.fromKeyArgs(keyArgs) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| OzoneTestUtils.expectOmException(OMException.ResultCodes.KEY_NOT_FOUND, |
| () -> keyManager.checkAccess(nonExistentKey, currentUserReads())); |
| } |
| |
| @Test |
| public void testCheckAccessForDirectoryKey() throws Exception { |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName("some/dir") |
| .build(); |
| keyManager.createDirectory(keyArgs); |
| |
| OzoneObj dirKey = OzoneObjInfo.Builder.fromKeyArgs(keyArgs) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| Assert.assertTrue(keyManager.checkAccess(dirKey, currentUserReads())); |
| } |
| |
| @Test |
| public void testPrefixAclOps() throws IOException { |
| String volumeName = "vol1"; |
| String bucketName = "bucket1"; |
| String prefix1 = "pf1/"; |
| |
| OzoneObj ozPrefix1 = new OzoneObjInfo.Builder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .setPrefixName(prefix1) |
| .setResType(OzoneObj.ResourceType.PREFIX) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| |
| OzoneAcl ozAcl1 = new OzoneAcl(ACLIdentityType.USER, "user1", |
| ACLType.READ, ACCESS); |
| prefixManager.addAcl(ozPrefix1, ozAcl1); |
| |
| List<OzoneAcl> ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(1, ozAclGet.size()); |
| Assert.assertEquals(ozAcl1, ozAclGet.get(0)); |
| |
| List<OzoneAcl> acls = new ArrayList<>(); |
| OzoneAcl ozAcl2 = new OzoneAcl(ACLIdentityType.USER, "admin", |
| ACLType.ALL, ACCESS); |
| |
| BitSet rwRights = new BitSet(); |
| rwRights.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); |
| rwRights.set(IAccessAuthorizer.ACLType.READ.ordinal()); |
| OzoneAcl ozAcl3 = new OzoneAcl(ACLIdentityType.GROUP, "dev", |
| rwRights, ACCESS); |
| |
| BitSet wRights = new BitSet(); |
| wRights.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); |
| OzoneAcl ozAcl4 = new OzoneAcl(ACLIdentityType.GROUP, "dev", |
| wRights, ACCESS); |
| |
| BitSet rRights = new BitSet(); |
| rRights.set(IAccessAuthorizer.ACLType.READ.ordinal()); |
| OzoneAcl ozAcl5 = new OzoneAcl(ACLIdentityType.GROUP, "dev", |
| rRights, ACCESS); |
| |
| acls.add(ozAcl2); |
| acls.add(ozAcl3); |
| |
| prefixManager.setAcl(ozPrefix1, acls); |
| ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(2, ozAclGet.size()); |
| |
| int matchEntries = 0; |
| for (OzoneAcl acl : ozAclGet) { |
| if (acl.getType() == ACLIdentityType.GROUP) { |
| Assert.assertEquals(ozAcl3, acl); |
| matchEntries++; |
| } |
| if (acl.getType() == ACLIdentityType.USER) { |
| Assert.assertEquals(ozAcl2, acl); |
| matchEntries++; |
| } |
| } |
| Assert.assertEquals(2, matchEntries); |
| |
| boolean result = prefixManager.removeAcl(ozPrefix1, ozAcl4); |
| Assert.assertEquals(true, result); |
| |
| ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(2, ozAclGet.size()); |
| |
| result = prefixManager.removeAcl(ozPrefix1, ozAcl3); |
| Assert.assertEquals(true, result); |
| ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(1, ozAclGet.size()); |
| |
| Assert.assertEquals(ozAcl2, ozAclGet.get(0)); |
| |
| // add dev:w |
| prefixManager.addAcl(ozPrefix1, ozAcl4); |
| ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(2, ozAclGet.size()); |
| |
| // add dev:r and validate the acl bitset combined |
| prefixManager.addAcl(ozPrefix1, ozAcl5); |
| ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(2, ozAclGet.size()); |
| |
| matchEntries = 0; |
| for (OzoneAcl acl : ozAclGet) { |
| if (acl.getType() == ACLIdentityType.GROUP) { |
| Assert.assertEquals(ozAcl3, acl); |
| matchEntries++; |
| } |
| if (acl.getType() == ACLIdentityType.USER) { |
| Assert.assertEquals(ozAcl2, acl); |
| matchEntries++; |
| } |
| } |
| Assert.assertEquals(2, matchEntries); |
| } |
| |
| @Test |
| public void testInvalidPrefixAcl() throws IOException { |
| String volumeName = "vol1"; |
| String bucketName = "bucket1"; |
| String prefix1 = "pf1/"; |
| |
| // Invalid prefix not ending with "/" |
| String invalidPrefix = "invalid/pf"; |
| OzoneAcl ozAcl1 = new OzoneAcl(ACLIdentityType.USER, "user1", |
| ACLType.READ, ACCESS); |
| |
| OzoneObj ozInvalidPrefix = new OzoneObjInfo.Builder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .setPrefixName(invalidPrefix) |
| .setResType(OzoneObj.ResourceType.PREFIX) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| |
| // add acl with invalid prefix name |
| exception.expect(OMException.class); |
| exception.expectMessage("Invalid prefix name"); |
| prefixManager.addAcl(ozInvalidPrefix, ozAcl1); |
| |
| OzoneObj ozPrefix1 = new OzoneObjInfo.Builder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .setPrefixName(prefix1) |
| .setResType(OzoneObj.ResourceType.PREFIX) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| |
| |
| List<OzoneAcl> ozAclGet = prefixManager.getAcl(ozPrefix1); |
| Assert.assertEquals(1, ozAclGet.size()); |
| Assert.assertEquals(ozAcl1, ozAclGet.get(0)); |
| |
| // get acl with invalid prefix name |
| exception.expect(OMException.class); |
| exception.expectMessage("Invalid prefix name"); |
| ozAclGet = prefixManager.getAcl(ozInvalidPrefix); |
| Assert.assertEquals(null, ozAcl1); |
| |
| // set acl with invalid prefix name |
| List<OzoneAcl> ozoneAcls = new ArrayList<OzoneAcl>(); |
| ozoneAcls.add(ozAcl1); |
| exception.expect(OMException.class); |
| exception.expectMessage("Invalid prefix name"); |
| prefixManager.setAcl(ozInvalidPrefix, ozoneAcls); |
| |
| // remove acl with invalid prefix name |
| exception.expect(OMException.class); |
| exception.expectMessage("Invalid prefix name"); |
| prefixManager.removeAcl(ozInvalidPrefix, ozAcl1); |
| } |
| |
| @Test |
| public void testLongestPrefixPath() throws IOException { |
| String volumeName = "vol1"; |
| String bucketName = "bucket1"; |
| String prefix1 = "pf1/pf11/pf111/pf1111/"; |
| String file1 = "pf1/pf11/file1"; |
| String file2 = "pf1/pf11/pf111/pf1111/file2"; |
| |
| OzoneObj ozPrefix1 = new OzoneObjInfo.Builder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .setPrefixName(prefix1) |
| .setResType(OzoneObj.ResourceType.PREFIX) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| |
| OzoneAcl ozAcl1 = new OzoneAcl(ACLIdentityType.USER, "user1", |
| ACLType.READ, ACCESS); |
| prefixManager.addAcl(ozPrefix1, ozAcl1); |
| |
| OzoneObj ozFile1 = new OzoneObjInfo.Builder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .setKeyName(file1) |
| .setResType(OzoneObj.ResourceType.KEY) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| |
| List<OmPrefixInfo> prefixInfos = |
| prefixManager.getLongestPrefixPath(ozFile1.getPath()); |
| Assert.assertEquals(5, prefixInfos.size()); |
| |
| OzoneObj ozFile2 = new OzoneObjInfo.Builder() |
| .setVolumeName(volumeName) |
| .setBucketName(bucketName) |
| .setPrefixName(file2) |
| .setResType(OzoneObj.ResourceType.KEY) |
| .setStoreType(OzoneObj.StoreType.OZONE) |
| .build(); |
| |
| prefixInfos = |
| prefixManager.getLongestPrefixPath(ozFile2.getPath()); |
| Assert.assertEquals(7, prefixInfos.size()); |
| // Only the last node has acl on it |
| Assert.assertEquals(ozAcl1, prefixInfos.get(6).getAcls().get(0)); |
| // All other nodes don't have acl value associate with it |
| for (int i = 0; i < 6; i++) { |
| Assert.assertEquals(null, prefixInfos.get(i)); |
| } |
| } |
| |
| @Test |
| public void testLookupFile() throws IOException { |
| String keyName = RandomStringUtils.randomAlphabetic(5); |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .build(); |
| |
| // lookup for a non-existent file |
| try { |
| keyManager.lookupFile(keyArgs, null); |
| Assert.fail("Lookup file should fail for non existent file"); |
| } catch (OMException ex) { |
| if (ex.getResult() != OMException.ResultCodes.FILE_NOT_FOUND) { |
| throw ex; |
| } |
| } |
| |
| // create a file |
| OpenKeySession keySession = keyManager.createFile(keyArgs, false, false); |
| keyArgs.setLocationInfoList( |
| keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| Assert.assertEquals(keyManager.lookupFile(keyArgs, null).getKeyName(), |
| keyName); |
| |
| // lookup for created file |
| keyArgs = createBuilder() |
| .setKeyName("") |
| .build(); |
| try { |
| keyManager.lookupFile(keyArgs, null); |
| Assert.fail("Lookup file should fail for a directory"); |
| } catch (OMException ex) { |
| if (ex.getResult() != OMException.ResultCodes.NOT_A_FILE) { |
| throw ex; |
| } |
| } |
| } |
| |
| private OmKeyArgs createKeyArgs(String toKeyName) throws IOException { |
| return createBuilder().setKeyName(toKeyName).build(); |
| } |
| |
| @Test |
| public void testLookupKeyWithLocation() throws IOException { |
| String keyName = RandomStringUtils.randomAlphabetic(5); |
| OmKeyArgs keyArgs = createBuilder() |
| .setKeyName(keyName) |
| .setSortDatanodesInPipeline(true) |
| .build(); |
| |
| // lookup for a non-existent key |
| try { |
| keyManager.lookupKey(keyArgs, null); |
| Assert.fail("Lookup key should fail for non existent key"); |
| } catch (OMException ex) { |
| if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) { |
| throw ex; |
| } |
| } |
| |
| // create a key |
| OpenKeySession keySession = keyManager.createFile(keyArgs, false, false); |
| // randomly select 3 datanodes |
| List<DatanodeDetails> nodeList = new ArrayList<>(); |
| nodeList.add((DatanodeDetails)scm.getClusterMap().getNode( |
| 0, null, null, null, null, 0)); |
| nodeList.add((DatanodeDetails)scm.getClusterMap().getNode( |
| 1, null, null, null, null, 0)); |
| nodeList.add((DatanodeDetails)scm.getClusterMap().getNode( |
| 2, null, null, null, null, 0)); |
| Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(1))); |
| Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(2))); |
| // create a pipeline using 3 datanodes |
| Pipeline pipeline = scm.getPipelineManager().createPipeline( |
| ReplicationType.RATIS, ReplicationFactor.THREE, nodeList); |
| List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); |
| locationInfoList.add( |
| new OmKeyLocationInfo.Builder().setPipeline(pipeline) |
| .setBlockID(new BlockID(1L, 1L)).build()); |
| keyArgs.setLocationInfoList(locationInfoList); |
| |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| |
| OmKeyInfo key = keyManager.lookupKey(keyArgs, null); |
| Assert.assertEquals(key.getKeyName(), keyName); |
| List<OmKeyLocationInfo> keyLocations = |
| key.getLatestVersionLocations().getLocationList(); |
| DatanodeDetails leader = |
| keyLocations.get(0).getPipeline().getFirstNode(); |
| DatanodeDetails follower1 = |
| keyLocations.get(0).getPipeline().getNodes().get(1); |
| DatanodeDetails follower2 = |
| keyLocations.get(0).getPipeline().getNodes().get(2); |
| Assert.assertNotEquals(leader, follower1); |
| Assert.assertNotEquals(follower1, follower2); |
| |
| // lookup key, leader as client |
| OmKeyInfo key1 = keyManager.lookupKey(keyArgs, leader.getIpAddress()); |
| Assert.assertEquals(leader, key1.getLatestVersionLocations() |
| .getLocationList().get(0).getPipeline().getClosestNode()); |
| |
| // lookup key, follower1 as client |
| OmKeyInfo key2 = keyManager.lookupKey(keyArgs, follower1.getIpAddress()); |
| Assert.assertEquals(follower1, key2.getLatestVersionLocations() |
| .getLocationList().get(0).getPipeline().getClosestNode()); |
| |
| // lookup key, follower2 as client |
| OmKeyInfo key3 = keyManager.lookupKey(keyArgs, follower2.getIpAddress()); |
| Assert.assertEquals(follower2, key3.getLatestVersionLocations() |
| .getLocationList().get(0).getPipeline().getClosestNode()); |
| |
| // lookup key, random node as client |
| OmKeyInfo key4 = keyManager.lookupKey(keyArgs, |
| "/d=default-drack/127.0.0.1"); |
| Assert.assertEquals(leader, key4.getLatestVersionLocations() |
| .getLocationList().get(0).getPipeline().getClosestNode()); |
| } |
| |
| @Test |
| public void testListStatus() throws IOException { |
| String superDir = RandomStringUtils.randomAlphabetic(5); |
| |
| int numDirectories = 5; |
| int numFiles = 5; |
| // set of directory descendants of root |
| Set<String> directorySet = new TreeSet<>(); |
| // set of file descendants of root |
| Set<String> fileSet = new TreeSet<>(); |
| createDepthTwoDirectory(superDir, numDirectories, numFiles, directorySet, |
| fileSet); |
| // set of all descendants of root |
| Set<String> children = new TreeSet<>(directorySet); |
| children.addAll(fileSet); |
| // number of entries in the filesystem |
| int numEntries = directorySet.size() + fileSet.size(); |
| |
| OmKeyArgs rootDirArgs = createKeyArgs(""); |
| List<OzoneFileStatus> fileStatuses = |
| keyManager.listStatus(rootDirArgs, true, "", 100); |
| // verify the number of status returned is same as number of entries |
| Assert.assertEquals(numEntries, fileStatuses.size()); |
| |
| fileStatuses = keyManager.listStatus(rootDirArgs, false, "", 100); |
| // the number of immediate children of root is 1 |
| Assert.assertEquals(1, fileStatuses.size()); |
| |
| // if startKey is the first descendant of the root then listStatus should |
| // return all the entries. |
| String startKey = children.iterator().next(); |
| fileStatuses = keyManager.listStatus(rootDirArgs, true, |
| startKey.substring(0, startKey.length() - 1), 100); |
| Assert.assertEquals(numEntries, fileStatuses.size()); |
| |
| for (String directory : directorySet) { |
| // verify status list received for each directory with recursive flag set |
| // to false |
| OmKeyArgs dirArgs = createKeyArgs(directory); |
| fileStatuses = keyManager.listStatus(dirArgs, false, "", 100); |
| verifyFileStatus(directory, fileStatuses, directorySet, fileSet, false); |
| |
| // verify status list received for each directory with recursive flag set |
| // to true |
| fileStatuses = keyManager.listStatus(dirArgs, true, "", 100); |
| verifyFileStatus(directory, fileStatuses, directorySet, fileSet, true); |
| |
| // verify list status call with using the startKey parameter and |
| // recursive flag set to false. After every call to listStatus use the |
| // latest received file status as the startKey until no more entries are |
| // left to list. |
| List<OzoneFileStatus> tempFileStatus = null; |
| Set<OzoneFileStatus> tmpStatusSet = new HashSet<>(); |
| do { |
| tempFileStatus = keyManager.listStatus(dirArgs, false, |
| tempFileStatus != null ? OzoneFSUtils.pathToKey( |
| tempFileStatus.get(tempFileStatus.size() - 1).getPath()) : null, |
| 2); |
| tmpStatusSet.addAll(tempFileStatus); |
| } while (tempFileStatus.size() == 2); |
| verifyFileStatus(directory, new ArrayList<>(tmpStatusSet), directorySet, |
| fileSet, false); |
| |
| // verify list status call with using the startKey parameter and |
| // recursive flag set to true. After every call to listStatus use the |
| // latest received file status as the startKey until no more entries are |
| // left to list. |
| tempFileStatus = null; |
| tmpStatusSet = new HashSet<>(); |
| do { |
| tempFileStatus = keyManager.listStatus(dirArgs, true, |
| tempFileStatus != null ? OzoneFSUtils.pathToKey( |
| tempFileStatus.get(tempFileStatus.size() - 1).getPath()) : null, |
| 2); |
| tmpStatusSet.addAll(tempFileStatus); |
| } while (tempFileStatus.size() == 2); |
| verifyFileStatus(directory, new ArrayList<>(tmpStatusSet), directorySet, |
| fileSet, true); |
| } |
| } |
| |
| /** |
| * Creates a depth two directory. |
| * |
| * @param superDir Super directory to create |
| * @param numDirectories number of directory children |
| * @param numFiles number of file children |
| * @param directorySet set of descendant directories for the super directory |
| * @param fileSet set of descendant files for the super directory |
| */ |
| private void createDepthTwoDirectory(String superDir, int numDirectories, |
| int numFiles, Set<String> directorySet, Set<String> fileSet) |
| throws IOException { |
| // create super directory |
| OmKeyArgs superDirArgs = createKeyArgs(superDir); |
| keyManager.createDirectory(superDirArgs); |
| directorySet.add(superDir); |
| |
| // add directory children to super directory |
| Set<String> childDirectories = |
| createDirectories(superDir, new HashMap<>(), numDirectories); |
| directorySet.addAll(childDirectories); |
| // add file to super directory |
| fileSet.addAll(createFiles(superDir, new HashMap<>(), numFiles)); |
| |
| // for each child directory create files and directories |
| for (String child : childDirectories) { |
| fileSet.addAll(createFiles(child, new HashMap<>(), numFiles)); |
| directorySet |
| .addAll(createDirectories(child, new HashMap<>(), numDirectories)); |
| } |
| } |
| |
| private void verifyFileStatus(String directory, |
| List<OzoneFileStatus> fileStatuses, Set<String> directorySet, |
| Set<String> fileSet, boolean recursive) { |
| |
| for (OzoneFileStatus fileStatus : fileStatuses) { |
| String keyName = OzoneFSUtils.pathToKey(fileStatus.getPath()); |
| String parent = Paths.get(keyName).getParent().toString(); |
| if (!recursive) { |
| // if recursive is false, verify all the statuses have the input |
| // directory as parent |
| Assert.assertEquals(parent, directory); |
| } |
| // verify filestatus is present in directory or file set accordingly |
| if (fileStatus.isDirectory()) { |
| Assert.assertTrue(directorySet.contains(keyName)); |
| } else { |
| Assert.assertTrue(fileSet.contains(keyName)); |
| } |
| } |
| |
| // count the number of entries which should be present in the directory |
| int numEntries = 0; |
| Set<String> entrySet = new TreeSet<>(directorySet); |
| entrySet.addAll(fileSet); |
| for (String entry : entrySet) { |
| if (OzoneFSUtils.getParent(entry) |
| .startsWith(OzoneFSUtils.addTrailingSlashIfNeeded(directory))) { |
| if (recursive) { |
| numEntries++; |
| } else if (OzoneFSUtils.getParent(entry) |
| .equals(OzoneFSUtils.addTrailingSlashIfNeeded(directory))) { |
| numEntries++; |
| } |
| } |
| } |
| // verify the number of entries match the status list size |
| Assert.assertEquals(fileStatuses.size(), numEntries); |
| } |
| |
| private Set<String> createDirectories(String parent, |
| Map<String, List<String>> directoryMap, int numDirectories) |
| throws IOException { |
| Set<String> keyNames = new TreeSet<>(); |
| for (int i = 0; i < numDirectories; i++) { |
| String keyName = parent + "/" + RandomStringUtils.randomAlphabetic(5); |
| OmKeyArgs keyArgs = createBuilder().setKeyName(keyName).build(); |
| keyManager.createDirectory(keyArgs); |
| keyNames.add(keyName); |
| } |
| directoryMap.put(parent, new ArrayList<>(keyNames)); |
| return keyNames; |
| } |
| |
| private List<String> createFiles(String parent, |
| Map<String, List<String>> fileMap, int numFiles) throws IOException { |
| List<String> keyNames = new ArrayList<>(); |
| for (int i = 0; i < numFiles; i++) { |
| String keyName = parent + "/" + RandomStringUtils.randomAlphabetic(5); |
| OmKeyArgs keyArgs = createBuilder().setKeyName(keyName).build(); |
| OpenKeySession keySession = keyManager.createFile(keyArgs, false, false); |
| keyArgs.setLocationInfoList( |
| keySession.getKeyInfo().getLatestVersionLocations() |
| .getLocationList()); |
| keyManager.commitKey(keyArgs, keySession.getId()); |
| keyNames.add(keyName); |
| } |
| fileMap.put(parent, keyNames); |
| return keyNames; |
| } |
| |
| private OmKeyArgs.Builder createBuilder() throws IOException { |
| UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); |
| return new OmKeyArgs.Builder() |
| .setBucketName(BUCKET_NAME) |
| .setFactor(ReplicationFactor.ONE) |
| .setDataSize(0) |
| .setType(ReplicationType.STAND_ALONE) |
| .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(), |
| ALL, ALL)) |
| .setVolumeName(VOLUME_NAME); |
| } |
| |
| private RequestContext currentUserReads() throws IOException { |
| return RequestContext.newBuilder() |
| .setClientUgi(UserGroupInformation.getCurrentUser()) |
| .setAclRights(ACLType.READ_ACL) |
| .setAclType(ACLIdentityType.USER) |
| .build(); |
| } |
| } |
