org.apache.hadoop.hdfs.client.HdfsAdmin Java Examples

The following examples show how to use org.apache.hadoop.hdfs.client.HdfsAdmin. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestEncryptionZones.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  conf = new HdfsConfiguration();
  fsHelper = new FileSystemTestHelper();
  // Set up java key store
  String testRoot = fsHelper.getTestRootDir();
  testRootDir = new File(testRoot).getAbsoluteFile();
  conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, getKeyProviderURI());
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
  // Lower the batch size for testing
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
      2);
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
  fs = cluster.getFileSystem();
  fsWrapper = new FileSystemTestWrapper(fs);
  fcWrapper = new FileContextTestWrapper(
      FileContext.getFileContext(cluster.getURI(), conf));
  dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
  setProvider();
  // Create a test key
  DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}
 
Example #2
Source File: TestReservedRawPaths.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  conf = new HdfsConfiguration();
  fsHelper = new FileSystemTestHelper();
  // Set up java key store
  String testRoot = fsHelper.getTestRootDir();
  File testRootDir = new File(testRoot).getAbsoluteFile();
  final Path jksPath = new Path(testRootDir.toString(), "test.jks");
  conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
      JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
  );
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
  fs = cluster.getFileSystem();
  fsWrapper = new FileSystemTestWrapper(cluster.getFileSystem());
  fcWrapper = new FileContextTestWrapper(
      FileContext.getFileContext(cluster.getURI(), conf));
  dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
  // Need to set the client's KeyProvider to the NN's for JKS,
  // else the updates do not get flushed properly
  fs.getClient().setKeyProvider(cluster.getNameNode().getNamesystem()
      .getProvider());
  DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}
 
Example #3
Source File: TestReservedRawPaths.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  conf = new HdfsConfiguration();
  fsHelper = new FileSystemTestHelper();
  // Set up java key store
  String testRoot = fsHelper.getTestRootDir();
  File testRootDir = new File(testRoot).getAbsoluteFile();
  final Path jksPath = new Path(testRootDir.toString(), "test.jks");
  conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
      JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
  );
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
  fs = cluster.getFileSystem();
  fsWrapper = new FileSystemTestWrapper(cluster.getFileSystem());
  fcWrapper = new FileContextTestWrapper(
      FileContext.getFileContext(cluster.getURI(), conf));
  dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
  // Need to set the client's KeyProvider to the NN's for JKS,
  // else the updates do not get flushed properly
  fs.getClient().setKeyProvider(cluster.getNameNode().getNamesystem()
      .getProvider());
  DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}
 
Example #4
Source File: TestEncryptionZones.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  conf = new HdfsConfiguration();
  fsHelper = new FileSystemTestHelper();
  // Set up java key store
  String testRoot = fsHelper.getTestRootDir();
  testRootDir = new File(testRoot).getAbsoluteFile();
  conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, getKeyProviderURI());
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
  // Lower the batch size for testing
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
      2);
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
  fs = cluster.getFileSystem();
  fsWrapper = new FileSystemTestWrapper(fs);
  fcWrapper = new FileContextTestWrapper(
      FileContext.getFileContext(cluster.getURI(), conf));
  dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
  setProvider();
  // Create a test key
  DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}
 
Example #5
Source File: TestEncryptionZones.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 120000)
public void testReadWrite() throws Exception {
  final HdfsAdmin dfsAdmin =
      new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
  // Create a base file for comparison
  final Path baseFile = new Path("/base");
  final int len = 8192;
  DFSTestUtil.createFile(fs, baseFile, len, (short) 1, 0xFEED);
  // Create the first enc file
  final Path zone = new Path("/zone");
  fs.mkdirs(zone);
  dfsAdmin.createEncryptionZone(zone, TEST_KEY);
  final Path encFile1 = new Path(zone, "myfile");
  DFSTestUtil.createFile(fs, encFile1, len, (short) 1, 0xFEED);
  // Read them back in and compare byte-by-byte
  verifyFilesEqual(fs, baseFile, encFile1, len);
  // Roll the key of the encryption zone
  assertNumZones(1);
  String keyName = dfsAdmin.listEncryptionZones().next().getKeyName();
  cluster.getNamesystem().getProvider().rollNewVersion(keyName);
  // Read them back in and compare byte-by-byte
  verifyFilesEqual(fs, baseFile, encFile1, len);
  // Write a new enc file and validate
  final Path encFile2 = new Path(zone, "myfile2");
  DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED);
  // FEInfos should be different
  FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1);
  FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2);
  assertFalse("EDEKs should be different", Arrays
      .equals(feInfo1.getEncryptedDataEncryptionKey(),
          feInfo2.getEncryptedDataEncryptionKey()));
  assertNotEquals("Key was rolled, versions should be different",
      feInfo1.getEzKeyVersionName(), feInfo2.getEzKeyVersionName());
  // Contents still equal
  verifyFilesEqual(fs, encFile1, encFile2, len);
}
 
Example #6
Source File: TestEncryptionZones.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 120000)
public void testReadWrite() throws Exception {
  final HdfsAdmin dfsAdmin =
      new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
  // Create a base file for comparison
  final Path baseFile = new Path("/base");
  final int len = 8192;
  DFSTestUtil.createFile(fs, baseFile, len, (short) 1, 0xFEED);
  // Create the first enc file
  final Path zone = new Path("/zone");
  fs.mkdirs(zone);
  dfsAdmin.createEncryptionZone(zone, TEST_KEY);
  final Path encFile1 = new Path(zone, "myfile");
  DFSTestUtil.createFile(fs, encFile1, len, (short) 1, 0xFEED);
  // Read them back in and compare byte-by-byte
  verifyFilesEqual(fs, baseFile, encFile1, len);
  // Roll the key of the encryption zone
  assertNumZones(1);
  String keyName = dfsAdmin.listEncryptionZones().next().getKeyName();
  cluster.getNamesystem().getProvider().rollNewVersion(keyName);
  // Read them back in and compare byte-by-byte
  verifyFilesEqual(fs, baseFile, encFile1, len);
  // Write a new enc file and validate
  final Path encFile2 = new Path(zone, "myfile2");
  DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED);
  // FEInfos should be different
  FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1);
  FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2);
  assertFalse("EDEKs should be different", Arrays
      .equals(feInfo1.getEncryptedDataEncryptionKey(),
          feInfo2.getEncryptedDataEncryptionKey()));
  assertNotEquals("Key was rolled, versions should be different",
      feInfo1.getEzKeyVersionName(), feInfo2.getEzKeyVersionName());
  // Contents still equal
  verifyFilesEqual(fs, encFile1, encFile2, len);
}
 
Example #7
Source File: TestEncryptionZones.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test listing encryption zones as a non super user.
 */
@Test(timeout = 60000)
public void testListEncryptionZonesAsNonSuperUser() throws Exception {

  final UserGroupInformation user = UserGroupInformation.
      createUserForTesting("user", new String[] { "mygroup" });

  final Path testRoot = new Path("/tmp/TestEncryptionZones");
  final Path superPath = new Path(testRoot, "superuseronly");
  final Path allPath = new Path(testRoot, "accessall");

  fsWrapper.mkdir(superPath, new FsPermission((short) 0700), true);
  dfsAdmin.createEncryptionZone(superPath, TEST_KEY);

  fsWrapper.mkdir(allPath, new FsPermission((short) 0707), true);
  dfsAdmin.createEncryptionZone(allPath, TEST_KEY);

  user.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws Exception {
      final HdfsAdmin userAdmin =
          new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
      try {
        userAdmin.listEncryptionZones();
      } catch (AccessControlException e) {
        assertExceptionContains("Superuser privilege is required", e);
      }
      return null;
    }
  });
}
 
Example #8
Source File: TestEncryptionZonesWithHA.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setupCluster() throws Exception {
  conf = new Configuration();
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
  HAUtil.setAllowStandbyReads(conf, true);
  fsHelper = new FileSystemTestHelper();
  String testRoot = fsHelper.getTestRootDir();
  testRootDir = new File(testRoot).getAbsoluteFile();
  conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
      JavaKeyStoreProvider.SCHEME_NAME + "://file" +
      new Path(testRootDir.toString(), "test.jks").toUri()
  );

  cluster = new MiniDFSCluster.Builder(conf)
    .nnTopology(MiniDFSNNTopology.simpleHATopology())
    .numDataNodes(1)
    .build();
  cluster.waitActive();
  cluster.transitionToActive(0);

  fs = (DistributedFileSystem)HATestUtil.configureFailoverFs(cluster, conf);
  DFSTestUtil.createKey(TEST_KEY, cluster, 0, conf);
  DFSTestUtil.createKey(TEST_KEY, cluster, 1, conf);
  nn0 = cluster.getNameNode(0);
  nn1 = cluster.getNameNode(1);
  dfsAdmin0 = new HdfsAdmin(cluster.getURI(0), conf);
  dfsAdmin1 = new HdfsAdmin(cluster.getURI(1), conf);
  KeyProviderCryptoExtension nn0Provider =
      cluster.getNameNode(0).getNamesystem().getProvider();
  fs.getClient().setKeyProvider(nn0Provider);
}
 
Example #9
Source File: TestHdfsAdmin.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test that we can set and clear quotas via {@link HdfsAdmin}.
 */
@Test
public void testHdfsAdminSetQuota() throws Exception {
  HdfsAdmin dfsAdmin = new HdfsAdmin(
      FileSystem.getDefaultUri(conf), conf);
  FileSystem fs = null;
  try {
    fs = FileSystem.get(conf);
    assertTrue(fs.mkdirs(TEST_PATH));
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.setSpaceQuota(TEST_PATH, 10);
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(10, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.setQuota(TEST_PATH, 10);
    assertEquals(10, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(10, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.clearSpaceQuota(TEST_PATH);
    assertEquals(10, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.clearQuota(TEST_PATH);
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getSpaceQuota());
  } finally {
    if (fs != null) {
      fs.close();
    }
  }
}
 
Example #10
Source File: TestHdfsAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test that we can set and clear quotas via {@link HdfsAdmin}.
 */
@Test
public void testHdfsAdminSetQuota() throws Exception {
  HdfsAdmin dfsAdmin = new HdfsAdmin(
      FileSystem.getDefaultUri(conf), conf);
  FileSystem fs = null;
  try {
    fs = FileSystem.get(conf);
    assertTrue(fs.mkdirs(TEST_PATH));
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.setSpaceQuota(TEST_PATH, 10);
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(10, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.setQuota(TEST_PATH, 10);
    assertEquals(10, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(10, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.clearSpaceQuota(TEST_PATH);
    assertEquals(10, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getSpaceQuota());
    
    dfsAdmin.clearQuota(TEST_PATH);
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1, fs.getContentSummary(TEST_PATH).getSpaceQuota());
  } finally {
    if (fs != null) {
      fs.close();
    }
  }
}
 
Example #11
Source File: TestEncryptionZones.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test listing encryption zones as a non super user.
 */
@Test(timeout = 60000)
public void testListEncryptionZonesAsNonSuperUser() throws Exception {

  final UserGroupInformation user = UserGroupInformation.
      createUserForTesting("user", new String[] { "mygroup" });

  final Path testRoot = new Path("/tmp/TestEncryptionZones");
  final Path superPath = new Path(testRoot, "superuseronly");
  final Path allPath = new Path(testRoot, "accessall");

  fsWrapper.mkdir(superPath, new FsPermission((short) 0700), true);
  dfsAdmin.createEncryptionZone(superPath, TEST_KEY);

  fsWrapper.mkdir(allPath, new FsPermission((short) 0707), true);
  dfsAdmin.createEncryptionZone(allPath, TEST_KEY);

  user.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws Exception {
      final HdfsAdmin userAdmin =
          new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
      try {
        userAdmin.listEncryptionZones();
      } catch (AccessControlException e) {
        assertExceptionContains("Superuser privilege is required", e);
      }
      return null;
    }
  });
}
 
Example #12
Source File: TestEncryptionZonesWithHA.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setupCluster() throws Exception {
  conf = new Configuration();
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
  HAUtil.setAllowStandbyReads(conf, true);
  fsHelper = new FileSystemTestHelper();
  String testRoot = fsHelper.getTestRootDir();
  testRootDir = new File(testRoot).getAbsoluteFile();
  conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
      JavaKeyStoreProvider.SCHEME_NAME + "://file" +
      new Path(testRootDir.toString(), "test.jks").toUri()
  );

  cluster = new MiniDFSCluster.Builder(conf)
    .nnTopology(MiniDFSNNTopology.simpleHATopology())
    .numDataNodes(1)
    .build();
  cluster.waitActive();
  cluster.transitionToActive(0);

  fs = (DistributedFileSystem)HATestUtil.configureFailoverFs(cluster, conf);
  DFSTestUtil.createKey(TEST_KEY, cluster, 0, conf);
  DFSTestUtil.createKey(TEST_KEY, cluster, 1, conf);
  nn0 = cluster.getNameNode(0);
  nn1 = cluster.getNameNode(1);
  dfsAdmin0 = new HdfsAdmin(cluster.getURI(0), conf);
  dfsAdmin1 = new HdfsAdmin(cluster.getURI(1), conf);
  KeyProviderCryptoExtension nn0Provider =
      cluster.getNameNode(0).getNamesystem().getProvider();
  fs.getClient().setKeyProvider(nn0Provider);
}
 
Example #13
Source File: GetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
protected HdfsAdmin getHdfsAdmin() {
    try {
        // Currently HdfsAdmin is the only public API that allows access to the inotify API. Because of this we need to have super user rights in HDFS.
        return new HdfsAdmin(getFileSystem().getUri(), getFileSystem().getConf());
    } catch (IOException e) {
        getLogger().error("Unable to get and instance of HDFS admin. You must be an HDFS super user to view HDFS events.");
        throw new ProcessException(e);
    }
}
 
Example #14
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
    mockNiFiProperties = mock(NiFiProperties.class);
    when(mockNiFiProperties.getKerberosConfigurationFile()).thenReturn(null);
    kerberosProperties = new KerberosProperties(null);
    inotifyEventInputStream = mock(DFSInotifyEventInputStream.class);
    hdfsAdmin = mock(HdfsAdmin.class);
}
 
Example #15
Source File: GetHDFSEvents.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
protected HdfsAdmin getHdfsAdmin() {
    try {
        // Currently HdfsAdmin is the only public API that allows access to the inotify API. Because of this we need to have super user rights in HDFS.
        return new HdfsAdmin(getFileSystem().getUri(), getFileSystem().getConf());
    } catch (IOException e) {
        getLogger().error("Unable to get and instance of HDFS admin. You must be an HDFS super user to view HDFS events.");
        throw new ProcessException(e);
    }
}
 
Example #16
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
    mockNiFiProperties = mock(NiFiProperties.class);
    when(mockNiFiProperties.getKerberosConfigurationFile()).thenReturn(null);
    kerberosProperties = new KerberosProperties(null);
    inotifyEventInputStream = mock(DFSInotifyEventInputStream.class);
    hdfsAdmin = mock(HdfsAdmin.class);
}
 
Example #17
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
protected HdfsAdmin getHdfsAdmin() {
    return hdfsAdmin;
}
 
Example #18
Source File: TestHdfsAdmin.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Make sure that a non-HDFS URI throws a helpful error.
 */
@Test(expected = IllegalArgumentException.class)
public void testHdfsAdminWithBadUri() throws IOException, URISyntaxException {
  new HdfsAdmin(new URI("file:///bad-scheme"), conf);
}
 
Example #19
Source File: TestEncryptionZones.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 120000)
public void testReadWriteUsingWebHdfs() throws Exception {
  final HdfsAdmin dfsAdmin =
      new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
  final FileSystem webHdfsFs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
      WebHdfsFileSystem.SCHEME);

  final Path zone = new Path("/zone");
  fs.mkdirs(zone);
  dfsAdmin.createEncryptionZone(zone, TEST_KEY);

  /* Create an unencrypted file for comparison purposes. */
  final Path unencFile = new Path("/unenc");
  final int len = 8192;
  DFSTestUtil.createFile(webHdfsFs, unencFile, len, (short) 1, 0xFEED);

  /*
   * Create the same file via webhdfs, but this time encrypted. Compare it
   * using both webhdfs and DFS.
   */
  final Path encFile1 = new Path(zone, "myfile");
  DFSTestUtil.createFile(webHdfsFs, encFile1, len, (short) 1, 0xFEED);
  verifyFilesEqual(webHdfsFs, unencFile, encFile1, len);
  verifyFilesEqual(fs, unencFile, encFile1, len);

  /*
   * Same thing except this time create the encrypted file using DFS.
   */
  final Path encFile2 = new Path(zone, "myfile2");
  DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED);
  verifyFilesEqual(webHdfsFs, unencFile, encFile2, len);
  verifyFilesEqual(fs, unencFile, encFile2, len);

  /* Verify appending to files works correctly. */
  appendOneByte(fs, unencFile);
  appendOneByte(webHdfsFs, encFile1);
  appendOneByte(fs, encFile2);
  verifyFilesEqual(webHdfsFs, unencFile, encFile1, len);
  verifyFilesEqual(fs, unencFile, encFile1, len);
  verifyFilesEqual(webHdfsFs, unencFile, encFile2, len);
  verifyFilesEqual(fs, unencFile, encFile2, len);
}
 
Example #20
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 4 votes vote down vote up
TestableGetHDFSEvents(KerberosProperties testKerberosProperties, HdfsAdmin hdfsAdmin) {
    this.testKerberosProperties = testKerberosProperties;
    this.hdfsAdmin = hdfsAdmin;
}
 
Example #21
Source File: TestRpcProgramNfs3.java    From big-c with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  String currentUser = System.getProperty("user.name");

  config.set("fs.permissions.umask-mode", "u=rwx,g=,o=");
  config.set(DefaultImpersonationProvider.getTestProvider()
      .getProxySuperuserGroupConfKey(currentUser), "*");
  config.set(DefaultImpersonationProvider.getTestProvider()
      .getProxySuperuserIpConfKey(currentUser), "*");
  fsHelper = new FileSystemTestHelper();
  // Set up java key store
  String testRoot = fsHelper.getTestRootDir();
  testRootDir = new File(testRoot).getAbsoluteFile();
  final Path jksPath = new Path(testRootDir.toString(), "test.jks");
  config.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
      JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
  ProxyUsers.refreshSuperUserGroupsConfiguration(config);

  cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
  cluster.waitActive();
  hdfs = cluster.getFileSystem();
  nn = cluster.getNameNode();
  dfsAdmin = new HdfsAdmin(cluster.getURI(), config);

  // Use ephemeral ports in case tests are running in parallel
  config.setInt("nfs3.mountd.port", 0);
  config.setInt("nfs3.server.port", 0);

  // Start NFS with allowed.hosts set to "* rw"
  config.set("dfs.nfs.exports.allowed.hosts", "* rw");
  nfs = new Nfs3(config);
  nfs.startServiceInternal(false);
  nfsd = (RpcProgramNfs3) nfs.getRpcProgram();

  hdfs.getClient().setKeyProvider(nn.getNamesystem().getProvider());
  DFSTestUtil.createKey(TEST_KEY, cluster, config);

  // Mock SecurityHandler which returns system user.name
  securityHandler = Mockito.mock(SecurityHandler.class);
  Mockito.when(securityHandler.getUser()).thenReturn(currentUser);

  // Mock SecurityHandler which returns a dummy username "harry"
  securityHandlerUnpriviledged = Mockito.mock(SecurityHandler.class);
  Mockito.when(securityHandlerUnpriviledged.getUser()).thenReturn("harry");
}
 
Example #22
Source File: TestHdfsAdmin.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Make sure that a non-HDFS URI throws a helpful error.
 */
@Test(expected = IllegalArgumentException.class)
public void testHdfsAdminWithBadUri() throws IOException, URISyntaxException {
  new HdfsAdmin(new URI("file:///bad-scheme"), conf);
}
 
Example #23
Source File: TestEncryptionZones.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 120000)
public void testReadWriteUsingWebHdfs() throws Exception {
  final HdfsAdmin dfsAdmin =
      new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
  final FileSystem webHdfsFs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
      WebHdfsFileSystem.SCHEME);

  final Path zone = new Path("/zone");
  fs.mkdirs(zone);
  dfsAdmin.createEncryptionZone(zone, TEST_KEY);

  /* Create an unencrypted file for comparison purposes. */
  final Path unencFile = new Path("/unenc");
  final int len = 8192;
  DFSTestUtil.createFile(webHdfsFs, unencFile, len, (short) 1, 0xFEED);

  /*
   * Create the same file via webhdfs, but this time encrypted. Compare it
   * using both webhdfs and DFS.
   */
  final Path encFile1 = new Path(zone, "myfile");
  DFSTestUtil.createFile(webHdfsFs, encFile1, len, (short) 1, 0xFEED);
  verifyFilesEqual(webHdfsFs, unencFile, encFile1, len);
  verifyFilesEqual(fs, unencFile, encFile1, len);

  /*
   * Same thing except this time create the encrypted file using DFS.
   */
  final Path encFile2 = new Path(zone, "myfile2");
  DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED);
  verifyFilesEqual(webHdfsFs, unencFile, encFile2, len);
  verifyFilesEqual(fs, unencFile, encFile2, len);

  /* Verify appending to files works correctly. */
  appendOneByte(fs, unencFile);
  appendOneByte(webHdfsFs, encFile1);
  appendOneByte(fs, encFile2);
  verifyFilesEqual(webHdfsFs, unencFile, encFile1, len);
  verifyFilesEqual(fs, unencFile, encFile1, len);
  verifyFilesEqual(webHdfsFs, unencFile, encFile2, len);
  verifyFilesEqual(fs, unencFile, encFile2, len);
}
 
Example #24
Source File: TestRpcProgramNfs3.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  String currentUser = System.getProperty("user.name");

  config.set("fs.permissions.umask-mode", "u=rwx,g=,o=");
  config.set(DefaultImpersonationProvider.getTestProvider()
      .getProxySuperuserGroupConfKey(currentUser), "*");
  config.set(DefaultImpersonationProvider.getTestProvider()
      .getProxySuperuserIpConfKey(currentUser), "*");
  fsHelper = new FileSystemTestHelper();
  // Set up java key store
  String testRoot = fsHelper.getTestRootDir();
  testRootDir = new File(testRoot).getAbsoluteFile();
  final Path jksPath = new Path(testRootDir.toString(), "test.jks");
  config.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
      JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
  ProxyUsers.refreshSuperUserGroupsConfiguration(config);

  cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
  cluster.waitActive();
  hdfs = cluster.getFileSystem();
  nn = cluster.getNameNode();
  dfsAdmin = new HdfsAdmin(cluster.getURI(), config);

  // Use ephemeral ports in case tests are running in parallel
  config.setInt("nfs3.mountd.port", 0);
  config.setInt("nfs3.server.port", 0);

  // Start NFS with allowed.hosts set to "* rw"
  config.set("dfs.nfs.exports.allowed.hosts", "* rw");
  nfs = new Nfs3(config);
  nfs.startServiceInternal(false);
  nfsd = (RpcProgramNfs3) nfs.getRpcProgram();

  hdfs.getClient().setKeyProvider(nn.getNamesystem().getProvider());
  DFSTestUtil.createKey(TEST_KEY, cluster, config);

  // Mock SecurityHandler which returns system user.name
  securityHandler = Mockito.mock(SecurityHandler.class);
  Mockito.when(securityHandler.getUser()).thenReturn(currentUser);

  // Mock SecurityHandler which returns a dummy username "harry"
  securityHandlerUnpriviledged = Mockito.mock(SecurityHandler.class);
  Mockito.when(securityHandlerUnpriviledged.getUser()).thenReturn("harry");
}
 
Example #25
Source File: HdfsFileWatcherPolicy.java    From kafka-connect-fs with Apache License 2.0 4 votes vote down vote up
EventStreamThread(FileSystem fs, HdfsAdmin admin, long retrySleepMs) {
    this.fs = fs;
    this.admin = admin;
    this.retrySleepMs = retrySleepMs;
    this.time = new SystemTime();
}
 
Example #26
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
protected HdfsAdmin getHdfsAdmin() {
    return hdfsAdmin;
}
 
Example #27
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
TestableGetHDFSEvents(KerberosProperties testKerberosProperties, HdfsAdmin hdfsAdmin) {
    this.testKerberosProperties = testKerberosProperties;
    this.hdfsAdmin = hdfsAdmin;
}