Java Code Examples for org.apache.hadoop.hdfs.MiniDFSCluster#Builder

The following examples show how to use org.apache.hadoop.hdfs.MiniDFSCluster#Builder . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RollingSinkITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void setup() throws Exception {

	LOG.info("In RollingSinkITCase: Starting MiniDFSCluster ");

	dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	hdfsURI = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/";

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(4)
			.build());

	miniClusterResource.before();
}

Example 2

Source File: BucketingSinkFaultToleranceITCase.java From flink with Apache License 2.0

6 votes

@Before
public void createHDFS() throws IOException {
	Configuration conf = new Configuration();

	File dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	outPath = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/string-non-rolling-out";
}

Example 3

Source File: TajoTestingCluster.java From tajo with Apache License 2.0

6 votes

/**
 * Start a minidfscluster.
 * Can only create one.
 * @param servers How many DNs to start.
 * @param dir Where to home your dfs cluster.
 * @param hosts hostnames DNs to run on.
 * @throws Exception
 * @see {@link #shutdownMiniDFSCluster()}
 * @return The mini dfs cluster created.
 * @throws java.io.IOException
 */
public MiniDFSCluster startMiniDFSCluster(int servers,
                                          File dir,
                                          final String hosts[])
    throws IOException {

  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.getAbsolutePath());
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false);
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 0);

  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf));
  builder.hosts(hosts);
  builder.numDataNodes(servers);
  builder.format(true);
  builder.waitSafeMode(true);
  this.dfsCluster = builder.build();

  // Set this just-started cluster as our filesystem.
  this.defaultFS = this.dfsCluster.getFileSystem();
  this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString());
  this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo");
  isDFSRunning = true;
  return this.dfsCluster;
}

Example 4

Source File: HDFSFileStreamSourceTest.java From incubator-samoa with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
  // Start MiniDFSCluster
  config = new Configuration();
  config.set("hdfs.minidfs.basedir", "target/build/test/data/dfs");
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(config).hosts(HOSTS).numDataNodes(1)
      .format(true);
  hdfsCluster = builder.build();
  hdfsCluster.waitActive();
  hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort();

  // Construct stream source
  streamSource = new HDFSFileStreamSource();

  // General config
  config.set("fs.defaultFS", hdfsURI);
}

Example 5

Source File: TestBinaryTokenFile.java From hadoop with Apache License 2.0

6 votes

@BeforeClass
public static void setUp() throws Exception {
  final Configuration conf = new Configuration();
  
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  conf.set(YarnConfiguration.RM_PRINCIPAL, "jt_id/" + SecurityUtil.HOSTNAME_PATTERN + "@APACHE.ORG");
  
  final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
  builder.checkExitOnShutdown(true);
  builder.numDataNodes(numSlaves);
  builder.format(true);
  builder.racks(null);
  dfsCluster = builder.build();
  
  mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(), noOfNMs);
  mrCluster.init(conf);
  mrCluster.start();

  NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); 
  
  FileSystem fs = dfsCluster.getFileSystem(); 
  p1 = new Path("file1");
  p1 = fs.makeQualified(p1);
}

Example 6

Source File: FileSystemFunctionsTest.java From metron with Apache License 2.0

6 votes

@BeforeAll
public static void setupFS() throws IOException {
  {
    hdfsBaseDir = Files.createTempDirectory("test_hdfs").toFile().getAbsoluteFile();
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsBaseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    hdfsCluster = builder.build();
    hdfsPrefix = "/";
  }
  {
    localPrefix = "target/fsTest/";
    if (new File(localPrefix).exists()) {
      new File(localPrefix).delete();
    }
    new File(localPrefix).mkdirs();
  }
}

Example 7

Source File: HDFSTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Before
public void createHDFS() {
	try {
		Configuration hdConf = new Configuration();

		File baseDir = new File("./target/hdfs/hdfsTest").getAbsoluteFile();
		FileUtil.fullyDelete(baseDir);
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";

		hdPath = new org.apache.hadoop.fs.Path("/test");
		hdfs = hdPath.getFileSystem(hdConf);
		FSDataOutputStream stream = hdfs.create(hdPath);
		for (int i = 0; i < 10; i++) {
			stream.write("Hello HDFS\n".getBytes(ConfigConstants.DEFAULT_CHARSET));
		}
		stream.close();

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}

Example 8

Source File: DistributedCacheDfsTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void setup() throws Exception {
	File dataDir = TEMP_FOLDER.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	String hdfsURI = "hdfs://"
		+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
		+ "/";

	FileSystem dfs = FileSystem.get(new URI(hdfsURI));
	testFile = writeFile(dfs, dfs.getHomeDirectory(), "testFile");

	testDir = new Path(dfs.getHomeDirectory(), "testDir");
	dfs.mkdirs(testDir);
	writeFile(dfs, testDir, "testFile1");
	writeFile(dfs, testDir, "testFile2");
}

Example 9

Source File: RollingSinkFaultToleranceITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void createHDFS() throws IOException {
	Configuration conf = new Configuration();

	File dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	outPath = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/string-non-rolling-out";
}

Example 10

Source File: TestDocUseCases.java From jsr203-hadoop with Apache License 2.0

5 votes

private static MiniDFSCluster startMini(String testName) throws IOException {
	File baseDir = new File("./target/hdfs/" + testName).getAbsoluteFile();
	FileUtil.fullyDelete(baseDir);
	Configuration conf = new Configuration();
	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	MiniDFSCluster hdfsCluster = builder.clusterId(testName).build();
	hdfsCluster.waitActive();
	return hdfsCluster;
}

Example 11

Source File: CreateHDFSStoreTest.java From gemfirexd-oss with Apache License 2.0

5 votes

public static MiniDFSCluster initMiniCluster(int port, int numDN, HashMap<String, String> map) throws Exception {
  System.setProperty("test.build.data", "hdfs-test-cluster");
  Configuration hconf = new HdfsConfiguration();
  for (Entry<String, String> entry : map.entrySet()) {
    hconf.set(entry.getKey(), entry.getValue());
  }

  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDN);
  builder.nameNodePort(port);
  MiniDFSCluster cluster = builder.build();
  return cluster;
}

Example 12

Source File: YarnFileStageTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@BeforeClass
public static void createHDFS() throws Exception {
	Assume.assumeTrue(!OperatingSystem.isWindows());

	final File tempDir = CLASS_TEMP_DIR.newFolder();

	org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
	hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath());

	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
	hdfsCluster = builder.build();
	hdfsRootPath = new Path(hdfsCluster.getURI());
}

Example 13

Source File: TestBlockScanner.java From big-c with Apache License 2.0

5 votes

TestContext(Configuration conf, int numNameServices) throws Exception {
  this.numNameServices = numNameServices;
  MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).
      numDataNodes(1).
      storagesPerDatanode(1);
  if (numNameServices > 1) {
    bld.nnTopology(MiniDFSNNTopology.
          simpleFederatedTopology(numNameServices));
  }
  cluster = bld.build();
  cluster.waitActive();
  dfs = new DistributedFileSystem[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    dfs[i] = cluster.getFileSystem(i);
  }
  bpids = new String[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    bpids[i] = cluster.getNamesystem(i).getBlockPoolId();
  }
  datanode = cluster.getDataNodes().get(0);
  blockScanner = datanode.getBlockScanner();
  for (int i = 0; i < numNameServices; i++) {
    dfs[i].mkdirs(new Path("/test"));
  }
  data = datanode.getFSDataset();
  volumes = data.getVolumes();
}

Example 14

Source File: FileStoreTestBase.java From pulsar with Apache License 2.0

5 votes

@BeforeMethod
public void start() throws Exception {
    File baseDir = Files.createTempDirectory(basePath).toFile().getAbsoluteFile();
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    hdfsCluster = builder.build();

    hdfsURI = "hdfs://localhost:"+ hdfsCluster.getNameNodePort() + "/";
    Properties properties = new Properties();
    fileSystemManagedLedgerOffloader = new FileSystemManagedLedgerOffloader(
            OffloadPolicies.create(properties),
            scheduler, hdfsURI, basePath);
}

Example 15

Source File: HadoopRecoverableWriterOldHadoopWithNoTruncateSupportTest.java From flink with Apache License 2.0

5 votes

@BeforeClass
public static void createHDFS() throws Exception {
	final File baseDir = TEMP_FOLDER.newFolder();

	final Configuration hdConf = new Configuration();
	hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());

	final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
	hdfsCluster = builder.build();

	final org.apache.hadoop.fs.FileSystem hdfs = hdfsCluster.getFileSystem();

	fileSystem = new HadoopFileSystem(hdfs);
	basePath = new Path(hdfs.getUri() + "/tests");
}

Example 16

Source File: TestPath.java From jsr203-hadoop with Apache License 2.0

5 votes

private static MiniDFSCluster startMini(String testName) throws IOException {
  File baseDir = new File("./target/hdfs/" + testName).getAbsoluteFile();
  FileUtil.fullyDelete(baseDir);
  Configuration conf = new Configuration();
  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
  MiniDFSCluster hdfsCluster = builder.clusterId(testName).build();
  hdfsCluster.waitActive();
  return hdfsCluster;
}

Example 17

Source File: TestBlockScanner.java From hadoop with Apache License 2.0

5 votes

TestContext(Configuration conf, int numNameServices) throws Exception {
  this.numNameServices = numNameServices;
  MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).
      numDataNodes(1).
      storagesPerDatanode(1);
  if (numNameServices > 1) {
    bld.nnTopology(MiniDFSNNTopology.
          simpleFederatedTopology(numNameServices));
  }
  cluster = bld.build();
  cluster.waitActive();
  dfs = new DistributedFileSystem[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    dfs[i] = cluster.getFileSystem(i);
  }
  bpids = new String[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    bpids[i] = cluster.getNamesystem(i).getBlockPoolId();
  }
  datanode = cluster.getDataNodes().get(0);
  blockScanner = datanode.getBlockScanner();
  for (int i = 0; i < numNameServices; i++) {
    dfs[i].mkdirs(new Path("/test"));
  }
  data = datanode.getFSDataset();
  volumes = data.getVolumes();
}

Example 18

Source File: YarnPreConfiguredMasterHaServicesTest.java From flink with Apache License 2.0

5 votes

@BeforeClass
public static void createHDFS() throws Exception {
	Assume.assumeTrue(!OperatingSystem.isWindows());

	final File tempDir = TEMP_DIR.newFolder();

	org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
	hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath());

	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
	hdfsCluster = builder.build();
	hdfsRootPath = new Path(hdfsCluster.getURI());
}

Example 19

Source File: TestBalancerWithNodeGroup.java From hadoop with Apache License 2.0

4 votes

/**
 * Create a cluster with even distribution, and a new empty node is added to
 * the cluster, then test node-group locality for balancer policy.
 */
@Test(timeout=60000)
public void testBalancerWithNodeGroup() throws Exception {
  Configuration conf = createConf();
  long[] capacities = new long[]{CAPACITY, CAPACITY, CAPACITY, CAPACITY};
  String[] racks = new String[]{RACK0, RACK0, RACK1, RACK1};
  String[] nodeGroups = new String[]{NODEGROUP0, NODEGROUP0, NODEGROUP1, NODEGROUP2};
  
  int numOfDatanodes = capacities.length;
  assertEquals(numOfDatanodes, racks.length);
  assertEquals(numOfDatanodes, nodeGroups.length);
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
                              .numDataNodes(capacities.length)
                              .racks(racks)
                              .simulatedCapacities(capacities);
  MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups);
  cluster = new MiniDFSClusterWithNodeGroup(builder);
  try {
    cluster.waitActive();
    client = NameNodeProxies.createProxy(conf, 
        cluster.getFileSystem(0).getUri(),
        ClientProtocol.class).getProxy();

    long totalCapacity = TestBalancer.sum(capacities);
    // fill up the cluster to be 20% full
    long totalUsedSpace = totalCapacity * 2 / 10;
    TestBalancer.createFile(cluster, filePath, totalUsedSpace / (numOfDatanodes/2),
        (short) (numOfDatanodes/2), 0);
    
    long newCapacity = CAPACITY;
    String newRack = RACK1;
    String newNodeGroup = NODEGROUP2;
    // start up an empty node with the same capacity and on NODEGROUP2
    cluster.startDataNodes(conf, 1, true, null, new String[]{newRack},
        new long[] {newCapacity}, new String[]{newNodeGroup});

    totalCapacity += newCapacity;

    // run balancer and validate results
    runBalancer(conf, totalUsedSpace, totalCapacity);

  } finally {
    cluster.shutdown();
  }
}

Example 20

Source File: MiniQJMHACluster.java From big-c with Apache License 2.0

4 votes

public MiniDFSCluster.Builder getDfsBuilder() {
  return dfsBuilder;
}