Java Code Examples for org.apache.hadoop.fs.FileSystem#setReplication()

private void testSetReplication() throws Exception {
  FileSystem fs = FileSystem.get(getProxiedFSConf());
  Path path = new Path(getProxiedFSTestDir(), "foo.txt");
  OutputStream os = fs.create(path);
  os.write(1);
  os.close();
  fs.close();
  fs.setReplication(path, (short) 2);

  fs = getHttpFSFileSystem();
  fs.setReplication(path, (short) 1);
  fs.close();

  fs = FileSystem.get(getProxiedFSConf());
  FileStatus status1 = fs.getFileStatus(path);
  fs.close();
  Assert.assertEquals(status1.getReplication(), (short) 1);
}
 

/**
 * Test over replicated block should get invalidated when decreasing the
 * replication for a partial block.
 */
@Test
public void testInvalidateOverReplicatedBlock() throws Exception {
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3)
      .build();
  try {
    final FSNamesystem namesystem = cluster.getNamesystem();
    final BlockManager bm = namesystem.getBlockManager();
    FileSystem fs = cluster.getFileSystem();
    Path p = new Path(MiniDFSCluster.getBaseDirectory(), "/foo1");
    FSDataOutputStream out = fs.create(p, (short) 2);
    out.writeBytes("HDFS-3119: " + p);
    out.hsync();
    fs.setReplication(p, (short) 1);
    out.close();
    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, p);
    assertEquals("Expected only one live replica for the block", 1, bm
        .countNodes(block.getLocalBlock()).liveReplicas());
  } finally {
    cluster.shutdown();
  }
}
 

private static FSDataOutputStream createFile(FileSystem fs, Path splitFile, 
    Configuration job)  throws IOException {
  FSDataOutputStream out = FileSystem.create(fs, splitFile, 
      new FsPermission(JobSubmissionFiles.JOB_FILE_PERMISSION));
  int replication = job.getInt(Job.SUBMIT_REPLICATION, 10);
  fs.setReplication(splitFile, (short)replication);
  writeSplitHeader(out);
  return out;
}
 

/**
 * Stages the source file or folder to a Hadoop file system and sets their permission and replication value
 * appropriately to be used with the Distributed Cache. WARNING: This will delete the contents of dest before staging
 * the archive.
 *
 * @param source    File or folder to copy to the file system. If it is a folder all contents will be copied into
 *                  dest.
 * @param fs        Hadoop file system to store the contents of the archive in
 * @param dest      Destination to copy source into. If source is a file, the new file name will be exactly dest. If
 *                  source is a folder its contents will be copied into dest. For more info see {@link
 *                  FileSystem#copyFromLocalFile(org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path)}.
 * @param overwrite Should an existing file or folder be overwritten? If not an exception will be thrown.
 * @throws IOException         Destination exists is not a directory
 * @throws KettleFileException Source does not exist or destination exists and overwrite is false.
 */
public void stageForCache( FileObject source, FileSystem fs, Path dest, boolean overwrite, boolean isPublic )
  throws IOException, KettleFileException {
  if ( !source.exists() ) {
    throw new KettleFileException(
      BaseMessages.getString( DistributedCacheUtilImpl.class, "DistributedCacheUtil.SourceDoesNotExist", source ) );
  }

  if ( fs.exists( dest ) ) {
    if ( overwrite ) {
      // It is a directory, clear it out
      fs.delete( dest, true );
    } else {
      throw new KettleFileException( BaseMessages
        .getString( DistributedCacheUtilImpl.class, "DistributedCacheUtil.DestinationExists",
          dest.toUri().getPath() ) );
    }
  }

  // Use the same replication we'd use for submitting jobs
  short replication = (short) fs.getConf().getInt( "mapred.submit.replication", 10 );

  if ( source.getURL().toString().endsWith( CONFIG_PROPERTIES ) ) {
    copyConfigProperties( source, fs, dest );
  } else {
    Path local = new Path( source.getURL().getPath() );
    fs.copyFromLocalFile( local, dest );
  }

  if ( isPublic ) {
    fs.setPermission( dest, PUBLIC_CACHED_FILE_PERMISSION );
  } else {
    fs.setPermission( dest, CACHED_FILE_PERMISSION );
  }
  fs.setReplication( dest, replication );
}
 

@Test
public void testPreserveGroupOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 

@Test
public void testPreserveReplicationOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  // Replication shouldn't apply to dirs so this should still be 0 == 0
  Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
 

@Test
public void testPreserveDefaults() throws IOException {
  FileSystem fs = FileSystem.get(config);
  
  // preserve replication, block size, user, group, permission, 
  // checksum type and timestamps    
  EnumSet<FileAttribute> attributes = 
      DistCpUtils.unpackAttributes(
          DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.substring(1));

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);
  
  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
 

@Test
public void testPreserveTimestampOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 

@Test
public void testPreserveGroupOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 

@Test
public void testPreserveNothingOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 

static void writeFile(NameNode namenode, Configuration conf, Path name, 
                      short replication)
    throws IOException, TimeoutException, InterruptedException {
  FileSystem fileSys = FileSystem.get(conf);
  SequenceFile.Writer writer = 
    SequenceFile.createWriter(fileSys, conf, name, 
                              BytesWritable.class, BytesWritable.class,
                              CompressionType.NONE);
  writer.append(new BytesWritable(), new BytesWritable());
  writer.close();
  fileSys.setReplication(name, replication);
  DFSTestUtil.waitReplication(fileSys, name, replication);
}
 

@Test
public void testNodeDecomissionWithOverreplicationRespectsRackPolicy() 
    throws Exception {
  Configuration conf = getConf();
  short REPLICATION_FACTOR = 5;
  final Path filePath = new Path("/testFile");

  // Configure an excludes file
  FileSystem localFileSys = FileSystem.getLocal(conf);
  Path workingDir = localFileSys.getWorkingDirectory();
  Path dir = new Path(workingDir, "build/test/data/temp/decommission");
  Path excludeFile = new Path(dir, "exclude");
  Path includeFile = new Path(dir, "include");
  assertTrue(localFileSys.mkdirs(dir));
  DFSTestUtil.writeFile(localFileSys, excludeFile, "");
  DFSTestUtil.writeFile(localFileSys, includeFile, "");
  conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());

  // All hosts are on two racks, only one host on /rack2
  String racks[] = {"/rack1", "/rack2", "/rack1", "/rack1", "/rack1"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns = cluster.getNameNode().getNamesystem();

  try {
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Lower the replication factor so the blocks are over replicated
    REPLICATION_FACTOR = 2;
    fs.setReplication(filePath, REPLICATION_FACTOR);

    // Decommission one of the hosts with the block that is not on
    // the lone host on rack2 (if we decomission that host it would
    // be impossible to respect the rack policy).
    BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(filePath), 0, Long.MAX_VALUE);
    for (String top : locs[0].getTopologyPaths()) {
      if (!top.startsWith("/rack2")) {
        String name = top.substring("/rack1".length()+1);
        DFSTestUtil.writeFile(localFileSys, excludeFile, name);
        ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
        DFSTestUtil.waitForDecommission(fs, name);
        break;
      }
    }

    // Check the block still has sufficient # replicas across racks,
    // ie we didn't remove the replica on the host on /rack1.
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
  } finally {
    cluster.shutdown();
  }
}
 

private void testBadBlockReportOnTransfer(
    boolean corruptBlockByDeletingBlockFile) throws Exception {
  Configuration conf = new HdfsConfiguration();
  FileSystem fs = null;
  DFSClient dfsClient = null;
  LocatedBlocks blocks = null;
  int replicaCount = 0;
  short replFactor = 1;
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  dfsClient = new DFSClient(new InetSocketAddress("localhost",
                            cluster.getNameNodePort()), conf);

  // Create file with replication factor of 1
  Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");
  DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0);
  DFSTestUtil.waitReplication(fs, file1, replFactor);

  // Corrupt the block belonging to the created file
  ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);

  int blockFilesCorrupted =
      corruptBlockByDeletingBlockFile?
          cluster.corruptBlockOnDataNodesByDeletingBlockFile(block) :
            cluster.corruptBlockOnDataNodes(block);       

  assertEquals("Corrupted too few blocks", replFactor, blockFilesCorrupted); 

  // Increase replication factor, this should invoke transfer request
  // Receiving datanode fails on checksum and reports it to namenode
  replFactor = 2;
  fs.setReplication(file1, replFactor);

  // Now get block details and check if the block is corrupt
  blocks = dfsClient.getNamenode().
            getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
  while (blocks.get(0).isCorrupt() != true) {
    try {
      LOG.info("Waiting until block is marked as corrupt...");
      Thread.sleep(1000);
    } catch (InterruptedException ie) {
    }
    blocks = dfsClient.getNamenode().
              getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
  }
  replicaCount = blocks.get(0).getLocations().length;
  assertTrue(replicaCount == 1);
  cluster.shutdown();
}
 

@Test
public void testNodeDecomissionWithOverreplicationRespectsRackPolicy() 
    throws Exception {
  Configuration conf = getConf();
  short REPLICATION_FACTOR = 5;
  final Path filePath = new Path("/testFile");

  // Configure an excludes file
  FileSystem localFileSys = FileSystem.getLocal(conf);
  Path workingDir = localFileSys.getWorkingDirectory();
  Path dir = new Path(workingDir, "build/test/data/temp/decommission");
  Path excludeFile = new Path(dir, "exclude");
  Path includeFile = new Path(dir, "include");
  assertTrue(localFileSys.mkdirs(dir));
  DFSTestUtil.writeFile(localFileSys, excludeFile, "");
  DFSTestUtil.writeFile(localFileSys, includeFile, "");
  conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());

  // All hosts are on two racks, only one host on /rack2
  String racks[] = {"/rack1", "/rack2", "/rack1", "/rack1", "/rack1"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns = cluster.getNameNode().getNamesystem();

  try {
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Lower the replication factor so the blocks are over replicated
    REPLICATION_FACTOR = 2;
    fs.setReplication(filePath, REPLICATION_FACTOR);

    // Decommission one of the hosts with the block that is not on
    // the lone host on rack2 (if we decomission that host it would
    // be impossible to respect the rack policy).
    BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(filePath), 0, Long.MAX_VALUE);
    for (String top : locs[0].getTopologyPaths()) {
      if (!top.startsWith("/rack2")) {
        String name = top.substring("/rack1".length()+1);
        DFSTestUtil.writeFile(localFileSys, excludeFile, name);
        ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
        DFSTestUtil.waitForDecommission(fs, name);
        break;
      }
    }

    // Check the block still has sufficient # replicas across racks,
    // ie we didn't remove the replica on the host on /rack1.
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
  } finally {
    cluster.shutdown();
  }
}
 

@Test(timeout=60000)
public void testStandbyIsHot() throws Exception {
  Configuration conf = new Configuration();
  // We read from the standby to watch block locations
  HAUtil.setAllowStandbyReads(conf, true);
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .nnTopology(MiniDFSNNTopology.simpleHATopology())
    .numDataNodes(3)
    .build();
  try {
    cluster.waitActive();
    cluster.transitionToActive(0);
    
    NameNode nn1 = cluster.getNameNode(0);
    NameNode nn2 = cluster.getNameNode(1);
    
    FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
    
    Thread.sleep(1000);
    System.err.println("==================================");
    DFSTestUtil.writeFile(fs, TEST_FILE_PATH, TEST_FILE_DATA);
    // Have to force an edit log roll so that the standby catches up
    nn1.getRpcServer().rollEditLog();
    System.err.println("==================================");

    // Block locations should show up on standby.
    LOG.info("Waiting for block locations to appear on standby node");
    waitForBlockLocations(cluster, nn2, TEST_FILE, 3);

    // Trigger immediate heartbeats and block reports so
    // that the active "trusts" all of the DNs
    cluster.triggerHeartbeats();
    cluster.triggerBlockReports();

    // Change replication
    LOG.info("Changing replication to 1");
    fs.setReplication(TEST_FILE_PATH, (short)1);
    BlockManagerTestUtil.computeAllPendingWork(
        nn1.getNamesystem().getBlockManager());
    waitForBlockLocations(cluster, nn1, TEST_FILE, 1);

    nn1.getRpcServer().rollEditLog();
    
    LOG.info("Waiting for lowered replication to show up on standby");
    waitForBlockLocations(cluster, nn2, TEST_FILE, 1);
    
    // Change back to 3
    LOG.info("Changing replication to 3");
    fs.setReplication(TEST_FILE_PATH, (short)3);
    BlockManagerTestUtil.computeAllPendingWork(
        nn1.getNamesystem().getBlockManager());
    nn1.getRpcServer().rollEditLog();
    
    LOG.info("Waiting for higher replication to show up on standby");
    waitForBlockLocations(cluster, nn2, TEST_FILE, 3);
    
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * Preserve attribute on file matching that of the file status being sent
 * as argument. Barring the block size, all the other attributes are preserved
 * by this function
 *
 * @param targetFS - File system
 * @param path - Path that needs to preserve original file status
 * @param srcFileStatus - Original file status
 * @param attributes - Attribute set that needs to be preserved
 * @param preserveRawXattrs if true, raw.* xattrs should be preserved
 * @throws IOException - Exception if any (particularly relating to group/owner
 *                       change or any transient error)
 */
public static void preserve(FileSystem targetFS, Path path,
                            CopyListingFileStatus srcFileStatus,
                            EnumSet<FileAttribute> attributes,
                            boolean preserveRawXattrs) throws IOException {

  FileStatus targetFileStatus = targetFS.getFileStatus(path);
  String group = targetFileStatus.getGroup();
  String user = targetFileStatus.getOwner();
  boolean chown = false;

  if (attributes.contains(FileAttribute.ACL)) {
    List<AclEntry> srcAcl = srcFileStatus.getAclEntries();
    List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus);
    if (!srcAcl.equals(targetAcl)) {
      targetFS.setAcl(path, srcAcl);
    }
    // setAcl doesn't preserve sticky bit, so also call setPermission if needed.
    if (srcFileStatus.getPermission().getStickyBit() !=
        targetFileStatus.getPermission().getStickyBit()) {
      targetFS.setPermission(path, srcFileStatus.getPermission());
    }
  } else if (attributes.contains(FileAttribute.PERMISSION) &&
    !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) {
    targetFS.setPermission(path, srcFileStatus.getPermission());
  }

  final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR);
  if (preserveXAttrs || preserveRawXattrs) {
    final String rawNS =
        StringUtils.toLowerCase(XAttr.NameSpace.RAW.name());
    Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs();
    Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path);
    if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) {
      for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) {
        String xattrName = entry.getKey();
        if (xattrName.startsWith(rawNS) || preserveXAttrs) {
          targetFS.setXAttr(path, xattrName, entry.getValue());
        }
      }
    }
  }

  if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() &&
      (srcFileStatus.getReplication() != targetFileStatus.getReplication())) {
    targetFS.setReplication(path, srcFileStatus.getReplication());
  }

  if (attributes.contains(FileAttribute.GROUP) &&
      !group.equals(srcFileStatus.getGroup())) {
    group = srcFileStatus.getGroup();
    chown = true;
  }

  if (attributes.contains(FileAttribute.USER) &&
      !user.equals(srcFileStatus.getOwner())) {
    user = srcFileStatus.getOwner();
    chown = true;
  }

  if (chown) {
    targetFS.setOwner(path, user, group);
  }
  
  if (attributes.contains(FileAttribute.TIMES)) {
    targetFS.setTimes(path, 
        srcFileStatus.getModificationTime(), 
        srcFileStatus.getAccessTime());
  }
}
 

public void testReplicationError() throws Exception {
  // bring up a cluster of 1
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  FileSystem fs = cluster.getFileSystem();

  try {
    // create a file of replication factor of 1
    final Path fileName = new Path("/test.txt");
    final int fileLen = 1;
    DFSTestUtil.createFile(fs, fileName, 1, (short)1, 1L);
    DFSTestUtil.waitReplication(fs, fileName, (short)1);

    // get the block belonged to the created file
    LocatedBlocks blocks = cluster.getNameNode().namesystem.getBlockLocations(
        fileName.toString(), 0, (long)fileLen);
    assertEquals(blocks.locatedBlockCount(), 1);
    LocatedBlock block = blocks.get(0);

    // bring up a second datanode
    cluster.startDataNodes(conf, 1, true, null, null);
    cluster.waitActive();
    final int sndNode = 1;
    DataNode datanode = cluster.getDataNodes().get(sndNode);

    // replicate the block to the second datanode
    InetSocketAddress target = datanode.getSelfAddr();
    Socket s = new Socket(target.getAddress(), target.getPort());
      //write the header.
    DataOutputStream out = new DataOutputStream(
        s.getOutputStream());

    out.writeShort( DataTransferProtocol.DATA_TRANSFER_VERSION );
    out.write( DataTransferProtocol.OP_WRITE_BLOCK );
    out.writeLong( block.getBlock().getBlockId());
    out.writeLong( block.getBlock().getGenerationStamp() );
    out.writeInt(1);
    out.writeBoolean( false );       // recovery flag
    Text.writeString( out, "" );
    out.writeBoolean(false); // Not sending src node information
    out.writeInt(0);
    
    // write check header
    out.writeByte( 1 );
    out.writeInt( 512 );

    out.flush();

    // close the connection before sending the content of the block
    out.close();
    
    // the temporary block & meta files should be deleted
    File dir1 = new File(cluster.getBlockDirectory(
          "data"+(2*sndNode+1)).getParentFile().getParent(), "tmp");
    File dir2 = new File(cluster.getBlockDirectory(
          "data"+(2*sndNode+2)).getParentFile().getParent(), "tmp");
    while (dir1.listFiles().length != 0 || dir2.listFiles().length != 0) {
      Thread.sleep(100);
    }
    
    // then increase the file's replication factor
    fs.setReplication(fileName, (short)2);
    // replication should succeed
    DFSTestUtil.waitReplication(fs, fileName, (short)1);
    
    // clean up the file
    fs.delete(fileName, false);
  } finally {
    cluster.shutdown();
  }
}
 

private void setReplication(FileSystem fileSystem, Path p, int rep) throws IOException, InterruptedException {
  fileSystem.setReplication(p, (short) rep);
  waitForReplication(fileSystem, p, rep);
}
 

/**
 * set up input file which has the list of input files.
 * 
 * @return boolean
 * @throws IOException
 */
private boolean setup() throws IOException {
  estimateSavings();

  final String randomId = getRandomId();
  JobClient jClient = new JobClient(jobconf);
  Path jobdir = new Path(jClient.getSystemDir(), NAME + "_" + randomId);

  LOG.info(JOB_DIR_LABEL + "=" + jobdir);
  jobconf.set(JOB_DIR_LABEL, jobdir.toString());
  Path log = new Path(jobdir, "_logs");

  // The control file should have small size blocks. This helps
  // in spreading out the load from mappers that will be spawned.
  jobconf.setInt("dfs.blocks.size",  OP_LIST_BLOCK_SIZE);

  FileOutputFormat.setOutputPath(jobconf, log);
  LOG.info("log=" + log);

  // create operation list
  FileSystem fs = jobdir.getFileSystem(jobconf);
  Path opList = new Path(jobdir, "_" + OP_LIST_LABEL);
  jobconf.set(OP_LIST_LABEL, opList.toString());
  int opCount = 0, synCount = 0;
  SequenceFile.Writer opWriter = null;

  try {
    opWriter = SequenceFile.createWriter(fs, jobconf, opList, Text.class,
        PolicyInfo.class, SequenceFile.CompressionType.NONE);
    for (RaidPolicyPathPair p : raidPolicyPathPairList) {
      // If a large set of files are Raided for the first time, files
      // in the same directory that tend to have the same size will end up
      // with the same map. This shuffle mixes things up, allowing a better
      // mix of files.
      java.util.Collections.shuffle(p.srcPaths);
      for (FileStatus st : p.srcPaths) {
        opWriter.append(new Text(st.getPath().toString()), p.policy);
        opCount++;
        if (++synCount > SYNC_FILE_MAX) {
          opWriter.sync();
          synCount = 0;
        }
      }
    }

  } finally {
    if (opWriter != null) {
      opWriter.close();
    }
    fs.setReplication(opList, OP_LIST_REPLICATION); // increase replication for control file
  }
  raidPolicyPathPairList.clear();
  
  jobconf.setInt(OP_COUNT_LABEL, opCount);
  LOG.info("Number of files=" + opCount);
  jobconf.setNumMapTasks(getMapCount(opCount));
  LOG.info("jobName= " + jobName + " numMapTasks=" + jobconf.getNumMapTasks());
  return opCount != 0;

}
 

private void testBadBlockReportOnTransfer(
    boolean corruptBlockByDeletingBlockFile) throws Exception {
  Configuration conf = new HdfsConfiguration();
  FileSystem fs = null;
  DFSClient dfsClient = null;
  LocatedBlocks blocks = null;
  int replicaCount = 0;
  short replFactor = 1;
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  dfsClient = new DFSClient(new InetSocketAddress("localhost",
                            cluster.getNameNodePort()), conf);

  // Create file with replication factor of 1
  Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");
  DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0);
  DFSTestUtil.waitReplication(fs, file1, replFactor);

  // Corrupt the block belonging to the created file
  ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);

  int blockFilesCorrupted =
      corruptBlockByDeletingBlockFile?
          cluster.corruptBlockOnDataNodesByDeletingBlockFile(block) :
            cluster.corruptBlockOnDataNodes(block);       

  assertEquals("Corrupted too few blocks", replFactor, blockFilesCorrupted); 

  // Increase replication factor, this should invoke transfer request
  // Receiving datanode fails on checksum and reports it to namenode
  replFactor = 2;
  fs.setReplication(file1, replFactor);

  // Now get block details and check if the block is corrupt
  blocks = dfsClient.getNamenode().
            getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
  while (blocks.get(0).isCorrupt() != true) {
    try {
      LOG.info("Waiting until block is marked as corrupt...");
      Thread.sleep(1000);
    } catch (InterruptedException ie) {
    }
    blocks = dfsClient.getNamenode().
              getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
  }
  replicaCount = blocks.get(0).getLocations().length;
  assertTrue(replicaCount == 1);
  cluster.shutdown();
}