Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#hsync()

private void doFlush(FSDataOutputStream outputStream,
    long currentTime, boolean sync) throws IOException {
  if (sync) {
    outputStream.hsync();
  } else {
    outputStream.hflush();
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("Flushing output stream"
        + ", lastTimeSinceFLush=" + lastFlushTime
        + ", timeSinceLastFlush=" + (currentTime - lastFlushTime)
        + ", unflushedEventsCount=" + unflushedEventsCount
        + ", maxUnflushedEvents=" + maxUnflushedEvents);
  }

  unflushedEventsCount = 0;
  lastFlushTime = currentTime;
}
 

/**
 * Ensure that reserved space is released when the client goes away
 * unexpectedly.
 *
 * The verification is done for each replica in the write pipeline.
 *
 * @throws IOException
 */
@Test(timeout=300000)
public void testSpaceReleasedOnUnexpectedEof()
    throws IOException, InterruptedException, TimeoutException {
  final short replication = 3;
  startCluster(BLOCK_SIZE, replication, -1);

  final String methodName = GenericTestUtils.getMethodName();
  final Path file = new Path("/" + methodName + ".01.dat");

  // Write 1 byte to the file and kill the writer.
  FSDataOutputStream os = fs.create(file, replication);
  os.write(new byte[1]);
  os.hsync();
  DFSTestUtil.abortStream((DFSOutputStream) os.getWrappedStream());

  // Ensure all space reserved for the replica was released on each
  // DataNode.
  for (DataNode dn : cluster.getDataNodes()) {
    final FsVolumeImpl volume = (FsVolumeImpl) dn.getFSDataset().getVolumes().get(0);
    GenericTestUtils.waitFor(new Supplier<Boolean>() {
      @Override
      public Boolean get() {
        return (volume.getReservedForRbw() == 0);
      }
    }, 500, Integer.MAX_VALUE); // Wait until the test times out.
  }
}
 

private void writeTestFile(String testFileName) throws Exception {
  Path filePath = new Path(testFileName);
  FSDataOutputStream stream = dfs.create(filePath);
  for (int i = 0; i < 10; i++) {
    byte[] data = RandomStringUtils.randomAlphabetic(102400).getBytes();
    stream.write(data);
  }
  stream.hsync();
  stream.close();
}
 

private void testDeleteAddBlockRace(boolean hasSnapshot) throws Exception {
  try {
    conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
        SlowBlockPlacementPolicy.class, BlockPlacementPolicy.class);
    cluster = new MiniDFSCluster.Builder(conf).build();
    FileSystem fs = cluster.getFileSystem();
    final String fileName = "/testDeleteAddBlockRace";
    Path filePath = new Path(fileName);

    FSDataOutputStream out = null;
    out = fs.create(filePath);
    if (hasSnapshot) {
      SnapshotTestHelper.createSnapshot((DistributedFileSystem) fs, new Path(
          "/"), "s1");
    }

    Thread deleteThread = new DeleteThread(fs, filePath);
    deleteThread.start();

    try {
      // write data and syn to make sure a block is allocated.
      out.write(new byte[32], 0, 32);
      out.hsync();
      Assert.fail("Should have failed.");
    } catch (FileNotFoundException e) {
      GenericTestUtils.assertExceptionContains(filePath.getName(), e);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/** Test that syncBlock is correctly performed at replicas */
@Test
public void testHSyncWithReplication() throws Exception {
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  final FileSystem fs = cluster.getFileSystem();

  final Path p = new Path("/testHSyncWithReplication/foo");
  final int len = 1 << 16;
  FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
      4096, (short) 3, len, null);
  out.write(1);
  out.hflush();
  checkSyncMetric(cluster, 0, 0);
  checkSyncMetric(cluster, 1, 0);
  checkSyncMetric(cluster, 2, 0);
  out.hsync();
  checkSyncMetric(cluster, 0, 1);
  checkSyncMetric(cluster, 1, 1);
  checkSyncMetric(cluster, 2, 1);
  out.hsync();
  checkSyncMetric(cluster, 0, 2);
  checkSyncMetric(cluster, 1, 2);
  checkSyncMetric(cluster, 2, 2);
  cluster.shutdown();
}
 

/** Test hsync on an exact block boundary */
@Test
public void testHSyncBlockBoundary() throws Exception {
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  final FileSystem fs = cluster.getFileSystem();
  
  final Path p = new Path("/testHSyncBlockBoundary/foo");
  final int len = 1 << 16;
  final byte[] fileContents = AppendTestUtil.initBuffer(len);
  FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
      4096, (short) 1, len, null);
  // fill exactly one block (tests the SYNC_BLOCK case) and flush
  out.write(fileContents, 0, len);
  out.hflush();
  // the full block should have caused a sync
  checkSyncMetric(cluster, 1);
  out.hsync();
  // first on block again
  checkSyncMetric(cluster, 1);
  // write one more byte and sync again
  out.write(1);
  out.hsync();
  checkSyncMetric(cluster, 2);
  out.close();
  checkSyncMetric(cluster, 3);
  cluster.shutdown();
}
 

@SuppressWarnings("unchecked")
@Test(timeout = 30000)
public void testRBWFileCreationError() throws Exception {

  final short replication = 1;
  startCluster(BLOCK_SIZE, replication, -1);

  final FsVolumeImpl fsVolumeImpl = (FsVolumeImpl) cluster.getDataNodes()
      .get(0).getFSDataset().getVolumes().get(0);
  final String methodName = GenericTestUtils.getMethodName();
  final Path file = new Path("/" + methodName + ".01.dat");

  // Mock BlockPoolSlice so that RBW file creation gives IOExcception
  BlockPoolSlice blockPoolSlice = Mockito.mock(BlockPoolSlice.class);
  Mockito.when(blockPoolSlice.createRbwFile((Block) Mockito.any()))
      .thenThrow(new IOException("Synthetic IO Exception Throgh MOCK"));

  Field field = FsVolumeImpl.class.getDeclaredField("bpSlices");
  field.setAccessible(true);
  Map<String, BlockPoolSlice> bpSlices = (Map<String, BlockPoolSlice>) field
      .get(fsVolumeImpl);
  bpSlices.put(fsVolumeImpl.getBlockPoolList()[0], blockPoolSlice);

  try {
    // Write 1 byte to the file
    FSDataOutputStream os = fs.create(file, replication);
    os.write(new byte[1]);
    os.hsync();
    os.close();
    fail("Expecting IOException file creation failure");
  } catch (IOException e) {
    // Exception can be ignored (expected)
  }

  // Ensure RBW space reserved is released
  assertTrue("Expected ZERO but got " + fsVolumeImpl.getReservedForRbw(),
      fsVolumeImpl.getReservedForRbw() == 0);
}
 

/**
 * Ensure that reserved space is released when the client goes away
 * unexpectedly.
 *
 * The verification is done for each replica in the write pipeline.
 *
 * @throws IOException
 */
@Test(timeout=300000)
public void testSpaceReleasedOnUnexpectedEof()
    throws IOException, InterruptedException, TimeoutException {
  final short replication = 3;
  startCluster(BLOCK_SIZE, replication, -1);

  final String methodName = GenericTestUtils.getMethodName();
  final Path file = new Path("/" + methodName + ".01.dat");

  // Write 1 byte to the file and kill the writer.
  FSDataOutputStream os = fs.create(file, replication);
  os.write(new byte[1]);
  os.hsync();
  DFSTestUtil.abortStream((DFSOutputStream) os.getWrappedStream());

  // Ensure all space reserved for the replica was released on each
  // DataNode.
  for (DataNode dn : cluster.getDataNodes()) {
    final FsVolumeImpl volume = (FsVolumeImpl) dn.getFSDataset().getVolumes().get(0);
    GenericTestUtils.waitFor(new Supplier<Boolean>() {
      @Override
      public Boolean get() {
        return (volume.getReservedForRbw() == 0);
      }
    }, 500, Integer.MAX_VALUE); // Wait until the test times out.
  }
}
 

@Override
public void sync(boolean forceSync) throws IOException {
  FSDataOutputStream fsdos = this.output;
  if (fsdos == null) {
    return; // Presume closed
  }
  fsdos.flush();
  if (forceSync) {
    fsdos.hsync();
  } else {
    fsdos.hflush();
  }
}
 

@Test
public void testReceivePacketMetrics() throws Exception {
  Configuration conf = new HdfsConfiguration();
  final int interval = 1;
  conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  try {
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();

    Path testFile = new Path("/testFlushNanosMetric.txt");
    FSDataOutputStream fout = fs.create(testFile);
    fout.write(new byte[1]);
    fout.hsync();
    fout.close();
    List<DataNode> datanodes = cluster.getDataNodes();
    DataNode datanode = datanodes.get(0);
    MetricsRecordBuilder dnMetrics = getMetrics(datanode.getMetrics().name());
    // Expect two flushes, 1 for the flush that occurs after writing, 
    // 1 that occurs on closing the data and metadata files.
    assertCounter("FlushNanosNumOps", 2L, dnMetrics);
    // Expect two syncs, one from the hsync, one on close.
    assertCounter("FsyncNanosNumOps", 2L, dnMetrics);
    // Wait for at least 1 rollover
    Thread.sleep((interval + 1) * 1000);
    // Check the receivePacket percentiles that should be non-zero
    String sec = interval + "s";
    assertQuantileGauges("FlushNanos" + sec, dnMetrics);
    assertQuantileGauges("FsyncNanos" + sec, dnMetrics);
  } finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 

protected final void makeTestFile(Path path, long length,
    boolean isLazyPersist) throws IOException {

  EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);

  if (isLazyPersist) {
    createFlags.add(LAZY_PERSIST);
  }

  FSDataOutputStream fos = null;
  try {
    fos =
        fs.create(path,
            FsPermission.getFileDefault(),
            createFlags,
            BUFFER_LENGTH,
            REPL_FACTOR,
            BLOCK_SIZE,
            null);

    // Allocate a block.
    byte[] buffer = new byte[BUFFER_LENGTH];
    for (int bytesWritten = 0; bytesWritten < length; ) {
      fos.write(buffer, 0, buffer.length);
      bytesWritten += buffer.length;
    }
    if (length > 0) {
      fos.hsync();
    }
  } finally {
    IOUtils.closeQuietly(fos);
  }
}
 

@BeforeClass
public static void setup() throws IOException {
  try {
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
        .format(true).racks(null).build();
    remoteFs = dfsCluster.getFileSystem();
  } catch (IOException io) {
    throw new RuntimeException("problem starting mini dfs cluster", io);
  }

  Configuration testConf = new YarnConfiguration(
      dfsCluster.getFileSystem().getConf());


  FSDataOutputStream dataOutputStream = null;
  try {
    dataOutputStream = remoteFs.create(new Path("/tmp/input/test.xml"), true);
    testConf.writeXml(dataOutputStream);
    dataOutputStream.hsync();
  } catch (IOException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    throw new RuntimeException(e);
  } finally {
    if (dataOutputStream != null) {
      dataOutputStream.close();
    }
  }

  remoteFs.mkdirs(new Path("/tmp/input/"));
  remoteFs.mkdirs(new Path("/tmp/splitsDirNew/"));
  remoteFs.mkdirs(new Path("/tmp/splitsDirOld/"));
  testFilePath = remoteFs.makeQualified(new Path("/tmp/input/test.xml"));
  FileStatus fsStatus = remoteFs.getFileStatus(testFilePath);
  Assert.assertTrue(fsStatus.getLen() > 0);

  oldSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirOld/"));
  newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/"));
}
 

private void writeTestFile(String testFileName) throws Exception {
  Path filePath = new Path(testFileName);
  FSDataOutputStream stream = dfs.create(filePath);
  for (int i = 0; i < 10; i++) {
    byte[] data = RandomStringUtils.randomAlphabetic(102400).getBytes();
    stream.write(data);
  }
  stream.hsync();
  stream.close();
}
 

private void hsync(FSDataOutputStream fsOut) throws Exception {
    // 调用hsync时，必须设置SyncFlag.UPDATE_LENGTH，否则RDD或者MR任务读取不到写入的数据
    // 参见：
    // https://issues.cloudera.org/browse/DISTRO-696;
    // http://www.hypertable.com/documentation/administrator_guide/hdfs_and_durability
    // https://blog.csdn.net/leen0304/article/details/77854052?locationNum=10&fps=1
    // https://issues.apache.org/jira/browse/HDFS-11915
    if (fsOut instanceof HdfsDataOutputStream) {
        ((HdfsDataOutputStream) fsOut).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
    } else if (fsOut.getWrappedStream() instanceof DFSOutputStream) {
        ((DFSOutputStream) fsOut.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
    } else {
        fsOut.hsync();
    }
}
 

/**
 * Test when a block's replica is removed from RBW folder in one of the
 * datanode, namenode should ask to invalidate that corrupted block and
 * schedule replication for one more replica for that under replicated block.
 */
@Test(timeout=600000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN()
    throws IOException, InterruptedException {
  // This test cannot pass on Windows due to file locking enforcement.  It will
  // reject the attempt to delete the block file from the RBW folder.
  assumeTrue(!Path.WINDOWS);

  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 300);
  conf.setLong(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
      .build();
  FSDataOutputStream out = null;
  try {
    final FSNamesystem namesystem = cluster.getNamesystem();
    FileSystem fs = cluster.getFileSystem();
    Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1");
    out = fs.create(testPath, (short) 2);
    out.writeBytes("HDFS-3157: " + testPath);
    out.hsync();
    cluster.startDataNodes(conf, 1, true, null, null, null);
    String bpid = namesystem.getBlockPoolId();
    ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, testPath);
    Block block = blk.getLocalBlock();
    DataNode dn = cluster.getDataNodes().get(0);

    // Delete partial block and its meta information from the RBW folder
    // of first datanode.
    File blockFile = DataNodeTestUtils.getBlockFile(dn, bpid, block);
    File metaFile = DataNodeTestUtils.getMetaFile(dn, bpid, block);
    assertTrue("Could not delete the block file from the RBW folder",
        blockFile.delete());
    assertTrue("Could not delete the block meta file from the RBW folder",
        metaFile.delete());

    out.close();
    
    int liveReplicas = 0;
    while (true) {
      if ((liveReplicas = countReplicas(namesystem, blk).liveReplicas()) < 2) {
        // This confirms we have a corrupt replica
        LOG.info("Live Replicas after corruption: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be less than 2 replicas in the "
        + "liveReplicasMap", 1, liveReplicas);
    
    while (true) {
      if ((liveReplicas =
            countReplicas(namesystem, blk).liveReplicas()) > 1) {
        //Wait till the live replica count becomes equal to Replication Factor
        LOG.info("Live Replicas after Rereplication: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be two live replicas", 2, liveReplicas);

    while (true) {
      Thread.sleep(100);
      if (countReplicas(namesystem, blk).corruptReplicas() == 0) {
        LOG.info("Corrupt Replicas becomes 0");
        break;
      }
    }
  } finally {
    if (out != null) {
      out.close();
    }
    cluster.shutdown();
  }
}
 

private void createFileAndTestSpaceReservation(
    final String fileNamePrefix, final int fileBlockSize)
    throws IOException, InterruptedException {
  // Enough for 1 block + meta files + some delta.
  final long configuredCapacity = fileBlockSize * 2 - 1;
  startCluster(BLOCK_SIZE, 1, configuredCapacity);
  FSDataOutputStream out = null;
  Path path = new Path("/" + fileNamePrefix + ".dat");

  try {
    out = fs.create(path, false, 4096, (short) 1, fileBlockSize);

    byte[] buffer = new byte[rand.nextInt(fileBlockSize / 4)];
    out.write(buffer);
    out.hsync();
    int bytesWritten = buffer.length;

    // Check that space was reserved for a full block minus the bytesWritten.
    assertThat(singletonVolume.getReservedForRbw(),
               is((long) fileBlockSize - bytesWritten));
    out.close();
    out = null;

    // Check that the reserved space has been released since we closed the
    // file.
    assertThat(singletonVolume.getReservedForRbw(), is(0L));

    // Reopen the file for appends and write 1 more byte.
    out = fs.append(path);
    out.write(buffer);
    out.hsync();
    bytesWritten += buffer.length;

    // Check that space was again reserved for a full block minus the
    // bytesWritten so far.
    assertThat(singletonVolume.getReservedForRbw(),
               is((long) fileBlockSize - bytesWritten));

    // Write once again and again verify the available space. This ensures
    // that the reserved space is progressively adjusted to account for bytes
    // written to disk.
    out.write(buffer);
    out.hsync();
    bytesWritten += buffer.length;
    assertThat(singletonVolume.getReservedForRbw(),
               is((long) fileBlockSize - bytesWritten));
  } finally {
    if (out != null) {
      out.close();
    }
  }
}
 

/**
 * Block Recovery when the meta file not having crcs for all chunks in block
 * file
 */
@Test
public void testBlockRecoveryWithLessMetafile() throws Exception {
  Configuration conf = new Configuration();
  conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
      UserGroupInformation.getCurrentUser().getShortUserName());
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Path file = new Path("/testRecoveryFile");
  DistributedFileSystem dfs = cluster.getFileSystem();
  FSDataOutputStream out = dfs.create(file);
  int count = 0;
  while (count < 2 * 1024 * 1024) {
    out.writeBytes("Data");
    count += 4;
  }
  out.hsync();
  // abort the original stream
  ((DFSOutputStream) out.getWrappedStream()).abort();

  LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations(
      file.toString(), 0, count);
  ExtendedBlock block = locations.get(0).getBlock();
  DataNode dn = cluster.getDataNodes().get(0);
  BlockLocalPathInfo localPathInfo = dn.getBlockLocalPathInfo(block, null);
  File metafile = new File(localPathInfo.getMetaPath());
  assertTrue(metafile.exists());

  // reduce the block meta file size
  RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
  raf.setLength(metafile.length() - 20);
  raf.close();

  // restart DN to make replica to RWR
  DataNodeProperties dnProp = cluster.stopDataNode(0);
  cluster.restartDataNode(dnProp, true);

  // try to recover the lease
  DistributedFileSystem newdfs = (DistributedFileSystem) FileSystem
      .newInstance(cluster.getConfiguration(0));
  count = 0;
  while (++count < 10 && !newdfs.recoverLease(file)) {
    Thread.sleep(1000);
  }
  assertTrue("File should be closed", newdfs.recoverLease(file));

}
 

/**
 * Test NN restart if a failure happens in between creating the fsimage
 * MD5 file and renaming the fsimage.
 */
@Test(timeout=30000)
public void testFailureBeforeRename () throws IOException {
  Configuration conf = new HdfsConfiguration();
  FSDataOutputStream fos = null;
  SecondaryNameNode secondary = null;
  MiniDFSCluster cluster = null;
  FileSystem fs = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
        .build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    secondary = startSecondaryNameNode(conf);
    fos = fs.create(new Path("tmpfile0"));
    fos.write(new byte[] { 0, 1, 2, 3 });
    secondary.doCheckpoint();
    fos.write(new byte[] { 0, 1, 2, 3 });
    fos.hsync();

    // Cause merge to fail in next checkpoint.
    Mockito.doThrow(new IOException(
        "Injecting failure after MD5Rename"))
        .when(faultInjector).afterMD5Rename();

    try {
      secondary.doCheckpoint();
      fail("Fault injection failed.");
    } catch (IOException ioe) {
      // This is expected.
    }
    Mockito.reset(faultInjector);
    // Namenode should still restart successfully
    cluster.restartNameNode();
  } finally {
    if (fs != null) {
      fs.close();
    }
    cleanup(secondary);
    secondary = null;
    cleanup(cluster);
    cluster = null;
    Mockito.reset(faultInjector);
  }
}
 

private void writeOkFile(String reqString, JSONObject reqJson) throws Exception {
    DBConfiguration dbConf = FullPullHelper.getDbConfiguration(reqString);
    String sinkType = dbConf.getString(DBConfiguration.SINK_TYPE);
    if (FullPullConstants.SINK_TYPE_KAFKA.equals(sinkType)) {
        //kafka 无需写ok文件
        return;
    }

    FSDataOutputStream outputStream = null;
    try {
        // 防止多个线程并发写hdfs同一个文件
        if (context.getThisTaskIndex() != 0) {
            logger.info("[pull bolt] 任务index:{},忽略写ok文件请求,仅index为0的任务负责写ok文件.", context.getThisTaskId());
            return;
        }
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");

        String opTs = dbConf.getString(DBConfiguration.DATA_IMPORT_OP_TS);
        String dbType = dbConf.getString(DBConfiguration.DataSourceInfo.DS_TYPE);
        // 这里oracle的ums_ts到微秒2020-04-21 12:46:45.461281,需要去掉后三位
        if (dbType.equals("oracle")) {
            opTs = opTs.substring(0, opTs.length() - 3);
        }
        long time = sdf.parse(opTs).getTime();
        String path = dbConf.getString(DBConfiguration.HDFS_TABLE_PATH) + "ok_" + time;
        logger.info("[pull bolt] will write ok file {}", path);

        outputStream = getFileSystem().create(new Path(path));
        JSONObject data = new JSONObject();
        data.put("ums_ts_", opTs);
        data.put("id", FullPullHelper.getSeqNo(reqJson));
        data.put("end_time", new Date());
        outputStream.write(data.toJSONString().getBytes());
        outputStream.hsync();
        logger.info("[pull bolt] write ok file success.{},{}", path, data);
    } catch (Exception e) {
        logger.error("Exception when write ok file to hdfs");
        throw e;
    } finally {
        if (outputStream != null) {
            outputStream.close();
        }
    }
}
 

/**
 * Test hsync with END_BLOCK flag.
 */
@Test
public void hSyncEndBlock_00() throws IOException {
  final int preferredBlockSize = 1024;
  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, preferredBlockSize);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
      .build();
  DistributedFileSystem fileSystem = cluster.getFileSystem();
  FSDataOutputStream stm = null;
  try {
    Path path = new Path("/" + fName);
    stm = fileSystem.create(path, true, 4096, (short) 2,
        AppendTestUtil.BLOCK_SIZE);
    System.out.println("Created file " + path.toString());
    ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet
        .of(SyncFlag.END_BLOCK));
    long currentFileLength = fileSystem.getFileStatus(path).getLen();
    assertEquals(0L, currentFileLength);
    LocatedBlocks blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
    assertEquals(0, blocks.getLocatedBlocks().size());

    // write a block and call hsync(end_block) at the block boundary
    stm.write(new byte[preferredBlockSize]);
    ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet
        .of(SyncFlag.END_BLOCK));
    currentFileLength = fileSystem.getFileStatus(path).getLen();
    assertEquals(preferredBlockSize, currentFileLength);
    blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
    assertEquals(1, blocks.getLocatedBlocks().size());

    // call hsync then call hsync(end_block) immediately
    stm.write(new byte[preferredBlockSize / 2]);
    stm.hsync();
    ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet
        .of(SyncFlag.END_BLOCK));
    currentFileLength = fileSystem.getFileStatus(path).getLen();
    assertEquals(preferredBlockSize + preferredBlockSize / 2,
        currentFileLength);
    blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
    assertEquals(2, blocks.getLocatedBlocks().size());

    stm.write(new byte[preferredBlockSize / 4]);
    stm.hsync();
    currentFileLength = fileSystem.getFileStatus(path).getLen();
    assertEquals(preferredBlockSize + preferredBlockSize / 2
        + preferredBlockSize / 4, currentFileLength);
    blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
    assertEquals(3, blocks.getLocatedBlocks().size());
  } finally {
    IOUtils.cleanup(null, stm, fileSystem);
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}