Java Code Examples for org.apache.hadoop.hdfs.protocol.Block#filename2id()

The following examples show how to use org.apache.hadoop.hdfs.protocol.Block#filename2id() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FsVolumeImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Get the next block.<p/>
 *
 * Each volume has a hierarchical structure.<p/>
 *
 * <code>
 * BPID B0
 *   finalized/
 *     subdir0
 *       subdir0
 *         blk_000
 *         blk_001
 *       ...
 *     subdir1
 *       subdir0
 *         ...
 *   rbw/
 * </code>
 *
 * When we run out of entries at one level of the structure, we search
 * progressively higher levels.  For example, when we run out of blk_
 * entries in a subdirectory, we search for the next subdirectory.
 * And so on.
 */
@Override
public ExtendedBlock nextBlock() throws IOException {
  if (state.atEnd) {
    return null;
  }
  try {
    while (true) {
      List<String> entries = getSubdirEntries();
      if (entries != null) {
        state.curEntry = nextSorted(entries, state.curEntry);
        if (state.curEntry == null) {
          LOG.trace("nextBlock({}, {}): advancing from {} to next " +
              "subdirectory.", storageID, bpid, state.curFinalizedSubDir);
        } else {
          ExtendedBlock block =
              new ExtendedBlock(bpid, Block.filename2id(state.curEntry));
          LOG.trace("nextBlock({}, {}): advancing to {}",
              storageID, bpid, block);
          return block;
        }
      }
      state.curFinalizedSubDir = getNextFinalizedSubDir();
      if (state.curFinalizedSubDir == null) {
        state.curFinalizedDir = getNextFinalizedDir();
        if (state.curFinalizedDir == null) {
          state.atEnd = true;
          return null;
        }
      }
    }
  } catch (IOException e) {
    state.atEnd = true;
    LOG.error("nextBlock({}, {}): I/O error", storageID, bpid, e);
    throw e;
  }
}
Example 2
Source File: FsVolumeImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Get the next block.<p/>
 *
 * Each volume has a hierarchical structure.<p/>
 *
 * <code>
 * BPID B0
 *   finalized/
 *     subdir0
 *       subdir0
 *         blk_000
 *         blk_001
 *       ...
 *     subdir1
 *       subdir0
 *         ...
 *   rbw/
 * </code>
 *
 * When we run out of entries at one level of the structure, we search
 * progressively higher levels.  For example, when we run out of blk_
 * entries in a subdirectory, we search for the next subdirectory.
 * And so on.
 */
@Override
public ExtendedBlock nextBlock() throws IOException {
  if (state.atEnd) {
    return null;
  }
  try {
    while (true) {
      List<String> entries = getSubdirEntries();
      if (entries != null) {
        state.curEntry = nextSorted(entries, state.curEntry);
        if (state.curEntry == null) {
          LOG.trace("nextBlock({}, {}): advancing from {} to next " +
              "subdirectory.", storageID, bpid, state.curFinalizedSubDir);
        } else {
          ExtendedBlock block =
              new ExtendedBlock(bpid, Block.filename2id(state.curEntry));
          LOG.trace("nextBlock({}, {}): advancing to {}",
              storageID, bpid, block);
          return block;
        }
      }
      state.curFinalizedSubDir = getNextFinalizedSubDir();
      if (state.curFinalizedSubDir == null) {
        state.curFinalizedDir = getNextFinalizedDir();
        if (state.curFinalizedDir == null) {
          state.atEnd = true;
          return null;
        }
      }
    }
  } catch (IOException e) {
    state.atEnd = true;
    LOG.error("nextBlock({}, {}): I/O error", storageID, bpid, e);
    throw e;
  }
}
Example 3
Source File: BlockPoolSlice.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
/**
 * Add replicas under the given directory to the volume map
 * @param volumeMap the replicas map
 * @param dir an input directory
 * @param lazyWriteReplicaMap Map of replicas on transient
 *                                storage.
 * @param isFinalized true if the directory has finalized replicas;
 *                    false if the directory has rbw replicas
 * @param exceptions list of exception which need to return to parent thread.
 * @param subTaskQueue queue of sub tasks
 */
void addToReplicasMap(ReplicaMap volumeMap, File dir,
                      final RamDiskReplicaTracker lazyWriteReplicaMap, boolean isFinalized,
                      List<IOException> exceptions, Queue<RecursiveAction> subTaskQueue)
    throws IOException {
  File[] files = fileIoProvider.listFiles(volume, dir);
  Arrays.sort(files, FILE_COMPARATOR);
  for (int i = 0; i < files.length; i++) {
    File file = files[i];
    if (file.isDirectory()) {
      // Launch new sub task.
      AddReplicaProcessor subTask = new AddReplicaProcessor(volumeMap, file,
          lazyWriteReplicaMap, isFinalized, exceptions, subTaskQueue);
      subTask.fork();
      subTaskQueue.add(subTask);
    }

    if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
      file = recoverTempUnlinkedBlock(file);
      if (file == null) { // the original block still exists, so we cover it
        // in another iteration and can continue here
        continue;
      }
    }
    if (!Block.isBlockFilename(file)) {
      continue;
    }

    long genStamp = FsDatasetUtil.getGenerationStampFromFile(
        files, file, i);
    long blockId = Block.filename2id(file.getName());
    Block block = new Block(blockId, file.length(), genStamp);
    addReplicaToReplicasMap(block, volumeMap, lazyWriteReplicaMap,
        isFinalized);
  }
}