Java Code Examples for org.apache.hadoop.fs.BlockLocation#getOffset()

The following examples show how to use org.apache.hadoop.fs.BlockLocation#getOffset() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Metadata.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Get the host affinity for a row group.
 *
 * @param fileStatus the parquet file
 * @param start      the start of the row group
 * @param length     the length of the row group
 * @return host affinity for the row group
 */
private Map<String, Float> getHostAffinity(FileStatus fileStatus, FileSystem fs, long start, long length)
    throws IOException {
  BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, start, length);
  Map<String, Float> hostAffinityMap = Maps.newHashMap();
  for (BlockLocation blockLocation : blockLocations) {
    for (String host : blockLocation.getHosts()) {
      Float currentAffinity = hostAffinityMap.get(host);
      float blockStart = blockLocation.getOffset();
      float blockEnd = blockStart + blockLocation.getLength();
      float rowGroupEnd = start + length;
      Float newAffinity = (blockLocation.getLength() - (blockStart < start ? start - blockStart : 0) -
          (blockEnd > rowGroupEnd ? blockEnd - rowGroupEnd : 0)) / length;
      if (currentAffinity != null) {
        hostAffinityMap.put(host, currentAffinity + newAffinity);
      } else {
        hostAffinityMap.put(host, newAffinity);
      }
    }
  }
  return hostAffinityMap;
}
 
Example 2
Source File: BlockMapBuilder.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Builds a mapping of block locations to file byte range
 */
private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(FileStatus status) throws IOException {
  final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time();
  BlockLocation[] blocks;
  ImmutableRangeMap<Long,BlockLocation> blockMap;
  blocks = fs.getFileBlockLocations(status, 0, status.getLen());
  ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>();
  for (BlockLocation block : blocks) {
    long start = block.getOffset();
    long end = start + block.getLength();
    Range<Long> range = Range.closedOpen(start, end);
    blockMapBuilder = blockMapBuilder.put(range, block);
  }
  blockMap = blockMapBuilder.build();
  blockMapMap.put(status.getPath(), blockMap);
  context.stop();
  return blockMap;
}
 
Example 3
Source File: FileInputFormat.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 4
Source File: FileInputFormat.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 5
Source File: RaidShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
void collectFileCorruptBlocksInStripe(final DistributedFileSystem dfs, 
    final RaidInfo raidInfo, final Path filePath, 
    final HashMap<Integer, Integer> corruptBlocksPerStripe)
        throws IOException {
  // read conf
  final int stripeBlocks = raidInfo.codec.stripeLength;

  // figure out which blocks are missing/corrupted
  final FileStatus fileStatus = dfs.getFileStatus(filePath);
  final long blockSize = fileStatus.getBlockSize();
  final long fileLength = fileStatus.getLen();
  final long fileLengthInBlocks = RaidNode.numBlocks(fileStatus); 
  final long fileStripes = RaidNode.numStripes(fileLengthInBlocks,
      stripeBlocks);
  final BlockLocation[] fileBlocks = 
    dfs.getFileBlockLocations(fileStatus, 0, fileLength);
  
  // figure out which stripes these corrupted blocks belong to
  for (BlockLocation fileBlock: fileBlocks) {
    int blockNo = (int) (fileBlock.getOffset() / blockSize);
    final int stripe = blockNo / stripeBlocks;
    if (this.isBlockCorrupt(fileBlock)) {
      this.incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe);
      if (LOG.isDebugEnabled()) {
        LOG.debug("file " + filePath.toString() + " corrupt in block " + 
                 blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe +
                 "/" + fileStripes);
      }
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug("file " + filePath.toString() + " OK in block " + blockNo +
                 "/" + fileLengthInBlocks + ", stripe " + stripe + "/" +
                 fileStripes);
      }
    }
  }
  checkParityBlocks(filePath, corruptBlocksPerStripe, blockSize, 0, fileStripes,
                    fileStripes, raidInfo);
}
 
Example 6
Source File: PlacementMonitor.java    From RDFS with Apache License 2.0 5 votes vote down vote up
List<BlockInfo> getBlockInfos(
  FileSystem fs, Path path, long start, long length)
    throws IOException {
  LocatedFileStatus stat = getLocatedFileStatus(fs, path);
  List<BlockInfo> result = new ArrayList<BlockInfo>();
  long end = start + length;
  if (stat != null) {
    for (BlockLocation loc : stat.getBlockLocations()) {
      if (loc.getOffset() >= start && loc.getOffset() < end) {
        result.add(new BlockInfo(loc, path));
      }
    }
  }
  return result;
}
 
Example 7
Source File: FileInputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 8
Source File: FileInputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 9
Source File: HadoopIgfsSecondaryFileSystemDelegateImpl.java    From ignite with Apache License 2.0 5 votes vote down vote up
/**
 * Convert IGFS affinity block location into Hadoop affinity block location.
 *
 * @param block IGFS affinity block location.
 * @return Hadoop affinity block location.
 */
private IgfsBlockLocation convertBlockLocation(BlockLocation block) {
    try {
        String[] names = block.getNames();
        String[] hosts = block.getHosts();

        return new IgfsBlockLocationImpl(
            block.getOffset(), block.getLength(),
            Arrays.asList(names), Arrays.asList(hosts));
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed convert block location: " + block);
    }
}
 
Example 10
Source File: FileInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 11
Source File: FileInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 12
Source File: TestAffinityCalculator.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Test
public void testBuildRangeMap() {
  BlockLocation[] blocks = buildBlockLocations(new String[4], 256*1024*1024);
  long tA = System.nanoTime();
  ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<>();
  for (BlockLocation block : blocks) {
    long start = block.getOffset();
    long end = start + block.getLength();
    Range<Long> range = Range.closedOpen(start, end);
    blockMapBuilder = blockMapBuilder.put(range, block);
  }
  ImmutableRangeMap<Long,BlockLocation> map = blockMapBuilder.build();
  long tB = System.nanoTime();
  System.out.println(String.format("Took %f ms to build range map", (tB - tA) / 1e6));
}
 
Example 13
Source File: FileInputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 14
Source File: FileInputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 
Example 15
Source File: RubixInputSplit.java    From Cubert with Apache License 2.0 4 votes vote down vote up
@Override
public String[] getLocations() throws IOException,
        InterruptedException
{
    if (hostnames == null)
    {
        /* Obtain the FileSystem object and get the FileStatus objects for the split */
        FileSystem fileSystem = FileSystem.get(conf);
        FileStatus fileStatus = fileSystem.getFileStatus(filename);
        /*
         * Obtain the Block locations for the split. This also provides the offset and
         * length information for each block
         */
        final BlockLocation[] blockLocations =
                fileSystem.getFileBlockLocations(fileStatus, offset, length);
        /**
         * Collect all hosts in a map and populate the number of bytes to be read from
         * each host
         */
        Long l;
        Map<String, Long> hostMap = new HashMap<String, Long>();
        for (BlockLocation bl : blockLocations)
        {
            final long start = bl.getOffset() < offset ? offset : bl.getOffset();
            final long end =
                    (offset + length) < (bl.getOffset() + bl.getLength()) ? offset
                            + length : bl.getOffset() + bl.getLength();
            final long nRelevantBytes = end - start;
            for (String host : bl.getHosts())
            {
                hostMap.put(host, ((l = hostMap.get(host)) == null ? 0 : l)
                        + nRelevantBytes);
            }
        }
        /* Sort them in decreasing order of maximum number of relevant bytes */
        final Set<Map.Entry<String, Long>> entries = hostMap.entrySet();
        final Map.Entry<String, Long>[] hostLengthPairs =
                entries.toArray(new Map.Entry[entries.size()]);

        Arrays.sort(hostLengthPairs, new Comparator<Map.Entry<String, Long>>()
        {
            @Override
            public int compare(Map.Entry<String, Long> e1, Map.Entry<String, Long> e2)
            {
                return (int) (e2.getValue() - e1.getValue());
            }
        });

        /* Populate the hostnames object */
        final int nHost = Math.min(hostLengthPairs.length, MAX_LOCATIONS);
        hostnames = new String[nHost];
        for (int i = 0; i < nHost; ++i)
        {
            hostnames[i] = hostLengthPairs[i].getKey();
        }
    }
    return hostnames;
}
 
Example 16
Source File: FileFragment.java    From tajo with Apache License 2.0 4 votes vote down vote up
public FileFragment(String tableName, Path uri, BlockLocation blockLocation)
    throws IOException {
  this(tableName, uri, blockLocation.getOffset(), blockLocation.getLength(), blockLocation.getHosts(), null);
}
 
Example 17
Source File: ParquetInputFormat.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
private long getHDFSBlockEndingPosition(int hdfsBlockIndex) {
  BlockLocation hdfsBlock = hdfsBlocks[hdfsBlockIndex];
  return hdfsBlock.getOffset() + hdfsBlock.getLength() - 1;
}
 
Example 18
Source File: RaidShell.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * checks the parity blocks for a given file and modifies
 * corruptBlocksPerStripe accordingly
 */
private void checkParityBlocks(final Path filePath,
                               final HashMap<Integer, Integer>
                               corruptBlocksPerStripe,
                               final long blockSize,
                               final long startStripeIdx,
                               final long endStripeIdx,
                               final long numStripes,
                               final RaidInfo raidInfo)
  throws IOException {

  // get the blocks of the parity file
  // because of har, multiple blocks may be returned as one container block
  BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize,
      numStripes, raidInfo);

  long parityStripeLength = blockSize *
    ((long) raidInfo.parityBlocksPerStripe);

  long parityBlocksFound = 0L;

  for (BlockLocation cb: containerBlocks) {
    if (cb.getLength() % blockSize != 0) {
      throw new IOException("container block size is not " +
                            "multiple of parity block size");
    }
    LOG.debug("found container with offset " + cb.getOffset() +
             ", length " + cb.getLength());

    for (long offset = cb.getOffset();
         offset < cb.getOffset() + cb.getLength();
         offset += blockSize) {
      long block = offset / blockSize;
      
      int stripe = (int) (offset / parityStripeLength);

      if (stripe < 0) {
        // before the beginning of the parity file
        continue;
      }
      if (stripe >= numStripes) {
        // past the end of the parity file
        break;
      }

      parityBlocksFound++;
      
      if (stripe < startStripeIdx || stripe > endStripeIdx) {
        continue;
      }

      if (this.isBlockCorrupt(cb)) {
        LOG.info("parity file for " + filePath.toString() + 
                 " corrupt in block " + block +
                 ", stripe " + stripe + "/" + numStripes);
        this.incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe);
      } else {
        LOG.debug("parity file for " + filePath.toString() + 
                 " OK in block " + block +
                 ", stripe " + stripe + "/" + numStripes);
      }
    }
  }

  long parityBlocksExpected = raidInfo.parityBlocksPerStripe * numStripes;
  if (parityBlocksFound != parityBlocksExpected ) {
    throw new IOException("expected " + parityBlocksExpected + 
                          " parity blocks but got " + parityBlocksFound);
  }
}