Java Code Examples for org.apache.hadoop.fs.BlockLocation#getOffset()

/**
 * Get the host affinity for a row group.
 *
 * @param fileStatus the parquet file
 * @param start      the start of the row group
 * @param length     the length of the row group
 * @return host affinity for the row group
 */
private Map<String, Float> getHostAffinity(FileStatus fileStatus, FileSystem fs, long start, long length)
    throws IOException {
  BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, start, length);
  Map<String, Float> hostAffinityMap = Maps.newHashMap();
  for (BlockLocation blockLocation : blockLocations) {
    for (String host : blockLocation.getHosts()) {
      Float currentAffinity = hostAffinityMap.get(host);
      float blockStart = blockLocation.getOffset();
      float blockEnd = blockStart + blockLocation.getLength();
      float rowGroupEnd = start + length;
      Float newAffinity = (blockLocation.getLength() - (blockStart < start ? start - blockStart : 0) -
          (blockEnd > rowGroupEnd ? blockEnd - rowGroupEnd : 0)) / length;
      if (currentAffinity != null) {
        hostAffinityMap.put(host, currentAffinity + newAffinity);
      } else {
        hostAffinityMap.put(host, newAffinity);
      }
    }
  }
  return hostAffinityMap;
}
 

/**
 * Builds a mapping of block locations to file byte range
 */
private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(FileStatus status) throws IOException {
  final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time();
  BlockLocation[] blocks;
  ImmutableRangeMap<Long,BlockLocation> blockMap;
  blocks = fs.getFileBlockLocations(status, 0, status.getLen());
  ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>();
  for (BlockLocation block : blocks) {
    long start = block.getOffset();
    long end = start + block.getLength();
    Range<Long> range = Range.closedOpen(start, end);
    blockMapBuilder = blockMapBuilder.put(range, block);
  }
  blockMap = blockMapBuilder.build();
  blockMapMap.put(status.getPath(), blockMap);
  context.stop();
  return blockMap;
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

void collectFileCorruptBlocksInStripe(final DistributedFileSystem dfs, 
    final RaidInfo raidInfo, final Path filePath, 
    final HashMap<Integer, Integer> corruptBlocksPerStripe)
        throws IOException {
  // read conf
  final int stripeBlocks = raidInfo.codec.stripeLength;

  // figure out which blocks are missing/corrupted
  final FileStatus fileStatus = dfs.getFileStatus(filePath);
  final long blockSize = fileStatus.getBlockSize();
  final long fileLength = fileStatus.getLen();
  final long fileLengthInBlocks = RaidNode.numBlocks(fileStatus); 
  final long fileStripes = RaidNode.numStripes(fileLengthInBlocks,
      stripeBlocks);
  final BlockLocation[] fileBlocks = 
    dfs.getFileBlockLocations(fileStatus, 0, fileLength);
  
  // figure out which stripes these corrupted blocks belong to
  for (BlockLocation fileBlock: fileBlocks) {
    int blockNo = (int) (fileBlock.getOffset() / blockSize);
    final int stripe = blockNo / stripeBlocks;
    if (this.isBlockCorrupt(fileBlock)) {
      this.incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe);
      if (LOG.isDebugEnabled()) {
        LOG.debug("file " + filePath.toString() + " corrupt in block " + 
                 blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe +
                 "/" + fileStripes);
      }
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug("file " + filePath.toString() + " OK in block " + blockNo +
                 "/" + fileLengthInBlocks + ", stripe " + stripe + "/" +
                 fileStripes);
      }
    }
  }
  checkParityBlocks(filePath, corruptBlocksPerStripe, blockSize, 0, fileStripes,
                    fileStripes, raidInfo);
}
 

List<BlockInfo> getBlockInfos(
  FileSystem fs, Path path, long start, long length)
    throws IOException {
  LocatedFileStatus stat = getLocatedFileStatus(fs, path);
  List<BlockInfo> result = new ArrayList<BlockInfo>();
  long end = start + length;
  if (stat != null) {
    for (BlockLocation loc : stat.getBlockLocations()) {
      if (loc.getOffset() >= start && loc.getOffset() < end) {
        result.add(new BlockInfo(loc, path));
      }
    }
  }
  return result;
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

/**
 * Convert IGFS affinity block location into Hadoop affinity block location.
 *
 * @param block IGFS affinity block location.
 * @return Hadoop affinity block location.
 */
private IgfsBlockLocation convertBlockLocation(BlockLocation block) {
    try {
        String[] names = block.getNames();
        String[] hosts = block.getHosts();

        return new IgfsBlockLocationImpl(
            block.getOffset(), block.getLength(),
            Arrays.asList(names), Arrays.asList(hosts));
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed convert block location: " + block);
    }
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

@Test
public void testBuildRangeMap() {
  BlockLocation[] blocks = buildBlockLocations(new String[4], 256*1024*1024);
  long tA = System.nanoTime();
  ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<>();
  for (BlockLocation block : blocks) {
    long start = block.getOffset();
    long end = start + block.getLength();
    Range<Long> range = Range.closedOpen(start, end);
    blockMapBuilder = blockMapBuilder.put(range, block);
  }
  ImmutableRangeMap<Long,BlockLocation> map = blockMapBuilder.build();
  long tB = System.nanoTime();
  System.out.println(String.format("Took %f ms to build range map", (tB - tA) / 1e6));
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

protected int getBlockIndex(BlockLocation[] blkLocations, 
                            long offset) {
  for (int i = 0 ; i < blkLocations.length; i++) {
    // is the offset inside this block?
    if ((blkLocations[i].getOffset() <= offset) &&
        (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
      return i;
    }
  }
  BlockLocation last = blkLocations[blkLocations.length -1];
  long fileLength = last.getOffset() + last.getLength() -1;
  throw new IllegalArgumentException("Offset " + offset + 
                                     " is outside of file (0.." +
                                     fileLength + ")");
}
 

@Override
public String[] getLocations() throws IOException,
        InterruptedException
{
    if (hostnames == null)
    {
        /* Obtain the FileSystem object and get the FileStatus objects for the split */
        FileSystem fileSystem = FileSystem.get(conf);
        FileStatus fileStatus = fileSystem.getFileStatus(filename);
        /*
         * Obtain the Block locations for the split. This also provides the offset and
         * length information for each block
         */
        final BlockLocation[] blockLocations =
                fileSystem.getFileBlockLocations(fileStatus, offset, length);
        /**
         * Collect all hosts in a map and populate the number of bytes to be read from
         * each host
         */
        Long l;
        Map<String, Long> hostMap = new HashMap<String, Long>();
        for (BlockLocation bl : blockLocations)
        {
            final long start = bl.getOffset() < offset ? offset : bl.getOffset();
            final long end =
                    (offset + length) < (bl.getOffset() + bl.getLength()) ? offset
                            + length : bl.getOffset() + bl.getLength();
            final long nRelevantBytes = end - start;
            for (String host : bl.getHosts())
            {
                hostMap.put(host, ((l = hostMap.get(host)) == null ? 0 : l)
                        + nRelevantBytes);
            }
        }
        /* Sort them in decreasing order of maximum number of relevant bytes */
        final Set<Map.Entry<String, Long>> entries = hostMap.entrySet();
        final Map.Entry<String, Long>[] hostLengthPairs =
                entries.toArray(new Map.Entry[entries.size()]);

        Arrays.sort(hostLengthPairs, new Comparator<Map.Entry<String, Long>>()
        {
            @Override
            public int compare(Map.Entry<String, Long> e1, Map.Entry<String, Long> e2)
            {
                return (int) (e2.getValue() - e1.getValue());
            }
        });

        /* Populate the hostnames object */
        final int nHost = Math.min(hostLengthPairs.length, MAX_LOCATIONS);
        hostnames = new String[nHost];
        for (int i = 0; i < nHost; ++i)
        {
            hostnames[i] = hostLengthPairs[i].getKey();
        }
    }
    return hostnames;
}
 

public FileFragment(String tableName, Path uri, BlockLocation blockLocation)
    throws IOException {
  this(tableName, uri, blockLocation.getOffset(), blockLocation.getLength(), blockLocation.getHosts(), null);
}
 

private long getHDFSBlockEndingPosition(int hdfsBlockIndex) {
  BlockLocation hdfsBlock = hdfsBlocks[hdfsBlockIndex];
  return hdfsBlock.getOffset() + hdfsBlock.getLength() - 1;
}
 

/**
 * checks the parity blocks for a given file and modifies
 * corruptBlocksPerStripe accordingly
 */
private void checkParityBlocks(final Path filePath,
                               final HashMap<Integer, Integer>
                               corruptBlocksPerStripe,
                               final long blockSize,
                               final long startStripeIdx,
                               final long endStripeIdx,
                               final long numStripes,
                               final RaidInfo raidInfo)
  throws IOException {

  // get the blocks of the parity file
  // because of har, multiple blocks may be returned as one container block
  BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize,
      numStripes, raidInfo);

  long parityStripeLength = blockSize *
    ((long) raidInfo.parityBlocksPerStripe);

  long parityBlocksFound = 0L;

  for (BlockLocation cb: containerBlocks) {
    if (cb.getLength() % blockSize != 0) {
      throw new IOException("container block size is not " +
                            "multiple of parity block size");
    }
    LOG.debug("found container with offset " + cb.getOffset() +
             ", length " + cb.getLength());

    for (long offset = cb.getOffset();
         offset < cb.getOffset() + cb.getLength();
         offset += blockSize) {
      long block = offset / blockSize;
      
      int stripe = (int) (offset / parityStripeLength);

      if (stripe < 0) {
        // before the beginning of the parity file
        continue;
      }
      if (stripe >= numStripes) {
        // past the end of the parity file
        break;
      }

      parityBlocksFound++;
      
      if (stripe < startStripeIdx || stripe > endStripeIdx) {
        continue;
      }

      if (this.isBlockCorrupt(cb)) {
        LOG.info("parity file for " + filePath.toString() + 
                 " corrupt in block " + block +
                 ", stripe " + stripe + "/" + numStripes);
        this.incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe);
      } else {
        LOG.debug("parity file for " + filePath.toString() + 
                 " OK in block " + block +
                 ", stripe " + stripe + "/" + numStripes);
      }
    }
  }

  long parityBlocksExpected = raidInfo.parityBlocksPerStripe * numStripes;
  if (parityBlocksFound != parityBlocksExpected ) {
    throw new IOException("expected " + parityBlocksExpected + 
                          " parity blocks but got " + parityBlocksFound);
  }
}