Java Code Examples for org.apache.hadoop.fs.BlockLocation#getOffset()
The following examples show how to use
org.apache.hadoop.fs.BlockLocation#getOffset() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Metadata.java From Bats with Apache License 2.0 | 6 votes |
/** * Get the host affinity for a row group. * * @param fileStatus the parquet file * @param start the start of the row group * @param length the length of the row group * @return host affinity for the row group */ private Map<String, Float> getHostAffinity(FileStatus fileStatus, FileSystem fs, long start, long length) throws IOException { BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, start, length); Map<String, Float> hostAffinityMap = Maps.newHashMap(); for (BlockLocation blockLocation : blockLocations) { for (String host : blockLocation.getHosts()) { Float currentAffinity = hostAffinityMap.get(host); float blockStart = blockLocation.getOffset(); float blockEnd = blockStart + blockLocation.getLength(); float rowGroupEnd = start + length; Float newAffinity = (blockLocation.getLength() - (blockStart < start ? start - blockStart : 0) - (blockEnd > rowGroupEnd ? blockEnd - rowGroupEnd : 0)) / length; if (currentAffinity != null) { hostAffinityMap.put(host, currentAffinity + newAffinity); } else { hostAffinityMap.put(host, newAffinity); } } } return hostAffinityMap; }
Example 2
Source File: BlockMapBuilder.java From Bats with Apache License 2.0 | 6 votes |
/** * Builds a mapping of block locations to file byte range */ private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(FileStatus status) throws IOException { final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time(); BlockLocation[] blocks; ImmutableRangeMap<Long,BlockLocation> blockMap; blocks = fs.getFileBlockLocations(status, 0, status.getLen()); ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>(); for (BlockLocation block : blocks) { long start = block.getOffset(); long end = start + block.getLength(); Range<Long> range = Range.closedOpen(start, end); blockMapBuilder = blockMapBuilder.put(range, block); } blockMap = blockMapBuilder.build(); blockMapMap.put(status.getPath(), blockMap); context.stop(); return blockMap; }
Example 3
Source File: FileInputFormat.java From hadoop-gpu with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 4
Source File: FileInputFormat.java From hadoop-gpu with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 5
Source File: RaidShell.java From RDFS with Apache License 2.0 | 5 votes |
void collectFileCorruptBlocksInStripe(final DistributedFileSystem dfs, final RaidInfo raidInfo, final Path filePath, final HashMap<Integer, Integer> corruptBlocksPerStripe) throws IOException { // read conf final int stripeBlocks = raidInfo.codec.stripeLength; // figure out which blocks are missing/corrupted final FileStatus fileStatus = dfs.getFileStatus(filePath); final long blockSize = fileStatus.getBlockSize(); final long fileLength = fileStatus.getLen(); final long fileLengthInBlocks = RaidNode.numBlocks(fileStatus); final long fileStripes = RaidNode.numStripes(fileLengthInBlocks, stripeBlocks); final BlockLocation[] fileBlocks = dfs.getFileBlockLocations(fileStatus, 0, fileLength); // figure out which stripes these corrupted blocks belong to for (BlockLocation fileBlock: fileBlocks) { int blockNo = (int) (fileBlock.getOffset() / blockSize); final int stripe = blockNo / stripeBlocks; if (this.isBlockCorrupt(fileBlock)) { this.incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe); if (LOG.isDebugEnabled()) { LOG.debug("file " + filePath.toString() + " corrupt in block " + blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe + "/" + fileStripes); } } else { if (LOG.isDebugEnabled()) { LOG.debug("file " + filePath.toString() + " OK in block " + blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe + "/" + fileStripes); } } } checkParityBlocks(filePath, corruptBlocksPerStripe, blockSize, 0, fileStripes, fileStripes, raidInfo); }
Example 6
Source File: PlacementMonitor.java From RDFS with Apache License 2.0 | 5 votes |
List<BlockInfo> getBlockInfos( FileSystem fs, Path path, long start, long length) throws IOException { LocatedFileStatus stat = getLocatedFileStatus(fs, path); List<BlockInfo> result = new ArrayList<BlockInfo>(); long end = start + length; if (stat != null) { for (BlockLocation loc : stat.getBlockLocations()) { if (loc.getOffset() >= start && loc.getOffset() < end) { result.add(new BlockInfo(loc, path)); } } } return result; }
Example 7
Source File: FileInputFormat.java From RDFS with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 8
Source File: FileInputFormat.java From RDFS with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 9
Source File: HadoopIgfsSecondaryFileSystemDelegateImpl.java From ignite with Apache License 2.0 | 5 votes |
/** * Convert IGFS affinity block location into Hadoop affinity block location. * * @param block IGFS affinity block location. * @return Hadoop affinity block location. */ private IgfsBlockLocation convertBlockLocation(BlockLocation block) { try { String[] names = block.getNames(); String[] hosts = block.getHosts(); return new IgfsBlockLocationImpl( block.getOffset(), block.getLength(), Arrays.asList(names), Arrays.asList(hosts)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed convert block location: " + block); } }
Example 10
Source File: FileInputFormat.java From big-c with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 11
Source File: FileInputFormat.java From big-c with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 12
Source File: TestAffinityCalculator.java From dremio-oss with Apache License 2.0 | 5 votes |
@Test public void testBuildRangeMap() { BlockLocation[] blocks = buildBlockLocations(new String[4], 256*1024*1024); long tA = System.nanoTime(); ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<>(); for (BlockLocation block : blocks) { long start = block.getOffset(); long end = start + block.getLength(); Range<Long> range = Range.closedOpen(start, end); blockMapBuilder = blockMapBuilder.put(range, block); } ImmutableRangeMap<Long,BlockLocation> map = blockMapBuilder.build(); long tB = System.nanoTime(); System.out.println(String.format("Took %f ms to build range map", (tB - tA) / 1e6)); }
Example 13
Source File: FileInputFormat.java From hadoop with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 14
Source File: FileInputFormat.java From hadoop with Apache License 2.0 | 5 votes |
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { for (int i = 0 ; i < blkLocations.length; i++) { // is the offset inside this block? if ((blkLocations[i].getOffset() <= offset) && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){ return i; } } BlockLocation last = blkLocations[blkLocations.length -1]; long fileLength = last.getOffset() + last.getLength() -1; throw new IllegalArgumentException("Offset " + offset + " is outside of file (0.." + fileLength + ")"); }
Example 15
Source File: RubixInputSplit.java From Cubert with Apache License 2.0 | 4 votes |
@Override public String[] getLocations() throws IOException, InterruptedException { if (hostnames == null) { /* Obtain the FileSystem object and get the FileStatus objects for the split */ FileSystem fileSystem = FileSystem.get(conf); FileStatus fileStatus = fileSystem.getFileStatus(filename); /* * Obtain the Block locations for the split. This also provides the offset and * length information for each block */ final BlockLocation[] blockLocations = fileSystem.getFileBlockLocations(fileStatus, offset, length); /** * Collect all hosts in a map and populate the number of bytes to be read from * each host */ Long l; Map<String, Long> hostMap = new HashMap<String, Long>(); for (BlockLocation bl : blockLocations) { final long start = bl.getOffset() < offset ? offset : bl.getOffset(); final long end = (offset + length) < (bl.getOffset() + bl.getLength()) ? offset + length : bl.getOffset() + bl.getLength(); final long nRelevantBytes = end - start; for (String host : bl.getHosts()) { hostMap.put(host, ((l = hostMap.get(host)) == null ? 0 : l) + nRelevantBytes); } } /* Sort them in decreasing order of maximum number of relevant bytes */ final Set<Map.Entry<String, Long>> entries = hostMap.entrySet(); final Map.Entry<String, Long>[] hostLengthPairs = entries.toArray(new Map.Entry[entries.size()]); Arrays.sort(hostLengthPairs, new Comparator<Map.Entry<String, Long>>() { @Override public int compare(Map.Entry<String, Long> e1, Map.Entry<String, Long> e2) { return (int) (e2.getValue() - e1.getValue()); } }); /* Populate the hostnames object */ final int nHost = Math.min(hostLengthPairs.length, MAX_LOCATIONS); hostnames = new String[nHost]; for (int i = 0; i < nHost; ++i) { hostnames[i] = hostLengthPairs[i].getKey(); } } return hostnames; }
Example 16
Source File: FileFragment.java From tajo with Apache License 2.0 | 4 votes |
public FileFragment(String tableName, Path uri, BlockLocation blockLocation) throws IOException { this(tableName, uri, blockLocation.getOffset(), blockLocation.getLength(), blockLocation.getHosts(), null); }
Example 17
Source File: ParquetInputFormat.java From parquet-mr with Apache License 2.0 | 4 votes |
private long getHDFSBlockEndingPosition(int hdfsBlockIndex) { BlockLocation hdfsBlock = hdfsBlocks[hdfsBlockIndex]; return hdfsBlock.getOffset() + hdfsBlock.getLength() - 1; }
Example 18
Source File: RaidShell.java From RDFS with Apache License 2.0 | 4 votes |
/** * checks the parity blocks for a given file and modifies * corruptBlocksPerStripe accordingly */ private void checkParityBlocks(final Path filePath, final HashMap<Integer, Integer> corruptBlocksPerStripe, final long blockSize, final long startStripeIdx, final long endStripeIdx, final long numStripes, final RaidInfo raidInfo) throws IOException { // get the blocks of the parity file // because of har, multiple blocks may be returned as one container block BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize, numStripes, raidInfo); long parityStripeLength = blockSize * ((long) raidInfo.parityBlocksPerStripe); long parityBlocksFound = 0L; for (BlockLocation cb: containerBlocks) { if (cb.getLength() % blockSize != 0) { throw new IOException("container block size is not " + "multiple of parity block size"); } LOG.debug("found container with offset " + cb.getOffset() + ", length " + cb.getLength()); for (long offset = cb.getOffset(); offset < cb.getOffset() + cb.getLength(); offset += blockSize) { long block = offset / blockSize; int stripe = (int) (offset / parityStripeLength); if (stripe < 0) { // before the beginning of the parity file continue; } if (stripe >= numStripes) { // past the end of the parity file break; } parityBlocksFound++; if (stripe < startStripeIdx || stripe > endStripeIdx) { continue; } if (this.isBlockCorrupt(cb)) { LOG.info("parity file for " + filePath.toString() + " corrupt in block " + block + ", stripe " + stripe + "/" + numStripes); this.incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe); } else { LOG.debug("parity file for " + filePath.toString() + " OK in block " + block + ", stripe " + stripe + "/" + numStripes); } } } long parityBlocksExpected = raidInfo.parityBlocksPerStripe * numStripes; if (parityBlocksFound != parityBlocksExpected ) { throw new IOException("expected " + parityBlocksExpected + " parity blocks but got " + parityBlocksFound); } }