Java Code Examples for org.apache.hadoop.hdfs.protocol.LocatedBlock#getBlockSize()
The following examples show how to use
org.apache.hadoop.hdfs.protocol.LocatedBlock#getBlockSize() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileFixer.java From RDFS with Apache License 2.0 | 6 votes |
/** * Returns the corrupt blocks in a file. **/ List<LocatedBlock> corruptBlocksInFile( DistributedFileSystem fs, String uriPath, FileStatus stat) throws IOException { List<LocatedBlock> corrupt = new LinkedList<LocatedBlock>(); LocatedBlocks locatedBlocks = fs.getClient().namenode.getBlockLocations( uriPath, 0, stat.getLen()); for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) { if (b.isCorrupt() || (b.getLocations().length == 0 && b.getBlockSize() > 0)) { LOG.info("Adding bad block for file " + uriPath); corrupt.add(b); } } return corrupt; }
Example 2
Source File: DFSOutputStream.java From big-c with Apache License 2.0 | 6 votes |
/** Construct a new output stream for append. */ private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException { this(dfsClient, src, progress, stat, checksum); initialFileSize = stat.getLen(); // length of file when opened this.shouldSyncBlock = flags.contains(CreateFlag.SYNC_BLOCK); boolean toNewBlock = flags.contains(CreateFlag.NEW_BLOCK); // The last partial block of the file has to be filled. if (!toNewBlock && lastBlock != null) { // indicate that we are appending to an existing block bytesCurBlock = lastBlock.getBlockSize(); streamer = new DataStreamer(lastBlock, stat, bytesPerChecksum); } else { computePacketChunkSize(dfsClient.getConf().writePacketSize, bytesPerChecksum); streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null); } this.fileEncryptionInfo = stat.getFileEncryptionInfo(); }
Example 3
Source File: DFSLocatedBlocks.java From RDFS with Apache License 2.0 | 6 votes |
/** * Determine whether the input block is the block under-construction * for the file. If the current file is not under-construction, always * false is returned. * * The result is the best guess based on unknown * information. The bottom line is, when the position equals to file length, * the block selected will be return true. This has to be guaranteed to make * sure the available size updating logic will always be triggered when * reading to the end of a under-construction file. * * @param block * @return */ public boolean isUnderConstructionBlock(Block block) { if (!isUnderConstruction()) { return false; } LocatedBlock lastBlock = this.get(this.locatedBlockCount() - 1); // There are potential inconsistency when counting the size of the // last block, but fileLength is not likely to be under-estimated // the size, unless the last block size is 0. if ((this.fileLength <= lastBlock.getStartOffset() + lastBlock.getBlockSize()) && lastBlock.getBlock().equals(block)) { return true; } return false; }
Example 4
Source File: DistributedRaidFileSystem.java From RDFS with Apache License 2.0 | 5 votes |
private static long getBlockSize(LocatedBlocks lbs) throws IOException { List<LocatedBlock> locatedBlocks = lbs.getLocatedBlocks(); long bs = -1; for (LocatedBlock lb: locatedBlocks) { if (lb.getBlockSize() > bs) { bs = lb.getBlockSize(); } } return bs; }
Example 5
Source File: TestPlacementMonitor.java From RDFS with Apache License 2.0 | 5 votes |
private BlockInfo createBlockInfo(Path file, LocatedBlock b) { DatanodeInfo[] locations = b.getLocations(); String[] hosts = new String[locations.length]; String[] names = new String[locations.length]; for (int i = 0; i < locations.length; ++i) { DatanodeInfo d = locations[i]; hosts[i] = d.getHost(); names[i] = d.getName(); } BlockLocation loc = new BlockLocation( names, hosts, b.getStartOffset(), b.getBlockSize()); return new BlockInfo(loc, file); }
Example 6
Source File: RaidDFSUtil.java From RDFS with Apache License 2.0 | 5 votes |
/** * Returns the corrupt blocks in a file. */ public static List<LocatedBlock> corruptBlocksInFile( DistributedFileSystem dfs, String path, long offset, long length) throws IOException { List<LocatedBlock> corrupt = new LinkedList<LocatedBlock>(); LocatedBlocks locatedBlocks = getBlockLocations(dfs, path, offset, length); for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) { if (b.isCorrupt() || (b.getLocations().length == 0 && b.getBlockSize() > 0)) { corrupt.add(b); } } return corrupt; }
Example 7
Source File: DistributedRaidFileSystem.java From RDFS with Apache License 2.0 | 5 votes |
private static long getFileSize(LocatedBlocks lbs) throws IOException { List<LocatedBlock> locatedBlocks = lbs.getLocatedBlocks(); long fileSize = 0; for (LocatedBlock lb: locatedBlocks) { fileSize += lb.getBlockSize(); } if (fileSize != lbs.getFileLength()) { throw new IOException("lbs.getFileLength() " + lbs.getFileLength() + " does not match sum of block sizes " + fileSize); } return fileSize; }
Example 8
Source File: BlockReconstructor.java From RDFS with Apache License 2.0 | 5 votes |
List<LocatedBlockWithMetaInfo> lostBlocksInFile( DistributedFileSystem fs, String uriPath, FileStatus stat) throws IOException { List<LocatedBlockWithMetaInfo> corrupt = new LinkedList<LocatedBlockWithMetaInfo>(); VersionedLocatedBlocks locatedBlocks; int namespaceId = 0; int methodFingerprint = 0; if (DFSClient .isMetaInfoSuppoted(fs.getClient().namenodeProtocolProxy)) { LocatedBlocksWithMetaInfo lbksm = fs.getClient().namenode .openAndFetchMetaInfo(uriPath, 0, stat.getLen()); namespaceId = lbksm.getNamespaceID(); locatedBlocks = lbksm; methodFingerprint = lbksm.getMethodFingerPrint(); fs.getClient().getNewNameNodeIfNeeded(methodFingerprint); } else { locatedBlocks = fs.getClient().namenode.open(uriPath, 0, stat.getLen()); } final int dataTransferVersion = locatedBlocks .getDataProtocolVersion(); for (LocatedBlock b : locatedBlocks.getLocatedBlocks()) { if (b.isCorrupt() || (b.getLocations().length == 0 && b.getBlockSize() > 0)) { corrupt.add(new LocatedBlockWithMetaInfo(b.getBlock(), b .getLocations(), b.getStartOffset(), dataTransferVersion, namespaceId, methodFingerprint)); } } return corrupt; }
Example 9
Source File: BlockReconstructor.java From RDFS with Apache License 2.0 | 5 votes |
boolean isBlockCorrupt(LocatedBlock block) { if (block.isCorrupt() || (block.getLocations().length == 0 && block.getBlockSize() > 0)) { return true; } return false; }
Example 10
Source File: Decoder.java From RDFS with Apache License 2.0 | 5 votes |
boolean isBlockCorrupt(LocatedBlock block) { if (block.isCorrupt() || (block.getLocations().length == 0 && block.getBlockSize() > 0)) { return true; } return false; }
Example 11
Source File: TestDFSLocatedBlocks.java From RDFS with Apache License 2.0 | 5 votes |
@Test public void testBlockContainingOffset() { for (long blockSizeDelta = -1; blockSizeDelta <= 0; ++blockSizeDelta) { DFSLocatedBlocks locatedBlocks = randomDFSLocatedBlocks(1000, blockSizeDelta); LOG.info("Located blocks: " + locatedBlocks); List<LocatedBlock> allBlocks = locatedBlocks.getLocatedBlocks(); for (LocatedBlock b : allBlocks) { long startOffset = b.getStartOffset(); long endOffset = startOffset + b.getBlockSize(); assertTrue( locatedBlocks.getBlockContainingOffset(startOffset - 1) != b); assertTrue(locatedBlocks.getBlockContainingOffset(startOffset) == b); assertTrue(locatedBlocks.getBlockContainingOffset(endOffset - 1) == b); assertTrue(locatedBlocks.getBlockContainingOffset(endOffset) != b); if (blockSizeDelta < 0) { // We have left gaps between blocks. Check that the byte immediately // before and the byte immediately after the block are not in any // block. assertTrue("b=" + b, locatedBlocks.getBlockContainingOffset(startOffset - 1) == null); assertTrue("b=" + b, locatedBlocks.getBlockContainingOffset(endOffset) == null); } } } }
Example 12
Source File: BlockReconstructor.java From RDFS with Apache License 2.0 | 4 votes |
List<LocatedBlockWithMetaInfo> lostBlocksInFile( DistributedFileSystem fs, String uriPath, FileStatus stat) throws IOException { List<LocatedBlockWithMetaInfo> decommissioning = new LinkedList<LocatedBlockWithMetaInfo>(); VersionedLocatedBlocks locatedBlocks; int namespaceId = 0; int methodFingerprint = 0; if (DFSClient .isMetaInfoSuppoted(fs.getClient().namenodeProtocolProxy)) { LocatedBlocksWithMetaInfo lbksm = fs.getClient().namenode .openAndFetchMetaInfo(uriPath, 0, stat.getLen()); namespaceId = lbksm.getNamespaceID(); locatedBlocks = lbksm; methodFingerprint = lbksm.getMethodFingerPrint(); fs.getClient().getNewNameNodeIfNeeded(methodFingerprint); } else { locatedBlocks = fs.getClient().namenode.open(uriPath, 0, stat.getLen()); } final int dataTransferVersion = locatedBlocks .getDataProtocolVersion(); for (LocatedBlock b : locatedBlocks.getLocatedBlocks()) { if (b.isCorrupt() || (b.getLocations().length == 0 && b.getBlockSize() > 0)) { // If corrupt, this block is the responsibility of the // CorruptBlockReconstructor continue; } // Copy this block iff all good copies are being decommissioned boolean allDecommissioning = true; for (DatanodeInfo i : b.getLocations()) { allDecommissioning &= i.isDecommissionInProgress(); } if (allDecommissioning) { decommissioning .add(new LocatedBlockWithMetaInfo(b.getBlock(), b .getLocations(), b.getStartOffset(), dataTransferVersion, namespaceId, methodFingerprint)); } } return decommissioning; }
Example 13
Source File: FastCopy.java From RDFS with Apache License 2.0 | 4 votes |
private boolean copy() throws Exception { // Get source file information and create empty destination file. FileStatus srcFileStatus = srcNamenode.getFileInfo(src); if (srcFileStatus == null) { throw new FileNotFoundException("File : " + src + " does not exist"); } dstNamenode.create(destination, srcFileStatus.getPermission(), clientName, true, true, srcFileStatus.getReplication(), srcFileStatus.getBlockSize()); try { LocatedBlocks srcLocatedBlks; int srcNamespaceId = 0; boolean supportFederation = false; if (srcNamenodeProtocolProxy.isMethodSupported( "openAndFetchMetaInfo", String.class, long.class, long.class)) { supportFederation = true; LocatedBlocksWithMetaInfo srcBlockWithMetaInfo = srcNamenode.openAndFetchMetaInfo(src, 0, Long.MAX_VALUE); srcNamespaceId = srcBlockWithMetaInfo.getNamespaceID(); srcLocatedBlks = srcBlockWithMetaInfo; } else { srcLocatedBlks = srcNamenode.getBlockLocations(src, 0, Long.MAX_VALUE); } List<LocatedBlock> locatedBlocks = srcLocatedBlks.getLocatedBlocks(); this.totalBlocks = locatedBlocks.size(); LOG.debug("FastCopy : Block locations retrieved for : " + src); // Instruct each datanode to create a copy of the respective block. int blocksAdded = 0; long startPos = 0; Block lastBlock = null; // Loop through each block and create copies. for (LocatedBlock srcLocatedBlock : locatedBlocks) { DatanodeInfo[] favoredNodes = srcLocatedBlock.getLocations(); LocatedBlock destinationLocatedBlock = getBlockFromNameNode( supportFederation, favoredNodes, startPos); if (destinationLocatedBlock == null) { throw new IOException("get null located block from namendoe"); } int dstNamespaceId = 0; if (destinationLocatedBlock instanceof LocatedBlockWithMetaInfo) { dstNamespaceId = ((LocatedBlockWithMetaInfo) destinationLocatedBlock) .getNamespaceID(); } blocksAdded++; startPos += srcLocatedBlock.getBlockSize(); if (LOG.isDebugEnabled()) { LOG.debug("Fast Copy : Block " + destinationLocatedBlock.getBlock() + " added to namenode"); } copyBlock(srcLocatedBlock, destinationLocatedBlock, srcNamespaceId, dstNamespaceId, supportFederation); // Wait for the block copies to reach a threshold. waitForBlockCopy(blocksAdded); checkAndThrowException(); lastBlock = destinationLocatedBlock.getBlock(); } terminateExecutor(); // Wait for all blocks of the file to be copied. waitForFile(src, destination, srcFileStatus.getLen(), lastBlock); } catch (IOException e) { LOG.error("failed to copy src : " + src + " dst : " + destination, e); // If we fail to copy, cleanup destination. dstNamenode.delete(destination, false); throw e; } finally { shutdown(); } return true; }
Example 14
Source File: TestDFSLocatedBlocks.java From RDFS with Apache License 2.0 | 4 votes |
private static long getLastBlockEnd(List<LocatedBlock> blocks) { LocatedBlock lastBlk = blocks.get(blocks.size() - 1); return lastBlk.getStartOffset() + lastBlk.getBlockSize(); }
Example 15
Source File: DFSOutputStream.java From RDFS with Apache License 2.0 | 4 votes |
/** * Create a new output stream to the given DataNode with namespace id. */ DFSOutputStream(DFSClient dfsClient, String src, int buffersize, Progressable progress, LocatedBlock lastBlock, FileStatus stat, int bytesPerChecksum, int namespaceId) throws IOException { this(dfsClient, src, stat.getBlockSize(), progress, bytesPerChecksum, stat.getReplication(), false, false, null); initialFileSize = stat.getLen(); // length of file when opened this.namespaceId = namespaceId; // // The last partial block of the file has to be filled. // if (lastBlock != null) { block = lastBlock.getBlock(); long usedInLastBlock = stat.getLen() % blockSize; int freeInLastBlock = (int)(blockSize - usedInLastBlock); // calculate the amount of free space in the pre-existing // last crc chunk int usedInCksum = (int)(stat.getLen() % bytesPerChecksum); int freeInCksum = bytesPerChecksum - usedInCksum; // if there is space in the last block, then we have to // append to that block if (freeInLastBlock > blockSize) { throw new IOException("The last block for file " + src + " is full."); } // indicate that we are appending to an existing block bytesCurBlock = lastBlock.getBlockSize(); if (usedInCksum > 0 && freeInCksum > 0) { // if there is space in the last partial chunk, then // setup in such a way that the next packet will have only // one chunk that fills up the partial chunk. // computePacketChunkSize(0, freeInCksum); resetChecksumChunk(freeInCksum); this.appendChunk = true; } else { // if the remaining space in the block is smaller than // that expected size of of a packet, then create // smaller size packet. // computePacketChunkSize(Math.min(dfsClient.writePacketSize, freeInLastBlock), bytesPerChecksum); } // setup pipeline to append to the last block nodes = lastBlock.getLocations(); errorIndex = -1; // no errors yet. if (nodes.length < 1) { throw new IOException("Unable to retrieve blocks locations" + " for append to last block " + block + " of file " + src); } // keep trying to setup a pipeline until you know all DNs are dead while (processDatanodeError(true, true)) { try { Thread.sleep(1000); } catch (InterruptedException e) { } } if (lastException != null) { throw lastException; } } else { computePacketChunkSize(dfsClient.writePacketSize, bytesPerChecksum); } long blockOffset = stat.getLen(); blockOffset -= blockOffset % blockSize; setOffsets(blockOffset); streamer.start(); }
Example 16
Source File: DFSInputStream.java From RDFS with Apache License 2.0 | 4 votes |
/** * Get blocks in the specified range. The locations of all blocks * overlapping with the given segment of the file are retrieved. Fetch them * from the namenode if not cached. * * @param offset the offset of the segment to read * @param length the length of the segment to read * @return consequent segment of located blocks * @throws IOException */ private List<LocatedBlock> getBlockRange(final long offset, final long length) throws IOException { List<LocatedBlock> blockRange = new ArrayList<LocatedBlock>(); // Zero length. Not sure this ever happens in practice. if (length == 0) return blockRange; // A defensive measure to ensure that we never loop here eternally. // With a 256 M block size, 10000 blocks will correspond to 2.5 TB. // No one should read this much data at once in practice. int maxLoops = 10000; // Copy locatedBlocks to a local data structure. This ensures that // a concurrent invocation of openInfo() works OK, the reason being // that openInfo may completely replace locatedBlocks. DFSLocatedBlocks locatedBlocks = this.locatedBlocks; if (locatedBlocks == null) { // Make this an IO exception because this is input/output code error. throw new IOException("locatedBlocks is null"); } long remaining = length; long curOff = offset; while (remaining > 0) { // a defensive check to bail out of this loop at all costs if (--maxLoops < 0) { String msg = "Failed to getBlockRange at offset " + offset + ", length=" + length + ", curOff=" + curOff + ", remaining=" + remaining + ". Aborting..."; DFSClient.LOG.warn(msg); throw new IOException(msg); } LocatedBlock blk = locatedBlocks.getBlockContainingOffset(curOff); if (blk == null) { LocatedBlocks newBlocks; newBlocks = getLocatedBlocks(src, curOff, remaining); if (newBlocks == null) { throw new IOException("Could not get block locations for curOff=" + curOff + ", remaining=" + remaining + " (offset=" + offset + ")"); } locatedBlocks.insertRange(newBlocks.getLocatedBlocks()); continue; } blockRange.add(blk); long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff; remaining -= bytesRead; curOff += bytesRead; } DFSClient.checkBlockRange(blockRange, offset, length); return blockRange; }
Example 17
Source File: DFSLocatedBlocks.java From RDFS with Apache License 2.0 | 4 votes |
public LocatedBlock getBlockContainingOffset(long offset) { readLock(); try { int blockIdx = super.binarySearchBlockStartOffsets(offset); List<LocatedBlock> locatedBlocks = super.getLocatedBlocks(); if (blockIdx >= 0) return locatedBlocks.get(blockIdx); // exact match blockIdx = LocatedBlocks.getInsertIndex(blockIdx); // Here, blockIdx is the "insertion point" of the queried offset in // the array (the index of the first element greater than the offset), // which by definition means that // // locatedBlocks.get(blockIdx - 1).getStartOffset() < offset && // offset < locatedBlocks.get(blockIdx).getStartOffset(). // // In particular, if blockIdx == 0, then // offset < locatedBlocks.get(0).getStartOffset(). if (blockIdx == 0) return null; // The offset is not found in known blocks. LocatedBlock blk = locatedBlocks.get(blockIdx - 1); long blkStartOffset = blk.getStartOffset(); if (offset < blkStartOffset) { // By definition of insertion point, // locatedBlocks.get(blockIdx - 1).getStartOffset() < offset. throw new AssertionError("Invalid insertion point: " + blockIdx + " for offset " + offset + " (located blocks: " + locatedBlocks + ")"); } long blkLen = blk.getBlockSize(); if (offset < blkStartOffset + blkLen || (offset == blkStartOffset + blkLen && isUnderConstruction() && blockIdx == locatedBlocks.size())) { return blk; } // Block not found in the location cache, the caller should ask the // namenode instead. return null; } finally { readUnlock(); } }
Example 18
Source File: DFSInputStream.java From big-c with Apache License 2.0 | 4 votes |
private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException { final LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("newInfo = " + newInfo); } if (newInfo == null) { throw new IOException("Cannot open filename " + src); } if (locatedBlocks != null) { Iterator<LocatedBlock> oldIter = locatedBlocks.getLocatedBlocks().iterator(); Iterator<LocatedBlock> newIter = newInfo.getLocatedBlocks().iterator(); while (oldIter.hasNext() && newIter.hasNext()) { if (! oldIter.next().getBlock().equals(newIter.next().getBlock())) { throw new IOException("Blocklist for " + src + " has changed!"); } } } locatedBlocks = newInfo; long lastBlockBeingWrittenLength = 0; if (!locatedBlocks.isLastBlockComplete()) { final LocatedBlock last = locatedBlocks.getLastLocatedBlock(); if (last != null) { if (last.getLocations().length == 0) { if (last.getBlockSize() == 0) { // if the length is zero, then no data has been written to // datanode. So no need to wait for the locations. return 0; } return -1; } final long len = readBlockLength(last); last.getBlock().setNumBytes(len); lastBlockBeingWrittenLength = len; } } fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo(); return lastBlockBeingWrittenLength; }
Example 19
Source File: DFSClient.java From RDFS with Apache License 2.0 | 4 votes |
/** * Checks that the given block range covers the given file segment and * consists of contiguous blocks. This function assumes that the length * of the queried segment is non-zero, and a non-empty block list is * expected. * @param blockRange the set of blocks obtained for the given file segment * @param offset the start offset of the file segment * @param length the length of the file segment. Assumed to be positive. */ static void checkBlockRange(List<LocatedBlock> blockRange, long offset, long length) throws IOException { boolean isValid = false; if (!blockRange.isEmpty()) { int numBlocks = blockRange.size(); LocatedBlock firstBlock = blockRange.get(0); LocatedBlock lastBlock = blockRange.get(numBlocks - 1); long segmentEnd = offset + length; // Check that the queried segment is between the beginning of the first // block and the end of the last block in the block range. if (firstBlock.getStartOffset() <= offset && (segmentEnd <= lastBlock.getStartOffset() + lastBlock.getBlockSize())) { isValid = true; // There is a chance the block list is valid LocatedBlock prevBlock = firstBlock; for (int i = 1; i < numBlocks; ++i) { // In this loop, prevBlock is always the block #(i - 1) and curBlock // is the block #i. long prevBlkEnd = prevBlock.getStartOffset() + prevBlock.getBlockSize(); LocatedBlock curBlock = blockRange.get(i); long curBlkOffset = curBlock.getStartOffset(); if (prevBlkEnd != curBlkOffset || // Blocks are not contiguous prevBlkEnd <= offset || // Previous block is redundant segmentEnd <= curBlkOffset) { // Current block is redundant isValid = false; break; } prevBlock = curBlock; } } } if (!isValid) { throw new IOException("Got incorrect block range for " + "offset=" + offset + ", length=" + length + ": " + blockRange); } }
Example 20
Source File: DFSInputStream.java From hadoop with Apache License 2.0 | 4 votes |
private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException { final LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("newInfo = " + newInfo); } if (newInfo == null) { throw new IOException("Cannot open filename " + src); } if (locatedBlocks != null) { Iterator<LocatedBlock> oldIter = locatedBlocks.getLocatedBlocks().iterator(); Iterator<LocatedBlock> newIter = newInfo.getLocatedBlocks().iterator(); while (oldIter.hasNext() && newIter.hasNext()) { if (! oldIter.next().getBlock().equals(newIter.next().getBlock())) { throw new IOException("Blocklist for " + src + " has changed!"); } } } locatedBlocks = newInfo; long lastBlockBeingWrittenLength = 0; if (!locatedBlocks.isLastBlockComplete()) { final LocatedBlock last = locatedBlocks.getLastLocatedBlock(); if (last != null) { if (last.getLocations().length == 0) { if (last.getBlockSize() == 0) { // if the length is zero, then no data has been written to // datanode. So no need to wait for the locations. return 0; } return -1; } final long len = readBlockLength(last); last.getBlock().setNumBytes(len); lastBlockBeingWrittenLength = len; } } fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo(); return lastBlockBeingWrittenLength; }