org.apache.hadoop.hdfs.BlockReader Java Examples

The following examples show how to use org.apache.hadoop.hdfs.BlockReader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DFSByteBufferReader.java    From indexr with Apache License 2.0 6 votes vote down vote up
private void tryGetLocalFile() {
    if (tryGetLocalFileTimes >= TRY_GET_LOCAL_FILE_LIMIT) {
        return;
    }
    if (isSingleBlock && HDFS_READ_HACK_ENABLE) {
        try {
            InputStream is = input.getWrappedStream();
            if (is instanceof DFSInputStream) {
                BlockReader blockReader = MemoryUtil.getDFSInputStream_blockReader(is);
                if (blockReader != null && blockReader.isShortCircuit()) {
                    localFile = MemoryUtil.getBlockReaderLocal_dataIn(blockReader);
                }
            }
        } catch (Throwable e) {
            logger.debug("HDFS READ HACK failed.", e);
        }
    }
    tryGetLocalFileTimes++;
}
 
Example #2
Source File: NamenodeFsck.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
                       OutputStream fos) throws Exception {
  int failures = 0;
  InetSocketAddress targetAddr = null;
  TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
  BlockReader blockReader = null; 
  ExtendedBlock block = lblock.getBlock(); 

  while (blockReader == null) {
    DatanodeInfo chosenNode;
    
    try {
      chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
      targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
    }  catch (IOException ie) {
      if (failures >= DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
        throw new IOException("Could not obtain block " + lblock, ie);
      }
      LOG.info("Could not obtain block from any node:  " + ie);
      try {
        Thread.sleep(10000);
      }  catch (InterruptedException iex) {
      }
      deadNodes.clear();
      failures++;
      continue;
    }
    try {
      String file = BlockReaderFactory.getFileName(targetAddr,
          block.getBlockPoolId(), block.getBlockId());
      blockReader = new BlockReaderFactory(dfs.getConf()).
          setFileName(file).
          setBlock(block).
          setBlockToken(lblock.getBlockToken()).
          setStartOffset(0).
          setLength(-1).
          setVerifyChecksum(true).
          setClientName("fsck").
          setDatanodeInfo(chosenNode).
          setInetSocketAddress(targetAddr).
          setCachingStrategy(CachingStrategy.newDropBehind()).
          setClientCacheContext(dfs.getClientContext()).
          setConfiguration(namenode.conf).
          setRemotePeerFactory(new RemotePeerFactory() {
            @Override
            public Peer newConnectedPeer(InetSocketAddress addr,
                Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
                throws IOException {
              Peer peer = null;
              Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
              try {
                s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
                s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
                peer = TcpPeerServer.peerFromSocketAndKey(
                      dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
                      blockToken, datanodeId);
              } finally {
                if (peer == null) {
                  IOUtils.closeQuietly(s);
                }
              }
              return peer;
            }
          }).
          build();
    }  catch (IOException ex) {
      // Put chosen node into dead list, continue
      LOG.info("Failed to connect to " + targetAddr + ":" + ex);
      deadNodes.add(chosenNode);
    }
  }
  byte[] buf = new byte[1024];
  int cnt = 0;
  boolean success = true;
  long bytesRead = 0;
  try {
    while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
      fos.write(buf, 0, cnt);
      bytesRead += cnt;
    }
    if ( bytesRead != block.getNumBytes() ) {
      throw new IOException("Recorded block size is " + block.getNumBytes() + 
                            ", but datanode returned " +bytesRead+" bytes");
    }
  } catch (Exception e) {
    LOG.error("Error reading block", e);
    success = false;
  } finally {
    blockReader.close();
  }
  if (!success) {
    throw new Exception("Could not copy block data for " + lblock.getBlock());
  }
}
 
Example #3
Source File: TestBlockTokenWithDFS.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private static void tryRead(final Configuration conf, LocatedBlock lblock,
    boolean shouldSucceed) {
  InetSocketAddress targetAddr = null;
  IOException ioe = null;
  BlockReader blockReader = null;
  ExtendedBlock block = lblock.getBlock();
  try {
    DatanodeInfo[] nodes = lblock.getLocations();
    targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr());

    blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).
        setFileName(BlockReaderFactory.getFileName(targetAddr, 
                      "test-blockpoolid", block.getBlockId())).
        setBlock(block).
        setBlockToken(lblock.getBlockToken()).
        setInetSocketAddress(targetAddr).
        setStartOffset(0).
        setLength(-1).
        setVerifyChecksum(true).
        setClientName("TestBlockTokenWithDFS").
        setDatanodeInfo(nodes[0]).
        setCachingStrategy(CachingStrategy.newDefaultStrategy()).
        setClientCacheContext(ClientContext.getFromConf(conf)).
        setConfiguration(conf).
        setRemotePeerFactory(new RemotePeerFactory() {
          @Override
          public Peer newConnectedPeer(InetSocketAddress addr,
              Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
              throws IOException {
            Peer peer = null;
            Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
            try {
              sock.connect(addr, HdfsServerConstants.READ_TIMEOUT);
              sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
              peer = TcpPeerServer.peerFromSocket(sock);
            } finally {
              if (peer == null) {
                IOUtils.closeSocket(sock);
              }
            }
            return peer;
          }
        }).
        build();
  } catch (IOException ex) {
    ioe = ex;
  } finally {
    if (blockReader != null) {
      try {
        blockReader.close();
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  }
  if (shouldSucceed) {
    Assert.assertNotNull("OP_READ_BLOCK: access token is invalid, "
          + "when it is expected to be valid", blockReader);
  } else {
    Assert.assertNotNull("OP_READ_BLOCK: access token is valid, "
        + "when it is expected to be invalid", ioe);
    Assert.assertTrue(
        "OP_READ_BLOCK failed due to reasons other than access token: ",
        ioe instanceof InvalidBlockTokenException);
  }
}
 
Example #4
Source File: TestDataNodeVolumeFailure.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * try to access a block on a data node. If fails - throws exception
 * @param datanode
 * @param lblock
 * @throws IOException
 */
private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock)
  throws IOException {
  InetSocketAddress targetAddr = null;
  ExtendedBlock block = lblock.getBlock(); 
 
  targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr());

  BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).
    setInetSocketAddress(targetAddr).
    setBlock(block).
    setFileName(BlockReaderFactory.getFileName(targetAddr,
                  "test-blockpoolid", block.getBlockId())).
    setBlockToken(lblock.getBlockToken()).
    setStartOffset(0).
    setLength(-1).
    setVerifyChecksum(true).
    setClientName("TestDataNodeVolumeFailure").
    setDatanodeInfo(datanode).
    setCachingStrategy(CachingStrategy.newDefaultStrategy()).
    setClientCacheContext(ClientContext.getFromConf(conf)).
    setConfiguration(conf).
    setRemotePeerFactory(new RemotePeerFactory() {
      @Override
      public Peer newConnectedPeer(InetSocketAddress addr,
          Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
          throws IOException {
        Peer peer = null;
        Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
        try {
          sock.connect(addr, HdfsServerConstants.READ_TIMEOUT);
          sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
          peer = TcpPeerServer.peerFromSocket(sock);
        } finally {
          if (peer == null) {
            IOUtils.closeSocket(sock);
          }
        }
        return peer;
      }
    }).
    build();
  blockReader.close();
}
 
Example #5
Source File: NamenodeFsck.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
                       OutputStream fos) throws Exception {
  int failures = 0;
  InetSocketAddress targetAddr = null;
  TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
  BlockReader blockReader = null; 
  ExtendedBlock block = lblock.getBlock(); 

  while (blockReader == null) {
    DatanodeInfo chosenNode;
    
    try {
      chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
      targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
    }  catch (IOException ie) {
      if (failures >= DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
        throw new IOException("Could not obtain block " + lblock, ie);
      }
      LOG.info("Could not obtain block from any node:  " + ie);
      try {
        Thread.sleep(10000);
      }  catch (InterruptedException iex) {
      }
      deadNodes.clear();
      failures++;
      continue;
    }
    try {
      String file = BlockReaderFactory.getFileName(targetAddr,
          block.getBlockPoolId(), block.getBlockId());
      blockReader = new BlockReaderFactory(dfs.getConf()).
          setFileName(file).
          setBlock(block).
          setBlockToken(lblock.getBlockToken()).
          setStartOffset(0).
          setLength(-1).
          setVerifyChecksum(true).
          setClientName("fsck").
          setDatanodeInfo(chosenNode).
          setInetSocketAddress(targetAddr).
          setCachingStrategy(CachingStrategy.newDropBehind()).
          setClientCacheContext(dfs.getClientContext()).
          setConfiguration(namenode.conf).
          setRemotePeerFactory(new RemotePeerFactory() {
            @Override
            public Peer newConnectedPeer(InetSocketAddress addr,
                Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
                throws IOException {
              Peer peer = null;
              Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
              try {
                s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
                s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
                peer = TcpPeerServer.peerFromSocketAndKey(
                      dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
                      blockToken, datanodeId);
              } finally {
                if (peer == null) {
                  IOUtils.closeQuietly(s);
                }
              }
              return peer;
            }
          }).
          build();
    }  catch (IOException ex) {
      // Put chosen node into dead list, continue
      LOG.info("Failed to connect to " + targetAddr + ":" + ex);
      deadNodes.add(chosenNode);
    }
  }
  byte[] buf = new byte[1024];
  int cnt = 0;
  boolean success = true;
  long bytesRead = 0;
  try {
    while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
      fos.write(buf, 0, cnt);
      bytesRead += cnt;
    }
    if ( bytesRead != block.getNumBytes() ) {
      throw new IOException("Recorded block size is " + block.getNumBytes() + 
                            ", but datanode returned " +bytesRead+" bytes");
    }
  } catch (Exception e) {
    LOG.error("Error reading block", e);
    success = false;
  } finally {
    blockReader.close();
  }
  if (!success) {
    throw new Exception("Could not copy block data for " + lblock.getBlock());
  }
}
 
Example #6
Source File: TestBlockTokenWithDFS.java    From big-c with Apache License 2.0 4 votes vote down vote up
private static void tryRead(final Configuration conf, LocatedBlock lblock,
    boolean shouldSucceed) {
  InetSocketAddress targetAddr = null;
  IOException ioe = null;
  BlockReader blockReader = null;
  ExtendedBlock block = lblock.getBlock();
  try {
    DatanodeInfo[] nodes = lblock.getLocations();
    targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr());

    blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).
        setFileName(BlockReaderFactory.getFileName(targetAddr, 
                      "test-blockpoolid", block.getBlockId())).
        setBlock(block).
        setBlockToken(lblock.getBlockToken()).
        setInetSocketAddress(targetAddr).
        setStartOffset(0).
        setLength(-1).
        setVerifyChecksum(true).
        setClientName("TestBlockTokenWithDFS").
        setDatanodeInfo(nodes[0]).
        setCachingStrategy(CachingStrategy.newDefaultStrategy()).
        setClientCacheContext(ClientContext.getFromConf(conf)).
        setConfiguration(conf).
        setRemotePeerFactory(new RemotePeerFactory() {
          @Override
          public Peer newConnectedPeer(InetSocketAddress addr,
              Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
              throws IOException {
            Peer peer = null;
            Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
            try {
              sock.connect(addr, HdfsServerConstants.READ_TIMEOUT);
              sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
              peer = TcpPeerServer.peerFromSocket(sock);
            } finally {
              if (peer == null) {
                IOUtils.closeSocket(sock);
              }
            }
            return peer;
          }
        }).
        build();
  } catch (IOException ex) {
    ioe = ex;
  } finally {
    if (blockReader != null) {
      try {
        blockReader.close();
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  }
  if (shouldSucceed) {
    Assert.assertNotNull("OP_READ_BLOCK: access token is invalid, "
          + "when it is expected to be valid", blockReader);
  } else {
    Assert.assertNotNull("OP_READ_BLOCK: access token is valid, "
        + "when it is expected to be invalid", ioe);
    Assert.assertTrue(
        "OP_READ_BLOCK failed due to reasons other than access token: ",
        ioe instanceof InvalidBlockTokenException);
  }
}
 
Example #7
Source File: TestDataNodeVolumeFailure.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * try to access a block on a data node. If fails - throws exception
 * @param datanode
 * @param lblock
 * @throws IOException
 */
private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock)
  throws IOException {
  InetSocketAddress targetAddr = null;
  ExtendedBlock block = lblock.getBlock(); 
 
  targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr());

  BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).
    setInetSocketAddress(targetAddr).
    setBlock(block).
    setFileName(BlockReaderFactory.getFileName(targetAddr,
                  "test-blockpoolid", block.getBlockId())).
    setBlockToken(lblock.getBlockToken()).
    setStartOffset(0).
    setLength(-1).
    setVerifyChecksum(true).
    setClientName("TestDataNodeVolumeFailure").
    setDatanodeInfo(datanode).
    setCachingStrategy(CachingStrategy.newDefaultStrategy()).
    setClientCacheContext(ClientContext.getFromConf(conf)).
    setConfiguration(conf).
    setRemotePeerFactory(new RemotePeerFactory() {
      @Override
      public Peer newConnectedPeer(InetSocketAddress addr,
          Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
          throws IOException {
        Peer peer = null;
        Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
        try {
          sock.connect(addr, HdfsServerConstants.READ_TIMEOUT);
          sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
          peer = TcpPeerServer.peerFromSocket(sock);
        } finally {
          if (peer == null) {
            IOUtils.closeSocket(sock);
          }
        }
        return peer;
      }
    }).
    build();
  blockReader.close();
}
 
Example #8
Source File: TestReadSlowDataNode.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Test that copy on write for blocks works correctly
 * 
 * @throws NoSuchFieldException
 * @throws SecurityException
 * @throws IllegalAccessException
 * @throws IllegalArgumentException
 */
public void testSlowDn() throws IOException, SecurityException,
    NoSuchFieldException, IllegalArgumentException, IllegalAccessException {
  Configuration conf = new Configuration();
  conf.setLong("dfs.min.read.speed.bps", 1024 * 200);

  MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
  FileSystem fs = cluster.getFileSystem();
  FSDataInputStream in = null;
  try {

    // create a new file, write to it and close it.
    //
    Path file1 = new Path("/filestatus.dat");
    FSDataOutputStream stm = createFile(fs, file1, 2);
    writeFile(stm);
    stm.close();

    in = fs.open(file1);
    in.readByte();

    DFSInputStream dfsClientIn = findDFSClientInputStream(in);      
    Field blockReaderField = DFSInputStream.class.getDeclaredField("blockReader");
    blockReaderField.setAccessible(true);
    BlockReader blockReader = (BlockReader) blockReaderField.get(dfsClientIn);

    blockReader.setArtificialSlowdown(1000);
    blockReader.isReadLocal = false;
    blockReader.isReadRackLocal = false;
    blockReader.ENABLE_THROW_FOR_SLOW = true;
    for (int i = 0; i < 1024; i++) {
      in.readByte();
    }

    blockReader.setArtificialSlowdown(0);
    for (int i = 1024; i < fileSize - 1; i++) {
      in.readByte();
    }

    ConcurrentHashMap<DatanodeInfo, DatanodeInfo> deadNodes = getDeadNodes(dfsClientIn);
    TestCase.assertEquals(1, deadNodes.size());
  } finally {
    if (in != null) {
      in.close();
    }
    fs.close();
    cluster.shutdown();
  }
}
 
Example #9
Source File: JspHelper.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public void streamBlockInAscii(InetSocketAddress addr, int namespaceId,
                               long blockId, long genStamp, long blockSize,
                               long offsetIntoBlock, long chunkSizeToView, JspWriter out) 
  throws IOException {
  if (chunkSizeToView == 0) return;
  Socket s = new Socket();
  s.connect(addr, HdfsConstants.READ_TIMEOUT);
  s.setSoTimeout(HdfsConstants.READ_TIMEOUT);
    
    long amtToRead = Math.min(chunkSizeToView, blockSize - offsetIntoBlock);     
    
    // Use the block name for file name. 
    BlockReader blockReader = 
      BlockReader.newBlockReader(DataTransferProtocol.DATA_TRANSFER_VERSION,
                                  namespaceId,
                                  s, addr.toString() + ":" + blockId,
                                  blockId, genStamp ,offsetIntoBlock, 
                                  amtToRead, 
                                  conf.getInt("io.file.buffer.size", 4096));
      
  byte[] buf = new byte[(int)amtToRead];
  int readOffset = 0;
  int retries = 2;
  while ( amtToRead > 0 ) {
    int numRead;
    try {
      numRead = blockReader.readAll(buf, readOffset, (int)amtToRead);
    }
    catch (IOException e) {
      retries--;
      if (retries == 0)
        throw new IOException("Could not read data from datanode");
      continue;
    }
    amtToRead -= numRead;
    readOffset += numRead;
  }
  blockReader = null;
  s.close();
  out.print(new String(buf));
}
 
Example #10
Source File: MemoryUtil.java    From indexr with Apache License 2.0 votes vote down vote up
public static BlockReader getDFSInputStream_blockReader(Object br) {return (BlockReader) unsafe.getObject(br, DFSInputStream_blockReader);}