Java Code Examples for org.apache.hadoop.hdfs.protocol.LocatedBlock#setBlockToken()

The following examples show how to use org.apache.hadoop.hdfs.protocol.LocatedBlock#setBlockToken() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestPBHelper.java From hadoop with Apache License 2.0

6 votes

private LocatedBlock createLocatedBlock() {
  DatanodeInfo[] dnInfos = {
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h1",
          AdminStates.DECOMMISSION_INPROGRESS),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2",
          AdminStates.DECOMMISSIONED),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3", 
          AdminStates.NORMAL),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h4",
          AdminStates.NORMAL),
  };
  String[] storageIDs = {"s1", "s2", "s3", "s4"};
  StorageType[] media = {
      StorageType.DISK,
      StorageType.SSD,
      StorageType.DISK,
      StorageType.RAM_DISK
  };
  LocatedBlock lb = new LocatedBlock(
      new ExtendedBlock("bp12", 12345, 10, 53),
      dnInfos, storageIDs, media, 5, false, new DatanodeInfo[]{});
  lb.setBlockToken(new Token<BlockTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(), new Text("kind"),
      new Text("service")));
  return lb;
}

Example 2

Source File: TestPBHelper.java From hadoop with Apache License 2.0

6 votes

private LocatedBlock createLocatedBlockNoStorageMedia() {
  DatanodeInfo[] dnInfos = {
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h1",
                                       AdminStates.DECOMMISSION_INPROGRESS),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2",
                                       AdminStates.DECOMMISSIONED),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3",
                                       AdminStates.NORMAL)
  };
  LocatedBlock lb = new LocatedBlock(
      new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false);
  lb.setBlockToken(new Token<BlockTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(), new Text("kind"),
      new Text("service")));
  return lb;
}

Example 3

Source File: TestPBHelper.java From big-c with Apache License 2.0

6 votes

private LocatedBlock createLocatedBlock() {
  DatanodeInfo[] dnInfos = {
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h1",
          AdminStates.DECOMMISSION_INPROGRESS),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2",
          AdminStates.DECOMMISSIONED),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3", 
          AdminStates.NORMAL),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h4",
          AdminStates.NORMAL),
  };
  String[] storageIDs = {"s1", "s2", "s3", "s4"};
  StorageType[] media = {
      StorageType.DISK,
      StorageType.SSD,
      StorageType.DISK,
      StorageType.RAM_DISK
  };
  LocatedBlock lb = new LocatedBlock(
      new ExtendedBlock("bp12", 12345, 10, 53),
      dnInfos, storageIDs, media, 5, false, new DatanodeInfo[]{});
  lb.setBlockToken(new Token<BlockTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(), new Text("kind"),
      new Text("service")));
  return lb;
}

Example 4

Source File: TestPBHelper.java From big-c with Apache License 2.0

6 votes

private LocatedBlock createLocatedBlockNoStorageMedia() {
  DatanodeInfo[] dnInfos = {
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h1",
                                       AdminStates.DECOMMISSION_INPROGRESS),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2",
                                       AdminStates.DECOMMISSIONED),
      DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3",
                                       AdminStates.NORMAL)
  };
  LocatedBlock lb = new LocatedBlock(
      new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false);
  lb.setBlockToken(new Token<BlockTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(), new Text("kind"),
      new Text("service")));
  return lb;
}

Example 5

Source File: BlockManager.java From hadoop with Apache License 2.0

5 votes

/** Generate a block token for the located block. */
public void setBlockToken(final LocatedBlock b,
    final BlockTokenSecretManager.AccessMode mode) throws IOException {
  if (isBlockTokenEnabled()) {
    // Use cached UGI if serving RPC calls.
    b.setBlockToken(blockTokenSecretManager.generateToken(
        NameNode.getRemoteUser().getShortUserName(),
        b.getBlock(), EnumSet.of(mode)));
  }    
}

Example 6

Source File: PBHelper.java From hadoop with Apache License 2.0

5 votes

public static LocatedBlock convert(LocatedBlockProto proto) {
  if (proto == null) return null;
  List<DatanodeInfoProto> locs = proto.getLocsList();
  DatanodeInfo[] targets = new DatanodeInfo[locs.size()];
  for (int i = 0; i < locs.size(); i++) {
    targets[i] = PBHelper.convert(locs.get(i));
  }

  final StorageType[] storageTypes = convertStorageTypes(
      proto.getStorageTypesList(), locs.size());

  final int storageIDsCount = proto.getStorageIDsCount();
  final String[] storageIDs;
  if (storageIDsCount == 0) {
    storageIDs = null;
  } else {
    Preconditions.checkState(storageIDsCount == locs.size());
    storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
  }

  // Set values from the isCached list, re-using references from loc
  List<DatanodeInfo> cachedLocs = new ArrayList<DatanodeInfo>(locs.size());
  List<Boolean> isCachedList = proto.getIsCachedList();
  for (int i=0; i<isCachedList.size(); i++) {
    if (isCachedList.get(i)) {
      cachedLocs.add(targets[i]);
    }
  }

  LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets,
      storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
      cachedLocs.toArray(new DatanodeInfo[0]));
  lb.setBlockToken(PBHelper.convert(proto.getBlockToken()));

  return lb;
}

Example 7

Source File: BlockManager.java From big-c with Apache License 2.0

5 votes

/** Generate a block token for the located block. */
public void setBlockToken(final LocatedBlock b,
    final BlockTokenSecretManager.AccessMode mode) throws IOException {
  if (isBlockTokenEnabled()) {
    // Use cached UGI if serving RPC calls.
    b.setBlockToken(blockTokenSecretManager.generateToken(
        NameNode.getRemoteUser().getShortUserName(),
        b.getBlock(), EnumSet.of(mode)));
  }    
}

Example 8

Source File: PBHelper.java From big-c with Apache License 2.0

5 votes

public static LocatedBlock convert(LocatedBlockProto proto) {
  if (proto == null) return null;
  List<DatanodeInfoProto> locs = proto.getLocsList();
  DatanodeInfo[] targets = new DatanodeInfo[locs.size()];
  for (int i = 0; i < locs.size(); i++) {
    targets[i] = PBHelper.convert(locs.get(i));
  }

  final StorageType[] storageTypes = convertStorageTypes(
      proto.getStorageTypesList(), locs.size());

  final int storageIDsCount = proto.getStorageIDsCount();
  final String[] storageIDs;
  if (storageIDsCount == 0) {
    storageIDs = null;
  } else {
    Preconditions.checkState(storageIDsCount == locs.size());
    storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
  }

  // Set values from the isCached list, re-using references from loc
  List<DatanodeInfo> cachedLocs = new ArrayList<DatanodeInfo>(locs.size());
  List<Boolean> isCachedList = proto.getIsCachedList();
  for (int i=0; i<isCachedList.size(); i++) {
    if (isCachedList.get(i)) {
      cachedLocs.add(targets[i]);
    }
  }

  LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets,
      storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
      cachedLocs.toArray(new DatanodeInfo[0]));
  lb.setBlockToken(PBHelper.convert(proto.getBlockToken()));

  return lb;
}

Example 9

Source File: TestBlockToken.java From hadoop with Apache License 2.0

4 votes

/**
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test
public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(conf);
  
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm = new BlockTokenSecretManager(
      blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
  Token<BlockTokenIdentifier> token = sm.generateToken(block3,
      EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));

  final Server server = createMockDatanode(sm, token, conf);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());

  ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
  LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);

  // Create another RPC proxy with the same configuration - this will never
  // attempt to connect anywhere -- but it causes the refcount on the
  // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
  // actually close the TCP connections to the real target DN.
  ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
      ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
      new InetSocketAddress("1.1.1.1", 1),
      UserGroupInformation.createRemoteUser("junk"), conf,
      NetUtils.getDefaultSocketFactory(conf));

  ClientDatanodeProtocol proxy = null;

  int fdsAtStart = countOpenFileDescriptors();
  try {
    long endTime = Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
          false, fakeBlock);
      assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }

    int fdsAtEnd = countOpenFileDescriptors();

    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  } finally {
    server.stop();
  }

  RPC.stopProxy(proxyToNoWhere);
}

Example 10

Source File: TestBlockToken.java From big-c with Apache License 2.0

4 votes

/**
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test
public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(conf);
  
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm = new BlockTokenSecretManager(
      blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
  Token<BlockTokenIdentifier> token = sm.generateToken(block3,
      EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));

  final Server server = createMockDatanode(sm, token, conf);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());

  ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
  LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);

  // Create another RPC proxy with the same configuration - this will never
  // attempt to connect anywhere -- but it causes the refcount on the
  // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
  // actually close the TCP connections to the real target DN.
  ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
      ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
      new InetSocketAddress("1.1.1.1", 1),
      UserGroupInformation.createRemoteUser("junk"), conf,
      NetUtils.getDefaultSocketFactory(conf));

  ClientDatanodeProtocol proxy = null;

  int fdsAtStart = countOpenFileDescriptors();
  try {
    long endTime = Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
          false, fakeBlock);
      assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }

    int fdsAtEnd = countOpenFileDescriptors();

    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  } finally {
    server.stop();
  }

  RPC.stopProxy(proxyToNoWhere);
}