org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestIsMethodSupported.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testNamenodeProtocol() throws IOException {
  NamenodeProtocol np =
      NameNodeProxies.createNonHAProxy(conf,
          nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
          true).getProxy();

  boolean exists = RpcClientUtil.isMethodSupported(np,
      NamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
      RPC.getProtocolVersion(NamenodeProtocolPB.class), "rollEditLog");

  assertTrue(exists);
  exists = RpcClientUtil.isMethodSupported(np,
      NamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
      RPC.getProtocolVersion(NamenodeProtocolPB.class), "bogusMethod");
  assertFalse(exists);
}
 
Example #2
Source File: NameNodeProxies.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries) throws IOException {
  NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
      address, conf, ugi, NamenodeProtocolPB.class);
  if (withRetries) { // create the proxy with retries
    RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
            TimeUnit.MILLISECONDS);
    Map<String, RetryPolicy> methodNameToPolicyMap
         = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("getBlocks", timeoutPolicy);
    methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy);
    NamenodeProtocol translatorProxy =
        new NamenodeProtocolTranslatorPB(proxy);
    return (NamenodeProtocol) RetryProxy.create(
        NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap);
  } else {
    return new NamenodeProtocolTranslatorPB(proxy);
  }
}
 
Example #3
Source File: TestIsMethodSupported.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testNamenodeProtocol() throws IOException {
  NamenodeProtocol np =
      NameNodeProxies.createNonHAProxy(conf,
          nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
          true).getProxy();

  boolean exists = RpcClientUtil.isMethodSupported(np,
      NamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
      RPC.getProtocolVersion(NamenodeProtocolPB.class), "rollEditLog");

  assertTrue(exists);
  exists = RpcClientUtil.isMethodSupported(np,
      NamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
      RPC.getProtocolVersion(NamenodeProtocolPB.class), "bogusMethod");
  assertFalse(exists);
}
 
Example #4
Source File: BackupNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
private NamespaceInfo handshake(Configuration conf) throws IOException {
  // connect to name node
  InetSocketAddress nnAddress = NameNode.getServiceAddress(conf, true);
  this.namenode = NameNodeProxies.createNonHAProxy(conf, nnAddress,
      NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
      true).getProxy();
  this.nnRpcAddress = NetUtils.getHostPortString(nnAddress);
  this.nnHttpAddress = DFSUtil.getInfoServer(nnAddress, conf,
      DFSUtil.getHttpClientScheme(conf)).toURL();
  // get version and id info from the name-node
  NamespaceInfo nsInfo = null;
  while(!isStopRequested()) {
    try {
      nsInfo = handshake(namenode);
      break;
    } catch(SocketTimeoutException e) {  // name-node is busy
      LOG.info("Problem connecting to server: " + nnAddress);
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        LOG.warn("Encountered exception ", e);
      }
    }
  }
  return nsInfo;
}
 
Example #5
Source File: BackupNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static NamespaceInfo handshake(NamenodeProtocol namenode)
throws IOException, SocketTimeoutException {
  NamespaceInfo nsInfo;
  nsInfo = namenode.versionRequest();  // throws SocketTimeoutException 
  String errorMsg = null;
  // verify build version
  if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion())) {
    errorMsg = "Incompatible build versions: active name-node BV = " 
      + nsInfo.getBuildVersion() + "; backup node BV = "
      + Storage.getBuildVersion();
    LOG.error(errorMsg);
    throw new IOException(errorMsg);
  }
  assert HdfsConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() :
    "Active and backup node layout versions must be the same. Expected: "
    + HdfsConstants.NAMENODE_LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();
  return nsInfo;
}
 
Example #6
Source File: NameNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException {
  if (protocol.equals(ClientProtocol.class.getName())) {
    return ClientProtocol.versionID; 
  } else if (protocol.equals(DatanodeProtocol.class.getName())){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName())){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())){
    return RefreshUserMappingsProtocol.versionID;
  } else if (protocol.equals(RefreshCallQueueProtocol.class.getName())) {
    return RefreshCallQueueProtocol.versionID;
  } else if (protocol.equals(GetUserMappingsProtocol.class.getName())){
    return GetUserMappingsProtocol.versionID;
  } else if (protocol.equals(TraceAdminProtocol.class.getName())){
    return TraceAdminProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #7
Source File: NameNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException {
  if (protocol.equals(ClientProtocol.class.getName())) {
    return ClientProtocol.versionID; 
  } else if (protocol.equals(DatanodeProtocol.class.getName())){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName())){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())){
    return RefreshUserMappingsProtocol.versionID;
  } else if (protocol.equals(RefreshCallQueueProtocol.class.getName())) {
    return RefreshCallQueueProtocol.versionID;
  } else if (protocol.equals(GetUserMappingsProtocol.class.getName())){
    return GetUserMappingsProtocol.versionID;
  } else if (protocol.equals(TraceAdminProtocol.class.getName())){
    return TraceAdminProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #8
Source File: BackupNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static NamespaceInfo handshake(NamenodeProtocol namenode)
throws IOException, SocketTimeoutException {
  NamespaceInfo nsInfo;
  nsInfo = namenode.versionRequest();  // throws SocketTimeoutException 
  String errorMsg = null;
  // verify build version
  if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion())) {
    errorMsg = "Incompatible build versions: active name-node BV = " 
      + nsInfo.getBuildVersion() + "; backup node BV = "
      + Storage.getBuildVersion();
    LOG.error(errorMsg);
    throw new IOException(errorMsg);
  }
  assert HdfsConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() :
    "Active and backup node layout versions must be the same. Expected: "
    + HdfsConstants.NAMENODE_LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();
  return nsInfo;
}
 
Example #9
Source File: BackupNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private NamespaceInfo handshake(Configuration conf) throws IOException {
  // connect to name node
  InetSocketAddress nnAddress = NameNode.getServiceAddress(conf, true);
  this.namenode = NameNodeProxies.createNonHAProxy(conf, nnAddress,
      NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
      true).getProxy();
  this.nnRpcAddress = NetUtils.getHostPortString(nnAddress);
  this.nnHttpAddress = DFSUtil.getInfoServer(nnAddress, conf,
      DFSUtil.getHttpClientScheme(conf)).toURL();
  // get version and id info from the name-node
  NamespaceInfo nsInfo = null;
  while(!isStopRequested()) {
    try {
      nsInfo = handshake(namenode);
      break;
    } catch(SocketTimeoutException e) {  // name-node is busy
      LOG.info("Problem connecting to server: " + nnAddress);
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        LOG.warn("Encountered exception ", e);
      }
    }
  }
  return nsInfo;
}
 
Example #10
Source File: NameNodeProxies.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries) throws IOException {
  NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
      address, conf, ugi, NamenodeProtocolPB.class);
  if (withRetries) { // create the proxy with retries
    RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
            TimeUnit.MILLISECONDS);
    Map<String, RetryPolicy> methodNameToPolicyMap
         = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("getBlocks", timeoutPolicy);
    methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy);
    NamenodeProtocol translatorProxy =
        new NamenodeProtocolTranslatorPB(proxy);
    return (NamenodeProtocol) RetryProxy.create(
        NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap);
  } else {
    return new NamenodeProtocolTranslatorPB(proxy);
  }
}
 
Example #11
Source File: FSImage.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Start checkpoint.
 * <p>
 * If backup storage contains image that is newer than or incompatible with 
 * what the active name-node has, then the backup node should shutdown.<br>
 * If the backup image is older than the active one then it should 
 * be discarded and downloaded from the active node.<br>
 * If the images are the same then the backup image will be used as current.
 * 
 * @param bnReg the backup node registration.
 * @param nnReg this (active) name-node registration.
 * @return {@link NamenodeCommand} if backup node should shutdown or
 * {@link CheckpointCommand} prescribing what backup node should 
 *         do with its image.
 * @throws IOException
 */
NamenodeCommand startCheckpoint(NamenodeRegistration bnReg, // backup node
                                NamenodeRegistration nnReg) // active name-node
throws IOException {
  LOG.info("Start checkpoint at txid " + getEditLog().getLastWrittenTxId());
  String msg = null;
  // Verify that checkpoint is allowed
  if(bnReg.getNamespaceID() != storage.getNamespaceID())
    msg = "Name node " + bnReg.getAddress()
          + " has incompatible namespace id: " + bnReg.getNamespaceID()
          + " expected: " + storage.getNamespaceID();
  else if(bnReg.isRole(NamenodeRole.NAMENODE))
    msg = "Name node " + bnReg.getAddress()
          + " role " + bnReg.getRole() + ": checkpoint is not allowed.";
  else if(bnReg.getLayoutVersion() < storage.getLayoutVersion()
      || (bnReg.getLayoutVersion() == storage.getLayoutVersion()
          && bnReg.getCTime() > storage.getCTime()))
    // remote node has newer image age
    msg = "Name node " + bnReg.getAddress()
          + " has newer image layout version: LV = " +bnReg.getLayoutVersion()
          + " cTime = " + bnReg.getCTime()
          + ". Current version: LV = " + storage.getLayoutVersion()
          + " cTime = " + storage.getCTime();
  if(msg != null) {
    LOG.error(msg);
    return new NamenodeCommand(NamenodeProtocol.ACT_SHUTDOWN);
  }
  boolean needToReturnImg = true;
  if(storage.getNumStorageDirs(NameNodeDirType.IMAGE) == 0)
    // do not return image if there are no image directories
    needToReturnImg = false;
  CheckpointSignature sig = rollEditLog();
  return new CheckpointCommand(sig, needToReturnImg);
}
 
Example #12
Source File: Balancer.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
private static NamenodeProtocol createNamenode(Configuration conf)
  throws IOException {
  InetSocketAddress nameNodeAddr = NameNode.getAddress(conf);
  RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
      5, 200, TimeUnit.MILLISECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      timeoutPolicy, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap =
      new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("getBlocks", methodPolicy);

  UserGroupInformation ugi;
  try {
    ugi = UnixUserGroupInformation.login(conf);
  } catch (javax.security.auth.login.LoginException e) {
    throw new IOException(StringUtils.stringifyException(e));
  }

  return (NamenodeProtocol) RetryProxy.create(
      NamenodeProtocol.class,
      RPC.getProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID,
          nameNodeAddr,
          ugi,
          conf,
          NetUtils.getDefaultSocketFactory(conf)),
      methodNameToPolicyMap);
}
 
Example #13
Source File: EditLogTailer.java    From big-c with Apache License 2.0 5 votes vote down vote up
private NamenodeProtocol getActiveNodeProxy() throws IOException {
  if (cachedActiveProxy == null) {
    int rpcTimeout = conf.getInt(
        DFSConfigKeys.DFS_HA_LOGROLL_RPC_TIMEOUT_KEY,
        DFSConfigKeys.DFS_HA_LOGROLL_RPC_TIMEOUT_DEFAULT);
    NamenodeProtocolPB proxy = RPC.waitForProxy(NamenodeProtocolPB.class,
        RPC.getProtocolVersion(NamenodeProtocolPB.class), activeAddr, conf,
        rpcTimeout, Long.MAX_VALUE);
    cachedActiveProxy = new NamenodeProtocolTranslatorPB(proxy);
  }
  assert cachedActiveProxy != null;
  return cachedActiveProxy;
}
 
Example #14
Source File: BootstrapStandby.java    From big-c with Apache License 2.0 5 votes vote down vote up
private int downloadImage(NNStorage storage, NamenodeProtocol proxy)
    throws IOException {
  // Load the newly formatted image, using all of the directories
  // (including shared edits)
  final long imageTxId = proxy.getMostRecentCheckpointTxId();
  final long curTxId = proxy.getTransactionID();
  FSImage image = new FSImage(conf);
  try {
    image.getStorage().setStorageInfo(storage);
    image.initEditLog(StartupOption.REGULAR);
    assert image.getEditLog().isOpenForRead() :
        "Expected edit log to be open for read";

    // Ensure that we have enough edits already in the shared directory to
    // start up from the last checkpoint on the active.
    if (!skipSharedEditsCheck &&
        !checkLogsAvailableForRead(image, imageTxId, curTxId)) {
      return ERR_CODE_LOGS_UNAVAILABLE;
    }

    image.getStorage().writeTransactionIdFileToStorage(curTxId);

    // Download that checkpoint into our storage directories.
    MD5Hash hash = TransferFsImage.downloadImageToStorage(
        otherHttpAddr, imageTxId, storage, true);
    image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
        hash);
  } catch (IOException ioe) {
    image.close();
    throw ioe;
  }
  return 0;
}
 
Example #15
Source File: BackupNode.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override // NameNode
public void stop() {
  if(checkpointManager != null) {
    // Prevent from starting a new checkpoint.
    // Checkpoints that has already been started may proceed until 
    // the error reporting to the name-node is complete.
    // Checkpoint manager should not be interrupted yet because it will
    // close storage file channels and the checkpoint may fail with 
    // ClosedByInterruptException.
    checkpointManager.shouldRun = false;
  }
  if(namenode != null && getRegistration() != null) {
    // Exclude this node from the list of backup streams on the name-node
    try {
      namenode.errorReport(getRegistration(), NamenodeProtocol.FATAL,
          "Shutting down.");
    } catch(IOException e) {
      LOG.error("Failed to report to name-node.", e);
    }
  }
  // Stop the RPC client
  if (namenode != null) {
    RPC.stopProxy(namenode);
  }
  namenode = null;
  // Stop the checkpoint manager
  if(checkpointManager != null) {
    checkpointManager.interrupt();
    checkpointManager = null;
  }

  // Abort current log segment - otherwise the NN shutdown code
  // will close it gracefully, which is incorrect.
  getFSImage().getEditLog().abortCurrentLogSegment();

  // Stop name-node threads
  super.stop();
}
 
Example #16
Source File: NameNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException { 
  if (protocol.equals(ClientProtocol.class.getName())) {
    return ClientProtocol.versionID; 
  } else if (protocol.equals(DatanodeProtocol.class.getName())){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName())){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #17
Source File: TestGetBlocks.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
private void getBlocksWithException(NamenodeProtocol namenode,
                                    DatanodeInfo datanode,
                                    long size) throws IOException {
  boolean getException = false;
  try {
      namenode.getBlocks(new DatanodeInfo(), 2);
  } catch(RemoteException e) {
    getException = true;
    assertTrue(e.getMessage().contains("IllegalArgumentException"));
  }
  assertTrue(getException);
}
 
Example #18
Source File: Balancer.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private static NamenodeProtocol createNamenode(InetSocketAddress nameNodeAddr, Configuration conf)
  throws IOException {
  RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
      5, 200, TimeUnit.MILLISECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      timeoutPolicy, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap =
      new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("getBlocks", methodPolicy);

  UserGroupInformation ugi;
  try {
    ugi = UnixUserGroupInformation.login(conf);
  } catch (javax.security.auth.login.LoginException e) {
    throw new IOException(StringUtils.stringifyException(e));
  }

  return (NamenodeProtocol) RetryProxy.create(
      NamenodeProtocol.class,
      RPC.getProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID,
          nameNodeAddr,
          ugi,
          conf,
          NetUtils.getDefaultSocketFactory(conf)),
      methodNameToPolicyMap);
}
 
Example #19
Source File: SnapshotNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Initialize SnapshotNode
 * @throws IOException
 */
private void init() throws IOException {
  ssDir = conf.get("fs.snapshot.dir", "/.SNAPSHOT");
  tempDir = conf.get("fs.snapshot.tempdir", "/tmp/snapshot");

  fileServer = getImageServer();
  dfs = FileSystem.get(conf);

  Path ssPath = new Path(ssDir);
  if (!dfs.exists(ssPath)) {
    dfs.mkdirs(ssPath);
  }

  maxLeaseUpdateThreads = conf.getInt("fs.snapshot.leaseupdatethreads", 100);

  // Waiting room purge thread
  purgeThread = new Daemon((new WaitingRoom(conf)).getPurger());
  purgeThread.start();

  // Get namenode rpc connection
  nameNodeAddr = NameNode.getAddress(conf);
  namenode = (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
                             NamenodeProtocol.versionID, nameNodeAddr, conf);

  // Snapshot RPC Server
  InetSocketAddress socAddr = SnapshotNode.getAddress(conf);
  int handlerCount = conf.getInt("fs.snapshot.handler.count", 10);
  server = RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(),
                         handlerCount, false, conf);
  // The rpc-server port can be ephemeral... ensure we have the correct info
  serverAddress = server.getListenerAddress();
  LOG.info("SnapshotNode up at: " + serverAddress);

  server.start(); // start rpc server
}
 
Example #20
Source File: NameNodeConnector.java    From big-c with Apache License 2.0 5 votes vote down vote up
public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
                         List<Path> targetPaths, Configuration conf,
                         int maxNotChangedIterations)
    throws IOException {
  this.nameNodeUri = nameNodeUri;
  this.idPath = idPath;
  this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays
      .asList(new Path("/")) : targetPaths;
  this.maxNotChangedIterations = maxNotChangedIterations;

  this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
      NamenodeProtocol.class).getProxy();
  this.client = NameNodeProxies.createProxy(conf, nameNodeUri,
      ClientProtocol.class, fallbackToSimpleAuth).getProxy();
  this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf);

  final NamespaceInfo namespaceinfo = namenode.versionRequest();
  this.blockpoolID = namespaceinfo.getBlockPoolID();

  final FsServerDefaults defaults = fs.getServerDefaults(new Path("/"));
  this.keyManager = new KeyManager(blockpoolID, namenode,
      defaults.getEncryptDataTransfer(), conf);
  // if it is for test, we do not create the id file
  out = checkAndMarkRunning();
  if (out == null) {
    // Exit if there is another one running.
    throw new IOException("Another " + name + " is running.");
  }
}
 
Example #21
Source File: KeyManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
public KeyManager(String blockpoolID, NamenodeProtocol namenode,
    boolean encryptDataTransfer, Configuration conf) throws IOException {
  this.namenode = namenode;
  this.encryptDataTransfer = encryptDataTransfer;

  final ExportedBlockKeys keys = namenode.getBlockKeys();
  this.isBlockTokenEnabled = keys.isBlockTokenEnabled();
  if (isBlockTokenEnabled) {
    long updateInterval = keys.getKeyUpdateInterval();
    long tokenLifetime = keys.getTokenLifetime();
    LOG.info("Block token params received from NN: update interval="
        + StringUtils.formatTime(updateInterval)
        + ", token lifetime=" + StringUtils.formatTime(tokenLifetime));
    String encryptionAlgorithm = conf.get(
        DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
    this.blockTokenSecretManager = new BlockTokenSecretManager(
        updateInterval, tokenLifetime, blockpoolID, encryptionAlgorithm);
    this.blockTokenSecretManager.addKeys(keys);

    // sync block keys with NN more frequently than NN updates its block keys
    this.blockKeyUpdater = new BlockKeyUpdater(updateInterval / 4);
    this.shouldRun = true;
  } else {
    this.blockTokenSecretManager = null;
    this.blockKeyUpdater = null;
  }
}
 
Example #22
Source File: TestGetBlocks.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void getBlocksWithException(NamenodeProtocol namenode,
    DatanodeInfo datanode, long size) throws IOException {
  boolean getException = false;
  try {
    namenode.getBlocks(DFSTestUtil.getLocalDatanodeInfo(), 2);
  } catch (RemoteException e) {
    getException = true;
    assertTrue(e.getClassName().contains("HadoopIllegalArgumentException"));
  }
  assertTrue(getException);
}
 
Example #23
Source File: BootstrapStandby.java    From big-c with Apache License 2.0 5 votes vote down vote up
private NamenodeProtocol createNNProtocolProxy()
    throws IOException {
  return NameNodeProxies.createNonHAProxy(getConf(),
      otherIpcAddr, NamenodeProtocol.class,
      UserGroupInformation.getLoginUser(), true)
      .getProxy();
}
 
Example #24
Source File: TestGetBlocks.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private void getBlocksWithException(NamenodeProtocol namenode,
                                    DatanodeInfo datanode,
                                    long size) throws IOException {
  boolean getException = false;
  try {
      namenode.getBlocks(new DatanodeInfo(), 2);
  } catch(RemoteException e) {
    getException = true;
    assertTrue(e.getMessage().contains("IllegalArgumentException"));
  }
  assertTrue(getException);
}
 
Example #25
Source File: NameNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException {
  InetSocketAddress requestAddress = Server.get().getListenerAddress();
  boolean dnRequest = false, clientRequest = false;
  // If dnProtocolAddress is null - there is only one server running
  // otherwise check the address of the incoming request.
  if (dnProtocolAddress == null ||
      dnProtocolAddress.equals(requestAddress)) {
    dnRequest = true;
  }
  if (dnProtocolAddress == null || requestAddress.equals(serverAddress)) {
    clientRequest = true;
  }
  if (protocol.equals(ClientProtocol.class.getName())) {
    long namenodeVersion = ClientProtocol.versionID;
    if (namenodeVersion > clientVersion &&
        !ProtocolCompatible.isCompatibleClientProtocol(
            clientVersion, namenodeVersion)) {
      throw new RPC.VersionIncompatible(
          protocol, clientVersion, namenodeVersion);
    }
    return namenodeVersion;
  } else if (protocol.equals(DatanodeProtocol.class.getName()) && dnRequest){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName()) && clientRequest){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName()) && clientRequest){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #26
Source File: Standby.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Initialize the webserver so that the primary namenode can fetch
 * transaction logs from standby via http.
 */
void initSecondary(Configuration conf) throws IOException {

  nameNodeAddr = avatarNode.getRemoteNamenodeAddress(conf);
  this.primaryNamenode =
      (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID, nameNodeAddr, conf);

  fsName = avatarNode.getRemoteNamenodeHttpName(conf);

  // Initialize other scheduling parameters from the configuration
  checkpointEnabled = conf.getBoolean("fs.checkpoint.enabled", false);
  checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
  checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

  // initialize the webserver for uploading files.
  String infoAddr = 
    NetUtils.getServerAddress(conf,
                              "dfs.secondary.info.bindAddress",
                              "dfs.secondary.info.port",
                              "dfs.secondary.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  infoBindAddress = infoSocAddr.getHostName();
  int tmpInfoPort = infoSocAddr.getPort();
  infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort,
      tmpInfoPort == 0, conf);
  infoServer.setAttribute("name.system.image", fsImage);
  this.infoServer.setAttribute("name.conf", conf);
  infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
  infoServer.start();

  // The web-server port can be ephemeral... ensure we have the correct info
  infoPort = infoServer.getPort();
  conf.set("dfs.secondary.http.address", infoBindAddress + ":" +infoPort);
  LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" +infoPort);
  LOG.warn("Checkpoint Period   :" + checkpointPeriod + " secs " +
           "(" + checkpointPeriod/60 + " min)");
  LOG.warn("Log Size Trigger    :" + checkpointSize + " bytes " +
           "(" + checkpointSize/1024 + " KB)");
}
 
Example #27
Source File: NameNodeProxies.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an explicitly non-HA-enabled proxy object. Most of the time you
 * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
 *
 * @param conf the configuration object
 * @param nnAddr address of the remote NN to connect to
 * @param xface the IPC interface which should be created
 * @param ugi the user who is making the calls on the proxy object
 * @param withRetries certain interfaces have a non-standard retry policy
 * @param fallbackToSimpleAuth - set to true or false during this method to
 *   indicate if a secure client falls back to simple auth
 * @return an object containing both the proxy and the associated
 *         delegation token service it corresponds to
 * @throws IOException
 */
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createNonHAProxy(
    Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
    UserGroupInformation ugi, boolean withRetries,
    AtomicBoolean fallbackToSimpleAuth) throws IOException {
  Text dtService = SecurityUtil.buildTokenService(nnAddr);

  T proxy;
  if (xface == ClientProtocol.class) {
    proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
        withRetries, fallbackToSimpleAuth);
  } else if (xface == JournalProtocol.class) {
    proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
  } else if (xface == NamenodeProtocol.class) {
    proxy = (T) createNNProxyWithNamenodeProtocol(nnAddr, conf, ugi,
        withRetries);
  } else if (xface == GetUserMappingsProtocol.class) {
    proxy = (T) createNNProxyWithGetUserMappingsProtocol(nnAddr, conf, ugi);
  } else if (xface == RefreshUserMappingsProtocol.class) {
    proxy = (T) createNNProxyWithRefreshUserMappingsProtocol(nnAddr, conf, ugi);
  } else if (xface == RefreshAuthorizationPolicyProtocol.class) {
    proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
        conf, ugi);
  } else if (xface == RefreshCallQueueProtocol.class) {
    proxy = (T) createNNProxyWithRefreshCallQueueProtocol(nnAddr, conf, ugi);
  } else {
    String message = "Unsupported protocol found when creating the proxy " +
        "connection to NameNode: " +
        ((xface != null) ? xface.getClass().getName() : "null");
    LOG.error(message);
    throw new IllegalStateException(message);
  }

  return new ProxyAndInfo<T>(proxy, dtService, nnAddr);
}
 
Example #28
Source File: FSImage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Start checkpoint.
 * <p>
 * If backup storage contains image that is newer than or incompatible with 
 * what the active name-node has, then the backup node should shutdown.<br>
 * If the backup image is older than the active one then it should 
 * be discarded and downloaded from the active node.<br>
 * If the images are the same then the backup image will be used as current.
 * 
 * @param bnReg the backup node registration.
 * @param nnReg this (active) name-node registration.
 * @return {@link NamenodeCommand} if backup node should shutdown or
 * {@link CheckpointCommand} prescribing what backup node should 
 *         do with its image.
 * @throws IOException
 */
NamenodeCommand startCheckpoint(NamenodeRegistration bnReg, // backup node
                                NamenodeRegistration nnReg) // active name-node
throws IOException {
  LOG.info("Start checkpoint at txid " + getEditLog().getLastWrittenTxId());
  String msg = null;
  // Verify that checkpoint is allowed
  if(bnReg.getNamespaceID() != storage.getNamespaceID())
    msg = "Name node " + bnReg.getAddress()
          + " has incompatible namespace id: " + bnReg.getNamespaceID()
          + " expected: " + storage.getNamespaceID();
  else if(bnReg.isRole(NamenodeRole.NAMENODE))
    msg = "Name node " + bnReg.getAddress()
          + " role " + bnReg.getRole() + ": checkpoint is not allowed.";
  else if(bnReg.getLayoutVersion() < storage.getLayoutVersion()
      || (bnReg.getLayoutVersion() == storage.getLayoutVersion()
          && bnReg.getCTime() > storage.getCTime()))
    // remote node has newer image age
    msg = "Name node " + bnReg.getAddress()
          + " has newer image layout version: LV = " +bnReg.getLayoutVersion()
          + " cTime = " + bnReg.getCTime()
          + ". Current version: LV = " + storage.getLayoutVersion()
          + " cTime = " + storage.getCTime();
  if(msg != null) {
    LOG.error(msg);
    return new NamenodeCommand(NamenodeProtocol.ACT_SHUTDOWN);
  }
  boolean needToReturnImg = true;
  if(storage.getNumStorageDirs(NameNodeDirType.IMAGE) == 0)
    // do not return image if there are no image directories
    needToReturnImg = false;
  CheckpointSignature sig = rollEditLog();
  return new CheckpointCommand(sig, needToReturnImg);
}
 
Example #29
Source File: BackupNode.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override // NameNode
public void stop() {
  if(checkpointManager != null) {
    // Prevent from starting a new checkpoint.
    // Checkpoints that has already been started may proceed until 
    // the error reporting to the name-node is complete.
    // Checkpoint manager should not be interrupted yet because it will
    // close storage file channels and the checkpoint may fail with 
    // ClosedByInterruptException.
    checkpointManager.shouldRun = false;
  }
  if(namenode != null && getRegistration() != null) {
    // Exclude this node from the list of backup streams on the name-node
    try {
      namenode.errorReport(getRegistration(), NamenodeProtocol.FATAL,
          "Shutting down.");
    } catch(IOException e) {
      LOG.error("Failed to report to name-node.", e);
    }
  }
  // Stop the RPC client
  if (namenode != null) {
    RPC.stopProxy(namenode);
  }
  namenode = null;
  // Stop the checkpoint manager
  if(checkpointManager != null) {
    checkpointManager.interrupt();
    checkpointManager = null;
  }

  // Abort current log segment - otherwise the NN shutdown code
  // will close it gracefully, which is incorrect.
  getFSImage().getEditLog().abortCurrentLogSegment();

  // Stop name-node threads
  super.stop();
}
 
Example #30
Source File: NameNodeProxies.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an explicitly non-HA-enabled proxy object. Most of the time you
 * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
 *
 * @param conf the configuration object
 * @param nnAddr address of the remote NN to connect to
 * @param xface the IPC interface which should be created
 * @param ugi the user who is making the calls on the proxy object
 * @param withRetries certain interfaces have a non-standard retry policy
 * @param fallbackToSimpleAuth - set to true or false during this method to
 *   indicate if a secure client falls back to simple auth
 * @return an object containing both the proxy and the associated
 *         delegation token service it corresponds to
 * @throws IOException
 */
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createNonHAProxy(
    Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
    UserGroupInformation ugi, boolean withRetries,
    AtomicBoolean fallbackToSimpleAuth) throws IOException {
  Text dtService = SecurityUtil.buildTokenService(nnAddr);

  T proxy;
  if (xface == ClientProtocol.class) {
    proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
        withRetries, fallbackToSimpleAuth);
  } else if (xface == JournalProtocol.class) {
    proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
  } else if (xface == NamenodeProtocol.class) {
    proxy = (T) createNNProxyWithNamenodeProtocol(nnAddr, conf, ugi,
        withRetries);
  } else if (xface == GetUserMappingsProtocol.class) {
    proxy = (T) createNNProxyWithGetUserMappingsProtocol(nnAddr, conf, ugi);
  } else if (xface == RefreshUserMappingsProtocol.class) {
    proxy = (T) createNNProxyWithRefreshUserMappingsProtocol(nnAddr, conf, ugi);
  } else if (xface == RefreshAuthorizationPolicyProtocol.class) {
    proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
        conf, ugi);
  } else if (xface == RefreshCallQueueProtocol.class) {
    proxy = (T) createNNProxyWithRefreshCallQueueProtocol(nnAddr, conf, ugi);
  } else {
    String message = "Unsupported protocol found when creating the proxy " +
        "connection to NameNode: " +
        ((xface != null) ? xface.getClass().getName() : "null");
    LOG.error(message);
    throw new IllegalStateException(message);
  }

  return new ProxyAndInfo<T>(proxy, dtService, nnAddr);
}