org.apache.hadoop.hdfs.server.common.IncorrectVersionException Java Examples

void readProperties(StorageDirectory sd, StartupOption startupOption)
    throws IOException {
  Properties props = readPropertiesFile(sd.getVersionFile());
  if (HdfsServerConstants.RollingUpgradeStartupOption.ROLLBACK.matches
      (startupOption)) {
    int lv = Integer.parseInt(getProperty(props, sd, "layoutVersion"));
    if (lv > getServiceLayoutVersion()) {
      // we should not use a newer version for rollingUpgrade rollback
      throw new IncorrectVersionException(getServiceLayoutVersion(), lv,
          "storage directory " + sd.getRoot().getAbsolutePath());
    }
    props.setProperty("layoutVersion",
        Integer.toString(HdfsConstants.NAMENODE_LAYOUT_VERSION));
  }
  setFieldsFromProperties(props, sd);
}
 

private void checkNNVersion(NamespaceInfo nsInfo)
    throws IncorrectVersionException {
  // build and layout versions should match
  String nnVersion = nsInfo.getSoftwareVersion();
  String minimumNameNodeVersion = dnConf.getMinimumNameNodeVersion();
  if (VersionUtil.compareVersions(nnVersion, minimumNameNodeVersion) < 0) {
    IncorrectVersionException ive = new IncorrectVersionException(
        minimumNameNodeVersion, nnVersion, "NameNode", "DataNode");
    LOG.warn(ive.getMessage());
    throw ive;
  }
  String dnVersion = VersionInfo.getVersion();
  if (!nnVersion.equals(dnVersion)) {
    LOG.info("Reported NameNode version '" + nnVersion + "' does not match " +
        "DataNode version '" + dnVersion + "' but is within acceptable " +
        "limits. Note: This is normal during a rolling upgrade.");
  }
}
 

/**
 * Ensure that during downgrade the NN fails to load a fsimage with newer
 * format.
 */
@Test(expected = IncorrectVersionException.class)
public void testRejectNewFsImage() throws IOException {
  final Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    NNStorage storage = spy(cluster.getNameNode().getFSImage().getStorage());
    int futureVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1;
    doReturn(futureVersion).when(storage).getServiceLayoutVersion();
    storage.writeAll();
    cluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade");
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testSoftwareVersionDifferences() throws Exception {
  // We expect no exception to be thrown when the software versions match.
  assertEquals(VersionInfo.getVersion(),
      actor.retrieveNamespaceInfo().getSoftwareVersion());
  
  // We expect no exception to be thrown when the min NN version is below the
  // reported NN version.
  doReturn("4.0.0").when(fakeNsInfo).getSoftwareVersion();
  doReturn("3.0.0").when(mockDnConf).getMinimumNameNodeVersion();
  assertEquals("4.0.0", actor.retrieveNamespaceInfo().getSoftwareVersion());
  
  // When the NN reports a version that's too low, throw an exception.
  doReturn("3.0.0").when(fakeNsInfo).getSoftwareVersion();
  doReturn("4.0.0").when(mockDnConf).getMinimumNameNodeVersion();
  try {
    actor.retrieveNamespaceInfo();
    fail("Should have thrown an exception for NN with too-low version");
  } catch (IncorrectVersionException ive) {
    GenericTestUtils.assertExceptionContains(
        "The reported NameNode version is too low", ive);
    LOG.info("Got expected exception", ive);
  }
}
 

void readProperties(StorageDirectory sd, StartupOption startupOption)
    throws IOException {
  Properties props = readPropertiesFile(sd.getVersionFile());
  if (HdfsServerConstants.RollingUpgradeStartupOption.ROLLBACK.matches
      (startupOption)) {
    int lv = Integer.parseInt(getProperty(props, sd, "layoutVersion"));
    if (lv > getServiceLayoutVersion()) {
      // we should not use a newer version for rollingUpgrade rollback
      throw new IncorrectVersionException(getServiceLayoutVersion(), lv,
          "storage directory " + sd.getRoot().getAbsolutePath());
    }
    props.setProperty("layoutVersion",
        Integer.toString(HdfsConstants.NAMENODE_LAYOUT_VERSION));
  }
  setFieldsFromProperties(props, sd);
}
 

private void checkNNVersion(NamespaceInfo nsInfo)
    throws IncorrectVersionException {
  // build and layout versions should match
  String nnVersion = nsInfo.getSoftwareVersion();
  String minimumNameNodeVersion = dnConf.getMinimumNameNodeVersion();
  if (VersionUtil.compareVersions(nnVersion, minimumNameNodeVersion) < 0) {
    IncorrectVersionException ive = new IncorrectVersionException(
        minimumNameNodeVersion, nnVersion, "NameNode", "DataNode");
    LOG.warn(ive.getMessage());
    throw ive;
  }
  String dnVersion = VersionInfo.getVersion();
  if (!nnVersion.equals(dnVersion)) {
    LOG.info("Reported NameNode version '" + nnVersion + "' does not match " +
        "DataNode version '" + dnVersion + "' but is within acceptable " +
        "limits. Note: This is normal during a rolling upgrade.");
  }
}
 

/**
 * Ensure that during downgrade the NN fails to load a fsimage with newer
 * format.
 */
@Test(expected = IncorrectVersionException.class)
public void testRejectNewFsImage() throws IOException {
  final Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    NNStorage storage = spy(cluster.getNameNode().getFSImage().getStorage());
    int futureVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1;
    doReturn(futureVersion).when(storage).getServiceLayoutVersion();
    storage.writeAll();
    cluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade");
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testSoftwareVersionDifferences() throws Exception {
  // We expect no exception to be thrown when the software versions match.
  assertEquals(VersionInfo.getVersion(),
      actor.retrieveNamespaceInfo().getSoftwareVersion());
  
  // We expect no exception to be thrown when the min NN version is below the
  // reported NN version.
  doReturn("4.0.0").when(fakeNsInfo).getSoftwareVersion();
  doReturn("3.0.0").when(mockDnConf).getMinimumNameNodeVersion();
  assertEquals("4.0.0", actor.retrieveNamespaceInfo().getSoftwareVersion());
  
  // When the NN reports a version that's too low, throw an exception.
  doReturn("3.0.0").when(fakeNsInfo).getSoftwareVersion();
  doReturn("4.0.0").when(mockDnConf).getMinimumNameNodeVersion();
  try {
    actor.retrieveNamespaceInfo();
    fail("Should have thrown an exception for NN with too-low version");
  } catch (IncorrectVersionException ive) {
    GenericTestUtils.assertExceptionContains(
        "The reported NameNode version is too low", ive);
    LOG.info("Got expected exception", ive);
  }
}
 

@Override
protected void setLayoutVersion(Properties props, StorageDirectory sd)
    throws IncorrectVersionException, InconsistentFSStateException {
  int lv = Integer.parseInt(getProperty(props, sd, "layoutVersion"));
  // For journal node, since it now does not decode but just scan through the
  // edits, it can handle edits with future version in most of the cases.
  // Thus currently we may skip the layoutVersion check here.
  layoutVersion = lv;
}
 

synchronized UpgradeCommand processUpgradeCommand(UpgradeCommand command
                                                  ) throws IOException {
  NameNode.LOG.debug("\n   Distributed upgrade for NameNode version " 
      + getUpgradeVersion() + " to current LV " 
      + FSConstants.LAYOUT_VERSION + " is processing upgrade command: "
      + command.getAction() + " status = " + getUpgradeStatus() + "%");
  if(currentUpgrades == null) {
    NameNode.LOG.info("Ignoring upgrade command: " 
        + command.getAction() + " version " + command.getVersion()
        + ". No distributed upgrades are currently running on the NameNode");
    return null;
  }
  UpgradeObjectNamenode curUO = (UpgradeObjectNamenode)currentUpgrades.first();
  if(command.getVersion() != curUO.getVersion())
    throw new IncorrectVersionException(command.getVersion(), 
        "UpgradeCommand", curUO.getVersion());
  UpgradeCommand reply = curUO.processUpgradeCommand(command);
  if(curUO.getUpgradeStatus() < 100) {
    return reply;
  }
  // current upgrade is done
  curUO.completeUpgrade();
  NameNode.LOG.info("\n   Distributed upgrade for NameNode version " 
      + curUO.getVersion() + " to current LV " 
      + FSConstants.LAYOUT_VERSION + " is complete.");
  // proceede with the next one
  currentUpgrades.remove(curUO);
  if(currentUpgrades.isEmpty()) { // all upgrades are done
    completeUpgrade();
  } else {  // start next upgrade
    curUO = (UpgradeObjectNamenode)currentUpgrades.first();
    this.broadcastCommand = curUO.startUpgrade();
  }
  return reply;
}
 

synchronized UpgradeCommand processUpgradeCommand(UpgradeCommand command
                                                  ) throws IOException {
  NameNode.LOG.debug("\n   Distributed upgrade for NameNode version " 
      + getUpgradeVersion() + " to current LV " 
      + FSConstants.LAYOUT_VERSION + " is processing upgrade command: "
      + command.getAction() + " status = " + getUpgradeStatus() + "%");
  if(currentUpgrades == null) {
    NameNode.LOG.info("Ignoring upgrade command: " 
        + command.getAction() + " version " + command.getVersion()
        + ". No distributed upgrades are currently running on the NameNode");
    return null;
  }
  UpgradeObjectNamenode curUO = (UpgradeObjectNamenode)currentUpgrades.first();
  if(command.getVersion() != curUO.getVersion())
    throw new IncorrectVersionException(command.getVersion(), 
        "UpgradeCommand", curUO.getVersion());
  UpgradeCommand reply = curUO.processUpgradeCommand(command);
  if(curUO.getUpgradeStatus() < 100) {
    return reply;
  }
  // current upgrade is done
  curUO.completeUpgrade();
  NameNode.LOG.info("\n   Distributed upgrade for NameNode version " 
      + curUO.getVersion() + " to current LV " 
      + FSConstants.LAYOUT_VERSION + " is complete.");
  // proceede with the next one
  currentUpgrades.remove(curUO);
  if(currentUpgrades.isEmpty()) { // all upgrades are done
    completeUpgrade();
  } else {  // start next upgrade
    curUO = (UpgradeObjectNamenode)currentUpgrades.first();
    this.broadcastCommand = curUO.startUpgrade();
  }
  return reply;
}
 

void handleRegistrationError(RemoteException re) {
  // If either the primary or standby NN throws these exceptions, this
  // datanode will exit. I think this is the right behaviour because
  // the excludes list on both namenode better be the same.
  String reClass = re.getClassName(); 
  if (UnregisteredDatanodeException.class.getName().equals(reClass) ||
      DisallowedDatanodeException.class.getName().equals(reClass) ||
      IncorrectVersionException.class.getName().equals(reClass)) {
    LOG.warn("DataNode is shutting down: ", re);
    shutdownDN();
  } else {
    LOG.warn(re);
  }
}
 

@Override
protected void setLayoutVersion(Properties props, StorageDirectory sd)
    throws IncorrectVersionException, InconsistentFSStateException {
  int lv = Integer.parseInt(getProperty(props, sd, "layoutVersion"));
  // For journal node, since it now does not decode but just scan through the
  // edits, it can handle edits with future version in most of the cases.
  // Thus currently we may skip the layoutVersion check here.
  layoutVersion = lv;
}
 

/**
 * Verify version.
 * @param version layout version
 * @throws IOException on layout version mismatch
 */
void verifyLayoutVersion(int version) throws IOException {
  if (version != HdfsConstants.NAMENODE_LAYOUT_VERSION)
    throw new IncorrectVersionException(
        HdfsConstants.NAMENODE_LAYOUT_VERSION, version, "data node");
}
 

@Test
public void testRegistrationWithDifferentSoftwareVersionsDuringUpgrade()
    throws Exception {
  Configuration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY, "1.0.0");
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf)
        .numDataNodes(0)
        .build();
    
    NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
    
    long nnCTime = cluster.getNamesystem().getFSImage().getStorage().getCTime();
    StorageInfo mockStorageInfo = mock(StorageInfo.class);
    doReturn(nnCTime).when(mockStorageInfo).getCTime();
    
    DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
    doReturn(HdfsConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion();
    doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
    doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo();
    
    // Should succeed when software versions are the same and CTimes are the
    // same.
    doReturn(VersionInfo.getVersion()).when(mockDnReg).getSoftwareVersion();
    doReturn("127.0.0.1").when(mockDnReg).getIpAddr();
    doReturn(123).when(mockDnReg).getXferPort();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should succeed when software versions are the same and CTimes are
    // different.
    doReturn(nnCTime + 1).when(mockStorageInfo).getCTime();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should fail when software version of DN is different from NN and CTimes
    // are different.
    doReturn(VersionInfo.getVersion() + ".1").when(mockDnReg).getSoftwareVersion();
    try {
      rpcServer.registerDatanode(mockDnReg);
      fail("Should not have been able to register DN with different software" +
          " versions and CTimes");
    } catch (IncorrectVersionException ive) {
      GenericTestUtils.assertExceptionContains(
          "does not match CTime of NN", ive);
      LOG.info("Got expected exception", ive);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testRegistrationWithDifferentSoftwareVersions() throws Exception {
  Configuration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY, "3.0.0");
  conf.set(DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY, "3.0.0");
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf)
        .numDataNodes(0)
        .build();
    
    NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
    
    long nnCTime = cluster.getNamesystem().getFSImage().getStorage().getCTime();
    StorageInfo mockStorageInfo = mock(StorageInfo.class);
    doReturn(nnCTime).when(mockStorageInfo).getCTime();
    
    DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
    doReturn(HdfsConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion();
    doReturn("127.0.0.1").when(mockDnReg).getIpAddr();
    doReturn(123).when(mockDnReg).getXferPort();
    doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
    doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo();
    
    // Should succeed when software versions are the same.
    doReturn("3.0.0").when(mockDnReg).getSoftwareVersion();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should succeed when software version of DN is above minimum required by NN.
    doReturn("4.0.0").when(mockDnReg).getSoftwareVersion();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should fail when software version of DN is below minimum required by NN.
    doReturn("2.0.0").when(mockDnReg).getSoftwareVersion();
    try {
      rpcServer.registerDatanode(mockDnReg);
      fail("Should not have been able to register DN with too-low version.");
    } catch (IncorrectVersionException ive) {
      GenericTestUtils.assertExceptionContains(
          "The reported DataNode version is too low", ive);
      LOG.info("Got expected exception", ive);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/**
 * Verify version.
 * @param version layout version
 * @throws IOException on layout version mismatch
 */
void verifyLayoutVersion(int version) throws IOException {
  if (version != HdfsConstants.NAMENODE_LAYOUT_VERSION)
    throw new IncorrectVersionException(
        HdfsConstants.NAMENODE_LAYOUT_VERSION, version, "data node");
}
 

@Test
public void testRegistrationWithDifferentSoftwareVersionsDuringUpgrade()
    throws Exception {
  Configuration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY, "1.0.0");
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf)
        .numDataNodes(0)
        .build();
    
    NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
    
    long nnCTime = cluster.getNamesystem().getFSImage().getStorage().getCTime();
    StorageInfo mockStorageInfo = mock(StorageInfo.class);
    doReturn(nnCTime).when(mockStorageInfo).getCTime();
    
    DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
    doReturn(HdfsConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion();
    doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
    doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo();
    
    // Should succeed when software versions are the same and CTimes are the
    // same.
    doReturn(VersionInfo.getVersion()).when(mockDnReg).getSoftwareVersion();
    doReturn("127.0.0.1").when(mockDnReg).getIpAddr();
    doReturn(123).when(mockDnReg).getXferPort();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should succeed when software versions are the same and CTimes are
    // different.
    doReturn(nnCTime + 1).when(mockStorageInfo).getCTime();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should fail when software version of DN is different from NN and CTimes
    // are different.
    doReturn(VersionInfo.getVersion() + ".1").when(mockDnReg).getSoftwareVersion();
    try {
      rpcServer.registerDatanode(mockDnReg);
      fail("Should not have been able to register DN with different software" +
          " versions and CTimes");
    } catch (IncorrectVersionException ive) {
      GenericTestUtils.assertExceptionContains(
          "does not match CTime of NN", ive);
      LOG.info("Got expected exception", ive);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testRegistrationWithDifferentSoftwareVersions() throws Exception {
  Configuration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY, "3.0.0");
  conf.set(DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY, "3.0.0");
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf)
        .numDataNodes(0)
        .build();
    
    NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
    
    long nnCTime = cluster.getNamesystem().getFSImage().getStorage().getCTime();
    StorageInfo mockStorageInfo = mock(StorageInfo.class);
    doReturn(nnCTime).when(mockStorageInfo).getCTime();
    
    DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
    doReturn(HdfsConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion();
    doReturn("127.0.0.1").when(mockDnReg).getIpAddr();
    doReturn(123).when(mockDnReg).getXferPort();
    doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
    doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo();
    
    // Should succeed when software versions are the same.
    doReturn("3.0.0").when(mockDnReg).getSoftwareVersion();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should succeed when software version of DN is above minimum required by NN.
    doReturn("4.0.0").when(mockDnReg).getSoftwareVersion();
    rpcServer.registerDatanode(mockDnReg);
    
    // Should fail when software version of DN is below minimum required by NN.
    doReturn("2.0.0").when(mockDnReg).getSoftwareVersion();
    try {
      rpcServer.registerDatanode(mockDnReg);
      fail("Should not have been able to register DN with too-low version.");
    } catch (IncorrectVersionException ive) {
      GenericTestUtils.assertExceptionContains(
          "The reported DataNode version is too low", ive);
      LOG.info("Got expected exception", ive);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/**
 * Verify version.
 * 
 * @param reportedVersion version reported by datanode
 * @param expectedVersion version expected by namenode
 * @param annotation explanation of the given version
 * @throws IncorrectVersionException
 */
public void verifyVersion(int reportedVersion,
    int expectedVersion, String annotation) throws IOException {
  if (reportedVersion != expectedVersion)
    throw new IncorrectVersionException(
        reportedVersion, "data node " + annotation, expectedVersion);
}
 

/**
 * Verify version.
 * 
 * @param version
 * @throws IOException
 */
public void verifyVersion(int version) throws IOException {
  if (version != LAYOUT_VERSION)
    throw new IncorrectVersionException(version, "data node");
}