Java Code Examples for org.apache.hadoop.hdfs.protocol.HdfsConstants#DATANODE_LAYOUT_VERSION

The following examples show how to use org.apache.hadoop.hdfs.protocol.HdfsConstants#DATANODE_LAYOUT_VERSION . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DataStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
void format(StorageDirectory sd, NamespaceInfo nsInfo,
            String datanodeUuid) throws IOException {
  sd.clearDirectory(); // create directory
  this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
  this.clusterID = nsInfo.getClusterID();
  this.namespaceID = nsInfo.getNamespaceID();
  this.cTime = 0;
  setDatanodeUuid(datanodeUuid);

  if (sd.getStorageUuid() == null) {
    // Assign a new Storage UUID.
    sd.setStorageUuid(DatanodeStorage.generateUuid());
  }

  writeProperties(sd);
}
Example 2
Source File: DataStorage.java    From big-c with Apache License 2.0 6 votes vote down vote up 
void format(StorageDirectory sd, NamespaceInfo nsInfo,
            String datanodeUuid) throws IOException {
  sd.clearDirectory(); // create directory
  this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
  this.clusterID = nsInfo.getClusterID();
  this.namespaceID = nsInfo.getNamespaceID();
  this.cTime = 0;
  setDatanodeUuid(datanodeUuid);

  if (sd.getStorageUuid() == null) {
    // Assign a new Storage UUID.
    sd.setStorageUuid(DatanodeStorage.generateUuid());
  }

  writeProperties(sd);
}
Example 3
Source File: BlockPoolSliceStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Format a block pool slice storage. 
 * @param bpSdir the block pool storage
 * @param nsInfo the name space info
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException {
  LOG.info("Formatting block pool " + blockpoolID + " directory "
      + bpSdir.getCurrentDir());
  bpSdir.clearDirectory(); // create directory
  this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
  this.cTime = nsInfo.getCTime();
  this.namespaceID = nsInfo.getNamespaceID();
  this.blockpoolID = nsInfo.getBlockPoolID();
  writeProperties(bpSdir);
}
Example 4
Source File: BlockPoolSliceStorage.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Format a block pool slice storage. 
 * @param bpSdir the block pool storage
 * @param nsInfo the name space info
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException {
  LOG.info("Formatting block pool " + blockpoolID + " directory "
      + bpSdir.getCurrentDir());
  bpSdir.clearDirectory(); // create directory
  this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
  this.cTime = nsInfo.getCTime();
  this.namespaceID = nsInfo.getNamespaceID();
  this.blockpoolID = nsInfo.getBlockPoolID();
  writeProperties(bpSdir);
}
Example 5
Source File: StorageInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public int getServiceLayoutVersion() {
  return storageType == NodeType.DATA_NODE ? HdfsConstants.DATANODE_LAYOUT_VERSION
      : HdfsConstants.NAMENODE_LAYOUT_VERSION;
}
Example 6
Source File: TestDFSStartupVersions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Initialize the versions array.  This array stores all combinations 
 * of cross product:
 *  {oldLayoutVersion,currentLayoutVersion,futureLayoutVersion} X
 *    {currentNamespaceId,incorrectNamespaceId} X
 *      {pastFsscTime,currentFsscTime,futureFsscTime}
 */
private StorageData[] initializeVersions() throws Exception {
  int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION;
  int layoutVersionCur = HdfsConstants.DATANODE_LAYOUT_VERSION;
  int layoutVersionNew = Integer.MIN_VALUE;
  int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID(null);
  int namespaceIdOld = Integer.MIN_VALUE;
  long fsscTimeOld = Long.MIN_VALUE;
  long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null);
  long fsscTimeNew = Long.MAX_VALUE;
  String clusterID = "testClusterID";
  String invalidClusterID = "testClusterID";
  String bpid = UpgradeUtilities.getCurrentBlockPoolID(null);
  String invalidBpid = "invalidBpid";
  
  return new StorageData[] {
      new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
          fsscTimeOld, bpid), // 0
      new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
          fsscTimeCur, bpid), // 1
      new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
          fsscTimeNew, bpid), // 2
      new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
          fsscTimeOld, bpid), // 3
      new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
          fsscTimeCur, bpid), // 4
      new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
          fsscTimeNew, bpid), // 5
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeOld, bpid), // 6
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeCur, bpid), // 7
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeNew, bpid), // 8
      new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
          fsscTimeOld, bpid), // 9
      new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
          fsscTimeCur, bpid), // 10
      new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
          fsscTimeNew, bpid), // 11
      new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
          fsscTimeOld, bpid), // 12
      new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
          fsscTimeCur, bpid), // 13
      new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
          fsscTimeNew, bpid), // 14
      new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
          fsscTimeOld, bpid), // 15
      new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
          fsscTimeCur, bpid), // 16
      new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
          fsscTimeNew, bpid), // 17
      // Test with invalid clusterId
      new StorageData(layoutVersionCur, namespaceIdCur, invalidClusterID,
          fsscTimeCur, bpid), // 18
      // Test with invalid block pool Id
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeCur, invalidBpid) // 19
  };
}
Example 7
Source File: TestDFSStartupVersions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Determines if the given Namenode version and Datanode version
 * are compatible with each other. Compatibility in this case mean
 * that the Namenode and Datanode will successfully start up and
 * will work together. The rules for compatibility,
 * taken from the DFS Upgrade Design, are as follows:
 * <pre>
 * <ol>
 * <li>Check 0: Datanode namespaceID != Namenode namespaceID the startup fails
 * </li>
 * <li>Check 1: Datanode clusterID != Namenode clusterID the startup fails
 * </li>
 * <li>Check 2: Datanode blockPoolID != Namenode blockPoolID the startup fails
 * </li>
 * <li>Check 3: The data-node does regular startup (no matter which options 
 *    it is started with) if
 *       softwareLV == storedLV AND 
 *       DataNode.FSSCTime == NameNode.FSSCTime
 * </li>
 * <li>Check 4: The data-node performs an upgrade if it is started without any 
 *    options and
 *       |softwareLV| > |storedLV| OR 
 *       (softwareLV == storedLV AND
 *        DataNode.FSSCTime < NameNode.FSSCTime)
 * </li>
 * <li>NOT TESTED: The data-node rolls back if it is started with
 *    the -rollback option and
 *       |softwareLV| >= |previous.storedLV| AND 
 *       DataNode.previous.FSSCTime <= NameNode.FSSCTime
 * </li>
 * <li>Check 5: In all other cases the startup fails.</li>
 * </ol>
 * </pre>
 */
boolean isVersionCompatible(StorageData namenodeSd, StorageData datanodeSd) {
  final StorageInfo namenodeVer = namenodeSd.storageInfo;
  final StorageInfo datanodeVer = datanodeSd.storageInfo;
  // check #0
  if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) {
    LOG.info("namespaceIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #1
  if (!namenodeVer.getClusterID().equals(datanodeVer.getClusterID())) {
    LOG.info("clusterIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #2
  if (!namenodeSd.blockPoolId.equals(datanodeSd.blockPoolId)) {
    LOG.info("blockPoolIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #3
  int softwareLV = HdfsConstants.DATANODE_LAYOUT_VERSION;
  int storedLV = datanodeVer.getLayoutVersion();
  if (softwareLV == storedLV &&  
      datanodeVer.getCTime() == namenodeVer.getCTime()) 
    {
      LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true");
      return true;
    }
  // check #4
  long absSoftwareLV = Math.abs((long)softwareLV);
  long absStoredLV = Math.abs((long)storedLV);
  if (absSoftwareLV > absStoredLV ||
      (softwareLV == storedLV &&
       datanodeVer.getCTime() < namenodeVer.getCTime())) 
    {
      LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true");
      return true;
    }
  // check #5
  LOG.info("default case: isVersionCompatible=false");
  return false;
}
Example 8
Source File: StorageInfo.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public int getServiceLayoutVersion() {
  return storageType == NodeType.DATA_NODE ? HdfsConstants.DATANODE_LAYOUT_VERSION
      : HdfsConstants.NAMENODE_LAYOUT_VERSION;
}
Example 9
Source File: TestDFSStartupVersions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Initialize the versions array.  This array stores all combinations 
 * of cross product:
 *  {oldLayoutVersion,currentLayoutVersion,futureLayoutVersion} X
 *    {currentNamespaceId,incorrectNamespaceId} X
 *      {pastFsscTime,currentFsscTime,futureFsscTime}
 */
private StorageData[] initializeVersions() throws Exception {
  int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION;
  int layoutVersionCur = HdfsConstants.DATANODE_LAYOUT_VERSION;
  int layoutVersionNew = Integer.MIN_VALUE;
  int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID(null);
  int namespaceIdOld = Integer.MIN_VALUE;
  long fsscTimeOld = Long.MIN_VALUE;
  long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null);
  long fsscTimeNew = Long.MAX_VALUE;
  String clusterID = "testClusterID";
  String invalidClusterID = "testClusterID";
  String bpid = UpgradeUtilities.getCurrentBlockPoolID(null);
  String invalidBpid = "invalidBpid";
  
  return new StorageData[] {
      new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
          fsscTimeOld, bpid), // 0
      new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
          fsscTimeCur, bpid), // 1
      new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
          fsscTimeNew, bpid), // 2
      new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
          fsscTimeOld, bpid), // 3
      new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
          fsscTimeCur, bpid), // 4
      new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
          fsscTimeNew, bpid), // 5
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeOld, bpid), // 6
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeCur, bpid), // 7
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeNew, bpid), // 8
      new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
          fsscTimeOld, bpid), // 9
      new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
          fsscTimeCur, bpid), // 10
      new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
          fsscTimeNew, bpid), // 11
      new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
          fsscTimeOld, bpid), // 12
      new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
          fsscTimeCur, bpid), // 13
      new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
          fsscTimeNew, bpid), // 14
      new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
          fsscTimeOld, bpid), // 15
      new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
          fsscTimeCur, bpid), // 16
      new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
          fsscTimeNew, bpid), // 17
      // Test with invalid clusterId
      new StorageData(layoutVersionCur, namespaceIdCur, invalidClusterID,
          fsscTimeCur, bpid), // 18
      // Test with invalid block pool Id
      new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
          fsscTimeCur, invalidBpid) // 19
  };
}
Example 10
Source File: TestDFSStartupVersions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Determines if the given Namenode version and Datanode version
 * are compatible with each other. Compatibility in this case mean
 * that the Namenode and Datanode will successfully start up and
 * will work together. The rules for compatibility,
 * taken from the DFS Upgrade Design, are as follows:
 * <pre>
 * <ol>
 * <li>Check 0: Datanode namespaceID != Namenode namespaceID the startup fails
 * </li>
 * <li>Check 1: Datanode clusterID != Namenode clusterID the startup fails
 * </li>
 * <li>Check 2: Datanode blockPoolID != Namenode blockPoolID the startup fails
 * </li>
 * <li>Check 3: The data-node does regular startup (no matter which options 
 *    it is started with) if
 *       softwareLV == storedLV AND 
 *       DataNode.FSSCTime == NameNode.FSSCTime
 * </li>
 * <li>Check 4: The data-node performs an upgrade if it is started without any 
 *    options and
 *       |softwareLV| > |storedLV| OR 
 *       (softwareLV == storedLV AND
 *        DataNode.FSSCTime < NameNode.FSSCTime)
 * </li>
 * <li>NOT TESTED: The data-node rolls back if it is started with
 *    the -rollback option and
 *       |softwareLV| >= |previous.storedLV| AND 
 *       DataNode.previous.FSSCTime <= NameNode.FSSCTime
 * </li>
 * <li>Check 5: In all other cases the startup fails.</li>
 * </ol>
 * </pre>
 */
boolean isVersionCompatible(StorageData namenodeSd, StorageData datanodeSd) {
  final StorageInfo namenodeVer = namenodeSd.storageInfo;
  final StorageInfo datanodeVer = datanodeSd.storageInfo;
  // check #0
  if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) {
    LOG.info("namespaceIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #1
  if (!namenodeVer.getClusterID().equals(datanodeVer.getClusterID())) {
    LOG.info("clusterIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #2
  if (!namenodeSd.blockPoolId.equals(datanodeSd.blockPoolId)) {
    LOG.info("blockPoolIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #3
  int softwareLV = HdfsConstants.DATANODE_LAYOUT_VERSION;
  int storedLV = datanodeVer.getLayoutVersion();
  if (softwareLV == storedLV &&  
      datanodeVer.getCTime() == namenodeVer.getCTime()) 
    {
      LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true");
      return true;
    }
  // check #4
  long absSoftwareLV = Math.abs((long)softwareLV);
  long absStoredLV = Math.abs((long)storedLV);
  if (absSoftwareLV > absStoredLV ||
      (softwareLV == storedLV &&
       datanodeVer.getCTime() < namenodeVer.getCTime())) 
    {
      LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true");
      return true;
    }
  // check #5
  LOG.info("default case: isVersionCompatible=false");
  return false;
}