Java Code Examples for org.apache.hadoop.hdfs.protocol.HdfsConstants#DATANODE_LAYOUT_VERSION
The following examples show how to use
org.apache.hadoop.hdfs.protocol.HdfsConstants#DATANODE_LAYOUT_VERSION .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DataStorage.java From hadoop with Apache License 2.0 | 6 votes |
void format(StorageDirectory sd, NamespaceInfo nsInfo, String datanodeUuid) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(datanodeUuid); if (sd.getStorageUuid() == null) { // Assign a new Storage UUID. sd.setStorageUuid(DatanodeStorage.generateUuid()); } writeProperties(sd); }
Example 2
Source File: DataStorage.java From big-c with Apache License 2.0 | 6 votes |
void format(StorageDirectory sd, NamespaceInfo nsInfo, String datanodeUuid) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(datanodeUuid); if (sd.getStorageUuid() == null) { // Assign a new Storage UUID. sd.setStorageUuid(DatanodeStorage.generateUuid()); } writeProperties(sd); }
Example 3
Source File: BlockPoolSliceStorage.java From hadoop with Apache License 2.0 | 5 votes |
/** * Format a block pool slice storage. * @param bpSdir the block pool storage * @param nsInfo the name space info * @throws IOException Signals that an I/O exception has occurred. */ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException { LOG.info("Formatting block pool " + blockpoolID + " directory " + bpSdir.getCurrentDir()); bpSdir.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); writeProperties(bpSdir); }
Example 4
Source File: BlockPoolSliceStorage.java From big-c with Apache License 2.0 | 5 votes |
/** * Format a block pool slice storage. * @param bpSdir the block pool storage * @param nsInfo the name space info * @throws IOException Signals that an I/O exception has occurred. */ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException { LOG.info("Formatting block pool " + blockpoolID + " directory " + bpSdir.getCurrentDir()); bpSdir.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); writeProperties(bpSdir); }
Example 5
Source File: StorageInfo.java From hadoop with Apache License 2.0 | 4 votes |
public int getServiceLayoutVersion() { return storageType == NodeType.DATA_NODE ? HdfsConstants.DATANODE_LAYOUT_VERSION : HdfsConstants.NAMENODE_LAYOUT_VERSION; }
Example 6
Source File: TestDFSStartupVersions.java From hadoop with Apache License 2.0 | 4 votes |
/** * Initialize the versions array. This array stores all combinations * of cross product: * {oldLayoutVersion,currentLayoutVersion,futureLayoutVersion} X * {currentNamespaceId,incorrectNamespaceId} X * {pastFsscTime,currentFsscTime,futureFsscTime} */ private StorageData[] initializeVersions() throws Exception { int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION; int layoutVersionCur = HdfsConstants.DATANODE_LAYOUT_VERSION; int layoutVersionNew = Integer.MIN_VALUE; int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID(null); int namespaceIdOld = Integer.MIN_VALUE; long fsscTimeOld = Long.MIN_VALUE; long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null); long fsscTimeNew = Long.MAX_VALUE; String clusterID = "testClusterID"; String invalidClusterID = "testClusterID"; String bpid = UpgradeUtilities.getCurrentBlockPoolID(null); String invalidBpid = "invalidBpid"; return new StorageData[] { new StorageData(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeOld, bpid), // 0 new StorageData(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeCur, bpid), // 1 new StorageData(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeNew, bpid), // 2 new StorageData(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeOld, bpid), // 3 new StorageData(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeCur, bpid), // 4 new StorageData(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeNew, bpid), // 5 new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeOld, bpid), // 6 new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeCur, bpid), // 7 new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeNew, bpid), // 8 new StorageData(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeOld, bpid), // 9 new StorageData(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeCur, bpid), // 10 new StorageData(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeNew, bpid), // 11 new StorageData(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeOld, bpid), // 12 new StorageData(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeCur, bpid), // 13 new StorageData(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeNew, bpid), // 14 new StorageData(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeOld, bpid), // 15 new StorageData(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeCur, bpid), // 16 new StorageData(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeNew, bpid), // 17 // Test with invalid clusterId new StorageData(layoutVersionCur, namespaceIdCur, invalidClusterID, fsscTimeCur, bpid), // 18 // Test with invalid block pool Id new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeCur, invalidBpid) // 19 }; }
Example 7
Source File: TestDFSStartupVersions.java From hadoop with Apache License 2.0 | 4 votes |
/** * Determines if the given Namenode version and Datanode version * are compatible with each other. Compatibility in this case mean * that the Namenode and Datanode will successfully start up and * will work together. The rules for compatibility, * taken from the DFS Upgrade Design, are as follows: * <pre> * <ol> * <li>Check 0: Datanode namespaceID != Namenode namespaceID the startup fails * </li> * <li>Check 1: Datanode clusterID != Namenode clusterID the startup fails * </li> * <li>Check 2: Datanode blockPoolID != Namenode blockPoolID the startup fails * </li> * <li>Check 3: The data-node does regular startup (no matter which options * it is started with) if * softwareLV == storedLV AND * DataNode.FSSCTime == NameNode.FSSCTime * </li> * <li>Check 4: The data-node performs an upgrade if it is started without any * options and * |softwareLV| > |storedLV| OR * (softwareLV == storedLV AND * DataNode.FSSCTime < NameNode.FSSCTime) * </li> * <li>NOT TESTED: The data-node rolls back if it is started with * the -rollback option and * |softwareLV| >= |previous.storedLV| AND * DataNode.previous.FSSCTime <= NameNode.FSSCTime * </li> * <li>Check 5: In all other cases the startup fails.</li> * </ol> * </pre> */ boolean isVersionCompatible(StorageData namenodeSd, StorageData datanodeSd) { final StorageInfo namenodeVer = namenodeSd.storageInfo; final StorageInfo datanodeVer = datanodeSd.storageInfo; // check #0 if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) { LOG.info("namespaceIDs are not equal: isVersionCompatible=false"); return false; } // check #1 if (!namenodeVer.getClusterID().equals(datanodeVer.getClusterID())) { LOG.info("clusterIDs are not equal: isVersionCompatible=false"); return false; } // check #2 if (!namenodeSd.blockPoolId.equals(datanodeSd.blockPoolId)) { LOG.info("blockPoolIDs are not equal: isVersionCompatible=false"); return false; } // check #3 int softwareLV = HdfsConstants.DATANODE_LAYOUT_VERSION; int storedLV = datanodeVer.getLayoutVersion(); if (softwareLV == storedLV && datanodeVer.getCTime() == namenodeVer.getCTime()) { LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true"); return true; } // check #4 long absSoftwareLV = Math.abs((long)softwareLV); long absStoredLV = Math.abs((long)storedLV); if (absSoftwareLV > absStoredLV || (softwareLV == storedLV && datanodeVer.getCTime() < namenodeVer.getCTime())) { LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true"); return true; } // check #5 LOG.info("default case: isVersionCompatible=false"); return false; }
Example 8
Source File: StorageInfo.java From big-c with Apache License 2.0 | 4 votes |
public int getServiceLayoutVersion() { return storageType == NodeType.DATA_NODE ? HdfsConstants.DATANODE_LAYOUT_VERSION : HdfsConstants.NAMENODE_LAYOUT_VERSION; }
Example 9
Source File: TestDFSStartupVersions.java From big-c with Apache License 2.0 | 4 votes |
/** * Initialize the versions array. This array stores all combinations * of cross product: * {oldLayoutVersion,currentLayoutVersion,futureLayoutVersion} X * {currentNamespaceId,incorrectNamespaceId} X * {pastFsscTime,currentFsscTime,futureFsscTime} */ private StorageData[] initializeVersions() throws Exception { int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION; int layoutVersionCur = HdfsConstants.DATANODE_LAYOUT_VERSION; int layoutVersionNew = Integer.MIN_VALUE; int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID(null); int namespaceIdOld = Integer.MIN_VALUE; long fsscTimeOld = Long.MIN_VALUE; long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null); long fsscTimeNew = Long.MAX_VALUE; String clusterID = "testClusterID"; String invalidClusterID = "testClusterID"; String bpid = UpgradeUtilities.getCurrentBlockPoolID(null); String invalidBpid = "invalidBpid"; return new StorageData[] { new StorageData(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeOld, bpid), // 0 new StorageData(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeCur, bpid), // 1 new StorageData(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeNew, bpid), // 2 new StorageData(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeOld, bpid), // 3 new StorageData(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeCur, bpid), // 4 new StorageData(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeNew, bpid), // 5 new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeOld, bpid), // 6 new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeCur, bpid), // 7 new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeNew, bpid), // 8 new StorageData(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeOld, bpid), // 9 new StorageData(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeCur, bpid), // 10 new StorageData(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeNew, bpid), // 11 new StorageData(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeOld, bpid), // 12 new StorageData(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeCur, bpid), // 13 new StorageData(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeNew, bpid), // 14 new StorageData(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeOld, bpid), // 15 new StorageData(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeCur, bpid), // 16 new StorageData(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeNew, bpid), // 17 // Test with invalid clusterId new StorageData(layoutVersionCur, namespaceIdCur, invalidClusterID, fsscTimeCur, bpid), // 18 // Test with invalid block pool Id new StorageData(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeCur, invalidBpid) // 19 }; }
Example 10
Source File: TestDFSStartupVersions.java From big-c with Apache License 2.0 | 4 votes |
/** * Determines if the given Namenode version and Datanode version * are compatible with each other. Compatibility in this case mean * that the Namenode and Datanode will successfully start up and * will work together. The rules for compatibility, * taken from the DFS Upgrade Design, are as follows: * <pre> * <ol> * <li>Check 0: Datanode namespaceID != Namenode namespaceID the startup fails * </li> * <li>Check 1: Datanode clusterID != Namenode clusterID the startup fails * </li> * <li>Check 2: Datanode blockPoolID != Namenode blockPoolID the startup fails * </li> * <li>Check 3: The data-node does regular startup (no matter which options * it is started with) if * softwareLV == storedLV AND * DataNode.FSSCTime == NameNode.FSSCTime * </li> * <li>Check 4: The data-node performs an upgrade if it is started without any * options and * |softwareLV| > |storedLV| OR * (softwareLV == storedLV AND * DataNode.FSSCTime < NameNode.FSSCTime) * </li> * <li>NOT TESTED: The data-node rolls back if it is started with * the -rollback option and * |softwareLV| >= |previous.storedLV| AND * DataNode.previous.FSSCTime <= NameNode.FSSCTime * </li> * <li>Check 5: In all other cases the startup fails.</li> * </ol> * </pre> */ boolean isVersionCompatible(StorageData namenodeSd, StorageData datanodeSd) { final StorageInfo namenodeVer = namenodeSd.storageInfo; final StorageInfo datanodeVer = datanodeSd.storageInfo; // check #0 if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) { LOG.info("namespaceIDs are not equal: isVersionCompatible=false"); return false; } // check #1 if (!namenodeVer.getClusterID().equals(datanodeVer.getClusterID())) { LOG.info("clusterIDs are not equal: isVersionCompatible=false"); return false; } // check #2 if (!namenodeSd.blockPoolId.equals(datanodeSd.blockPoolId)) { LOG.info("blockPoolIDs are not equal: isVersionCompatible=false"); return false; } // check #3 int softwareLV = HdfsConstants.DATANODE_LAYOUT_VERSION; int storedLV = datanodeVer.getLayoutVersion(); if (softwareLV == storedLV && datanodeVer.getCTime() == namenodeVer.getCTime()) { LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true"); return true; } // check #4 long absSoftwareLV = Math.abs((long)softwareLV); long absStoredLV = Math.abs((long)storedLV); if (absSoftwareLV > absStoredLV || (softwareLV == storedLV && datanodeVer.getCTime() < namenodeVer.getCTime())) { LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true"); return true; } // check #5 LOG.info("default case: isVersionCompatible=false"); return false; }