org.apache.hadoop.yarn.server.records.Version Java Examples

@VisibleForTesting
Version loadVersion() throws IOException {
  try {
    byte[] data = db.get(TIMELINE_STATE_STORE_VERSION_KEY);
    // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
    if (data == null || data.length == 0) {
      return getCurrentVersion();
    }
    Version version =
        new VersionPBImpl(
            YarnServerCommonProtos.VersionProto.parseFrom(data));
    return version;
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

/**
 * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of DB schema is a major upgrade, and any
 *    compatible change of DB schema is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade shuffle info or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded state DB schema version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing state DB schedma version info " + getCurrentVersion());
    storeVersion();
  } else {
    throw new IOException(
      "Incompatible version for state DB schema: expecting DB schema version " 
          + getCurrentVersion() + ", but loading version " + loadedVersion);
  }
}
 

/**
 * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of DB schema is a major upgrade, and any
 *    compatible change of DB schema is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade shuffle info or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded state DB schema version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing state DB schedma version info " + getCurrentVersion());
    storeVersion();
  } else {
    throw new IOException(
      "Incompatible version for state DB schema: expecting DB schema version " 
          + getCurrentVersion() + ", but loading version " + loadedVersion);
  }
}
 

/**
 * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of state-store is a major upgrade, and any
 *    compatible change of state-store is a minor upgrade.
 * 3) If theres's no version, treat it as CURRENT_VERSION_INFO.
 * 4) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 5) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade RM state.
 */
public void checkVersion() throws Exception {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded RM state version info " + loadedVersion);
  if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  // if there is no version info, treat it as CURRENT_VERSION_INFO;
  if (loadedVersion == null) {
    loadedVersion = getCurrentVersion();
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing RM state version info " + getCurrentVersion());
    storeVersion();
  } else {
    throw new RMStateVersionIncompatibleException(
      "Expecting RM state version " + getCurrentVersion()
          + ", but loading version " + loadedVersion);
  }
}
 

/**
 * 1) Versioning timeline store: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of TS-store is a major upgrade, and any
 *    compatible change of TS-store is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade timeline store or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded timeline store version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing timeline store version info " + getCurrentVersion());
    dbStoreVersion(CURRENT_VERSION_INFO);
  } else {
    String incompatibleMessage = 
        "Incompatible version for timeline store: expecting version " 
            + getCurrentVersion() + ", but loading version " + loadedVersion;
    LOG.fatal(incompatibleMessage);
    throw new IOException(incompatibleMessage);
  }
}
 

@VisibleForTesting
Version loadVersion() throws IOException {
  try {
    byte[] data = db.get(TIMELINE_STATE_STORE_VERSION_KEY);
    // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
    if (data == null || data.length == 0) {
      return getCurrentVersion();
    }
    Version version =
        new VersionPBImpl(
            YarnServerCommonProtos.VersionProto.parseFrom(data));
    return version;
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

/**
 * 1) Versioning timeline store: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of TS-store is a major upgrade, and any
 *    compatible change of TS-store is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade timeline store or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded timeline store version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing timeline store version info " + getCurrentVersion());
    dbStoreVersion(CURRENT_VERSION_INFO);
  } else {
    String incompatibleMessage = 
        "Incompatible version for timeline store: expecting version " 
            + getCurrentVersion() + ", but loading version " + loadedVersion;
    LOG.fatal(incompatibleMessage);
    throw new IOException(incompatibleMessage);
  }
}
 

/**
 * 1) Versioning timeline state store:
 * major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of TS-store is a major upgrade, and any
 * compatible change of TS-store is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 * overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 * throw exception and indicate user to use a separate upgrade tool to
 * upgrade timeline store or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded timeline state store version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing timeline state store version info " + getCurrentVersion());
    storeVersion(CURRENT_VERSION_INFO);
  } else {
    String incompatibleMessage =
        "Incompatible version for timeline state store: expecting version "
            + getCurrentVersion() + ", but loading version " + loadedVersion;
    LOG.fatal(incompatibleMessage);
    throw new IOException(incompatibleMessage);
  }
}
 

/**
 * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of state-store is a major upgrade, and any
 *    compatible change of state-store is a minor upgrade.
 * 3) If theres's no version, treat it as CURRENT_VERSION_INFO.
 * 4) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 5) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade RM state.
 */
public void checkVersion() throws Exception {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded RM state version info " + loadedVersion);
  if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  // if there is no version info, treat it as CURRENT_VERSION_INFO;
  if (loadedVersion == null) {
    loadedVersion = getCurrentVersion();
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing RM state version info " + getCurrentVersion());
    storeVersion();
  } else {
    throw new RMStateVersionIncompatibleException(
      "Expecting RM state version " + getCurrentVersion()
          + ", but loading version " + loadedVersion);
  }
}
 

/**
 * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of state-store is a major upgrade, and any
 *    compatible change of state-store is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 *    overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 *    throw exception and indicate user to use a separate upgrade tool to
 *    upgrade state or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded state version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing state version info " + getCurrentVersion());
    storeVersion();
  } else {
    throw new IOException(
      "Incompatible version for state: expecting state version "
          + getCurrentVersion() + ", but loading version " + loadedVersion);
  }
}
 

/**
 * 1) Versioning timeline state store:
 * major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
 * 2) Any incompatible change of TS-store is a major upgrade, and any
 * compatible change of TS-store is a minor upgrade.
 * 3) Within a minor upgrade, say 1.1 to 1.2:
 * overwrite the version info and proceed as normal.
 * 4) Within a major upgrade, say 1.2 to 2.0:
 * throw exception and indicate user to use a separate upgrade tool to
 * upgrade timeline store or remove incompatible old state.
 */
private void checkVersion() throws IOException {
  Version loadedVersion = loadVersion();
  LOG.info("Loaded timeline state store version info " + loadedVersion);
  if (loadedVersion.equals(getCurrentVersion())) {
    return;
  }
  if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
    LOG.info("Storing timeline state store version info " + getCurrentVersion());
    storeVersion(CURRENT_VERSION_INFO);
  } else {
    String incompatibleMessage =
        "Incompatible version for timeline state store: expecting version "
            + getCurrentVersion() + ", but loading version " + loadedVersion;
    LOG.fatal(incompatibleMessage);
    throw new IOException(incompatibleMessage);
  }
}
 

@VisibleForTesting
void storeVersion(Version state) throws IOException {
  byte[] data =
      ((VersionPBImpl) state).getProto().toByteArray();
  try {
    db.put(TIMELINE_STATE_STORE_VERSION_KEY, data);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@VisibleForTesting
Version loadVersion() throws IOException {
  byte[] data = stateDb.get(bytes(STATE_DB_SCHEMA_VERSION_KEY));
  // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
  if (data == null || data.length == 0) {
    return getCurrentVersion();
  }
  Version version =
      new VersionPBImpl(VersionProto.parseFrom(data));
  return version;
}
 

@Test
public void testCheckVersion() throws IOException {
  LeveldbTimelineStateStore store =
      initAndStartTimelineServiceStateStoreService();
  // default version
  Version defaultVersion = store.getCurrentVersion();
  Assert.assertEquals(defaultVersion, store.loadVersion());

  // compatible version
  Version compatibleVersion =
      Version.newInstance(defaultVersion.getMajorVersion(),
          defaultVersion.getMinorVersion() + 2);
  store.storeVersion(compatibleVersion);
  Assert.assertEquals(compatibleVersion, store.loadVersion());
  store.stop();

  // overwrite the compatible version
  store = initAndStartTimelineServiceStateStoreService();
  Assert.assertEquals(defaultVersion, store.loadVersion());

  // incompatible version
  Version incompatibleVersion =
      Version.newInstance(defaultVersion.getMajorVersion() + 1,
          defaultVersion.getMinorVersion());
  store.storeVersion(incompatibleVersion);
  store.stop();

  try {
    initAndStartTimelineServiceStateStoreService();
    Assert.fail("Incompatible version, should expect fail here.");
  } catch (ServiceStateException e) {
    Assert.assertTrue("Exception message mismatch",
        e.getMessage().contains("Incompatible version for timeline state store"));
  }
}
 

Version loadVersion() throws IOException {
  byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
  // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
  if (data == null || data.length == 0) {
    return getCurrentVersion();
  }
  Version version =
      new VersionPBImpl(VersionProto.parseFrom(data));
  return version;
}
 

void dbStoreVersion(Version state) throws IOException {
  String key = DB_SCHEMA_VERSION_KEY;
  byte[] data =
      ((VersionPBImpl) state).getProto().toByteArray();
  try {
    db.put(bytes(key), data);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

private void dbStoreVersion(Version state) throws IOException {
  String key = TIMELINE_STORE_VERSION_KEY;
  byte[] data = 
      ((VersionPBImpl) state).getProto().toByteArray();
  try {
    db.put(bytes(key), data);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

Version loadVersion() throws IOException {
  try {
    byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY));
    // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
    if (data == null || data.length == 0) {
      return getCurrentVersion();
    }
    Version version =
        new VersionPBImpl(VersionProto.parseFrom(data));
    return version;
  } catch(DBException e) {
    throw new IOException(e);    	
  }
}
 

Version loadVersion() throws IOException {
  try {
    byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY));
    // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
    if (data == null || data.length == 0) {
      return getCurrentVersion();
    }
    Version version =
        new VersionPBImpl(VersionProto.parseFrom(data));
    return version;
  } catch(DBException e) {
    throw new IOException(e);    	
  }
}
 

private void storeSchemaVersion(Version version) throws IOException {
  String key = STATE_DB_SCHEMA_VERSION_KEY;
  byte[] data = 
      ((VersionPBImpl) version).getProto().toByteArray();
  try {
    stateDb.put(bytes(key), data);
  } catch (DBException e) {
    throw new IOException(e.getMessage(), e);
  }
}
 

Version loadVersion() throws IOException {
  byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
  // if version is not stored previously, treat it as 1.0.
  if (data == null || data.length == 0) {
    return Version.newInstance(1, 0);
  }
  Version version =
      new VersionPBImpl(VersionProto.parseFrom(data));
  return version;
}
 

@Test
public void testCheckVersion() throws IOException {
  LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store;
  // default version
  Version defaultVersion = dbStore.getCurrentVersion();
  Assert.assertEquals(defaultVersion, dbStore.loadVersion());

  // compatible version
  Version compatibleVersion =
      Version.newInstance(defaultVersion.getMajorVersion(),
        defaultVersion.getMinorVersion() + 2);
  dbStore.storeVersion(compatibleVersion);
  Assert.assertEquals(compatibleVersion, dbStore.loadVersion());
  restartTimelineStore();
  dbStore = (LeveldbTimelineStore) store;
  // overwrite the compatible version
  Assert.assertEquals(defaultVersion, dbStore.loadVersion());

  // incompatible version
  Version incompatibleVersion =
    Version.newInstance(defaultVersion.getMajorVersion() + 1,
        defaultVersion.getMinorVersion());
  dbStore.storeVersion(incompatibleVersion);
  try {
    restartTimelineStore();
    Assert.fail("Incompatible version, should expect fail here.");
  } catch (ServiceStateException e) {
    Assert.assertTrue("Exception message mismatch", 
      e.getMessage().contains("Incompatible version for timeline store"));
  }
}
 

Version loadVersion() throws IOException {
  byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
  // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
  if (data == null || data.length == 0) {
    return getCurrentVersion();
  }
  Version version =
      new VersionPBImpl(VersionProto.parseFrom(data));
  return version;
}
 

@VisibleForTesting
void storeVersion(Version state) throws IOException {
  byte[] data =
      ((VersionPBImpl) state).getProto().toByteArray();
  try {
    db.put(TIMELINE_STATE_STORE_VERSION_KEY, data);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Test
public void testCheckVersion() throws IOException {
  // default version
  Version defaultVersion = stateStore.getCurrentVersion();
  Assert.assertEquals(defaultVersion, stateStore.loadVersion());

  // compatible version
  Version compatibleVersion =
      Version.newInstance(defaultVersion.getMajorVersion(),
        defaultVersion.getMinorVersion() + 2);
  stateStore.storeVersion(compatibleVersion);
  Assert.assertEquals(compatibleVersion, stateStore.loadVersion());
  restartStateStore();
  // overwrite the compatible version
  Assert.assertEquals(defaultVersion, stateStore.loadVersion());

  // incompatible version
  Version incompatibleVersion =
    Version.newInstance(defaultVersion.getMajorVersion() + 1,
        defaultVersion.getMinorVersion());
  stateStore.storeVersion(incompatibleVersion);
  try {
    restartStateStore();
    Assert.fail("Incompatible version, should expect fail here.");
  } catch (ServiceStateException e) {
    Assert.assertTrue("Exception message mismatch", 
      e.getMessage().contains("Incompatible version for NM state:"));
  }
}
 

void dbStoreVersion(Version state) throws IOException {
  String key = VERSION_NODE;
  byte[] data = ((VersionPBImpl) state).getProto().toByteArray();
  try {
    db.put(bytes(key), data);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

private void storeSchemaVersion(Version version) throws IOException {
  String key = STATE_DB_SCHEMA_VERSION_KEY;
  byte[] data =
      ((VersionPBImpl) version).getProto().toByteArray();
  try {
    stateDb.put(bytes(key), data);
  } catch (DBException e) {
    throw new IOException(e.getMessage(), e);
  }
}
 

@Override
protected synchronized Version loadVersion() throws Exception {
  Path versionNodePath = getNodePath(rootDirPath, VERSION_NODE);
  FileStatus status = getFileStatusWithRetries(versionNodePath);
  if (status != null) {
    byte[] data = readFileWithRetries(versionNodePath, status.getLen());
    Version version =
        new VersionPBImpl(VersionProto.parseFrom(data));
    return version;
  }
  return null;
}
 

@VisibleForTesting
Version loadVersion() throws IOException {
  byte[] data = stateDb.get(bytes(STATE_DB_SCHEMA_VERSION_KEY));
  // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
  if (data == null || data.length == 0) {
    return getCurrentVersion();
  }
  Version version =
      new VersionPBImpl(VersionProto.parseFrom(data));
  return version;
}
 

public void testCheckVersion(RMStateStoreHelper stateStoreHelper)
    throws Exception {
  RMStateStore store = stateStoreHelper.getRMStateStore();
  store.setRMDispatcher(new TestDispatcher());

  // default version
  Version defaultVersion = stateStoreHelper.getCurrentVersion();
  store.checkVersion();
  Assert.assertEquals(defaultVersion, store.loadVersion());

  // compatible version
  Version compatibleVersion =
      Version.newInstance(defaultVersion.getMajorVersion(),
        defaultVersion.getMinorVersion() + 2);
  stateStoreHelper.writeVersion(compatibleVersion);
  Assert.assertEquals(compatibleVersion, store.loadVersion());
  store.checkVersion();
  // overwrite the compatible version
  Assert.assertEquals(defaultVersion, store.loadVersion());

  // incompatible version
  Version incompatibleVersion =
      Version.newInstance(defaultVersion.getMajorVersion() + 2,
        defaultVersion.getMinorVersion());
  stateStoreHelper.writeVersion(incompatibleVersion);
  try {
    store.checkVersion();
    Assert.fail("Invalid version, should fail.");
  } catch (Throwable t) {
    Assert.assertTrue(t instanceof RMStateVersionIncompatibleException);
  }
}