Java Code Examples for org.apache.hadoop.hdfs.protocol.LayoutVersion

The following examples show how to use org.apache.hadoop.hdfs.protocol.LayoutVersion. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: FSEditLogLoader.java    License: Apache License 2.0 6 votes vote down vote up
private long getAndUpdateLastInodeId(long inodeIdFromOp, int logVersion,
    long lastInodeId) throws IOException {
  long inodeId = inodeIdFromOp;

  if (inodeId == INodeId.GRANDFATHER_INODE_ID) {
    if (NameNodeLayoutVersion.supports(
        LayoutVersion.Feature.ADD_INODE_ID, logVersion)) {
      throw new IOException("The layout version " + logVersion
          + " supports inodeId but gave bogus inodeId");
    }
    inodeId = fsNamesys.dir.allocateNewInodeId();
  } else {
    // need to reset lastInodeId. fsnamesys gets lastInodeId firstly from
    // fsimage but editlog captures more recent inodeId allocations
    if (inodeId > lastInodeId) {
      fsNamesys.dir.resetLastInodeId(inodeId);
    }
  }
  return inodeId;
}
 
Example 2
Source Project: hadoop   Source File: NNStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override // Storage
protected void setFieldsFromProperties(
    Properties props, StorageDirectory sd) throws IOException {
  super.setFieldsFromProperties(props, sd);
  if (layoutVersion == 0) {
    throw new IOException("NameNode directory "
                          + sd.getRoot() + " is not formatted.");
  }

  // Set Block pool ID in version with federation support
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, getLayoutVersion())) {
    String sbpid = props.getProperty("blockpoolID");
    setBlockPoolID(sd.getRoot(), sbpid);
  }
  setDeprecatedPropertiesForUpgrade(props);
}
 
Example 3
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. "
          + "Old rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example 4
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 2) {
      throw new IOException("Incorrect data format. " + "delete operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example 5
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (length != 3) {
      throw new IOException("Incorrect data format. " + "times operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.mtime = FSImageSerialization.readLong(in);
    this.atime = FSImageSerialization.readLong(in);
  } else {
    this.mtime = readLong(in);
    this.atime = readLong(in);
  }
}
 
Example 6
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. " + "Rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  this.options = readRenameOptions(in);
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example 7
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
  this.logVersion = logVersion;
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = DataChecksum.newCrc32();
  } else {
    this.checksum = null;
  }
  // It is possible that the logVersion is actually a future layoutversion
  // during the rolling upgrade (e.g., the NN gets upgraded first). We
  // assume future layout will also support length of editlog op.
  this.supportEditLogLength = NameNodeLayoutVersion.supports(
      NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion)
      || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
  this.limiter = limiter;
  this.cache = new OpInstanceCache();
  this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT;
}
 
Example 8
Source Project: hadoop   Source File: FSImageFormat.java    License: Apache License 2.0 6 votes vote down vote up
/** Load {@link INodeFileAttributes}. */
public INodeFileAttributes loadINodeFileAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asFile();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  final long accessTime = in.readLong();
  
  final short replication = namesystem.getBlockManager().adjustReplication(
      in.readShort());
  final long preferredBlockSize = in.readLong();

  return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
      accessTime, replication, preferredBlockSize, (byte) 0, null);
}
 
Example 9
Source Project: hadoop   Source File: FSImageFormat.java    License: Apache License 2.0 6 votes vote down vote up
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asDirectory();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  
  // Read quotas: quota by storage type does not need to be processed below.
  // It is handled only in protobuf based FsImagePBINode class for newer
  // fsImages. Tools using this class such as legacy-mode of offline image viewer
  // should only load legacy FSImages without newer features.
  final long nsQuota = in.readLong();
  final long dsQuota = in.readLong();

  return nsQuota == -1L && dsQuota == -1L ? new INodeDirectoryAttributes.SnapshotCopy(
      name, permissions, null, modificationTime, null)
    : new INodeDirectoryAttributes.CopyWithQuota(name, permissions,
        null, modificationTime, nsQuota, dsQuota, null, null);
}
 
Example 10
Source Project: hadoop   Source File: BlockPoolSliceStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir.
 * 
 * If the directory is not empty report an error; Otherwise remove the
 * directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion)
      && detachDir.exists() && detachDir.isDirectory()) {

    if (FileUtil.list(detachDir).length != 0) {
      throw new IOException("Detached directory " + detachDir
          + " is not empty. Please manually move each file under this "
          + "directory to the finalized directory if the finalized "
          + "directory tree does not have the file.");
    } else if (!detachDir.delete()) {
      throw new IOException("Cannot remove directory " + detachDir);
    }
  }
}
 
Example 11
Source Project: hadoop   Source File: DataStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void setPropertiesFromFields(Properties props, 
                         StorageDirectory sd 
                         ) throws IOException {
  props.setProperty("storageType", storageType.toString());
  props.setProperty("clusterID", clusterID);
  props.setProperty("cTime", String.valueOf(cTime));
  props.setProperty("layoutVersion", String.valueOf(layoutVersion));
  props.setProperty("storageID", sd.getStorageUuid());

  String datanodeUuid = getDatanodeUuid();
  if (datanodeUuid != null) {
    props.setProperty("datanodeUuid", datanodeUuid);
  }

  // Set NamespaceID in version before federation
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, layoutVersion)) {
    props.setProperty("namespaceID", String.valueOf(namespaceID));
  }
}
 
Example 12
Source Project: hadoop   Source File: DataStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir. 
 * 
 * If the directory is not empty report an error; 
 * Otherwise remove the directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion) &&
      detachDir.exists() && detachDir.isDirectory() ) {
    
      if (FileUtil.list(detachDir).length != 0 ) {
        throw new IOException("Detached directory " + detachDir +
            " is not empty. Please manually move each file under this " +
            "directory to the finalized directory if the finalized " +
            "directory tree does not have the file.");
      } else if (!detachDir.delete()) {
        throw new IOException("Cannot remove directory " + detachDir);
      }
  }
}
 
Example 13
Source Project: big-c   Source File: FSEditLogLoader.java    License: Apache License 2.0 6 votes vote down vote up
private long getAndUpdateLastInodeId(long inodeIdFromOp, int logVersion,
    long lastInodeId) throws IOException {
  long inodeId = inodeIdFromOp;

  if (inodeId == INodeId.GRANDFATHER_INODE_ID) {
    if (NameNodeLayoutVersion.supports(
        LayoutVersion.Feature.ADD_INODE_ID, logVersion)) {
      throw new IOException("The layout version " + logVersion
          + " supports inodeId but gave bogus inodeId");
    }
    inodeId = fsNamesys.dir.allocateNewInodeId();
  } else {
    // need to reset lastInodeId. fsnamesys gets lastInodeId firstly from
    // fsimage but editlog captures more recent inodeId allocations
    if (inodeId > lastInodeId) {
      fsNamesys.dir.resetLastInodeId(inodeId);
    }
  }
  return inodeId;
}
 
Example 14
Source Project: big-c   Source File: NNStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override // Storage
protected void setFieldsFromProperties(
    Properties props, StorageDirectory sd) throws IOException {
  super.setFieldsFromProperties(props, sd);
  if (layoutVersion == 0) {
    throw new IOException("NameNode directory "
                          + sd.getRoot() + " is not formatted.");
  }

  // Set Block pool ID in version with federation support
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, getLayoutVersion())) {
    String sbpid = props.getProperty("blockpoolID");
    setBlockPoolID(sd.getRoot(), sbpid);
  }
  setDeprecatedPropertiesForUpgrade(props);
}
 
Example 15
Source Project: big-c   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. "
          + "Old rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example 16
Source Project: big-c   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 2) {
      throw new IOException("Incorrect data format. " + "delete operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example 17
Source Project: big-c   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (length != 3) {
      throw new IOException("Incorrect data format. " + "times operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.mtime = FSImageSerialization.readLong(in);
    this.atime = FSImageSerialization.readLong(in);
  } else {
    this.mtime = readLong(in);
    this.atime = readLong(in);
  }
}
 
Example 18
Source Project: big-c   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. " + "Rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  this.options = readRenameOptions(in);
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example 19
Source Project: big-c   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
  this.logVersion = logVersion;
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = DataChecksum.newCrc32();
  } else {
    this.checksum = null;
  }
  // It is possible that the logVersion is actually a future layoutversion
  // during the rolling upgrade (e.g., the NN gets upgraded first). We
  // assume future layout will also support length of editlog op.
  this.supportEditLogLength = NameNodeLayoutVersion.supports(
      NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion)
      || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
  this.limiter = limiter;
  this.cache = new OpInstanceCache();
  this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT;
}
 
Example 20
Source Project: big-c   Source File: FSImageFormat.java    License: Apache License 2.0 6 votes vote down vote up
/** Load {@link INodeFileAttributes}. */
public INodeFileAttributes loadINodeFileAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asFile();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  final long accessTime = in.readLong();
  
  final short replication = namesystem.getBlockManager().adjustReplication(
      in.readShort());
  final long preferredBlockSize = in.readLong();

  return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
      accessTime, replication, preferredBlockSize, (byte) 0, null);
}
 
Example 21
Source Project: big-c   Source File: FSImageFormat.java    License: Apache License 2.0 6 votes vote down vote up
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asDirectory();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  
  // Read quotas: quota by storage type does not need to be processed below.
  // It is handled only in protobuf based FsImagePBINode class for newer
  // fsImages. Tools using this class such as legacy-mode of offline image viewer
  // should only load legacy FSImages without newer features.
  final long nsQuota = in.readLong();
  final long dsQuota = in.readLong();

  return nsQuota == -1L && dsQuota == -1L ? new INodeDirectoryAttributes.SnapshotCopy(
      name, permissions, null, modificationTime, null)
    : new INodeDirectoryAttributes.CopyWithQuota(name, permissions,
        null, modificationTime, nsQuota, dsQuota, null, null);
}
 
Example 22
Source Project: big-c   Source File: BlockPoolSliceStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir.
 * 
 * If the directory is not empty report an error; Otherwise remove the
 * directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion)
      && detachDir.exists() && detachDir.isDirectory()) {

    if (FileUtil.list(detachDir).length != 0) {
      throw new IOException("Detached directory " + detachDir
          + " is not empty. Please manually move each file under this "
          + "directory to the finalized directory if the finalized "
          + "directory tree does not have the file.");
    } else if (!detachDir.delete()) {
      throw new IOException("Cannot remove directory " + detachDir);
    }
  }
}
 
Example 23
Source Project: big-c   Source File: DataStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void setPropertiesFromFields(Properties props, 
                         StorageDirectory sd 
                         ) throws IOException {
  props.setProperty("storageType", storageType.toString());
  props.setProperty("clusterID", clusterID);
  props.setProperty("cTime", String.valueOf(cTime));
  props.setProperty("layoutVersion", String.valueOf(layoutVersion));
  props.setProperty("storageID", sd.getStorageUuid());

  String datanodeUuid = getDatanodeUuid();
  if (datanodeUuid != null) {
    props.setProperty("datanodeUuid", datanodeUuid);
  }

  // Set NamespaceID in version before federation
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, layoutVersion)) {
    props.setProperty("namespaceID", String.valueOf(namespaceID));
  }
}
 
Example 24
Source Project: big-c   Source File: DataStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir. 
 * 
 * If the directory is not empty report an error; 
 * Otherwise remove the directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion) &&
      detachDir.exists() && detachDir.isDirectory() ) {
    
      if (FileUtil.list(detachDir).length != 0 ) {
        throw new IOException("Detached directory " + detachDir +
            " is not empty. Please manually move each file under this " +
            "directory to the finalized directory if the finalized " +
            "directory tree does not have the file.");
      } else if (!detachDir.delete()) {
        throw new IOException("Cannot remove directory " + detachDir);
      }
  }
}
 
Example 25
Source Project: RDFS   Source File: FSEditLogOp.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
@SuppressWarnings("deprecation")
public Reader(DataInputStream in, int logVersion) {
  this.logVersion = logVersion;
  if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = FSEditLog.getChecksumForRead();
  } else {
    this.checksum = null;
  }

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
}
 
Example 26
Source Project: hadoop   Source File: NNStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write version file into the storage directory.
 *
 * The version file should always be written last.
 * Missing or corrupted version file indicates that
 * the checkpoint is not valid.
 *
 * @param sd storage directory
 * @throws IOException
 */
@Override // Storage
protected void setPropertiesFromFields(Properties props,
                         StorageDirectory sd
                         ) throws IOException {
  super.setPropertiesFromFields(props, sd);
  // Set blockpoolID in version with federation support
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, getLayoutVersion())) {
    props.setProperty("blockpoolID", blockpoolID);
  }
}
 
Example 27
Source Project: hadoop   Source File: NNStorage.java    License: Apache License 2.0 5 votes vote down vote up
/** 
 * Processes the startup options for the clusterid and blockpoolid 
 * for the upgrade. 
 * @param startOpt Startup options 
 * @param layoutVersion Layout version for the upgrade 
 * @throws IOException
 */
void processStartupOptionsForUpgrade(StartupOption startOpt, int layoutVersion)
    throws IOException {
  if (startOpt == StartupOption.UPGRADE || startOpt == StartupOption.UPGRADEONLY) {
    // If upgrade from a release that does not support federation,
    // if clusterId is provided in the startupOptions use it.
    // Else generate a new cluster ID      
    if (!NameNodeLayoutVersion.supports(
        LayoutVersion.Feature.FEDERATION, layoutVersion)) {
      if (startOpt.getClusterId() == null) {
        startOpt.setClusterId(newClusterID());
      }
      setClusterID(startOpt.getClusterId());
      setBlockPoolID(newBlockPoolID());
    } else {
      // Upgrade from one version of federation to another supported
      // version of federation doesn't require clusterID.
      // Warn the user if the current clusterid didn't match with the input
      // clusterid.
      if (startOpt.getClusterId() != null
          && !startOpt.getClusterId().equals(getClusterID())) {
        LOG.warn("Clusterid mismatch - current clusterid: " + getClusterID()
            + ", Ignoring given clusterid: " + startOpt.getClusterId());
      }
    }
    LOG.info("Using clusterid: " + getClusterID());
  }
}
 
Example 28
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 5 votes vote down vote up
void readRpcIds(DataInputStream in, int logVersion)
    throws IOException {
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_SUPPORT_RETRYCACHE, logVersion)) {
    this.rpcClientId = FSImageSerialization.readBytes(in);
    this.rpcCallId = FSImageSerialization.readInt(in);
  }
}
 
Example 29
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  this.path = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.replication = FSImageSerialization.readShort(in);
  } else {
    this.replication = readShort(in);
  }
}
 
Example 30
Source Project: hadoop   Source File: FSEditLogOp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 4) {
      throw new IOException("Incorrect data format. "
          + "symlink operation.");
    }
  }
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.ADD_INODE_ID, logVersion)) {
    this.inodeId = FSImageSerialization.readLong(in);
  } else {
    // This id should be updated when the editLogOp is applied
    this.inodeId = INodeId.GRANDFATHER_INODE_ID;
  }
  this.path = FSImageSerialization.readString(in);
  this.value = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.mtime = FSImageSerialization.readLong(in);
    this.atime = FSImageSerialization.readLong(in);
  } else {
    this.mtime = readLong(in);
    this.atime = readLong(in);
  }
  this.permissionStatus = PermissionStatus.read(in);
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}