org.apache.hadoop.hdfs.protocol.LayoutVersion Java Examples

The following examples show how to use org.apache.hadoop.hdfs.protocol.LayoutVersion. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. "
          + "Old rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #2
Source File: BlockPoolSliceStorage.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir.
 * 
 * If the directory is not empty report an error; Otherwise remove the
 * directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion)
      && detachDir.exists() && detachDir.isDirectory()) {

    if (FileUtil.list(detachDir).length != 0) {
      throw new IOException("Detached directory " + detachDir
          + " is not empty. Please manually move each file under this "
          + "directory to the finalized directory if the finalized "
          + "directory tree does not have the file.");
    } else if (!detachDir.delete()) {
      throw new IOException("Cannot remove directory " + detachDir);
    }
  }
}
 
Example #3
Source File: NNStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override // Storage
protected void setFieldsFromProperties(
    Properties props, StorageDirectory sd) throws IOException {
  super.setFieldsFromProperties(props, sd);
  if (layoutVersion == 0) {
    throw new IOException("NameNode directory "
                          + sd.getRoot() + " is not formatted.");
  }

  // Set Block pool ID in version with federation support
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, getLayoutVersion())) {
    String sbpid = props.getProperty("blockpoolID");
    setBlockPoolID(sd.getRoot(), sbpid);
  }
  setDeprecatedPropertiesForUpgrade(props);
}
 
Example #4
Source File: FSEditLogOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. "
          + "Old rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #5
Source File: FSEditLogOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 2) {
      throw new IOException("Incorrect data format. " + "delete operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #6
Source File: DataStorage.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir. 
 * 
 * If the directory is not empty report an error; 
 * Otherwise remove the directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion) &&
      detachDir.exists() && detachDir.isDirectory() ) {
    
      if (FileUtil.list(detachDir).length != 0 ) {
        throw new IOException("Detached directory " + detachDir +
            " is not empty. Please manually move each file under this " +
            "directory to the finalized directory if the finalized " +
            "directory tree does not have the file.");
      } else if (!detachDir.delete()) {
        throw new IOException("Cannot remove directory " + detachDir);
      }
  }
}
 
Example #7
Source File: FSEditLogOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (length != 3) {
      throw new IOException("Incorrect data format. " + "times operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.mtime = FSImageSerialization.readLong(in);
    this.atime = FSImageSerialization.readLong(in);
  } else {
    this.mtime = readLong(in);
    this.atime = readLong(in);
  }
}
 
Example #8
Source File: DataStorage.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void setPropertiesFromFields(Properties props, 
                         StorageDirectory sd 
                         ) throws IOException {
  props.setProperty("storageType", storageType.toString());
  props.setProperty("clusterID", clusterID);
  props.setProperty("cTime", String.valueOf(cTime));
  props.setProperty("layoutVersion", String.valueOf(layoutVersion));
  props.setProperty("storageID", sd.getStorageUuid());

  String datanodeUuid = getDatanodeUuid();
  if (datanodeUuid != null) {
    props.setProperty("datanodeUuid", datanodeUuid);
  }

  // Set NamespaceID in version before federation
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, layoutVersion)) {
    props.setProperty("namespaceID", String.valueOf(namespaceID));
  }
}
 
Example #9
Source File: FSEditLogOp.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
@SuppressWarnings("deprecation")
public Reader(DataInputStream in, int logVersion) {
  this.logVersion = logVersion;
  if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = FSEditLog.getChecksumForRead();
  } else {
    this.checksum = null;
  }

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
}
 
Example #10
Source File: FSEditLogOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. " + "Rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  this.options = readRenameOptions(in);
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #11
Source File: FSEditLogOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
  this.logVersion = logVersion;
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = DataChecksum.newCrc32();
  } else {
    this.checksum = null;
  }
  // It is possible that the logVersion is actually a future layoutversion
  // during the rolling upgrade (e.g., the NN gets upgraded first). We
  // assume future layout will also support length of editlog op.
  this.supportEditLogLength = NameNodeLayoutVersion.supports(
      NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion)
      || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
  this.limiter = limiter;
  this.cache = new OpInstanceCache();
  this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT;
}
 
Example #12
Source File: FSImageFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asDirectory();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  
  // Read quotas: quota by storage type does not need to be processed below.
  // It is handled only in protobuf based FsImagePBINode class for newer
  // fsImages. Tools using this class such as legacy-mode of offline image viewer
  // should only load legacy FSImages without newer features.
  final long nsQuota = in.readLong();
  final long dsQuota = in.readLong();

  return nsQuota == -1L && dsQuota == -1L ? new INodeDirectoryAttributes.SnapshotCopy(
      name, permissions, null, modificationTime, null)
    : new INodeDirectoryAttributes.CopyWithQuota(name, permissions,
        null, modificationTime, nsQuota, dsQuota, null, null);
}
 
Example #13
Source File: FSImageFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/** Load {@link INodeFileAttributes}. */
public INodeFileAttributes loadINodeFileAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asFile();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  final long accessTime = in.readLong();
  
  final short replication = namesystem.getBlockManager().adjustReplication(
      in.readShort());
  final long preferredBlockSize = in.readLong();

  return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
      accessTime, replication, preferredBlockSize, (byte) 0, null);
}
 
Example #14
Source File: FSImageFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
/** Load {@link INodeFileAttributes}. */
public INodeFileAttributes loadINodeFileAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asFile();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  final long accessTime = in.readLong();
  
  final short replication = namesystem.getBlockManager().adjustReplication(
      in.readShort());
  final long preferredBlockSize = in.readLong();

  return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
      accessTime, replication, preferredBlockSize, (byte) 0, null);
}
 
Example #15
Source File: FSEditLogLoader.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private long getAndUpdateLastInodeId(long inodeIdFromOp, int logVersion,
    long lastInodeId) throws IOException {
  long inodeId = inodeIdFromOp;

  if (inodeId == INodeId.GRANDFATHER_INODE_ID) {
    if (NameNodeLayoutVersion.supports(
        LayoutVersion.Feature.ADD_INODE_ID, logVersion)) {
      throw new IOException("The layout version " + logVersion
          + " supports inodeId but gave bogus inodeId");
    }
    inodeId = fsNamesys.dir.allocateNewInodeId();
  } else {
    // need to reset lastInodeId. fsnamesys gets lastInodeId firstly from
    // fsimage but editlog captures more recent inodeId allocations
    if (inodeId > lastInodeId) {
      fsNamesys.dir.resetLastInodeId(inodeId);
    }
  }
  return inodeId;
}
 
Example #16
Source File: NNStorage.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override // Storage
protected void setFieldsFromProperties(
    Properties props, StorageDirectory sd) throws IOException {
  super.setFieldsFromProperties(props, sd);
  if (layoutVersion == 0) {
    throw new IOException("NameNode directory "
                          + sd.getRoot() + " is not formatted.");
  }

  // Set Block pool ID in version with federation support
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, getLayoutVersion())) {
    String sbpid = props.getProperty("blockpoolID");
    setBlockPoolID(sd.getRoot(), sbpid);
  }
  setDeprecatedPropertiesForUpgrade(props);
}
 
Example #17
Source File: FSEditLogLoader.java    From big-c with Apache License 2.0 6 votes vote down vote up
private long getAndUpdateLastInodeId(long inodeIdFromOp, int logVersion,
    long lastInodeId) throws IOException {
  long inodeId = inodeIdFromOp;

  if (inodeId == INodeId.GRANDFATHER_INODE_ID) {
    if (NameNodeLayoutVersion.supports(
        LayoutVersion.Feature.ADD_INODE_ID, logVersion)) {
      throw new IOException("The layout version " + logVersion
          + " supports inodeId but gave bogus inodeId");
    }
    inodeId = fsNamesys.dir.allocateNewInodeId();
  } else {
    // need to reset lastInodeId. fsnamesys gets lastInodeId firstly from
    // fsimage but editlog captures more recent inodeId allocations
    if (inodeId > lastInodeId) {
      fsNamesys.dir.resetLastInodeId(inodeId);
    }
  }
  return inodeId;
}
 
Example #18
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 2) {
      throw new IOException("Incorrect data format. " + "delete operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #19
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (length != 3) {
      throw new IOException("Incorrect data format. " + "times operation.");
    }
  }
  this.path = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.mtime = FSImageSerialization.readLong(in);
    this.atime = FSImageSerialization.readLong(in);
  } else {
    this.mtime = readLong(in);
    this.atime = readLong(in);
  }
}
 
Example #20
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 3) {
      throw new IOException("Incorrect data format. " + "Rename operation.");
    }
  }
  this.src = FSImageSerialization.readString(in);
  this.dst = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.timestamp = FSImageSerialization.readLong(in);
  } else {
    this.timestamp = readLong(in);
  }
  this.options = readRenameOptions(in);
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #21
Source File: DataStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir. 
 * 
 * If the directory is not empty report an error; 
 * Otherwise remove the directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion) &&
      detachDir.exists() && detachDir.isDirectory() ) {
    
      if (FileUtil.list(detachDir).length != 0 ) {
        throw new IOException("Detached directory " + detachDir +
            " is not empty. Please manually move each file under this " +
            "directory to the finalized directory if the finalized " +
            "directory tree does not have the file.");
      } else if (!detachDir.delete()) {
        throw new IOException("Cannot remove directory " + detachDir);
      }
  }
}
 
Example #22
Source File: DataStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void setPropertiesFromFields(Properties props, 
                         StorageDirectory sd 
                         ) throws IOException {
  props.setProperty("storageType", storageType.toString());
  props.setProperty("clusterID", clusterID);
  props.setProperty("cTime", String.valueOf(cTime));
  props.setProperty("layoutVersion", String.valueOf(layoutVersion));
  props.setProperty("storageID", sd.getStorageUuid());

  String datanodeUuid = getDatanodeUuid();
  if (datanodeUuid != null) {
    props.setProperty("datanodeUuid", datanodeUuid);
  }

  // Set NamespaceID in version before federation
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, layoutVersion)) {
    props.setProperty("namespaceID", String.valueOf(namespaceID));
  }
}
 
Example #23
Source File: BlockPoolSliceStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Cleanup the detachDir.
 * 
 * If the directory is not empty report an error; Otherwise remove the
 * directory.
 * 
 * @param detachDir detach directory
 * @throws IOException if the directory is not empty or it can not be removed
 */
private void cleanupDetachDir(File detachDir) throws IOException {
  if (!DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.APPEND_RBW_DIR, layoutVersion)
      && detachDir.exists() && detachDir.isDirectory()) {

    if (FileUtil.list(detachDir).length != 0) {
      throw new IOException("Detached directory " + detachDir
          + " is not empty. Please manually move each file under this "
          + "directory to the finalized directory if the finalized "
          + "directory tree does not have the file.");
    } else if (!detachDir.delete()) {
      throw new IOException("Cannot remove directory " + detachDir);
    }
  }
}
 
Example #24
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
  this.logVersion = logVersion;
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = DataChecksum.newCrc32();
  } else {
    this.checksum = null;
  }
  // It is possible that the logVersion is actually a future layoutversion
  // during the rolling upgrade (e.g., the NN gets upgraded first). We
  // assume future layout will also support length of editlog op.
  this.supportEditLogLength = NameNodeLayoutVersion.supports(
      NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion)
      || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
  this.limiter = limiter;
  this.cache = new OpInstanceCache();
  this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT;
}
 
Example #25
Source File: FSImageFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
    throws IOException {
  final int layoutVersion = getLayoutVersion();
  
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
    return loadINodeWithLocalName(true, in, false).asDirectory();
  }
  
  final byte[] name = FSImageSerialization.readLocalName(in);
  final PermissionStatus permissions = PermissionStatus.read(in);
  final long modificationTime = in.readLong();
  
  // Read quotas: quota by storage type does not need to be processed below.
  // It is handled only in protobuf based FsImagePBINode class for newer
  // fsImages. Tools using this class such as legacy-mode of offline image viewer
  // should only load legacy FSImages without newer features.
  final long nsQuota = in.readLong();
  final long dsQuota = in.readLong();

  return nsQuota == -1L && dsQuota == -1L ? new INodeDirectoryAttributes.SnapshotCopy(
      name, permissions, null, modificationTime, null)
    : new INodeDirectoryAttributes.CopyWithQuota(name, permissions,
        null, modificationTime, nsQuota, dsQuota, null, null);
}
 
Example #26
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  if (!NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.length = in.readInt();
    if (this.length != 4) {
      throw new IOException("Incorrect data format. "
          + "symlink operation.");
    }
  }
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.ADD_INODE_ID, logVersion)) {
    this.inodeId = FSImageSerialization.readLong(in);
  } else {
    // This id should be updated when the editLogOp is applied
    this.inodeId = INodeId.GRANDFATHER_INODE_ID;
  }
  this.path = FSImageSerialization.readString(in);
  this.value = FSImageSerialization.readString(in);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.mtime = FSImageSerialization.readLong(in);
    this.atime = FSImageSerialization.readLong(in);
  } else {
    this.mtime = readLong(in);
    this.atime = readLong(in);
  }
  this.permissionStatus = PermissionStatus.read(in);
  
  // read RPC ids if necessary
  readRpcIds(in, logVersion);
}
 
Example #27
Source File: FSEditLogOp.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Read an operation from the input stream.
 * 
 * Note that the objects returned from this method may be re-used by future
 * calls to the same method.
 * 
 * @return the operation read from the stream, or null at the end of the file
 * @throws IOException on error.
 */
public FSEditLogOp readOp() throws IOException {
  if (checksum != null) {
    checksum.reset();
  }

  byte opCodeByte;
  try {
    opCodeByte = in.readByte();
  } catch (EOFException eof) {
    // EOF at an opcode boundary is expected
    return null;
  }

  FSEditLogOpCodes opCode = FSEditLogOpCodes.fromByte(opCodeByte);

  if (opCode == OP_INVALID) {
    LOG.info("Reached the end of the edit file...");
    return null;
  }

  FSEditLogOp op = opInstances.get().get(opCode);
  if (op == null) {
    throw new IOException("Read invalid opcode " + opCode);
  }

  if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
    // Read the txid
    op.setTransactionId(in.readLong());
  }

  op.readFields(in, logVersion);
  if (checksum != null)
    readChecksum = checksum.getValue();
  validateChecksum(in, checksum, op.txid);
  return op;
}
 
Example #28
Source File: ImageLoaderCurrent.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
  * Process an INode
  * 
  * @param in image stream
  * @param v visitor
  * @param skipBlocks skip blocks or not
  * @param parentName the name of its parent node
  * @throws IOException
  */
private void processINode(DataInputStream in, ImageVisitor v,
    boolean skipBlocks, String parentName) throws IOException {
  v.visitEnclosingElement(ImageElement.INODE);
  String pathName = FSImageSerialization.readString(in);
  if (parentName != null) {  // local name
    pathName = "/" + pathName;
    if (!"/".equals(parentName)) { // children of non-root directory
      pathName = parentName + pathName;
    }
  }

  v.visit(ImageElement.INODE_PATH, pathName);
  v.visit(ImageElement.REPLICATION, in.readShort());
  v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
  if(LayoutVersion.supports(Feature.FILE_ACCESS_TIME, imageVersion))
    v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
  v.visit(ImageElement.BLOCK_SIZE, in.readLong());
  int numBlocks = in.readInt();

  processBlocks(in, v, numBlocks, skipBlocks);

  // File or directory
  if (numBlocks > 0 || numBlocks == -1) {
    v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
    if (LayoutVersion.supports(Feature.DISKSPACE_QUOTA, imageVersion))
      v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
  }
  if (numBlocks == -2) {
    v.visit(ImageElement.SYMLINK, Text.readString(in));
  }

  processPermission(in, v);
  v.leaveEnclosingElement(); // INode
}
 
Example #29
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  this.path = FSImageSerialization.readString(in);
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
    this.replication = FSImageSerialization.readShort(in);
  } else {
    this.replication = readShort(in);
  }
}
 
Example #30
Source File: FSEditLogOp.java    From big-c with Apache License 2.0 5 votes vote down vote up
void readRpcIds(DataInputStream in, int logVersion)
    throws IOException {
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITLOG_SUPPORT_RETRYCACHE, logVersion)) {
    this.rpcClientId = FSImageSerialization.readBytes(in);
    this.rpcCallId = FSImageSerialization.readInt(in);
  }
}