Java Code Examples for org.apache.hadoop.hdfs.server.protocol.NamespaceInfo#getBlockPoolID()

The following examples show how to use org.apache.hadoop.hdfs.server.protocol.NamespaceInfo#getBlockPoolID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NNStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Format all available storage directories.
 */
public void format(NamespaceInfo nsInfo) throws IOException {
  Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 ||
      nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION,
      "Bad layout version: %s", nsInfo.getLayoutVersion());
  
  this.setStorageInfo(nsInfo);
  this.blockpoolID = nsInfo.getBlockPoolID();
  for (Iterator<StorageDirectory> it =
                         dirIterator(); it.hasNext();) {
    StorageDirectory sd = it.next();
    format(sd);
  }
}
 
Example 2
Source File: NameNodeConnector.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
                         List<Path> targetPaths, Configuration conf,
                         int maxNotChangedIterations)
    throws IOException {
  this.nameNodeUri = nameNodeUri;
  this.idPath = idPath;
  this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays
      .asList(new Path("/")) : targetPaths;
  this.maxNotChangedIterations = maxNotChangedIterations;

  this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
      NamenodeProtocol.class).getProxy();
  this.client = NameNodeProxies.createProxy(conf, nameNodeUri,
      ClientProtocol.class, fallbackToSimpleAuth).getProxy();
  this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf);

  final NamespaceInfo namespaceinfo = namenode.versionRequest();
  this.blockpoolID = namespaceinfo.getBlockPoolID();

  final FsServerDefaults defaults = fs.getServerDefaults(new Path("/"));
  this.keyManager = new KeyManager(blockpoolID, namenode,
      defaults.getEncryptDataTransfer(), conf);
  // if it is for test, we do not create the id file
  out = checkAndMarkRunning();
  if (out == null) {
    // Exit if there is another one running.
    throw new IOException("Another " + name + " is running.");
  }
}
 
Example 3
Source File: DataNode.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Initializes the {@link #data}. The initialization is done only once, when
 * handshake with the the first namenode is completed.
 */
private void initStorage(final NamespaceInfo nsInfo) throws IOException {
  final FsDatasetSpi.Factory<? extends FsDatasetSpi<?>> factory
      = FsDatasetSpi.Factory.getFactory(conf);
  
  if (!factory.isSimulated()) {
    final StartupOption startOpt = getStartupOption(conf);
    if (startOpt == null) {
      throw new IOException("Startup option not set.");
    }
    final String bpid = nsInfo.getBlockPoolID();
    //read storage info, lock data dirs and transition fs state if necessary
    synchronized (this) {
      storage.recoverTransitionRead(this, nsInfo, dataDirs, startOpt);
    }
    final StorageInfo bpStorage = storage.getBPStorage(bpid);
    LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID()
        + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion()
        + ";nsInfo=" + nsInfo + ";dnuuid=" + storage.getDatanodeUuid());
  }

  // If this is a newly formatted DataNode then assign a new DatanodeUuid.
  checkDatanodeUuid();

  synchronized(this)  {
    if (data == null) {
      data = factory.newInstance(this, storage, conf);
    }
  }
}
 
Example 4
Source File: BlockPoolSliceStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Format a block pool slice storage. 
 * @param bpSdir the block pool storage
 * @param nsInfo the name space info
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException {
  LOG.info("Formatting block pool " + blockpoolID + " directory "
      + bpSdir.getCurrentDir());
  bpSdir.clearDirectory(); // create directory
  this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
  this.cTime = nsInfo.getCTime();
  this.namespaceID = nsInfo.getNamespaceID();
  this.blockpoolID = nsInfo.getBlockPoolID();
  writeProperties(bpSdir);
}
 
Example 5
Source File: DataStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Prepare a storage directory. It creates a builder which can be used to add
 * to the volume. If the volume cannot be added, it is OK to discard the
 * builder later.
 *
 * @param datanode DataNode object.
 * @param volume the root path of a storage directory.
 * @param nsInfos an array of namespace infos.
 * @return a VolumeBuilder that holds the metadata of this storage directory
 * and can be added to DataStorage later.
 * @throws IOException if encounters I/O errors.
 *
 * Note that if there is IOException, the state of DataStorage is not modified.
 */
public VolumeBuilder prepareVolume(DataNode datanode, File volume,
    List<NamespaceInfo> nsInfos) throws IOException {
  if (containsStorageDir(volume)) {
    final String errorMessage = "Storage directory is in use";
    LOG.warn(errorMessage + ".");
    throw new IOException(errorMessage);
  }

  StorageDirectory sd = loadStorageDirectory(
      datanode, nsInfos.get(0), volume, StartupOption.HOTSWAP);
  VolumeBuilder builder =
      new VolumeBuilder(this, sd);
  for (NamespaceInfo nsInfo : nsInfos) {
    List<File> bpDataDirs = Lists.newArrayList();
    bpDataDirs.add(BlockPoolSliceStorage.getBpRoot(
        nsInfo.getBlockPoolID(), new File(volume, STORAGE_DIR_CURRENT)));
    makeBlockPoolDataDir(bpDataDirs, null);

    BlockPoolSliceStorage bpStorage;
    final String bpid = nsInfo.getBlockPoolID();
    synchronized (this) {
      bpStorage = this.bpStorageMap.get(bpid);
      if (bpStorage == null) {
        bpStorage = new BlockPoolSliceStorage(
            nsInfo.getNamespaceID(), bpid, nsInfo.getCTime(),
            nsInfo.getClusterID());
        addBlockPoolStorage(bpid, bpStorage);
      }
    }
    builder.addBpStorageDirectories(
        bpid, bpStorage.loadBpStorageDirectories(
            datanode, nsInfo, bpDataDirs, StartupOption.HOTSWAP));
  }
  return builder;
}
 
Example 6
Source File: BlockPoolRegistry.java    From nnproxy with Apache License 2.0 5 votes vote down vote up
void refreshBlockPools() throws ExecutionException, IOException {
    for (String fs : nnProxy.getMounts().getAllFs()) {
        NamespaceInfo nsInfo = upstreamManager.getUpstream(superuser, fs).nnProxyAndInfo.getProxy().versionRequest();
        String bpId = nsInfo.getBlockPoolID();
        bp2fs.put(bpId, fs);
    }
}
 
Example 7
Source File: NNStorage.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Format all available storage directories.
 */
public void format(NamespaceInfo nsInfo) throws IOException {
  Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 ||
      nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION,
      "Bad layout version: %s", nsInfo.getLayoutVersion());
  
  this.setStorageInfo(nsInfo);
  this.blockpoolID = nsInfo.getBlockPoolID();
  for (Iterator<StorageDirectory> it =
                         dirIterator(); it.hasNext();) {
    StorageDirectory sd = it.next();
    format(sd);
  }
}
 
Example 8
Source File: NameNodeConnector.java    From big-c with Apache License 2.0 5 votes vote down vote up
public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
                         List<Path> targetPaths, Configuration conf,
                         int maxNotChangedIterations)
    throws IOException {
  this.nameNodeUri = nameNodeUri;
  this.idPath = idPath;
  this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays
      .asList(new Path("/")) : targetPaths;
  this.maxNotChangedIterations = maxNotChangedIterations;

  this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
      NamenodeProtocol.class).getProxy();
  this.client = NameNodeProxies.createProxy(conf, nameNodeUri,
      ClientProtocol.class, fallbackToSimpleAuth).getProxy();
  this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf);

  final NamespaceInfo namespaceinfo = namenode.versionRequest();
  this.blockpoolID = namespaceinfo.getBlockPoolID();

  final FsServerDefaults defaults = fs.getServerDefaults(new Path("/"));
  this.keyManager = new KeyManager(blockpoolID, namenode,
      defaults.getEncryptDataTransfer(), conf);
  // if it is for test, we do not create the id file
  out = checkAndMarkRunning();
  if (out == null) {
    // Exit if there is another one running.
    throw new IOException("Another " + name + " is running.");
  }
}
 
Example 9
Source File: DataNode.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Initializes the {@link #data}. The initialization is done only once, when
 * handshake with the the first namenode is completed.
 */
private void initStorage(final NamespaceInfo nsInfo) throws IOException {
  final FsDatasetSpi.Factory<? extends FsDatasetSpi<?>> factory
      = FsDatasetSpi.Factory.getFactory(conf);
  
  if (!factory.isSimulated()) {
    final StartupOption startOpt = getStartupOption(conf);
    if (startOpt == null) {
      throw new IOException("Startup option not set.");
    }
    final String bpid = nsInfo.getBlockPoolID();
    //read storage info, lock data dirs and transition fs state if necessary
    synchronized (this) {
      storage.recoverTransitionRead(this, nsInfo, dataDirs, startOpt);
    }
    final StorageInfo bpStorage = storage.getBPStorage(bpid);
    LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID()
        + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion()
        + ";nsInfo=" + nsInfo + ";dnuuid=" + storage.getDatanodeUuid());
  }

  // If this is a newly formatted DataNode then assign a new DatanodeUuid.
  checkDatanodeUuid();

  synchronized(this)  {
    if (data == null) {
      data = factory.newInstance(this, storage, conf);
    }
  }
}
 
Example 10
Source File: BlockPoolSliceStorage.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Format a block pool slice storage. 
 * @param bpSdir the block pool storage
 * @param nsInfo the name space info
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException {
  LOG.info("Formatting block pool " + blockpoolID + " directory "
      + bpSdir.getCurrentDir());
  bpSdir.clearDirectory(); // create directory
  this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
  this.cTime = nsInfo.getCTime();
  this.namespaceID = nsInfo.getNamespaceID();
  this.blockpoolID = nsInfo.getBlockPoolID();
  writeProperties(bpSdir);
}
 
Example 11
Source File: DataStorage.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Prepare a storage directory. It creates a builder which can be used to add
 * to the volume. If the volume cannot be added, it is OK to discard the
 * builder later.
 *
 * @param datanode DataNode object.
 * @param volume the root path of a storage directory.
 * @param nsInfos an array of namespace infos.
 * @return a VolumeBuilder that holds the metadata of this storage directory
 * and can be added to DataStorage later.
 * @throws IOException if encounters I/O errors.
 *
 * Note that if there is IOException, the state of DataStorage is not modified.
 */
public VolumeBuilder prepareVolume(DataNode datanode, File volume,
    List<NamespaceInfo> nsInfos) throws IOException {
  if (containsStorageDir(volume)) {
    final String errorMessage = "Storage directory is in use";
    LOG.warn(errorMessage + ".");
    throw new IOException(errorMessage);
  }

  StorageDirectory sd = loadStorageDirectory(
      datanode, nsInfos.get(0), volume, StartupOption.HOTSWAP);
  VolumeBuilder builder =
      new VolumeBuilder(this, sd);
  for (NamespaceInfo nsInfo : nsInfos) {
    List<File> bpDataDirs = Lists.newArrayList();
    bpDataDirs.add(BlockPoolSliceStorage.getBpRoot(
        nsInfo.getBlockPoolID(), new File(volume, STORAGE_DIR_CURRENT)));
    makeBlockPoolDataDir(bpDataDirs, null);

    BlockPoolSliceStorage bpStorage;
    final String bpid = nsInfo.getBlockPoolID();
    synchronized (this) {
      bpStorage = this.bpStorageMap.get(bpid);
      if (bpStorage == null) {
        bpStorage = new BlockPoolSliceStorage(
            nsInfo.getNamespaceID(), bpid, nsInfo.getCTime(),
            nsInfo.getClusterID());
        addBlockPoolStorage(bpid, bpStorage);
      }
    }
    builder.addBpStorageDirectories(
        bpid, bpStorage.loadBpStorageDirectories(
            datanode, nsInfo, bpDataDirs, StartupOption.HOTSWAP));
  }
  return builder;
}