org.apache.hadoop.hdfs.server.protocol.NamespaceInfo Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.protocol.NamespaceInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BPServiceActor.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void connectToNNAndHandshake() throws IOException {
  // get NN proxy
  bpNamenode = dn.connectToNN(nnAddr);

  // First phase of the handshake with NN - get the namespace
  // info.
  NamespaceInfo nsInfo = retrieveNamespaceInfo();
  
  // Verify that this matches the other NN in this HA pair.
  // This also initializes our block pool in the DN if we are
  // the first NN connection for this BP.
  bpos.verifyAndSetNamespaceInfo(nsInfo);
  
  // Second phase of the handshake with the NN.
  register(nsInfo);
}
 
Example #2
Source File: TestBookKeeperJournalManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoWriters() throws Exception {
  long start = 1;
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);
  bkjm1.format(nsi);

  BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);


  EditLogOutputStream out1 = bkjm1.startLogSegment(start,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  try {
    bkjm2.startLogSegment(start,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
    fail("Shouldn't have been able to open the second writer");
  } catch (IOException ioe) {
    LOG.info("Caught exception as expected", ioe);
  }finally{
    out1.close();
  }
}
 
Example #3
Source File: GetJournalEditServlet.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static String buildPath(String journalId, long segmentTxId,
    NamespaceInfo nsInfo) {
  StringBuilder path = new StringBuilder("/getJournal?");
  try {
    path.append(JOURNAL_ID_PARAM).append("=")
        .append(URLEncoder.encode(journalId, "UTF-8"));
    path.append("&" + SEGMENT_TXID_PARAM).append("=")
        .append(segmentTxId);
    path.append("&" + STORAGEINFO_PARAM).append("=")
        .append(URLEncoder.encode(nsInfo.toColonSeparatedString(), "UTF-8"));
  } catch (UnsupportedEncodingException e) {
    // Never get here -- everyone supports UTF-8
    throw new RuntimeException(e);
  }
  return path.toString();
}
 
Example #4
Source File: DataNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
void setupNS(Configuration conf, AbstractList<File> dataDirs) 
throws IOException {
  // get NN proxy
  DatanodeProtocol dnp = 
    (DatanodeProtocol)RPC.waitForProxy(DatanodeProtocol.class,
        DatanodeProtocol.versionID, nnAddr, conf);
  setNameNode(dnp);

  // handshake with NN
  NamespaceInfo nsInfo = handshake();
  setNamespaceInfo(nsInfo);
  synchronized(DataNode.this){
    setupNSStorage();
  }
  
  nsRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
  nsRegistration.setInfoPort(infoServer.getPort());
}
 
Example #5
Source File: TestDatanodeRegister.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws IOException {
  mockDnConf = mock(DNConf.class);
  doReturn(VersionInfo.getVersion()).when(mockDnConf).getMinimumNameNodeVersion();
  
  DataNode mockDN = mock(DataNode.class);
  doReturn(true).when(mockDN).shouldRun();
  doReturn(mockDnConf).when(mockDN).getDnConf();
  
  BPOfferService mockBPOS = mock(BPOfferService.class);
  doReturn(mockDN).when(mockBPOS).getDataNode();
  
  actor = new BPServiceActor(INVALID_ADDR, mockBPOS);

  fakeNsInfo = mock(NamespaceInfo.class);
  // Return a a good software version.
  doReturn(VersionInfo.getVersion()).when(fakeNsInfo).getSoftwareVersion();
  // Return a good layout version for now.
  doReturn(HdfsConstants.NAMENODE_LAYOUT_VERSION).when(fakeNsInfo)
      .getLayoutVersion();
  
  DatanodeProtocolClientSideTranslatorPB fakeDnProt = 
      mock(DatanodeProtocolClientSideTranslatorPB.class);
  when(fakeDnProt.versionRequest()).thenReturn(fakeNsInfo);
  actor.setNameNode(fakeDnProt);
}
 
Example #6
Source File: BlockPoolSliceStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Analyze and load storage directories. Recover from previous transitions if
 * required.
 *
 * The block pool storages are either all analyzed or none of them is loaded.
 * Therefore, a failure on loading any block pool storage results a faulty
 * data volume.
 *
 * @param datanode Datanode to which this storage belongs to
 * @param nsInfo namespace information
 * @param dataDirs storage directories of block pool
 * @param startOpt startup option
 * @return an array of loaded block pool directories.
 * @throws IOException on error
 */
List<StorageDirectory> loadBpStorageDirectories(
    DataNode datanode, NamespaceInfo nsInfo,
    Collection<File> dataDirs, StartupOption startOpt) throws IOException {
  List<StorageDirectory> succeedDirs = Lists.newArrayList();
  try {
    for (File dataDir : dataDirs) {
      if (containsStorageDir(dataDir)) {
        throw new IOException(
            "BlockPoolSliceStorage.recoverTransitionRead: " +
                "attempt to load an used block storage: " + dataDir);
      }
      StorageDirectory sd =
          loadStorageDirectory(datanode, nsInfo, dataDir, startOpt);
      succeedDirs.add(sd);
    }
  } catch (IOException e) {
    LOG.warn("Failed to analyze storage directories for block pool "
        + nsInfo.getBlockPoolID(), e);
    throw e;
  }
  return succeedDirs;
}
 
Example #7
Source File: TestQuorumJournalManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
private QuorumJournalManager createSpyingQJM()
    throws IOException, URISyntaxException {
  AsyncLogger.Factory spyFactory = new AsyncLogger.Factory() {
    @Override
    public AsyncLogger createLogger(Configuration conf, NamespaceInfo nsInfo,
        String journalId, InetSocketAddress addr) {
      AsyncLogger logger = new IPCLoggerChannel(conf, nsInfo, journalId, addr) {
        protected ExecutorService createSingleThreadExecutor() {
          // Don't parallelize calls to the quorum in the tests.
          // This makes the tests more deterministic.
          return MoreExecutors.sameThreadExecutor();
        }
      };
      
      return Mockito.spy(logger);
    }
  };
  return closeLater(new QuorumJournalManager(
      conf, cluster.getQuorumJournalURI(JID), FAKE_NSINFO, spyFactory));
}
 
Example #8
Source File: BackupNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private NamespaceInfo handshake(Configuration conf) throws IOException {
  // connect to name node
  InetSocketAddress nnAddress = NameNode.getServiceAddress(conf, true);
  this.namenode = NameNodeProxies.createNonHAProxy(conf, nnAddress,
      NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
      true).getProxy();
  this.nnRpcAddress = NetUtils.getHostPortString(nnAddress);
  this.nnHttpAddress = DFSUtil.getInfoServer(nnAddress, conf,
      DFSUtil.getHttpClientScheme(conf)).toURL();
  // get version and id info from the name-node
  NamespaceInfo nsInfo = null;
  while(!isStopRequested()) {
    try {
      nsInfo = handshake(namenode);
      break;
    } catch(SocketTimeoutException e) {  // name-node is busy
      LOG.info("Problem connecting to server: " + nnAddress);
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        LOG.warn("Encountered exception ", e);
      }
    }
  }
  return nsInfo;
}
 
Example #9
Source File: TestBookKeeperJournalManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
 
Example #10
Source File: TestBookKeeperJournalManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);
 
  String zkpath = bkjm.finalizedLedgerZNode(1, 100);
  
  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
 
Example #11
Source File: BPServiceActor.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void connectToNNAndHandshake() throws IOException {
  // get NN proxy
  bpNamenode = dn.connectToNN(nnAddr);

  // First phase of the handshake with NN - get the namespace
  // info.
  NamespaceInfo nsInfo = retrieveNamespaceInfo();
  
  // Verify that this matches the other NN in this HA pair.
  // This also initializes our block pool in the DN if we are
  // the first NN connection for this BP.
  bpos.verifyAndSetNamespaceInfo(nsInfo);
  
  // Second phase of the handshake with the NN.
  register(nsInfo);
}
 
Example #12
Source File: DataNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * One of the Block Pools has successfully connected to its NN.
 * This initializes the local storage for that block pool,
 * checks consistency of the NN's cluster ID, etc.
 * 
 * If this is the first block pool to register, this also initializes
 * the datanode-scoped storage.
 * 
 * @param bpos Block pool offer service
 * @throws IOException if the NN is inconsistent with the local storage.
 */
void initBlockPool(BPOfferService bpos) throws IOException {
  NamespaceInfo nsInfo = bpos.getNamespaceInfo();
  if (nsInfo == null) {
    throw new IOException("NamespaceInfo not found: Block pool " + bpos
        + " should have retrieved namespace info before initBlockPool.");
  }
  
  setClusterId(nsInfo.clusterID, nsInfo.getBlockPoolID());

  // Register the new block pool with the BP manager.
  blockPoolManager.addBlockPool(bpos);
  
  // In the case that this is the first block pool to connect, initialize
  // the dataset, block scanners, etc.
  initStorage(nsInfo);

  // Exclude failed disks before initializing the block pools to avoid startup
  // failures.
  checkDiskError();

  data.addBlockPool(nsInfo.getBlockPoolID(), conf);
  blockScanner.enableBlockPoolId(bpos.getBlockPoolId());
  initDirectoryScanner(conf);
}
 
Example #13
Source File: NameSpaceSliceStorage.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Format a namespace slice storage. 
 * @param sd the namespace storage
 * @param nsInfo the name space info
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void format(StorageDirectory nsSdir, NamespaceInfo nsInfo) throws IOException {
  LOG.info("Formatting namespace " + namespaceID + " directory "
      + nsSdir.getCurrentDir());
  nsSdir.clearDirectory(); // create directory
  File rbwDir = new File(nsSdir.getCurrentDir(), STORAGE_DIR_RBW);
  File finalizedDir = new File(nsSdir.getCurrentDir(), STORAGE_DIR_FINALIZED);
  LOG.info("Creating Directories : " + rbwDir + ", " + finalizedDir);
  if (!rbwDir.mkdirs() || !finalizedDir.mkdirs()) {
    throw new IOException("Cannot create directories : " + rbwDir + ", "
        + finalizedDir);
  }
  this.layoutVersion = FSConstants.LAYOUT_VERSION;
  this.cTime = nsInfo.getCTime();
  this.namespaceID = nsInfo.getNamespaceID();
  this.storageType = NodeType.DATA_NODE;
  nsSdir.write();
}
 
Example #14
Source File: JNStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
void format(NamespaceInfo nsInfo) throws IOException {
  setStorageInfo(nsInfo);
  LOG.info("Formatting journal " + sd + " with nsid: " + getNamespaceID());
  // Unlock the directory before formatting, because we will
  // re-analyze it after format(). The analyzeStorage() call
  // below is reponsible for re-locking it. This is a no-op
  // if the storage is not currently locked.
  unlockAll();
  sd.clearDirectory();
  writeProperties(sd);
  createPaxosDir();
  analyzeStorage();
}
 
Example #15
Source File: TestQuorumJournalManagerUnit.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
  spyLoggers = ImmutableList.of(
      mockLogger(),
      mockLogger(),
      mockLogger());

  qjm = new QuorumJournalManager(conf, new URI("qjournal://host/jid"), FAKE_NSINFO) {
    @Override
    protected List<AsyncLogger> createLoggers(AsyncLogger.Factory factory) {
      return spyLoggers;
    }
  };

  for (AsyncLogger logger : spyLoggers) {
    futureReturns(GetJournalStateResponseProto.newBuilder()
        .setLastPromisedEpoch(0)
        .setHttpPort(-1)
        .build())
      .when(logger).getJournalState();
    
    futureReturns(
        NewEpochResponseProto.newBuilder().build()
        ).when(logger).newEpoch(Mockito.anyLong());
    
    futureReturns(null).when(logger).format(Mockito.<NamespaceInfo>any());
  }
  
  qjm.recoverUnfinalizedSegments();
}
 
Example #16
Source File: AvatarDataNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private void setupNS() throws IOException {
  // handshake with NN
  NamespaceInfo nsInfo;
  nsInfo = handshake(true);
  setNamespaceInfo(nsInfo);
  synchronized(AvatarDataNode.this){
    setupNSStorage();
  }
  
  nsRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
  nsRegistration.setInfoPort(infoServer.getPort());
}
 
Example #17
Source File: NameNodeConnector.java    From big-c with Apache License 2.0 5 votes vote down vote up
public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
                         List<Path> targetPaths, Configuration conf,
                         int maxNotChangedIterations)
    throws IOException {
  this.nameNodeUri = nameNodeUri;
  this.idPath = idPath;
  this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays
      .asList(new Path("/")) : targetPaths;
  this.maxNotChangedIterations = maxNotChangedIterations;

  this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
      NamenodeProtocol.class).getProxy();
  this.client = NameNodeProxies.createProxy(conf, nameNodeUri,
      ClientProtocol.class, fallbackToSimpleAuth).getProxy();
  this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf);

  final NamespaceInfo namespaceinfo = namenode.versionRequest();
  this.blockpoolID = namespaceinfo.getBlockPoolID();

  final FsServerDefaults defaults = fs.getServerDefaults(new Path("/"));
  this.keyManager = new KeyManager(blockpoolID, namenode,
      defaults.getEncryptDataTransfer(), conf);
  // if it is for test, we do not create the id file
  out = checkAndMarkRunning();
  if (out == null) {
    // Exit if there is another one running.
    throw new IOException("Another " + name + " is running.");
  }
}
 
Example #18
Source File: JNStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
void checkConsistentNamespace(NamespaceInfo nsInfo)
    throws IOException {
  if (nsInfo.getNamespaceID() != getNamespaceID()) {
    throw new IOException("Incompatible namespaceID for journal " +
        this.sd + ": NameNode has nsId " + nsInfo.getNamespaceID() +
        " but storage has nsId " + getNamespaceID());
  }
  
  if (!nsInfo.getClusterID().equals(getClusterID())) {
    throw new IOException("Incompatible clusterID for journal " +
        this.sd + ": NameNode has clusterId '" + nsInfo.getClusterID() +
        "' but storage has clusterId '" + getClusterID() + "'");
    
  }
}
 
Example #19
Source File: TestGenericJournalConf.java    From big-c with Apache License 2.0 5 votes vote down vote up
public DummyJournalManager(Configuration conf, URI u,
    NamespaceInfo nsInfo) {
  // Set static vars so the test case can verify them.
  DummyJournalManager.conf = conf;
  DummyJournalManager.uri = u;
  DummyJournalManager.nsInfo = nsInfo; 
}
 
Example #20
Source File: TestFsDatasetImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAddVolumeFailureReleasesInUseLock() throws IOException {
  FsDatasetImpl spyDataset = spy(dataset);
  FsVolumeImpl mockVolume = mock(FsVolumeImpl.class);
  File badDir = new File(BASE_DIR, "bad");
  badDir.mkdirs();
  doReturn(mockVolume).when(spyDataset)
      .createFsVolume(anyString(), any(File.class), any(StorageType.class));
  doThrow(new IOException("Failed to getVolumeMap()"))
    .when(mockVolume).getVolumeMap(
      anyString(),
      any(ReplicaMap.class),
      any(RamDiskReplicaLruTracker.class));

  Storage.StorageDirectory sd = createStorageDirectory(badDir);
  sd.lock();
  DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd);
  when(storage.prepareVolume(eq(datanode), eq(badDir.getAbsoluteFile()),
      Matchers.<List<NamespaceInfo>>any()))
      .thenReturn(builder);

  StorageLocation location = StorageLocation.parse(badDir.toString());
  List<NamespaceInfo> nsInfos = Lists.newArrayList();
  for (String bpid : BLOCK_POOL_IDS) {
    nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1));
  }

  try {
    spyDataset.addVolume(location, nsInfos);
    fail("Expect to throw MultipleIOException");
  } catch (MultipleIOException e) {
  }

  FsDatasetTestUtil.assertFileLockReleased(badDir.toString());
}
 
Example #21
Source File: FSEditLog.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Format all configured journals which are not file-based.
 * 
 * File-based journals are skipped, since they are formatted by the
 * Storage format code.
 */
synchronized void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException {
  Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS,
      "Bad state: %s", state);
  
  for (JournalManager jm : journalSet.getJournalManagers()) {
    if (!(jm instanceof FileJournalManager)) {
      jm.format(nsInfo);
    }
  }
}
 
Example #22
Source File: TestBookKeeperConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Verify the BKJM is creating the bookie available default path, when there
 * is no 'dfs.namenode.bookkeeperjournal.zk.availablebookies' configured
 */
@Test
public void testDefaultBKAvailablePath() throws Exception {
  Configuration conf = new Configuration();
  Assert.assertNull(BK_ROOT_PATH + " already exists", zkc.exists(
      BK_ROOT_PATH, false));
  NamespaceInfo nsi = newNSInfo();
  bkjm = new BookKeeperJournalManager(conf,
      URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-DefaultBKPath"),
      nsi);
  bkjm.format(nsi);
  Assert.assertNotNull("Bookie available path : " + BK_ROOT_PATH
      + " doesn't exists", zkc.exists(BK_ROOT_PATH, false));
}
 
Example #23
Source File: PBHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static NamespaceInfoProto convert(NamespaceInfo info) {
  return NamespaceInfoProto.newBuilder()
      .setBlockPoolID(info.getBlockPoolID())
      .setBuildVersion(info.getBuildVersion())
      .setUnused(0)
      .setStorageInfo(PBHelper.convert((StorageInfo)info))
      .setSoftwareVersion(info.getSoftwareVersion())
      .setCapabilities(info.getCapabilities())
      .build();
}
 
Example #24
Source File: PBHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static NamespaceInfoProto convert(NamespaceInfo info) {
  return NamespaceInfoProto.newBuilder()
      .setBlockPoolID(info.getBlockPoolID())
      .setBuildVersion(info.getBuildVersion())
      .setUnused(0)
      .setStorageInfo(PBHelper.convert((StorageInfo)info))
      .setSoftwareVersion(info.getSoftwareVersion())
      .setCapabilities(info.getCapabilities())
      .build();
}
 
Example #25
Source File: DatanodeProtocolServerSideTranslatorPB.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public VersionResponseProto versionRequest(RpcController controller,
    VersionRequestProto request) throws ServiceException {
  NamespaceInfo info;
  try {
    info = impl.versionRequest();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  return VersionResponseProto.newBuilder()
      .setInfo(PBHelper.convert(info)).build();
}
 
Example #26
Source File: DataStorage.java    From RDFS with Apache License 2.0 5 votes vote down vote up
void format(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException {
  sd.clearDirectory(); // create directory
  this.layoutVersion = FSConstants.LAYOUT_VERSION;
  this.namespaceID = nsInfo.getNamespaceID();  // mother namespaceid
  this.cTime = 0;
  // store storageID as it currently is
  sd.write();
}
 
Example #27
Source File: DataNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private NamespaceInfo handshake() throws IOException {
  NamespaceInfo nsInfo = new NamespaceInfo();
  while (shouldRun && shouldServiceRun) {
    try {
      nsInfo = nsNamenode.versionRequest();
      break;
    } catch(SocketTimeoutException e) {  // namenode is busy
      LOG.info("Problem connecting to server: " + nnAddr);
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {}
    }
  }
  String errorMsg = null;
  // verify build version
  if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion() )) {
    errorMsg = "Incompatible build versions: namenode BV = "
      + nsInfo.getBuildVersion() + "; datanode BV = "
      + Storage.getBuildVersion();
    LOG.warn( errorMsg );
    try {
      nsNamenode.errorReport( nsRegistration,
                            DatanodeProtocol.NOTIFY, errorMsg );
    } catch( SocketTimeoutException e ) {  // namenode is busy
      LOG.info("Problem connecting to server: " + nnAddr.toString());
    }
  }
  assert FSConstants.LAYOUT_VERSION == nsInfo.getLayoutVersion() :
    "Data-node and name-node layout versions must be the same."
    + "Expected: "+ FSConstants.LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();
  return nsInfo;
}
 
Example #28
Source File: QuorumJournalManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
QuorumJournalManager(Configuration conf,
    URI uri, NamespaceInfo nsInfo,
    AsyncLogger.Factory loggerFactory) throws IOException {
  Preconditions.checkArgument(conf != null, "must be configured");

  this.conf = conf;
  this.uri = uri;
  this.nsInfo = nsInfo;
  this.loggers = new AsyncLoggerSet(createLoggers(loggerFactory));
  this.connectionFactory = URLConnectionFactory
      .newDefaultURLConnectionFactory(conf);

  // Configure timeouts.
  this.startSegmentTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_START_SEGMENT_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_START_SEGMENT_TIMEOUT_DEFAULT);
  this.prepareRecoveryTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_PREPARE_RECOVERY_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_PREPARE_RECOVERY_TIMEOUT_DEFAULT);
  this.acceptRecoveryTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_ACCEPT_RECOVERY_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_ACCEPT_RECOVERY_TIMEOUT_DEFAULT);
  this.finalizeSegmentTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_FINALIZE_SEGMENT_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_FINALIZE_SEGMENT_TIMEOUT_DEFAULT);
  this.selectInputStreamsTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_SELECT_INPUT_STREAMS_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_SELECT_INPUT_STREAMS_TIMEOUT_DEFAULT);
  this.getJournalStateTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_GET_JOURNAL_STATE_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_GET_JOURNAL_STATE_TIMEOUT_DEFAULT);
  this.newEpochTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_NEW_EPOCH_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_NEW_EPOCH_TIMEOUT_DEFAULT);
  this.writeTxnsTimeoutMs = conf.getInt(
      DFSConfigKeys.DFS_QJOURNAL_WRITE_TXNS_TIMEOUT_KEY,
      DFSConfigKeys.DFS_QJOURNAL_WRITE_TXNS_TIMEOUT_DEFAULT);
}
 
Example #29
Source File: AsyncLoggerSet.java    From hadoop with Apache License 2.0 5 votes vote down vote up
QuorumCall<AsyncLogger,Void> format(NamespaceInfo nsInfo) {
  Map<AsyncLogger, ListenableFuture<Void>> calls =
      Maps.newHashMap();
  for (AsyncLogger logger : loggers) {
    ListenableFuture<Void> future =
        logger.format(nsInfo);
    calls.put(logger, future);
  }
  return QuorumCall.create(calls);
}
 
Example #30
Source File: FSImage.java    From big-c with Apache License 2.0 5 votes vote down vote up
void format(FSNamesystem fsn, String clusterId) throws IOException {
  long fileCount = fsn.getTotalFiles();
  // Expect 1 file, which is the root inode
  Preconditions.checkState(fileCount == 1,
      "FSImage.format should be called with an uninitialized namesystem, has " +
      fileCount + " files");
  NamespaceInfo ns = NNStorage.newNamespaceInfo();
  LOG.info("Allocated new BlockPoolId: " + ns.getBlockPoolID());
  ns.clusterID = clusterId;
  
  storage.format(ns);
  editLog.formatNonFileJournals(ns);
  saveFSImageInAllDirs(fsn, 0);
}