Java Code Examples for org.apache.hadoop.hdfs.server.namenode.CheckpointSignature

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.CheckpointSignature. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: PBHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static CheckpointSignatureProto convert(CheckpointSignature s) {
  return CheckpointSignatureProto.newBuilder()
      .setBlockPoolId(s.getBlockpoolID())
      .setCurSegmentTxId(s.getCurSegmentTxId())
      .setMostRecentCheckpointTxId(s.getMostRecentCheckpointTxId())
      .setStorageInfo(PBHelper.convert((StorageInfo) s)).build();
}
 
Example 2
@Override
public RollEditLogResponseProto rollEditLog(RpcController unused,
    RollEditLogRequestProto request) throws ServiceException {
  CheckpointSignature signature;
  try {
    signature = impl.rollEditLog();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  return RollEditLogResponseProto.newBuilder()
      .setSignature(PBHelper.convert(signature)).build();
}
 
Example 3
Source Project: hadoop   Source File: NamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointSignature rollEditLog() throws IOException {
  try {
    return PBHelper.convert(rpcProxy.rollEditLog(NULL_CONTROLLER,
        VOID_ROLL_EDITLOG_REQUEST).getSignature());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 4
Source Project: hadoop   Source File: NamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void endCheckpoint(NamenodeRegistration registration,
    CheckpointSignature sig) throws IOException {
  EndCheckpointRequestProto req = EndCheckpointRequestProto.newBuilder()
      .setRegistration(PBHelper.convert(registration))
      .setSignature(PBHelper.convert(sig)).build();
  try {
    rpcProxy.endCheckpoint(NULL_CONTROLLER, req);
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 5
Source Project: hadoop   Source File: TestBootstrapStandby.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test for the case where the shared edits dir doesn't have
 * all of the recent edit logs.
 */
@Test
public void testSharedEditsMissingLogs() throws Exception {
  removeStandbyNameDirs();
  
  CheckpointSignature sig = nn0.getRpcServer().rollEditLog();
  assertEquals(3, sig.getCurSegmentTxId());
  
  // Should have created edits_1-2 in shared edits dir
  URI editsUri = cluster.getSharedEditsDir(0, 1);
  File editsDir = new File(editsUri);
  File editsSegment = new File(new File(editsDir, "current"),
      NNStorage.getFinalizedEditsFileName(1, 2));
  GenericTestUtils.assertExists(editsSegment);

  // Delete the segment.
  assertTrue(editsSegment.delete());
  
  // Trying to bootstrap standby should now fail since the edit
  // logs aren't available in the shared dir.
  LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
      LogFactory.getLog(BootstrapStandby.class));
  try {
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(BootstrapStandby.ERR_CODE_LOGS_UNAVAILABLE, rc);
  } finally {
    logs.stopCapturing();
  }
  GenericTestUtils.assertMatches(logs.getOutput(),
      "FATAL.*Unable to read transaction ids 1-3 from the configured shared");
}
 
Example 6
Source Project: hadoop   Source File: TestPBHelper.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertCheckpointSignature() {
  CheckpointSignature s = new CheckpointSignature(
      getStorageInfo(NodeType.NAME_NODE), "bpid", 100, 1);
  CheckpointSignatureProto sProto = PBHelper.convert(s);
  CheckpointSignature s1 = PBHelper.convert(sProto);
  assertEquals(s.getBlockpoolID(), s1.getBlockpoolID());
  assertEquals(s.getClusterID(), s1.getClusterID());
  assertEquals(s.getCTime(), s1.getCTime());
  assertEquals(s.getCurSegmentTxId(), s1.getCurSegmentTxId());
  assertEquals(s.getLayoutVersion(), s1.getLayoutVersion());
  assertEquals(s.getMostRecentCheckpointTxId(),
      s1.getMostRecentCheckpointTxId());
  assertEquals(s.getNamespaceID(), s1.getNamespaceID());
}
 
Example 7
Source Project: big-c   Source File: PBHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static CheckpointSignatureProto convert(CheckpointSignature s) {
  return CheckpointSignatureProto.newBuilder()
      .setBlockPoolId(s.getBlockpoolID())
      .setCurSegmentTxId(s.getCurSegmentTxId())
      .setMostRecentCheckpointTxId(s.getMostRecentCheckpointTxId())
      .setStorageInfo(PBHelper.convert((StorageInfo) s)).build();
}
 
Example 8
@Override
public RollEditLogResponseProto rollEditLog(RpcController unused,
    RollEditLogRequestProto request) throws ServiceException {
  CheckpointSignature signature;
  try {
    signature = impl.rollEditLog();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  return RollEditLogResponseProto.newBuilder()
      .setSignature(PBHelper.convert(signature)).build();
}
 
Example 9
Source Project: big-c   Source File: NamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointSignature rollEditLog() throws IOException {
  try {
    return PBHelper.convert(rpcProxy.rollEditLog(NULL_CONTROLLER,
        VOID_ROLL_EDITLOG_REQUEST).getSignature());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 10
Source Project: big-c   Source File: NamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void endCheckpoint(NamenodeRegistration registration,
    CheckpointSignature sig) throws IOException {
  EndCheckpointRequestProto req = EndCheckpointRequestProto.newBuilder()
      .setRegistration(PBHelper.convert(registration))
      .setSignature(PBHelper.convert(sig)).build();
  try {
    rpcProxy.endCheckpoint(NULL_CONTROLLER, req);
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 11
Source Project: big-c   Source File: TestBootstrapStandby.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test for the case where the shared edits dir doesn't have
 * all of the recent edit logs.
 */
@Test
public void testSharedEditsMissingLogs() throws Exception {
  removeStandbyNameDirs();
  
  CheckpointSignature sig = nn0.getRpcServer().rollEditLog();
  assertEquals(3, sig.getCurSegmentTxId());
  
  // Should have created edits_1-2 in shared edits dir
  URI editsUri = cluster.getSharedEditsDir(0, 1);
  File editsDir = new File(editsUri);
  File editsSegment = new File(new File(editsDir, "current"),
      NNStorage.getFinalizedEditsFileName(1, 2));
  GenericTestUtils.assertExists(editsSegment);

  // Delete the segment.
  assertTrue(editsSegment.delete());
  
  // Trying to bootstrap standby should now fail since the edit
  // logs aren't available in the shared dir.
  LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
      LogFactory.getLog(BootstrapStandby.class));
  try {
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(BootstrapStandby.ERR_CODE_LOGS_UNAVAILABLE, rc);
  } finally {
    logs.stopCapturing();
  }
  GenericTestUtils.assertMatches(logs.getOutput(),
      "FATAL.*Unable to read transaction ids 1-3 from the configured shared");
}
 
Example 12
Source Project: big-c   Source File: TestPBHelper.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertCheckpointSignature() {
  CheckpointSignature s = new CheckpointSignature(
      getStorageInfo(NodeType.NAME_NODE), "bpid", 100, 1);
  CheckpointSignatureProto sProto = PBHelper.convert(s);
  CheckpointSignature s1 = PBHelper.convert(sProto);
  assertEquals(s.getBlockpoolID(), s1.getBlockpoolID());
  assertEquals(s.getClusterID(), s1.getClusterID());
  assertEquals(s.getCTime(), s1.getCTime());
  assertEquals(s.getCurSegmentTxId(), s1.getCurSegmentTxId());
  assertEquals(s.getLayoutVersion(), s1.getLayoutVersion());
  assertEquals(s.getMostRecentCheckpointTxId(),
      s1.getMostRecentCheckpointTxId());
  assertEquals(s.getNamespaceID(), s1.getNamespaceID());
}
 
Example 13
Source Project: RDFS   Source File: Standby.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Quiesces the ingest for the given file typ
 * 
 * @param type ingest to quiesce
 * @param sig signature for quiescing (checkpointing)
 */
private void quiesceIngest(IngestFile type, CheckpointSignature sig) 
    throws IOException {  
  File edits; 
  InjectionHandler.processEvent(InjectionEvent.STANDBY_QUIESCE_INGEST);
  synchronized (ingestStateLock) {
    if (type == IngestFile.EDITS) {
      assertState(StandbyIngestState.INGESTING_EDITS,
          StandbyIngestState.QUIESCING_EDITS);
    } else {
      assertState(StandbyIngestState.INGESTING_EDITS_NEW,
          StandbyIngestState.QUIESCING_EDITS_NEW);
    }
    edits = getIngestFile(type);
    currentIngestState = (type == IngestFile.EDITS)
        ? StandbyIngestState.QUIESCING_EDITS
        : StandbyIngestState.QUIESCING_EDITS_NEW;
    ingest.quiesce(sig);       
  } 
  try {
    ingestThread.join();
    currentIngestState = StandbyIngestState.NOT_INGESTING;
    LOG.info("Standby: Quiesce - Ingest thread for " 
        + edits.getName() + " exited.");
  } catch (InterruptedException e) {
    LOG.info("Standby: Quiesce - Ingest thread interrupted.");
    throw new IOException(e.getMessage());
  }
}
 
Example 14
Source Project: RDFS   Source File: Standby.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Copy the new fsimage into the NameNode
 */
private void putFSImage(CheckpointSignature sig) throws IOException {
  String fileid = "putimage=1&port=" + infoPort +
    "&machine=" +
    machineName +
    "&token=" + sig.toString();
  LOG.info("Standby: Posted URL " + fsName + fileid);
  TransferFsImage.getFileClient(fsName, fileid, (File[])null, false);
}
 
Example 15
Source Project: hadoop   Source File: CheckpointCommand.java    License: Apache License 2.0 4 votes vote down vote up
public CheckpointCommand(CheckpointSignature sig,
                         boolean needToReturnImg) {
  super(NamenodeProtocol.ACT_CHECKPOINT);
  this.cSig = sig;
  this.needToReturnImage = needToReturnImg;
}
 
Example 16
Source Project: hadoop   Source File: PBHelper.java    License: Apache License 2.0 4 votes vote down vote up
public static CheckpointSignature convert(CheckpointSignatureProto s) {
  StorageInfo si = PBHelper.convert(s.getStorageInfo(), NodeType.NAME_NODE);
  return new CheckpointSignature(si, s.getBlockPoolId(),
      s.getMostRecentCheckpointTxId(), s.getCurSegmentTxId());
}
 
Example 17
Source Project: big-c   Source File: CheckpointCommand.java    License: Apache License 2.0 4 votes vote down vote up
public CheckpointCommand(CheckpointSignature sig,
                         boolean needToReturnImg) {
  super(NamenodeProtocol.ACT_CHECKPOINT);
  this.cSig = sig;
  this.needToReturnImage = needToReturnImg;
}
 
Example 18
Source Project: big-c   Source File: PBHelper.java    License: Apache License 2.0 4 votes vote down vote up
public static CheckpointSignature convert(CheckpointSignatureProto s) {
  StorageInfo si = PBHelper.convert(s.getStorageInfo(), NodeType.NAME_NODE);
  return new CheckpointSignature(si, s.getBlockPoolId(),
      s.getMostRecentCheckpointTxId(), s.getCurSegmentTxId());
}
 
Example 19
Source Project: RDFS   Source File: Standby.java    License: Apache License 2.0 4 votes vote down vote up
private void finalizeCheckpoint(CheckpointSignature sig) 
    throws IOException{

  File[] imageFiles = fsImage.getImageFiles();
  if (imageFiles.length == 0) {
    throw new IOException("No good image is left");
  }
  File imageFile = imageFiles[0];
  InjectionHandler.processEvent(InjectionEvent.STANDBY_BEFORE_PUT_IMAGE,
      imageFile);
  
  // start a thread to validate image while uploading the image to primary
  ImageValidator imageValidator = new ImageValidator(imageFile);
  imageValidator.run();
  
  // copy image to primary namenode
  LOG.info("Standby: Checkpointing - Upload fsimage to remote namenode.");
  checkpointStatus("Image upload started");
  putFSImage(sig);

  // check if the image is valid
  try {
    imageValidator.join();
  } catch (InterruptedException ie) {
    throw (IOException)new InterruptedIOException().initCause(ie);
  }
  if (!imageValidator.succeeded) {
    throw new IOException("Image file validation failed", imageValidator.error);
  }
  
  // make transaction to primary namenode to switch edit logs
  LOG.info("Standby: Checkpointing - Roll fsimage on primary namenode.");
  InjectionHandler.processEventIO(InjectionEvent.STANDBY_BEFORE_ROLL_IMAGE);
    
  assertState(
      StandbyIngestState.NOT_INGESTING,
      StandbyIngestState.INGESTING_EDITS_NEW);
  
  // we might concurrently reopen ingested file because of 
  // checksum error
  synchronized (ingestStateLock) {
    boolean editsNewExisted = editsNewExists();
    try {
      primaryNamenode.rollFsImage(new CheckpointSignature(fsImage));
    } catch (IOException e) {
      if (editsNewExisted && !editsNewExists()
          && currentIngestState == StandbyIngestState.INGESTING_EDITS_NEW) {
        // we were ingesting edits.new
        // the roll did not succeed but edits.new does not exist anymore          
        // assume that the roll succeeded   
        LOG.warn("Roll did not succeed but edits.new does not exist!!! - assuming roll succeeded", e);
      } else {
        throw e;
      }
    }
    // after successful roll edits.new is rolled to edits
    // and we should be consuming it
    setCurrentIngestFile(editsFile);
    if (currentIngestState == StandbyIngestState.INGESTING_EDITS_NEW) {
      // 1) We currently consume edits.new - do the swap
      currentIngestState = StandbyIngestState.INGESTING_EDITS;
    } // 2) otherwise we don't consume anything - do not change the state
  }
  setLastRollSignature(null);
  lastFinalizeCheckpointFailed = false;
  
  LOG.info("Standby: Checkpointing - Checkpoint done. New Image Size: "
      + fsImage.getFsImageName().length());
  checkpointStatus("Completed");
}
 
Example 20
Source Project: RDFS   Source File: Standby.java    License: Apache License 2.0 4 votes vote down vote up
public void setLastRollSignature(CheckpointSignature sig) {
  this.sig = sig;
}
 
Example 21
Source Project: RDFS   Source File: Standby.java    License: Apache License 2.0 4 votes vote down vote up
public CheckpointSignature getLastRollSignature() {
  return this.sig;
}
 
Example 22
Source Project: hadoop   Source File: CheckpointCommand.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Checkpoint signature is used to ensure 
 * that nodes are talking about the same checkpoint.
 */
public CheckpointSignature getSignature() {
  return cSig;
}
 
Example 23
Source Project: hadoop   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
@Idempotent
public CheckpointSignature rollEditLog() throws IOException;
 
Example 24
Source Project: hadoop   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * A request to the active name-node to finalize
 * previously started checkpoint.
 * 
 * @param registration the requesting node
 * @param sig {@code CheckpointSignature} which identifies the checkpoint.
 * @throws IOException
 */
@AtMostOnce
public void endCheckpoint(NamenodeRegistration registration,
                          CheckpointSignature sig) throws IOException;
 
Example 25
Source Project: big-c   Source File: CheckpointCommand.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Checkpoint signature is used to ensure 
 * that nodes are talking about the same checkpoint.
 */
public CheckpointSignature getSignature() {
  return cSig;
}
 
Example 26
Source Project: big-c   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
@Idempotent
public CheckpointSignature rollEditLog() throws IOException;
 
Example 27
Source Project: big-c   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * A request to the active name-node to finalize
 * previously started checkpoint.
 * 
 * @param registration the requesting node
 * @param sig {@code CheckpointSignature} which identifies the checkpoint.
 * @throws IOException
 */
@AtMostOnce
public void endCheckpoint(NamenodeRegistration registration,
                          CheckpointSignature sig) throws IOException;
 
Example 28
Source Project: RDFS   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
public CheckpointSignature rollEditLog() throws IOException;
 
Example 29
Source Project: RDFS   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Rolls the fsImage log. It removes the old fsImage, copies the
 * new image to fsImage, removes the old edits and renames edits.new 
 * to edits. The call fails if any of the four files are missing.
 * 
 * @param newImageSignature the signature of the new fsimage
 * @throws IOException
 */
public void rollFsImage(CheckpointSignature newImageSignature)
throws IOException;
 
Example 30
Source Project: RDFS   Source File: NamenodeProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Gets the CheckpointSignature at the time the call was made
 * @return the CheckpointSignature
 */
public CheckpointSignature getCheckpointSignature();