org.apache.hadoop.hdfs.inotify.EventBatchList Java Examples

The following examples show how to use org.apache.hadoop.hdfs.inotify.EventBatchList. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ClientNamenodeProtocolTranslatorPB.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
      .setTxid(txid).build();
  try {
    return PBHelper.convert(rpcProxy.getEditsFromTxid(null, req));
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example #2
Source File: ClientNamenodeProtocolTranslatorPB.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
      .setTxid(txid).build();
  try {
    return PBHelper.convert(rpcProxy.getEditsFromTxid(null, req));
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example #3
Source File: DFSInotifyEventInputStream.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the next batch of events in the stream or null if no new
 * batches are currently available.
 *
 * @throws IOException because of network error or edit log
 * corruption. Also possible if JournalNodes are unresponsive in the
 * QJM setting (even one unresponsive JournalNode is enough in rare cases),
 * so catching this exception and retrying at least a few times is
 * recommended.
 * @throws MissingEventsException if we cannot return the next batch in the
 * stream because the data for the events (and possibly some subsequent
 * events) has been deleted (generally because this stream is a very large
 * number of transactions behind the current state of the NameNode). It is
 * safe to continue reading from the stream after this exception is thrown
 * The next available batch of events will be returned.
 */
public EventBatch poll() throws IOException, MissingEventsException {
  TraceScope scope =
      Trace.startSpan("inotifyPoll", traceSampler);
  try {
    // need to keep retrying until the NN sends us the latest committed txid
    if (lastReadTxid == -1) {
      LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
      lastReadTxid = namenode.getCurrentEditLogTxid();
      return null;
    }
    if (!it.hasNext()) {
      EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
      if (el.getLastTxid() != -1) {
        // we only want to set syncTxid when we were actually able to read some
        // edits on the NN -- otherwise it will seem like edits are being
        // generated faster than we can read them when the problem is really
        // that we are temporarily unable to read edits
        syncTxid = el.getSyncTxid();
        it = el.getBatches().iterator();
        long formerLastReadTxid = lastReadTxid;
        lastReadTxid = el.getLastTxid();
        if (el.getFirstTxid() != formerLastReadTxid + 1) {
          throw new MissingEventsException(formerLastReadTxid + 1,
              el.getFirstTxid());
        }
      } else {
        LOG.debug("poll(): read no edits from the NN when requesting edits " +
          "after txid {}", lastReadTxid);
        return null;
      }
    }

    if (it.hasNext()) { // can be empty if el.getLastTxid != -1 but none of the
      // newly seen edit log ops actually got converted to events
      return it.next();
    } else {
      return null;
    }
  } finally {
    scope.close();
  }
}
 
Example #4
Source File: NameNodeRpcServer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override // ClientProtocol
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  checkNNStartup();
  namesystem.checkOperation(OperationCategory.READ); // only active
  namesystem.checkSuperuserPrivilege();
  int maxEventsPerRPC = nn.conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY,
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
  FSEditLog log = namesystem.getFSImage().getEditLog();
  long syncTxid = log.getSyncTxId();
  // If we haven't synced anything yet, we can only read finalized
  // segments since we can't reliably determine which txns in in-progress
  // segments have actually been committed (e.g. written to a quorum of JNs).
  // If we have synced txns, we can definitely read up to syncTxid since
  // syncTxid is only updated after a transaction is committed to all
  // journals. (In-progress segments written by old writers are already
  // discarded for us, so if we read any in-progress segments they are
  // guaranteed to have been written by this NameNode.)
  boolean readInProgress = syncTxid > 0;

  List<EventBatch> batches = Lists.newArrayList();
  int totalEvents = 0;
  long maxSeenTxid = -1;
  long firstSeenTxid = -1;

  if (syncTxid > 0 && txid > syncTxid) {
    // we can't read past syncTxid, so there's no point in going any further
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  Collection<EditLogInputStream> streams = null;
  try {
    streams = log.selectInputStreams(txid, 0, null, readInProgress);
  } catch (IllegalStateException e) { // can happen if we have
    // transitioned out of active and haven't yet transitioned to standby
    // and are using QJM -- the edit log will be closed and this exception
    // will result
    LOG.info("NN is transitioning from active to standby and FSEditLog " +
    "is closed -- could not read edits");
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  boolean breakOuter = false;
  for (EditLogInputStream elis : streams) {
    // our assumption in this code is the EditLogInputStreams are ordered by
    // starting txid
    try {
      FSEditLogOp op = null;
      while ((op = readOp(elis)) != null) {
        // break out of here in the unlikely event that syncTxid is so
        // out of date that its segment has already been deleted, so the first
        // txid we get is greater than syncTxid
        if (syncTxid > 0 && op.getTransactionId() > syncTxid) {
          breakOuter = true;
          break;
        }

        EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
        if (eventBatch != null) {
          batches.add(eventBatch);
          totalEvents += eventBatch.getEvents().length;
        }
        if (op.getTransactionId() > maxSeenTxid) {
          maxSeenTxid = op.getTransactionId();
        }
        if (firstSeenTxid == -1) {
          firstSeenTxid = op.getTransactionId();
        }
        if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 &&
            op.getTransactionId() == syncTxid)) {
          // we're done
          breakOuter = true;
          break;
        }
      }
    } finally {
      elis.close();
    }
    if (breakOuter) {
      break;
    }
  }

  return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
 
Example #5
Source File: PBHelper.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
  IOException {
  final InotifyProtos.EventsListProto list = resp.getEventsList();
  final long firstTxid = list.getFirstTxid();
  final long lastTxid = list.getLastTxid();

  List<EventBatch> batches = Lists.newArrayList();
  if (list.getEventsList().size() > 0) {
    throw new IOException("Can't handle old inotify server response.");
  }
  for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
    long txid = bp.getTxid();
    if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
      throw new IOException("Error converting TxidResponseProto: got a " +
          "transaction id " + txid + " that was outside the range of [" +
          firstTxid + ", " + lastTxid + "].");
    }
    List<Event> events = Lists.newArrayList();
    for (InotifyProtos.EventProto p : bp.getEventsList()) {
      switch (p.getType()) {
        case EVENT_CLOSE:
          InotifyProtos.CloseEventProto close =
              InotifyProtos.CloseEventProto.parseFrom(p.getContents());
          events.add(new Event.CloseEvent(close.getPath(),
              close.getFileSize(), close.getTimestamp()));
          break;
        case EVENT_CREATE:
          InotifyProtos.CreateEventProto create =
              InotifyProtos.CreateEventProto.parseFrom(p.getContents());
          events.add(new Event.CreateEvent.Builder()
              .iNodeType(createTypeConvert(create.getType()))
              .path(create.getPath())
              .ctime(create.getCtime())
              .ownerName(create.getOwnerName())
              .groupName(create.getGroupName())
              .perms(convert(create.getPerms()))
              .replication(create.getReplication())
              .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
                  create.getSymlinkTarget())
              .defaultBlockSize(create.getDefaultBlockSize())
              .overwrite(create.getOverwrite()).build());
          break;
        case EVENT_METADATA:
          InotifyProtos.MetadataUpdateEventProto meta =
              InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
          events.add(new Event.MetadataUpdateEvent.Builder()
              .path(meta.getPath())
              .metadataType(metadataUpdateTypeConvert(meta.getType()))
              .mtime(meta.getMtime())
              .atime(meta.getAtime())
              .replication(meta.getReplication())
              .ownerName(
                  meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
              .groupName(
                  meta.getGroupName().isEmpty() ? null : meta.getGroupName())
              .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
              .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
                  meta.getAclsList()))
              .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
                  meta.getXAttrsList()))
              .xAttrsRemoved(meta.getXAttrsRemoved())
              .build());
          break;
        case EVENT_RENAME:
          InotifyProtos.RenameEventProto rename =
              InotifyProtos.RenameEventProto.parseFrom(p.getContents());
          events.add(new Event.RenameEvent.Builder()
                .srcPath(rename.getSrcPath())
                .dstPath(rename.getDestPath())
                .timestamp(rename.getTimestamp())
                .build());
          break;
        case EVENT_APPEND:
          InotifyProtos.AppendEventProto append =
              InotifyProtos.AppendEventProto.parseFrom(p.getContents());
          events.add(new Event.AppendEvent.Builder().path(append.getPath())
              .newBlock(append.hasNewBlock() && append.getNewBlock())
              .build());
          break;
        case EVENT_UNLINK:
          InotifyProtos.UnlinkEventProto unlink =
              InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
          events.add(new Event.UnlinkEvent.Builder()
                .path(unlink.getPath())
                .timestamp(unlink.getTimestamp())
                .build());
          break;
        default:
          throw new RuntimeException("Unexpected inotify event type: " +
              p.getType());
      }
    }
    batches.add(new EventBatch(txid, events.toArray(new Event[0])));
  }
  return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
      resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
 
Example #6
Source File: ProxyClientProtocolHandler.java    From nnproxy with Apache License 2.0 4 votes vote down vote up
@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
    throw new IOException("Invalid operation, do not use proxy");
}
 
Example #7
Source File: DFSInotifyEventInputStream.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the next batch of events in the stream or null if no new
 * batches are currently available.
 *
 * @throws IOException because of network error or edit log
 * corruption. Also possible if JournalNodes are unresponsive in the
 * QJM setting (even one unresponsive JournalNode is enough in rare cases),
 * so catching this exception and retrying at least a few times is
 * recommended.
 * @throws MissingEventsException if we cannot return the next batch in the
 * stream because the data for the events (and possibly some subsequent
 * events) has been deleted (generally because this stream is a very large
 * number of transactions behind the current state of the NameNode). It is
 * safe to continue reading from the stream after this exception is thrown
 * The next available batch of events will be returned.
 */
public EventBatch poll() throws IOException, MissingEventsException {
  TraceScope scope =
      Trace.startSpan("inotifyPoll", traceSampler);
  try {
    // need to keep retrying until the NN sends us the latest committed txid
    if (lastReadTxid == -1) {
      LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
      lastReadTxid = namenode.getCurrentEditLogTxid();
      return null;
    }
    if (!it.hasNext()) {
      EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
      if (el.getLastTxid() != -1) {
        // we only want to set syncTxid when we were actually able to read some
        // edits on the NN -- otherwise it will seem like edits are being
        // generated faster than we can read them when the problem is really
        // that we are temporarily unable to read edits
        syncTxid = el.getSyncTxid();
        it = el.getBatches().iterator();
        long formerLastReadTxid = lastReadTxid;
        lastReadTxid = el.getLastTxid();
        if (el.getFirstTxid() != formerLastReadTxid + 1) {
          throw new MissingEventsException(formerLastReadTxid + 1,
              el.getFirstTxid());
        }
      } else {
        LOG.debug("poll(): read no edits from the NN when requesting edits " +
          "after txid {}", lastReadTxid);
        return null;
      }
    }

    if (it.hasNext()) { // can be empty if el.getLastTxid != -1 but none of the
      // newly seen edit log ops actually got converted to events
      return it.next();
    } else {
      return null;
    }
  } finally {
    scope.close();
  }
}
 
Example #8
Source File: NameNodeRpcServer.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override // ClientProtocol
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  checkNNStartup();
  namesystem.checkOperation(OperationCategory.READ); // only active
  namesystem.checkSuperuserPrivilege();
  int maxEventsPerRPC = nn.conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY,
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
  FSEditLog log = namesystem.getFSImage().getEditLog();
  long syncTxid = log.getSyncTxId();
  // If we haven't synced anything yet, we can only read finalized
  // segments since we can't reliably determine which txns in in-progress
  // segments have actually been committed (e.g. written to a quorum of JNs).
  // If we have synced txns, we can definitely read up to syncTxid since
  // syncTxid is only updated after a transaction is committed to all
  // journals. (In-progress segments written by old writers are already
  // discarded for us, so if we read any in-progress segments they are
  // guaranteed to have been written by this NameNode.)
  boolean readInProgress = syncTxid > 0;

  List<EventBatch> batches = Lists.newArrayList();
  int totalEvents = 0;
  long maxSeenTxid = -1;
  long firstSeenTxid = -1;

  if (syncTxid > 0 && txid > syncTxid) {
    // we can't read past syncTxid, so there's no point in going any further
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  Collection<EditLogInputStream> streams = null;
  try {
    streams = log.selectInputStreams(txid, 0, null, readInProgress);
  } catch (IllegalStateException e) { // can happen if we have
    // transitioned out of active and haven't yet transitioned to standby
    // and are using QJM -- the edit log will be closed and this exception
    // will result
    LOG.info("NN is transitioning from active to standby and FSEditLog " +
    "is closed -- could not read edits");
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  boolean breakOuter = false;
  for (EditLogInputStream elis : streams) {
    // our assumption in this code is the EditLogInputStreams are ordered by
    // starting txid
    try {
      FSEditLogOp op = null;
      while ((op = readOp(elis)) != null) {
        // break out of here in the unlikely event that syncTxid is so
        // out of date that its segment has already been deleted, so the first
        // txid we get is greater than syncTxid
        if (syncTxid > 0 && op.getTransactionId() > syncTxid) {
          breakOuter = true;
          break;
        }

        EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
        if (eventBatch != null) {
          batches.add(eventBatch);
          totalEvents += eventBatch.getEvents().length;
        }
        if (op.getTransactionId() > maxSeenTxid) {
          maxSeenTxid = op.getTransactionId();
        }
        if (firstSeenTxid == -1) {
          firstSeenTxid = op.getTransactionId();
        }
        if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 &&
            op.getTransactionId() == syncTxid)) {
          // we're done
          breakOuter = true;
          break;
        }
      }
    } finally {
      elis.close();
    }
    if (breakOuter) {
      break;
    }
  }

  return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
 
Example #9
Source File: PBHelper.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
  IOException {
  final InotifyProtos.EventsListProto list = resp.getEventsList();
  final long firstTxid = list.getFirstTxid();
  final long lastTxid = list.getLastTxid();

  List<EventBatch> batches = Lists.newArrayList();
  if (list.getEventsList().size() > 0) {
    throw new IOException("Can't handle old inotify server response.");
  }
  for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
    long txid = bp.getTxid();
    if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
      throw new IOException("Error converting TxidResponseProto: got a " +
          "transaction id " + txid + " that was outside the range of [" +
          firstTxid + ", " + lastTxid + "].");
    }
    List<Event> events = Lists.newArrayList();
    for (InotifyProtos.EventProto p : bp.getEventsList()) {
      switch (p.getType()) {
        case EVENT_CLOSE:
          InotifyProtos.CloseEventProto close =
              InotifyProtos.CloseEventProto.parseFrom(p.getContents());
          events.add(new Event.CloseEvent(close.getPath(),
              close.getFileSize(), close.getTimestamp()));
          break;
        case EVENT_CREATE:
          InotifyProtos.CreateEventProto create =
              InotifyProtos.CreateEventProto.parseFrom(p.getContents());
          events.add(new Event.CreateEvent.Builder()
              .iNodeType(createTypeConvert(create.getType()))
              .path(create.getPath())
              .ctime(create.getCtime())
              .ownerName(create.getOwnerName())
              .groupName(create.getGroupName())
              .perms(convert(create.getPerms()))
              .replication(create.getReplication())
              .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
                  create.getSymlinkTarget())
              .defaultBlockSize(create.getDefaultBlockSize())
              .overwrite(create.getOverwrite()).build());
          break;
        case EVENT_METADATA:
          InotifyProtos.MetadataUpdateEventProto meta =
              InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
          events.add(new Event.MetadataUpdateEvent.Builder()
              .path(meta.getPath())
              .metadataType(metadataUpdateTypeConvert(meta.getType()))
              .mtime(meta.getMtime())
              .atime(meta.getAtime())
              .replication(meta.getReplication())
              .ownerName(
                  meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
              .groupName(
                  meta.getGroupName().isEmpty() ? null : meta.getGroupName())
              .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
              .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
                  meta.getAclsList()))
              .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
                  meta.getXAttrsList()))
              .xAttrsRemoved(meta.getXAttrsRemoved())
              .build());
          break;
        case EVENT_RENAME:
          InotifyProtos.RenameEventProto rename =
              InotifyProtos.RenameEventProto.parseFrom(p.getContents());
          events.add(new Event.RenameEvent.Builder()
                .srcPath(rename.getSrcPath())
                .dstPath(rename.getDestPath())
                .timestamp(rename.getTimestamp())
                .build());
          break;
        case EVENT_APPEND:
          InotifyProtos.AppendEventProto append =
              InotifyProtos.AppendEventProto.parseFrom(p.getContents());
          events.add(new Event.AppendEvent.Builder().path(append.getPath())
              .newBlock(append.hasNewBlock() && append.getNewBlock())
              .build());
          break;
        case EVENT_UNLINK:
          InotifyProtos.UnlinkEventProto unlink =
              InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
          events.add(new Event.UnlinkEvent.Builder()
                .path(unlink.getPath())
                .timestamp(unlink.getTimestamp())
                .build());
          break;
        default:
          throw new RuntimeException("Unexpected inotify event type: " +
              p.getType());
      }
    }
    batches.add(new EventBatch(txid, events.toArray(new Event[0])));
  }
  return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
      resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
 
Example #10
Source File: ClientProtocol.java    From hadoop with Apache License 2.0 2 votes vote down vote up
/**
 * Get an ordered list of batches of events corresponding to the edit log
 * transactions for txids equal to or greater than txid.
 */
@Idempotent
public EventBatchList getEditsFromTxid(long txid) throws IOException;
 
Example #11
Source File: ClientProtocol.java    From big-c with Apache License 2.0 2 votes vote down vote up
/**
 * Get an ordered list of batches of events corresponding to the edit log
 * transactions for txids equal to or greater than txid.
 */
@Idempotent
public EventBatchList getEditsFromTxid(long txid) throws IOException;