org.apache.zookeeper.Op Java Examples

The following examples show how to use org.apache.zookeeper.Op. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ZkController.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void removeEphemeralLiveNode() throws KeeperException, InterruptedException {
  if (zkRunOnly) {
    return;
  }
  String nodeName = getNodeName();
  String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
  String nodeAddedPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeName;
  log.info("Remove node as live in ZooKeeper:{}", nodePath);
  List<Op> ops = new ArrayList<>(2);
  ops.add(Op.delete(nodePath, -1));
  ops.add(Op.delete(nodeAddedPath, -1));

  try {
    zkClient.multi(ops, true);
  } catch (NoNodeException e) {

  }
}
 
Example #2
Source File: TestZKMulti.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void createZNodeTree(String rootZNode) throws KeeperException,
    InterruptedException {
  List<Op> opList = new ArrayList<>();
  opList.add(Op.create(rootZNode, new byte[0], Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT));
  int level = 0;
  String parentZNode = rootZNode;
  while (level < 10) {
    // define parent node
    parentZNode = parentZNode + "/" + level;
    opList.add(Op.create(parentZNode, new byte[0], Ids.OPEN_ACL_UNSAFE,
        CreateMode.PERSISTENT));
    int elements = 0;
    // add elements to the parent node
    while (elements < level) {
      opList.add(Op.create(parentZNode + "/" + elements, new byte[0],
          Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
      elements++;
    }
    level++;
  }
  zkw.getRecoverableZooKeeper().multi(opList);
}
 
Example #3
Source File: ZKRMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected synchronized void removeRMDelegationTokenState(
    RMDelegationTokenIdentifier rmDTIdentifier) throws Exception {
  String nodeRemovePath =
      getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
          + rmDTIdentifier.getSequenceNumber());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Removing RMDelegationToken_"
        + rmDTIdentifier.getSequenceNumber());
  }
  if (existsWithRetries(nodeRemovePath, false) != null) {
    ArrayList<Op> opList = new ArrayList<Op>();
    opList.add(Op.delete(nodeRemovePath, -1));
    doDeleteMultiWithRetries(opList);
  } else {
    LOG.debug("Attempted to delete a non-existing znode " + nodeRemovePath);
  }
}
 
Example #4
Source File: TestZKVersionedSetOp.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testAbortNullOpResult() throws Exception {
    final AtomicReference<Throwable> exception =
            new AtomicReference<Throwable>();
    final CountDownLatch latch = new CountDownLatch(1);
    ZKVersionedSetOp versionedSetOp =
            new ZKVersionedSetOp(mock(Op.class), new Transaction.OpListener<Version>() {
                @Override
                public void onCommit(Version r) {
                    // no-op
                }

                @Override
                public void onAbort(Throwable t) {
                    exception.set(t);
                    latch.countDown();
                }
            });
    KeeperException ke = KeeperException.create(KeeperException.Code.SESSIONEXPIRED);
    versionedSetOp.abortOpResult(ke, null);
    latch.await();
    assertTrue(ke == exception.get());
}
 
Example #5
Source File: TestZKVersionedSetOp.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testAbortOpResult() throws Exception {
    final AtomicReference<Throwable> exception =
            new AtomicReference<Throwable>();
    final CountDownLatch latch = new CountDownLatch(1);
    ZKVersionedSetOp versionedSetOp =
            new ZKVersionedSetOp(mock(Op.class), new Transaction.OpListener<Version>() {
                @Override
                public void onCommit(Version r) {
                    // no-op
                }

                @Override
                public void onAbort(Throwable t) {
                    exception.set(t);
                    latch.countDown();
                }
            });
    KeeperException ke = KeeperException.create(KeeperException.Code.SESSIONEXPIRED);
    OpResult opResult = new OpResult.ErrorResult(KeeperException.Code.NONODE.intValue());
    versionedSetOp.abortOpResult(ke, opResult);
    latch.await();
    assertTrue(exception.get() instanceof KeeperException.NoNodeException);
}
 
Example #6
Source File: TestLedgerAllocator.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testCloseAllocatorAfterAbort() throws Exception {
    String allocationPath = "/allocation3";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    // close during obtaining ledger.
    LedgerHandle lh = Utils.ioResult(allocator.tryObtain(txn, NULL_LISTENER));
    txn.addOp(DefaultZKOp.of(Op.setData("/unexistedpath", "data".getBytes(UTF_8), -1), null));
    try {
        Utils.ioResult(txn.execute());
        fail("Should fail the transaction when setting unexisted path");
    } catch (ZKException ke) {
        // expected
    }
    Utils.close(allocator);
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));
    // the ledger is not deleted.
    bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32,
            dlConf.getBKDigestPW().getBytes(UTF_8));
}
 
Example #7
Source File: ZKRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method that creates fencing node, executes the passed
 * delete related operations and deletes the fencing node.
 */
private synchronized void doDeleteMultiWithRetries(
    final List<Op> opList) throws Exception {
  final List<Op> execOpList = new ArrayList<Op>(opList.size() + 2);
  execOpList.add(createFencingNodePathOp);
  execOpList.addAll(opList);
  execOpList.add(deleteFencingNodePathOp);
  new ZKAction<Void>() {
    @Override
    public Void run() throws KeeperException, InterruptedException {
      setHasDeleteNodeOp(true);
      zkClient.multi(execOpList);
      return null;
    }
  }.runWithRetries();
}
 
Example #8
Source File: ZKRMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method that creates fencing node, executes the passed
 * delete related operations and deletes the fencing node.
 */
private synchronized void doDeleteMultiWithRetries(
    final List<Op> opList) throws Exception {
  final List<Op> execOpList = new ArrayList<Op>(opList.size() + 2);
  execOpList.add(createFencingNodePathOp);
  execOpList.addAll(opList);
  execOpList.add(deleteFencingNodePathOp);
  new ZKAction<Void>() {
    @Override
    public Void run() throws KeeperException, InterruptedException {
      setHasDeleteNodeOp(true);
      zkClient.multi(execOpList);
      return null;
    }
  }.runWithRetries();
}
 
Example #9
Source File: TestZKVersionedSetOp.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testAbortNullOpResult() throws Exception {
    final AtomicReference<Throwable> exception =
            new AtomicReference<Throwable>();
    final CountDownLatch latch = new CountDownLatch(1);
    ZKVersionedSetOp versionedSetOp =
            new ZKVersionedSetOp(mock(Op.class), new Transaction.OpListener<Version>() {
                @Override
                public void onCommit(Version r) {
                    // no-op
                }

                @Override
                public void onAbort(Throwable t) {
                    exception.set(t);
                    latch.countDown();
                }
            });
    KeeperException ke = KeeperException.create(KeeperException.Code.SESSIONEXPIRED);
    versionedSetOp.abortOpResult(ke, null);
    latch.await();
    assertTrue(ke == exception.get());
}
 
Example #10
Source File: ZKRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected synchronized void updateRMDelegationTokenState(
    RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
    throws Exception {
  ArrayList<Op> opList = new ArrayList<Op>();
  String nodeRemovePath =
      getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
          + rmDTIdentifier.getSequenceNumber());
  if (existsWithRetries(nodeRemovePath, false) == null) {
    // in case znode doesn't exist
    addStoreOrUpdateOps(opList, rmDTIdentifier, renewDate, false);
    LOG.debug("Attempted to update a non-existing znode " + nodeRemovePath);
  } else {
    // in case znode exists
    addStoreOrUpdateOps(opList, rmDTIdentifier, renewDate, true);
  }
  doStoreMultiWithRetries(opList);
}
 
Example #11
Source File: ZKRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected synchronized void removeRMDelegationTokenState(
    RMDelegationTokenIdentifier rmDTIdentifier) throws Exception {
  String nodeRemovePath =
      getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
          + rmDTIdentifier.getSequenceNumber());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Removing RMDelegationToken_"
        + rmDTIdentifier.getSequenceNumber());
  }
  if (existsWithRetries(nodeRemovePath, false) != null) {
    ArrayList<Op> opList = new ArrayList<Op>();
    opList.add(Op.delete(nodeRemovePath, -1));
    doDeleteMultiWithRetries(opList);
  } else {
    LOG.debug("Attempted to delete a non-existing znode " + nodeRemovePath);
  }
}
 
Example #12
Source File: TestZKVersionedSetOp.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testAbortOpResult() throws Exception {
    final AtomicReference<Throwable> exception =
            new AtomicReference<Throwable>();
    final CountDownLatch latch = new CountDownLatch(1);
    ZKVersionedSetOp versionedSetOp =
            new ZKVersionedSetOp(mock(Op.class), new Transaction.OpListener<Version>() {
                @Override
                public void onCommit(Version r) {
                    // no-op
                }

                @Override
                public void onAbort(Throwable t) {
                    exception.set(t);
                    latch.countDown();
                }
            });
    KeeperException ke = KeeperException.create(KeeperException.Code.SESSIONEXPIRED);
    OpResult opResult = new OpResult.ErrorResult(KeeperException.Code.NONODE.intValue());
    versionedSetOp.abortOpResult(ke, opResult);
    latch.await();
    assertTrue(exception.get() instanceof KeeperException.NoNodeException);
}
 
Example #13
Source File: ZKRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void removeApplicationStateInternal(
    ApplicationStateData  appState)
    throws Exception {
  String appId = appState.getApplicationSubmissionContext().getApplicationId()
      .toString();
  String appIdRemovePath = getNodePath(rmAppRoot, appId);
  ArrayList<Op> opList = new ArrayList<Op>();

  for (ApplicationAttemptId attemptId : appState.attempts.keySet()) {
    String attemptRemovePath = getNodePath(appIdRemovePath, attemptId.toString());
    opList.add(Op.delete(attemptRemovePath, -1));
  }
  opList.add(Op.delete(appIdRemovePath, -1));

  if (LOG.isDebugEnabled()) {
    LOG.debug("Removing info for app: " + appId + " at: " + appIdRemovePath
        + " and its attempts.");
  }
  doDeleteMultiWithRetries(opList);
}
 
Example #14
Source File: ZkController.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void createEphemeralLiveNode() throws KeeperException,
    InterruptedException {
  if (zkRunOnly) {
    return;
  }
  String nodeName = getNodeName();
  String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
  String nodeAddedPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeName;
  log.info("Register node as live in ZooKeeper:{}", nodePath);
  List<Op> ops = new ArrayList<>(2);
  ops.add(Op.create(nodePath, null, zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL));
  // if there are nodeAdded triggers don't create nodeAdded markers
  boolean createMarkerNode = zkStateReader.getAutoScalingConfig().hasTriggerForEvents(TriggerEventType.NODEADDED);
  if (createMarkerNode && !zkClient.exists(nodeAddedPath, true)) {
    // use EPHEMERAL so that it disappears if this node goes down
    // and no other action is taken
    byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", TimeSource.NANO_TIME.getEpochTimeNs()));
    ops.add(Op.create(nodeAddedPath, json, zkClient.getZkACLProvider().getACLsToAdd(nodeAddedPath), CreateMode.EPHEMERAL));
  }
  zkClient.multi(ops, true);
}
 
Example #15
Source File: TestLedgerAllocator.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testCloseAllocatorAfterAbort() throws Exception {
    String allocationPath = "/allocation3";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    // close during obtaining ledger.
    LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
    txn.addOp(DefaultZKOp.of(Op.setData("/unexistedpath", "data".getBytes(UTF_8), -1)));
    try {
        FutureUtils.result(txn.execute());
        fail("Should fail the transaction when setting unexisted path");
    } catch (ZKException ke) {
        // expected
    }
    Utils.close(allocator);
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));
    // the ledger is not deleted.
    bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32,
            dlConf.getBKDigestPW().getBytes(UTF_8));
}
 
Example #16
Source File: RecoverableZooKeeper.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Run multiple operations in a transactional manner. Retry before throwing exception
 */
public List<OpResult> multi(Iterable<Op> ops)
  throws KeeperException, InterruptedException {
  try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.multi")) {
    RetryCounter retryCounter = retryCounterFactory.create();
    Iterable<Op> multiOps = prepareZKMulti(ops);
    while (true) {
      try {
        return checkZk().multi(multiOps);
      } catch (KeeperException e) {
        switch (e.code()) {
          case CONNECTIONLOSS:
            retryOrThrow(retryCounter, e, "multi");
            break;
          case OPERATIONTIMEOUT:
            retryOrThrow(retryCounter, e, "multi");
            break;

          default:
            throw e;
        }
      }
      retryCounter.sleepUntilNextRetry();
    }
  }
}
 
Example #17
Source File: ZKUtil.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Convert from ZKUtilOp to ZKOp
 */
private static Op toZooKeeperOp(ZKWatcher zkw, ZKUtilOp op) throws UnsupportedOperationException {
  if(op == null) {
    return null;
  }

  if (op instanceof CreateAndFailSilent) {
    CreateAndFailSilent cafs = (CreateAndFailSilent)op;
    return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
      CreateMode.PERSISTENT);
  } else if (op instanceof DeleteNodeFailSilent) {
    DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
    return Op.delete(dnfs.getPath(), -1);
  } else if (op instanceof SetData) {
    SetData sd = (SetData) op;
    return Op.setData(sd.getPath(), sd.getData(), sd.getVersion());
  } else {
    throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
      + op.getClass().getName());
  }
}
 
Example #18
Source File: BKLogWriteHandler.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
void storeMaxSequenceNumber(final Transaction txn,
                            final MaxLogSegmentSequenceNo maxSeqNo,
                            final long seqNo,
                            final boolean isInprogress) {
    byte[] data = DLUtils.serializeLogSegmentSequenceNumber(seqNo);
    Op zkOp = Op.setData(logMetadata.getLogSegmentsPath(), data, maxSeqNo.getZkVersion());
    txn.addOp(new ZKVersionedSetOp(zkOp, new Transaction.OpListener<Version>() {
        @Override
        public void onCommit(Version version) {
            if (validateLogSegmentSequenceNumber) {
                synchronized (inprogressLSSNs) {
                    if (isInprogress) {
                        inprogressLSSNs.add(seqNo);
                    } else {
                        inprogressLSSNs.removeFirst();
                    }
                }
            }
            maxSeqNo.update((ZkVersion) version, seqNo);
        }

        @Override
        public void onAbort(Throwable t) {
            // no-op
        }
    }));
}
 
Example #19
Source File: TestLedgerAllocator.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
/**
 * {@link https://issues.apache.org/jira/browse/DL-43}
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testAllocation() throws Exception {
    String allocationPath = "/allocation1";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
    logger.info("Try obtaining ledger handle {}", lh.getId());
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));
    txn.addOp(DefaultZKOp.of(Op.setData("/unexistedpath", "data".getBytes(UTF_8), -1)));
    try {
        FutureUtils.result(txn.execute());
        fail("Should fail the transaction when setting unexisted path");
    } catch (ZKException ke) {
        // expected
        logger.info("Should fail on executing transaction when setting unexisted path", ke);
    }
    data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));

    // Create new transaction to obtain the ledger again.
    txn = newTxn();
    // we could obtain the ledger if it was obtained
    LedgerHandle newLh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
    assertEquals(lh.getId(), newLh.getId());
    FutureUtils.result(txn.execute());
    data = zkc.get().getData(allocationPath, false, null);
    assertEquals(0, data.length);
    Utils.close(allocator);
}
 
Example #20
Source File: ZKUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Use ZooKeeper's multi-update functionality.
 *
 * If all of the following are true:
 * - runSequentialOnMultiFailure is true
 * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
 * Then:
 * - we retry the operations one-by-one (sequentially)
 *
 * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
 * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
 * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
 * This will cause all operations in the multi to fail, however, because
 * the NodeExistsException that zk.create throws will fail the multi transaction.
 * In this case, if the previous conditions hold, the commands are run sequentially, which should
 * result in the correct final state, but means that the operations will not run atomically.
 *
 * @throws KeeperException if a ZooKeeper operation fails
 */
public static void multiOrSequential(ZKWatcher zkw, List<ZKUtilOp> ops,
                                     boolean runSequentialOnMultiFailure) throws KeeperException {
  if (zkw.getConfiguration().get("hbase.zookeeper.useMulti") != null) {
    LOG.warn("hbase.zookeeper.useMulti is deprecated. Default to true always.");
  }
  if (ops == null) {
    return;
  }

  List<Op> zkOps = new LinkedList<>();
  for (ZKUtilOp op : ops) {
    zkOps.add(toZooKeeperOp(zkw, op));
  }
  try {
    zkw.getRecoverableZooKeeper().multi(zkOps);
  } catch (KeeperException ke) {
    switch (ke.code()) {
      case NODEEXISTS:
      case NONODE:
      case BADVERSION:
      case NOAUTH:
      case NOTEMPTY:
        // if we get an exception that could be solved by running sequentially
        // (and the client asked us to), then break out and run sequentially
        if (runSequentialOnMultiFailure) {
          LOG.info("multi exception: {}; running operations sequentially " +
            "(runSequentialOnMultiFailure=true); {}", ke.toString(),
            ops.stream().map(o -> o.toString()).collect(Collectors.joining(",")));
          processSequentially(zkw, ops);
          break;
        }
      default:
        throw ke;
    }
  } catch (InterruptedException ie) {
    zkw.interruptedException(ie);
  }
}
 
Example #21
Source File: TestZKMulti.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchedDeletesOfWideZNodes() throws Exception {
  // Batch every 50bytes
  final int batchSize = 50;
  Configuration localConf = new Configuration(TEST_UTIL.getConfiguration());
  localConf.setInt("zookeeper.multi.max.size", batchSize);
  try (ZKWatcher customZkw = new ZKWatcher(localConf,
    "TestZKMulti_Custom", new ZKMultiAbortable(), true)) {

    // With a parent znode like this, we'll get batches of 2-3 elements
    final String parent1 = "/batchedDeletes1";
    final String parent2 = "/batchedDeletes2";
    final byte[] EMPTY_BYTES = new byte[0];

    // Write one node
    List<Op> ops = new ArrayList<>();
    ops.add(Op.create(parent1, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
    for (int i = 0; i < batchSize * 2; i++) {
      ops.add(Op.create(
          parent1 + "/" + i, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
    }
    customZkw.getRecoverableZooKeeper().multi(ops);

    // Write into a second node
    ops.clear();
    ops.add(Op.create(parent2, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
    for (int i = 0; i < batchSize * 4; i++) {
      ops.add(Op.create(
          parent2 + "/" + i, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
    }
    customZkw.getRecoverableZooKeeper().multi(ops);

    // These should return successfully
    ZKUtil.deleteChildrenRecursively(customZkw, parent1);
    ZKUtil.deleteChildrenRecursively(customZkw, parent2);
  }
}
 
Example #22
Source File: ZKLogSegmentMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public void createLogSegment(Transaction<Object> txn, LogSegmentMetadata segment) {
    byte[] finalisedData = segment.getFinalisedData().getBytes(UTF_8);
    Op createOp = Op.create(
            segment.getZkPath(),
            finalisedData,
            zkc.getDefaultACL(),
            CreateMode.PERSISTENT);
    txn.addOp(DefaultZKOp.of(createOp));
}
 
Example #23
Source File: ZKLogSegmentMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public void deleteLogSegment(Transaction<Object> txn, LogSegmentMetadata segment) {
    Op deleteOp = Op.delete(
            segment.getZkPath(),
            -1);
    txn.addOp(DefaultZKOp.of(deleteOp));
}
 
Example #24
Source File: SolrZkClient.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public List<OpResult> multi(final Iterable<Op> ops, boolean retryOnConnLoss) throws InterruptedException, KeeperException  {
  if (retryOnConnLoss) {
    return zkCmdExecutor.retryOperation(() -> keeper.multi(ops));
  } else {
    return keeper.multi(ops);
  }
}
 
Example #25
Source File: ZKLogSegmentMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public void createLogSegment(Transaction<Object> txn,
                             LogSegmentMetadata segment,
                             OpListener<Void> listener) {
    byte[] finalisedData = segment.getFinalisedData().getBytes(UTF_8);
    Op createOp = Op.create(
            segment.getZkPath(),
            finalisedData,
            zkc.getDefaultACL(),
            CreateMode.PERSISTENT);
    txn.addOp(DefaultZKOp.of(createOp, listener));
}
 
Example #26
Source File: ZKLogSegmentMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public void storeMaxTxnId(Transaction<Object> txn,
                          LogMetadataForWriter logMetadata,
                          Versioned<Long> transactionId,
                          Transaction.OpListener<Version> listener) {
    Version version = transactionId.getVersion();
    assert(version instanceof LongVersion);
    LongVersion zkVersion = (LongVersion) version;
    byte[] data = DLUtils.serializeTransactionId(transactionId.getValue());
    Op setDataOp = Op.setData(logMetadata.getMaxTxIdPath(), data, (int) zkVersion.getLongVersion());
    ZKOp zkOp = new ZKVersionedSetOp(setDataOp, listener);
    txn.addOp(zkOp);
}
 
Example #27
Source File: ZKLogSegmentMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public void storeMaxLogSegmentSequenceNumber(Transaction<Object> txn,
                                             LogMetadata logMetadata,
                                             Versioned<Long> lssn,
                                             Transaction.OpListener<Version> listener) {
    Version version = lssn.getVersion();
    assert(version instanceof LongVersion);
    LongVersion zkVersion = (LongVersion) version;
    byte[] data = DLUtils.serializeLogSegmentSequenceNumber(lssn.getValue());
    Op setDataOp = Op.setData(logMetadata.getLogSegmentsPath(), data, (int) zkVersion.getLongVersion());
    ZKOp zkOp = new ZKVersionedSetOp(setDataOp, listener);
    txn.addOp(zkOp);
}
 
Example #28
Source File: ZkDistributedQueue.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void remove(Collection<String> paths) throws KeeperException, InterruptedException {
  if (paths.isEmpty()) return;
  List<Op> ops = new ArrayList<>();
  for (String path : paths) {
    ops.add(Op.delete(dir + "/" + path, -1));
  }
  for (int from = 0; from < ops.size(); from += 1000) {
    int to = Math.min(from + 1000, ops.size());
    if (from < to) {
      try {
        zookeeper.multi(ops.subList(from, to), true);
      } catch (KeeperException.NoNodeException e) {
        // don't know which nodes are not exist, so try to delete one by one node
        for (int j = from; j < to; j++) {
          try {
            zookeeper.delete(ops.get(j).getPath(), -1, true);
          } catch (KeeperException.NoNodeException e2) {
            if (log.isDebugEnabled()) {
              log.debug("Can not remove node which is not exist : {}", ops.get(j).getPath());
            }
          }
        }
      }
    }
  }

  int cacheSizeBefore = knownChildren.size();
  knownChildren.removeAll(paths);
  if (cacheSizeBefore - paths.size() == knownChildren.size() && knownChildren.size() != 0) {
    stats.setQueueLength(knownChildren.size());
  } else {
    // There are elements get deleted but not present in the cache,
    // the cache seems not valid anymore
    knownChildren.clear();
    isDirty = true;
  }
}
 
Example #29
Source File: ZkTestServer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void buildZooKeeper(File solrhome, String config, String schema) throws Exception {

    Map<String,Object> props = new HashMap<>();
    props.put("configName", "conf1");
    final ZkNodeProps zkProps = new ZkNodeProps(props);


    List<Op> ops = new ArrayList<>(2);
    String path = "/collections";
    ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    path = "/collections/collection1";
    ops.add(Op.create(path, Utils.toJSON(zkProps), chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    path = "/collections/collection1/shards";
    ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    path = "/collections/control_collection";
    ops.add(Op.create(path, Utils.toJSON(zkProps), chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    path = "/collections/control_collection/shards";
    ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    path = "/configs";
    ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    path = "/configs/conf1";
    ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path),  CreateMode.PERSISTENT));
    chRootClient.multi(ops, true);

    // for now, always upload the config and schema to the canonical names
    putConfig("conf1", chRootClient, solrhome, config, "solrconfig.xml");
    putConfig("conf1", chRootClient, solrhome, schema, "schema.xml");

    putConfig("conf1", chRootClient, solrhome, "solrconfig.snippet.randomindexconfig.xml");
    putConfig("conf1", chRootClient, solrhome, "stopwords.txt");
    putConfig("conf1", chRootClient, solrhome, "protwords.txt");
    putConfig("conf1", chRootClient, solrhome, "currency.xml");
    putConfig("conf1", chRootClient, solrhome, "enumsConfig.xml");
    putConfig("conf1", chRootClient, solrhome, "open-exchange-rates.json");
    putConfig("conf1", chRootClient, solrhome, "mapping-ISOLatin1Accent.txt");
    putConfig("conf1", chRootClient, solrhome, "old_synonyms.txt");
    putConfig("conf1", chRootClient, solrhome, "synonyms.txt");
  }
 
Example #30
Source File: GenericDistributedQueue.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void remove(Collection<String> paths) throws Exception {
  if (paths.isEmpty()) return;
  List<Op> ops = new ArrayList<>();
  for (String path : paths) {
    ops.add(Op.delete(dir + "/" + path, -1));
  }
  for (int from = 0; from < ops.size(); from += 1000) {
    int to = Math.min(from + 1000, ops.size());
    if (from < to) {
      try {
        stateManager.multi(ops.subList(from, to));
      } catch (NoSuchElementException e) {
        // don't know which nodes are not exist, so try to delete one by one node
        for (int j = from; j < to; j++) {
          try {
            stateManager.removeData(ops.get(j).getPath(), -1);
          } catch (NoSuchElementException e2) {
            if (log.isDebugEnabled()) {
              log.debug("Can not remove node which is not exist : {}", ops.get(j).getPath());
            }
          }
        }
      }
    }
  }

  int cacheSizeBefore = knownChildren.size();
  knownChildren.removeAll(paths);
  if (cacheSizeBefore - paths.size() == knownChildren.size() && knownChildren.size() != 0) {
    stats.setQueueLength(knownChildren.size());
  } else {
    // There are elements get deleted but not present in the cache,
    // the cache seems not valid anymore
    knownChildren.clear();
    isDirty = true;
  }
}