org.apache.zookeeper.KeeperException.Code Java Examples

/**
 * Verify that, if there is a record of a prior active node, the
 * elector asks the application to fence it before becoming active.
 */
@Test
public void testFencesOldActive() throws Exception {
  byte[] fakeOldActiveData = new byte[0];
  mockPriorActive(fakeOldActiveData);
  
  elector.joinElection(data);
  elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  // Application fences active.
  Mockito.verify(mockApp, Mockito.times(1)).fenceOldActive(
      fakeOldActiveData);
  // Updates breadcrumb node to new data
  Mockito.verify(mockZK, Mockito.times(1)).setData(
      Mockito.eq(ZK_BREADCRUMB_NAME),
      Mockito.eq(data),
      Mockito.eq(0));
  // Then it becomes active itself
  Mockito.verify(mockApp, Mockito.times(1)).becomeActive();
}
 

private void createRootDir(final String rootPath) throws Exception {
  // For root dirs, we shouldn't use the doMulti helper methods
  new ZKAction<String>() {
    @Override
    public String run() throws KeeperException, InterruptedException {
      try {
        return zkClient.create(rootPath, null, zkAcl, CreateMode.PERSISTENT);
      } catch (KeeperException ke) {
        if (ke.code() == Code.NODEEXISTS) {
          LOG.debug(rootPath + "znode already exists!");
          return null;
        } else {
          throw ke;
        }
      }
    }
  }.runWithRetries();
}
 

/**
 * Verify that, when the callback fails to enter active state,
 * the elector rejoins the election after sleeping for a short period.
 */
@Test
public void testFailToBecomeActive() throws Exception {
  mockNoPriorActive();
  elector.joinElection(data);
  Assert.assertEquals(0, elector.sleptFor);
  
  Mockito.doThrow(new ServiceFailedException("failed to become active"))
      .when(mockApp).becomeActive();
  elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  // Should have tried to become active
  Mockito.verify(mockApp).becomeActive();
  
  // should re-join
  Mockito.verify(mockZK, Mockito.times(2)).create(ZK_LOCK_NAME, data,
      Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, elector, mockZK);
  Assert.assertEquals(2, count);
  Assert.assertTrue(elector.sleptFor > 0);
}
 

/**
 * Verify that, if there is a record of a prior active node, the
 * elector asks the application to fence it before becoming active.
 */
@Test
public void testFencesOldActive() throws Exception {
  byte[] fakeOldActiveData = new byte[0];
  mockPriorActive(fakeOldActiveData);
  
  elector.joinElection(data);
  elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  // Application fences active.
  Mockito.verify(mockApp, Mockito.times(1)).fenceOldActive(
      fakeOldActiveData);
  // Updates breadcrumb node to new data
  Mockito.verify(mockZK, Mockito.times(1)).setData(
      Mockito.eq(ZK_BREADCRUMB_NAME),
      Mockito.eq(data),
      Mockito.eq(0));
  // Then it becomes active itself
  Mockito.verify(mockApp, Mockito.times(1)).becomeActive();
}
 

@Test
public void testManagedLedgerWithCreateLedgerTimeOut() throws Exception {
    ManagedLedgerConfig config = new ManagedLedgerConfig().setMetadataOperationsTimeoutSeconds(3);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("timeout_ledger_test", config);

    BookKeeper bk = mock(BookKeeper.class);
    doNothing().when(bk).asyncCreateLedger(anyInt(), anyInt(), anyInt(), any(), any(), any(), any(), any());
    AtomicInteger response = new AtomicInteger(0);
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<Object> ctxHolder = new AtomicReference<>();
    ledger.asyncCreateLedger(bk, config, null, (rc, lh, ctx) -> {
        response.set(rc);
        latch.countDown();
        ctxHolder.set(ctx);
    }, Collections.emptyMap());

    latch.await(config.getMetadataOperationsTimeoutSeconds() + 2, TimeUnit.SECONDS);
    assertEquals(response.get(), BKException.Code.TimeoutException);
    assertTrue(ctxHolder.get() instanceof AtomicBoolean);
    AtomicBoolean ledgerCreated = (AtomicBoolean) ctxHolder.get();
    assertFalse(ledgerCreated.get());

    ledger.close();
}
 

/**
 * verify that if create znode results in nodeexists and that znode is deleted
 * before exists() watch is set then the return of the exists() method results
 * in attempt to re-create the znode and become active
 */
@Test
public void testCreateNodeResultRetryNoNode() {
  elector.joinElection(data);

  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  elector.processResult(Code.NODEEXISTS.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  verifyExistCall(1);

  elector.processResult(Code.NONODE.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  Mockito.verify(mockApp, Mockito.times(1)).enterNeutralMode();
  Mockito.verify(mockZK, Mockito.times(4)).create(ZK_LOCK_NAME, data,
      Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, elector, mockZK);
}
 

/**
 * verify that more than 3 network error retries result fatalError
 */
@Test
public void testStatNodeRetry() {
  elector.joinElection(data);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  Mockito
      .verify(mockApp, Mockito.times(1))
      .notifyFatalError(
          "Received stat error from Zookeeper. code:CONNECTIONLOSS. "+
          "Not retrying further znode monitoring connection errors.");
}
 

@Test(timeOut = 20000)
void errorRecoveringCursor3() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    Position p1 = ledger.addEntry("entry".getBytes());
    ledger.addEntry("entry".getBytes());
    ManagedCursor c1 = ledger.openCursor("c1");
    Position p3 = ledger.addEntry("entry".getBytes());

    assertEquals(c1.getReadPosition(), p3);

    ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());

    bkc.failAfter(4, BKException.Code.ReadException);

    ledger = factory2.open("my_test_ledger");
    c1 = ledger.openCursor("c1");

    // Verify the ManagedCursor was rewind back to the snapshotted position
    assertEquals(c1.getReadPosition(), p3);
    factory2.shutdown();
}
 

@Override
public CompletableFuture<Boolean> exists(String path) {
    CompletableFuture<Boolean> future = new CompletableFuture<>();

    try {
        zkc.exists(path, null, (rc, path1, ctx, stat) -> {
            executor.execute(() -> {
                Code code = Code.get(rc);
                if (code == Code.OK) {
                    future.complete(true);
                } else if (code == Code.NONODE) {
                    future.complete(false);
                } else {
                    future.completeExceptionally(getException(code, path));
                }
            });
        }, future);
    } catch (Throwable t) {
        future.completeExceptionally(new MetadataStoreException(t));
    }

    return future;
}
 

/**
 * get data set by the active leader
 * 
 * @return data set by the active instance
 * @throws ActiveNotFoundException
 *           when there is no active leader
 * @throws KeeperException
 *           other zookeeper operation errors
 * @throws InterruptedException
 * @throws IOException
 *           when ZooKeeper connection could not be established
 */
public synchronized byte[] getActiveData() throws ActiveNotFoundException,
    KeeperException, InterruptedException, IOException {
  try {
    if (zkClient == null) {
      createConnection();
    }
    Stat stat = new Stat();
    return getDataWithRetries(zkLockFilePath, false, stat);
  } catch(KeeperException e) {
    Code code = e.code();
    if (isNodeDoesNotExist(code)) {
      // handle the commonly expected cases that make sense for us
      throw new ActiveNotFoundException();
    } else {
      throw e;
    }
  }
}
 

/**
 * Waits for the next event from ZooKeeper to arrive.
 * 
 * @param connectionTimeoutMs zookeeper connection timeout in milliseconds
 * @throws KeeperException if the connection attempt times out. This will
 * be a ZooKeeper ConnectionLoss exception code.
 * @throws IOException if interrupted while connecting to ZooKeeper
 */
private void waitForZKConnectionEvent(int connectionTimeoutMs)
    throws KeeperException, IOException {
  try {
    if (!hasReceivedEvent.await(connectionTimeoutMs, TimeUnit.MILLISECONDS)) {
      LOG.error("Connection timed out: couldn't connect to ZooKeeper in "
          + connectionTimeoutMs + " milliseconds");
      zk.close();
      throw KeeperException.create(Code.CONNECTIONLOSS);
    }
  } catch (InterruptedException e) {
    Thread.currentThread().interrupt();
    throw new IOException(
        "Interrupted when connecting to zookeeper server", e);
  }
}
 

public void processResult(int rc, String path, Object ctx, List<String> children){    
    switch (Code.get(rc)) { 
    case CONNECTIONLOSS:
        getAssignedWorkers();
        
        break;
    case OK:  
        assignedWorkers = children;
        getWorkers(children);

        break;
    default:
        LOG.error("getChildren failed",  KeeperException.create(Code.get(rc), path));
        cb.recoveryComplete(RecoveryCallback.FAILED, null);
    }
}
 

@Test
public void errorInRecovering5() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ledger.addEntry("entry".getBytes());

    ledger.close();

    factory = new ManagedLedgerFactoryImpl(bkc, zkc);

    zkc.failConditional(Code.CONNECTIONLOSS, (op, path) -> {
            return path.equals("/managed-ledgers/my_test_ledger")
                && op == MockZooKeeper.Op.GET_CHILDREN;
        });

    try {
        ledger = factory.open("my_test_ledger");
        fail("should fail");
    } catch (ManagedLedgerException e) {
        // ok
    }

    // It should be fine now
    ledger = factory.open("my_test_ledger");
}
 

@Test
public void errorInRecovering6() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ledger.openCursor("c1");
    ledger.addEntry("entry".getBytes());

    ledger.close();

    factory = new ManagedLedgerFactoryImpl(bkc, zkc);

    zkc.failConditional(Code.CONNECTIONLOSS, (op, path) -> {
            return path.equals("/managed-ledgers/my_test_ledger/c1")
                && op == MockZooKeeper.Op.GET;
        });

    try {
        ledger = factory.open("my_test_ledger");
        fail("should fail");
    } catch (ManagedLedgerException e) {
        // ok
    }

    // It should be fine now
    ledger = factory.open("my_test_ledger");
}
 

/**
 * Verify that, when the callback fails to enter active state,
 * the elector rejoins the election after sleeping for a short period.
 */
@Test
public void testFailToBecomeActive() throws Exception {
  mockNoPriorActive();
  elector.joinElection(data);
  Assert.assertEquals(0, elector.sleptFor);
  
  Mockito.doThrow(new ServiceFailedException("failed to become active"))
      .when(mockApp).becomeActive();
  elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  // Should have tried to become active
  Mockito.verify(mockApp).becomeActive();
  
  // should re-join
  Mockito.verify(mockZK, Mockito.times(2)).create(ZK_LOCK_NAME, data,
      Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, elector, mockZK);
  Assert.assertEquals(2, count);
  Assert.assertTrue(elector.sleptFor > 0);
}
 

private void checkZnodePermsRecursive(ZKWatcher watcher,
    RecoverableZooKeeper zk, String znode) throws KeeperException, InterruptedException {

  boolean expectedWorldReadable = watcher.getZNodePaths().isClientReadable(znode);

  assertZnodePerms(zk, znode, expectedWorldReadable);

  try {
    List<String> children = zk.getChildren(znode, false);

    for (String child : children) {
      checkZnodePermsRecursive(watcher, zk, ZNodePaths.joinZNode(znode, child));
    }
  } catch (KeeperException ke) {
    // if we are not authenticated for listChildren, it is fine.
    if (ke.code() != Code.NOAUTH && ke.code() != Code.NONODE) {
      throw ke;
    }
  }
}
 

@Test
public void testQuitElectionRemovesBreadcrumbNode() throws Exception {
  mockNoPriorActive();
  elector.joinElection(data);
  elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  // Writes its own active info
  Mockito.verify(mockZK, Mockito.times(1)).create(
      Mockito.eq(ZK_BREADCRUMB_NAME), Mockito.eq(data),
      Mockito.eq(Ids.OPEN_ACL_UNSAFE),
      Mockito.eq(CreateMode.PERSISTENT));
  mockPriorActive(data);
  
  elector.quitElection(false);
  
  // Deletes its own active data
  Mockito.verify(mockZK, Mockito.times(1)).delete(
      Mockito.eq(ZK_BREADCRUMB_NAME), Mockito.eq(0));
}
 

@Override
public CompletableFuture<Void> delete(String path, Optional<Long> optExpectedVersion) {
    int expectedVersion = optExpectedVersion.orElse(-1L).intValue();

    CompletableFuture<Void> future = new CompletableFuture<>();

    try {
        zkc.delete(path, expectedVersion, (rc, path1, ctx) -> {
            executor.execute(() -> {
                Code code = Code.get(rc);
                if (code == Code.OK) {
                    future.complete(null);
                } else {
                    future.completeExceptionally(getException(code, path));
                }
            });
        }, null);
    } catch (Throwable t) {
        future.completeExceptionally(new MetadataStoreException(t));
    }

    return future;
}
 

/**
 * verify that if create znode results in nodeexists and that znode is deleted
 * before exists() watch is set then the return of the exists() method results
 * in attempt to re-create the znode and become active
 */
@Test
public void testCreateNodeResultRetryNoNode() {
  elector.joinElection(data);

  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  elector.processResult(Code.NODEEXISTS.intValue(), ZK_LOCK_NAME, mockZK,
      ZK_LOCK_NAME);
  verifyExistCall(1);

  elector.processResult(Code.NONODE.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  Mockito.verify(mockApp, Mockito.times(1)).enterNeutralMode();
  Mockito.verify(mockZK, Mockito.times(4)).create(ZK_LOCK_NAME, data,
      Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, elector, mockZK);
}
 

/**
 * verify that more than 3 network error retries result fatalError
 */
@Test
public void testStatNodeRetry() {
  elector.joinElection(data);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  elector.processResult(Code.CONNECTIONLOSS.intValue(), ZK_LOCK_NAME, mockZK,
      (Stat) null);
  Mockito
      .verify(mockApp, Mockito.times(1))
      .notifyFatalError(
          "Received stat error from Zookeeper. code:CONNECTIONLOSS. "+
          "Not retrying further znode monitoring connection errors.");
}
 

@Override
public CompletableFuture<Optional<GetResult>> get(String path) {
    CompletableFuture<Optional<GetResult>> future = new CompletableFuture<>();

    try {
        zkc.getData(path, null, (rc, path1, ctx, data, stat) -> {
            executor.execute(() -> {
                Code code = Code.get(rc);
                if (code == Code.OK) {
                    future.complete(Optional.of(new GetResult(data, getStat(stat))));
                } else if (code == Code.NONODE) {
                    future.complete(Optional.empty());
                } else {
                    future.completeExceptionally(getException(code, path));
                }
            });
        }, null);
    } catch (Throwable t) {
        future.completeExceptionally(new MetadataStoreException(t));
    }

    return future;
}
 

private static void existPath(ZooKeeper zk,
                              String path,
                              String basePath,
                              LinkedList<String> missingPaths,
                              CompletableFuture<List<String>> future) {
    if (basePath.equals(path)) {
        future.complete(missingPaths);
        return;
    }
    zk.exists(path, false, (rc, path1, ctx, stat) -> {
        if (Code.OK.intValue() != rc && Code.NONODE.intValue() != rc) {
            future.completeExceptionally(new ZKException("Failed to check existence of path " + path1,
                Code.get(rc)));
            return;
        }

        if (Code.OK.intValue() == rc) {
            future.complete(missingPaths);
            return;
        }

        missingPaths.addLast(path);
        String parentPath = Utils.getParent(path);
        existPath(zk, parentPath, basePath, missingPaths, future);
    }, null);
}
 

@Test
public void recoverAfterMarkDeleteError() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ManagedCursor cursor = ledger.openCursor("my-cursor");
    Position position = ledger.addEntry("entry".getBytes());

    bkc.failNow(BKException.Code.BookieHandleNotAvailableException);
    zkc.failConditional(Code.CONNECTIONLOSS, (op, path) -> {
            return path.equals("/managed-ledgers/my_test_ledger/my-cursor")
                && op == MockZooKeeper.Op.SET;
        });

    try {
        cursor.markDelete(position);
        fail("should fail");
    } catch (ManagedLedgerException e) {
        // ok
    }

    // The metadata ledger is reopened in background, until it's not reopened the mark-delete will fail
    Thread.sleep(100);

    // Next markDelete should succeed
    cursor.markDelete(position);
}
 

@Test(timeout = 60000)
public void testGetMissingPathsFailure() throws Exception {
    ZooKeeper mockZk = mock(ZooKeeper.class);
    ZooKeeperClient mockZkc = mock(ZooKeeperClient.class);
    when(mockZkc.get()).thenReturn(mockZk);
    doAnswer(invocationOnMock -> {
        String path = (String) invocationOnMock.getArguments()[0];
        StatCallback callback = (StatCallback) invocationOnMock.getArguments()[2];
        callback.processResult(Code.BADVERSION.intValue(), path, null, null);
        return null;
    }).when(mockZk).exists(anyString(), anyBoolean(), any(StatCallback.class), anyObject());

    try {
        FutureUtils.result(getMissingPaths(mockZkc, uri, "path/to/log"));
        fail("Should fail on getting missing paths on zookeeper exceptions.");
    } catch (ZKException zke) {
        assertEquals(Code.BADVERSION, zke.getKeeperExceptionCode());
    }
}
 

@Test(timeOut = 20000)
void errorRecoveringCursor2() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ledger.openCursor("c1");

    ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());

    bkc.failAfter(4, BKException.Code.MetadataVersionException);

    try {
        ledger = factory2.open("my_test_ledger");
        fail("should have failed");
    } catch (ManagedLedgerException e) {
        // ok
    }

    factory2.shutdown();
}
 

@NotNull
private CompletableFuture<LedgerHandle> createLedger(String schemaId) {
    Map<String, byte[]> metadata = LedgerMetadataUtils.buildMetadataForSchema(schemaId);
    final CompletableFuture<LedgerHandle> future = new CompletableFuture<>();
    try {
        bookKeeper.asyncCreateLedger(
                config.getManagedLedgerDefaultEnsembleSize(),
                config.getManagedLedgerDefaultWriteQuorum(),
                config.getManagedLedgerDefaultAckQuorum(),
                BookKeeper.DigestType.fromApiDigestType(config.getManagedLedgerDigestType()),
                LedgerPassword,
                (rc, handle, ctx) -> {
                    if (rc != BKException.Code.OK) {
                        future.completeExceptionally(bkException("Failed to create ledger", rc, -1, -1));
                    } else {
                        future.complete(handle);
                    }
                }, null, metadata);
    } catch (Throwable t) {
        log.error("[{}] Encountered unexpected error when creating schema ledger", schemaId, t);
        return FutureUtil.failedFuture(t);
    }
    return future;
}
 

@Test
public void errorInRecovering() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ledger.addEntry("entry".getBytes());

    ledger.close();

    factory = new ManagedLedgerFactoryImpl(bkc, zkc);

    bkc.failNow(BKException.Code.LedgerFencedException);

    try {
        ledger = factory.open("my_test_ledger");
        fail("should fail");
    } catch (ManagedLedgerException e) {
        // ok
    }

    // It should be fine now
    ledger = factory.open("my_test_ledger");
}
 

@Test
public void removingCursor2() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ledger.openCursor("c1");

    zkc.failConditional(Code.CONNECTIONLOSS, (op, path) -> {
            return op == MockZooKeeper.Op.DELETE
                && path.equals("/managed-ledgers/my_test_ledger/c1");
        });

    try {
        ledger.deleteCursor("c1");
        fail("should fail");
    } catch (ManagedLedgerException e) {
        // ok
    }
}
 

public static URI initializeDlogNamespace(InternalConfigurationData internalConf) throws IOException {
    String zookeeperServers = internalConf.getZookeeperServers();
    URI metadataServiceUri = URI.create(internalConf.getBookkeeperMetadataServiceUri());
    String ledgersStoreServers = metadataServiceUri.getAuthority().replace(";", ",");
    String ledgersRootPath = metadataServiceUri.getPath();
    BKDLConfig dlConfig = new BKDLConfig(ledgersStoreServers, ledgersRootPath);
    DLMetadata dlMetadata = DLMetadata.create(dlConfig);
    URI dlogUri = URI.create(String.format("distributedlog://%s/pulsar/functions", zookeeperServers));

    try {
        dlMetadata.create(dlogUri);
    } catch (ZKException e) {
        if (e.getKeeperExceptionCode() == Code.NODEEXISTS) {
            return dlogUri;
        }
        throw e;
    }
    return dlogUri;
}
 

@SuppressWarnings("deprecation")
public CompletableFuture<Boolean> existsAsync(String path, Watcher watcher) {
    return existsCache.get(path, (p, executor) -> {
        ZooKeeper zk = zkSession.get();
        if (zk == null) {
            return FutureUtil.failedFuture(new IOException("ZK session not ready"));
        }

        CompletableFuture<Boolean> future = new CompletableFuture<>();
        zk.exists(path, watcher, (rc, path1, ctx, stat) -> {
            if (rc == Code.OK.intValue()) {
                future.complete(true);
            } else if (rc == Code.NONODE.intValue()) {
                future.complete(false);
            } else {
                future.completeExceptionally(KeeperException.create(rc));
            }
        }, null);

        return future;
    });
}