/*
* Copyright (c) 2014-2015 VMware, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, without warranties or
* conditions of any kind, EITHER EXPRESS OR IMPLIED. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.vmware.xenon.common.test;
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
import static javax.xml.bind.DatatypeConverter.printBase64Binary;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.net.ssl.SSLContext;
import javax.xml.bind.DatatypeConverter;
import io.netty.handler.codec.http2.Http2SecurityUtil;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.ApplicationProtocolNames;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SupportedCipherSuiteFilter;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.lucene.store.LockObtainFailedException;
import org.junit.rules.TemporaryFolder;
import com.vmware.xenon.common.Claims;
import com.vmware.xenon.common.CommandLineArgumentParser;
import com.vmware.xenon.common.DeferredResult;
import com.vmware.xenon.common.NodeSelectorService;
import com.vmware.xenon.common.NodeSelectorState;
import com.vmware.xenon.common.Operation;
import com.vmware.xenon.common.Operation.AuthorizationContext;
import com.vmware.xenon.common.Operation.CompletionHandler;
import com.vmware.xenon.common.Service;
import com.vmware.xenon.common.Service.Action;
import com.vmware.xenon.common.Service.ServiceOption;
import com.vmware.xenon.common.ServiceClient;
import com.vmware.xenon.common.ServiceConfigUpdateRequest;
import com.vmware.xenon.common.ServiceConfiguration;
import com.vmware.xenon.common.ServiceDocument;
import com.vmware.xenon.common.ServiceDocumentDescription;
import com.vmware.xenon.common.ServiceDocumentDescription.Builder;
import com.vmware.xenon.common.ServiceDocumentQueryResult;
import com.vmware.xenon.common.ServiceErrorResponse;
import com.vmware.xenon.common.ServiceHost;
import com.vmware.xenon.common.ServiceStats;
import com.vmware.xenon.common.ServiceStats.ServiceStat;
import com.vmware.xenon.common.ServiceStats.ServiceStatLogHistogram;
import com.vmware.xenon.common.TaskState;
import com.vmware.xenon.common.TestResults;
import com.vmware.xenon.common.UriUtils;
import com.vmware.xenon.common.Utils;
import com.vmware.xenon.common.http.netty.NettyChannelContext;
import com.vmware.xenon.common.http.netty.NettyHttpServiceClient;
import com.vmware.xenon.common.serialization.KryoSerializers;
import com.vmware.xenon.common.test.TestRequestSender.FailureResponse;
import com.vmware.xenon.services.common.AuthorizationContextService;
import com.vmware.xenon.services.common.ConsistentHashingNodeSelectorService;
import com.vmware.xenon.services.common.ExampleService;
import com.vmware.xenon.services.common.ExampleService.ExampleServiceState;
import com.vmware.xenon.services.common.ExampleServiceHost;
import com.vmware.xenon.services.common.MinimalTestService.MinimalTestServiceErrorResponse;
import com.vmware.xenon.services.common.NodeGroupService;
import com.vmware.xenon.services.common.NodeGroupService.JoinPeerRequest;
import com.vmware.xenon.services.common.NodeGroupService.NodeGroupConfig;
import com.vmware.xenon.services.common.NodeGroupService.NodeGroupState;
import com.vmware.xenon.services.common.NodeGroupService.UpdateQuorumRequest;
import com.vmware.xenon.services.common.NodeGroupUtils;
import com.vmware.xenon.services.common.NodeState;
import com.vmware.xenon.services.common.NodeState.NodeOption;
import com.vmware.xenon.services.common.NodeState.NodeStatus;
import com.vmware.xenon.services.common.QueryTask;
import com.vmware.xenon.services.common.QueryTask.QuerySpecification;
import com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption;
import com.vmware.xenon.services.common.QueryTask.QueryTerm.MatchType;
import com.vmware.xenon.services.common.QueryValidationTestService.NestedType;
import com.vmware.xenon.services.common.QueryValidationTestService.QueryValidationServiceState;
import com.vmware.xenon.services.common.ServiceHostLogService.LogServiceState;
import com.vmware.xenon.services.common.ServiceHostManagementService;
import com.vmware.xenon.services.common.ServiceUriPaths;
import com.vmware.xenon.services.common.TaskService;
public class VerificationHost extends ExampleServiceHost {
public static final int FAST_MAINT_INTERVAL_MILLIS = 100;
public static final String LOCATION1 = "L1";
public static final String LOCATION2 = "L2";
private volatile TestContext context;
private int timeoutSeconds = 30;
private long testStartMicros;
private long testEndMicros;
private long expectedCompletionCount;
private Throwable failure;
private URI referer;
/**
* Command line argument. Comma separated list of one or more peer nodes to join through Nodes
* must be defined in URI form, e.g --peerNodes=http://192.168.1.59:8000,http://192.168.1.82
*/
public String[] peerNodes;
/**
* When {@link #peerNodes} is configured this flag will trigger join of the remote nodes.
*/
public boolean joinNodes;
/**
* Command line argument indicating this is a stress test
*/
public boolean isStressTest;
/**
* Command line argument indicating this is a multi-location test
*/
public boolean isMultiLocationTest;
/**
* Command line argument for test duration, set for long running tests
*/
public long testDurationSeconds;
/**
* Command line argument
*/
public long maintenanceIntervalMillis = FAST_MAINT_INTERVAL_MILLIS;
/**
* Command line argument
*/
public String connectionTag;
private String lastTestName;
private TemporaryFolder temporaryFolder;
private TestRequestSender sender;
public static AtomicInteger hostNumber = new AtomicInteger();
public static VerificationHost create() {
return new VerificationHost();
}
public static VerificationHost create(Integer port) throws Exception {
ServiceHost.Arguments args = buildDefaultServiceHostArguments(port);
return initialize(new VerificationHost(), args);
}
public static ServiceHost.Arguments buildDefaultServiceHostArguments(Integer port) {
ServiceHost.Arguments args = new ServiceHost.Arguments();
args.id = "host-" + hostNumber.incrementAndGet();
args.port = port;
args.sandbox = null;
args.bindAddress = ServiceHost.LOOPBACK_ADDRESS;
return args;
}
public static VerificationHost create(ServiceHost.Arguments args)
throws Exception {
return initialize(new VerificationHost(), args);
}
public static VerificationHost initialize(VerificationHost h, ServiceHost.Arguments args)
throws Exception {
if (args.sandbox == null) {
h.setTemporaryFolder(new TemporaryFolder());
h.getTemporaryFolder().create();
args.sandbox = h.getTemporaryFolder().getRoot().toPath();
}
try {
h.initialize(args);
} catch (Throwable e) {
throw new RuntimeException(e);
}
h.sender = new TestRequestSender(h);
return h;
}
public static void createAndAttachSSLClient(ServiceHost h) throws Throwable {
// we create a random userAgent string to validate host to host communication when
// the client appears to be from an external, non-Xenon source.
ServiceClient client = NettyHttpServiceClient.create(UUID.randomUUID().toString(),
null,
h.getScheduledExecutor(), h);
if (NettyChannelContext.isALPNEnabled()) {
SslContext http2ClientContext = SslContextBuilder.forClient()
.trustManager(InsecureTrustManagerFactory.INSTANCE)
.ciphers(Http2SecurityUtil.CIPHERS, SupportedCipherSuiteFilter.INSTANCE)
.applicationProtocolConfig(new ApplicationProtocolConfig(
ApplicationProtocolConfig.Protocol.ALPN,
ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,
ApplicationProtocolNames.HTTP_2))
.build();
((NettyHttpServiceClient) client).setHttp2SslContext(http2ClientContext);
}
SSLContext clientContext = SSLContext.getInstance(ServiceClient.TLS_PROTOCOL_NAME);
clientContext.init(null, InsecureTrustManagerFactory.INSTANCE.getTrustManagers(), null);
client.setSSLContext(clientContext);
h.setClient(client);
SelfSignedCertificate ssc = new SelfSignedCertificate();
h.setCertificateFileReference(ssc.certificate().toURI());
h.setPrivateKeyFileReference(ssc.privateKey().toURI());
}
public void tearDown() {
stop();
TemporaryFolder tempFolder = this.getTemporaryFolder();
if (tempFolder != null) {
tempFolder.delete();
}
}
public Operation createServiceStartPost(TestContext ctx) {
Operation post = Operation.createPost(null);
post.setUri(UriUtils.buildUri(this, "service/" + post.getId()));
return post.setCompletion(ctx.getCompletion());
}
public CompletionHandler getCompletion() {
return (o, e) -> {
if (e != null) {
failIteration(e);
return;
}
completeIteration();
};
}
public <T> BiConsumer<T, ? super Throwable> getCompletionDeferred() {
return (ignore, e) -> {
if (e != null) {
if (e instanceof CompletionException) {
e = e.getCause();
}
failIteration(e);
return;
}
completeIteration();
};
}
public CompletionHandler getExpectedFailureCompletion() {
return getExpectedFailureCompletion(null);
}
public CompletionHandler getExpectedFailureCompletion(Integer statusCode) {
return (o, e) -> {
if (e == null) {
failIteration(new IllegalStateException("Failure expected"));
return;
}
if (statusCode != null) {
if (!statusCode.equals(o.getStatusCode())) {
failIteration(new IllegalStateException(
"Expected different status code "
+ statusCode + " got " + o.getStatusCode()));
return;
}
}
if (e instanceof TimeoutException) {
if (o.getStatusCode() != Operation.STATUS_CODE_TIMEOUT) {
failIteration(new IllegalArgumentException(
"TImeout exception did not have timeout status code"));
return;
}
}
if (o.hasBody()) {
ServiceErrorResponse rsp = o.getErrorResponseBody();
if (rsp.message != null && rsp.message.toLowerCase().contains("timeout")
&& rsp.statusCode != Operation.STATUS_CODE_TIMEOUT) {
failIteration(new IllegalArgumentException(
"Service error response did not have timeout status code:"
+ Utils.toJsonHtml(rsp)));
return;
}
}
completeIteration();
};
}
public VerificationHost setTimeoutSeconds(int seconds) {
this.timeoutSeconds = seconds;
if (this.sender != null) {
this.sender.setTimeout(Duration.ofSeconds(seconds));
}
for (VerificationHost peer : this.localPeerHosts.values()) {
peer.setTimeoutSeconds(seconds);
}
return this;
}
public int getTimeoutSeconds() {
return this.timeoutSeconds;
}
public void send(Operation op) {
op.setReferer(getReferer());
super.sendRequest(op);
}
@Override
public DeferredResult<Operation> sendWithDeferredResult(Operation operation) {
operation.setReferer(getReferer());
return super.sendWithDeferredResult(operation);
}
@Override
public <T> DeferredResult<T> sendWithDeferredResult(Operation operation, Class<T> resultType) {
operation.setReferer(getReferer());
return super.sendWithDeferredResult(operation, resultType);
}
/**
* Creates a test wait context that can be nested and isolated from other wait contexts
*/
public TestContext testCreate(int c) {
return TestContext.create(c, TimeUnit.SECONDS.toMicros(this.timeoutSeconds));
}
/**
* Creates a test wait context that can be nested and isolated from other wait contexts
*/
public TestContext testCreate(long c) {
return testCreate((int) c);
}
/**
* Starts a test context used for a single synchronous test execution for the entire host
* @param c Expected completions
*/
public void testStart(long c) {
if (this.isSingleton) {
throw new IllegalStateException("Use testCreate on singleton, shared host instances");
}
String testName = buildTestNameFromStack();
testStart(
testName,
EnumSet.noneOf(TestProperty.class), c);
}
public String buildTestNameFromStack() {
StackTraceElement[] stack = new Exception().getStackTrace();
String rootTestMethod = "";
for (StackTraceElement s : stack) {
if (s.getClassName().contains("vmware")) {
rootTestMethod = s.getMethodName();
}
}
String testName = rootTestMethod + ":" + stack[2].getMethodName();
return testName;
}
public void testStart(String testName, EnumSet<TestProperty> properties, long c) {
if (this.isSingleton) {
throw new IllegalStateException("Use startTest on singleton, shared host instances");
}
if (this.context != null) {
throw new IllegalStateException("A test is already started");
}
String negative = properties != null && properties.contains(TestProperty.FORCE_FAILURE)
? "(NEGATIVE)"
: "";
if (c > 1) {
log("%sTest %s, iterations %d, started", negative, testName, c);
}
this.failure = null;
this.expectedCompletionCount = c;
this.testStartMicros = Utils.getSystemNowMicrosUtc();
this.context = TestContext.create((int) c, TimeUnit.SECONDS.toMicros(this.timeoutSeconds));
}
public void completeIteration() {
if (this.isSingleton) {
throw new IllegalStateException("Use startTest on singleton, shared host instances");
}
TestContext ctx = this.context;
if (ctx == null) {
String error = "testStart() and testWait() not paired properly" +
" or testStart(N) was called with N being less than actual completions";
log(error);
return;
}
ctx.completeIteration();
}
public void failIteration(Throwable e) {
if (this.isSingleton) {
throw new IllegalStateException("Use startTest on singleton, shared host instances");
}
if (isStopping()) {
log("Received completion after stop");
return;
}
TestContext ctx = this.context;
if (ctx == null) {
log("Test finished, ignoring completion. This might indicate wrong count was used in testStart(count)");
return;
}
log("test failed: %s", e.toString());
ctx.failIteration(e);
}
public void testWait(TestContext ctx) {
ctx.await();
}
public void testWait() {
testWait(new Exception().getStackTrace()[1].getMethodName(),
this.timeoutSeconds);
}
public void testWait(int timeoutSeconds) {
testWait(new Exception().getStackTrace()[1].getMethodName(), timeoutSeconds);
}
public void testWait(String testName, int timeoutSeconds) {
if (this.isSingleton) {
throw new IllegalStateException("Use startTest on singleton, shared host instances");
}
TestContext ctx = this.context;
if (ctx == null) {
throw new IllegalStateException("testStart() was not called before testWait()");
}
if (this.expectedCompletionCount > 1) {
log("Test %s, iterations %d, waiting ...", testName,
this.expectedCompletionCount);
}
try {
ctx.await();
this.testEndMicros = Utils.getSystemNowMicrosUtc();
if (this.expectedCompletionCount > 1) {
log("Test %s, iterations %d, complete!", testName,
this.expectedCompletionCount);
}
} finally {
this.context = null;
this.lastTestName = testName;
}
}
public double calculateThroughput() {
double t = this.testEndMicros - this.testStartMicros;
t /= 1000000.0;
t = this.expectedCompletionCount / t;
return t;
}
public long computeIterationsFromMemory(int serviceCount) {
return computeIterationsFromMemory(EnumSet.noneOf(TestProperty.class), serviceCount);
}
public long computeIterationsFromMemory(EnumSet<TestProperty> props, int serviceCount) {
long total = Runtime.getRuntime().totalMemory();
total /= 512;
total /= serviceCount;
if (props == null) {
props = EnumSet.noneOf(TestProperty.class);
}
if (props.contains(TestProperty.FORCE_REMOTE)) {
total /= 5;
}
if (props.contains(TestProperty.PERSISTED)) {
total /= 5;
}
if (props.contains(TestProperty.FORCE_FAILURE)
|| props.contains(TestProperty.EXPECT_FAILURE)) {
total = 10;
}
if (!this.isStressTest) {
total /= 100;
total = Math.max(Runtime.getRuntime().availableProcessors() * 16, total);
}
total = Math.max(1, total);
if (props.contains(TestProperty.SINGLE_ITERATION)) {
total = 1;
}
return total;
}
public void logThroughput() {
log("Test %s iterations per second: %f", this.lastTestName, calculateThroughput());
logMemoryInfo();
}
public void log(String fmt, Object... args) {
super.log(Level.INFO, 3, fmt, args);
}
public ServiceDocument buildMinimalTestState() {
return buildMinimalTestState(20);
}
public MinimalTestServiceState buildMinimalTestState(int bytes) {
MinimalTestServiceState minState = new MinimalTestServiceState();
minState.id = new Operation().getId() + "";
byte[] body = new byte[bytes];
new Random().nextBytes(body);
minState.stringValue = DatatypeConverter.printBase64Binary(body);
return minState;
}
public CompletableFuture<Operation> sendWithFuture(Operation op) {
if (op.getCompletion() != null) {
throw new IllegalStateException("completion handler must not be set");
}
CompletableFuture<Operation> res = new CompletableFuture<>();
op.setCompletion((o, e) -> {
if (e != null) {
res.completeExceptionally(e);
} else {
res.complete(o);
}
});
this.send(op);
return res;
}
/**
* Use built in Java synchronous HTTP client to verify DCP HttpListener is compliant
*/
public String sendWithJavaClient(URI serviceUri, String contentType, String body)
throws IOException {
URL url = serviceUri.toURL();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoInput(true);
connection.addRequestProperty(Operation.CONTENT_TYPE_HEADER, contentType);
if (body != null) {
connection.setDoOutput(true);
connection.getOutputStream().write(body.getBytes(Utils.CHARSET));
}
BufferedReader in = null;
try {
try {
in = new BufferedReader(
new InputStreamReader(
connection.getInputStream(), Utils.CHARSET));
} catch (Throwable e) {
InputStream errorStream = connection.getErrorStream();
if (errorStream != null) {
in = new BufferedReader(
new InputStreamReader(errorStream, Utils.CHARSET));
}
}
StringBuilder stringResponseBuilder = new StringBuilder();
if (in == null) {
return "";
}
do {
String line = in.readLine();
if (line == null) {
break;
}
stringResponseBuilder.append(line);
} while (true);
return stringResponseBuilder.toString();
} finally {
if (in != null) {
in.close();
}
}
}
public URI createQueryTaskService(QueryTask create) {
return createQueryTaskService(create, false);
}
public URI createQueryTaskService(QueryTask create, boolean forceRemote) {
return createQueryTaskService(create, forceRemote, false, null, null);
}
public URI createQueryTaskService(QueryTask create, boolean forceRemote, String sourceLink) {
return createQueryTaskService(create, forceRemote, false, null, sourceLink);
}
public URI createQueryTaskService(QueryTask create, boolean forceRemote, boolean isDirect,
QueryTask taskResult,
String sourceLink) {
return createQueryTaskService(null, create, forceRemote, isDirect, taskResult, sourceLink);
}
public URI createQueryTaskService(URI factoryUri, QueryTask create, boolean forceRemote,
boolean isDirect,
QueryTask taskResult,
String sourceLink) {
if (create.documentExpirationTimeMicros == 0) {
create.documentExpirationTimeMicros = Utils.fromNowMicrosUtc(
this.getOperationTimeoutMicros());
}
if (factoryUri == null) {
VerificationHost h = this;
if (!getInProcessHostMap().isEmpty()) {
// pick one host to create the query task
h = getInProcessHostMap().values().iterator().next();
}
factoryUri = UriUtils.buildUri(h, ServiceUriPaths.CORE_QUERY_TASKS);
}
create.documentSelfLink = UUID.randomUUID().toString();
create.documentSourceLink = sourceLink;
create.taskInfo.isDirect = isDirect;
Operation startPost = Operation.createPost(factoryUri).setBody(create);
if (forceRemote) {
startPost.forceRemote();
}
QueryTask result;
try {
result = this.sender.sendAndWait(startPost, QueryTask.class);
} catch (RuntimeException e) {
// throw original exception
throw ExceptionTestUtils.throwAsUnchecked(e.getSuppressed()[0]);
}
if (isDirect) {
taskResult.results = result.results;
taskResult.taskInfo.durationMicros = result.results.queryTimeMicros;
}
return UriUtils.extendUri(factoryUri, create.documentSelfLink);
}
public QueryTask waitForQueryTaskCompletion(QuerySpecification q, int totalDocuments,
int versionCount, URI u, boolean forceRemote, boolean deleteOnFinish) {
return waitForQueryTaskCompletion(q, totalDocuments, versionCount, u, forceRemote,
deleteOnFinish, true);
}
public boolean isOwner(String documentSelfLink, String nodeSelector) {
final boolean[] isOwner = new boolean[1];
log("Selecting owner for %s on %s", documentSelfLink, nodeSelector);
TestContext ctx = this.testCreate(1);
Operation op = Operation
.createPost(null)
.setExpiration(Utils.fromNowMicrosUtc(TimeUnit.SECONDS.toMicros(10)))
.setCompletion((o, e) -> {
if (e != null) {
ctx.failIteration(e);
return;
}
NodeSelectorService.SelectOwnerResponse rsp =
o.getBody(NodeSelectorService.SelectOwnerResponse.class);
log("Is owner: %s for %s", rsp.isLocalHostOwner, rsp.key);
isOwner[0] = rsp.isLocalHostOwner;
ctx.completeIteration();
});
this.selectOwner(nodeSelector, documentSelfLink, op);
ctx.await();
return isOwner[0];
}
public QueryTask waitForQueryTaskCompletion(QuerySpecification q, int totalDocuments,
int versionCount, URI u, boolean forceRemote, boolean deleteOnFinish,
boolean throwOnFailure) {
long startNanos = System.nanoTime();
if (q.options == null) {
q.options = EnumSet.noneOf(QueryOption.class);
}
EnumSet<TestProperty> props = EnumSet.noneOf(TestProperty.class);
if (forceRemote) {
props.add(TestProperty.FORCE_REMOTE);
}
waitFor("Query did not complete in time", () -> {
QueryTask taskState = getServiceState(props, QueryTask.class, u);
return taskState.taskInfo.stage == TaskState.TaskStage.FINISHED
|| taskState.taskInfo.stage == TaskState.TaskStage.FAILED
|| taskState.taskInfo.stage == TaskState.TaskStage.CANCELLED;
});
QueryTask latestTaskState = getServiceState(props, QueryTask.class, u);
// Throw if task was expected to be successful
if (throwOnFailure && (latestTaskState.taskInfo.stage == TaskState.TaskStage.FAILED)) {
throw new IllegalStateException(Utils.toJsonHtml(latestTaskState.taskInfo.failure));
}
if (totalDocuments * versionCount > 1) {
long endNanos = System.nanoTime();
double deltaSeconds = endNanos - startNanos;
deltaSeconds /= TimeUnit.SECONDS.toNanos(1);
double thpt = totalDocuments / deltaSeconds;
log("Options: %s. Throughput (documents / sec): %f", q.options.toString(), thpt);
}
// Delete task, if not direct
if (latestTaskState.taskInfo.isDirect) {
return latestTaskState;
}
if (deleteOnFinish) {
send(Operation.createDelete(u).setBody(new ServiceDocument()));
}
return latestTaskState;
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(
String fieldName, String fieldValue, long documentCount, long expectedResultCount,
TestResults testResults) {
return createAndWaitSimpleDirectQuery(this.getUri(), fieldName, fieldValue, documentCount,
expectedResultCount, testResults);
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(
String fieldName, String fieldValue, long documentCount, long expectedResultCount) {
return createAndWaitSimpleDirectQuery(fieldName, fieldValue, documentCount,
expectedResultCount, null);
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(URI hostUri,
String fieldName, String fieldValue, long documentCount, long expectedResultCount) {
return createAndWaitSimpleDirectQuery(hostUri, fieldName, fieldValue,
documentCount, expectedResultCount, null);
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(URI hostUri,
String fieldName, String fieldValue, long documentCount, long expectedResultCount,
TestResults testResults) {
QueryTask.QuerySpecification q = new QueryTask.QuerySpecification();
q.query.setTermPropertyName(fieldName).setTermMatchValue(fieldValue);
return createAndWaitSimpleDirectQuery(hostUri, q,
documentCount, expectedResultCount, testResults);
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(
QueryTask.QuerySpecification spec,
long documentCount, long expectedResultCount) {
return createAndWaitSimpleDirectQuery(spec,
documentCount, expectedResultCount, null);
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(
QuerySpecification spec,
long documentCount, long expectedResultCount, TestResults testResults) {
return createAndWaitSimpleDirectQuery(this.getUri(), spec,
documentCount, expectedResultCount, testResults);
}
public ServiceDocumentQueryResult createAndWaitSimpleDirectQuery(URI hostUri,
QuerySpecification spec, long documentCount, long expectedResultCount, TestResults testResults) {
long start = System.nanoTime() / 1000;
QueryTask[] tasks = new QueryTask[1];
waitFor("", () -> {
QueryTask task = QueryTask.create(spec).setDirect(true);
createQueryTaskService(UriUtils.buildUri(hostUri, ServiceUriPaths.CORE_QUERY_TASKS),
task, false, true, task, null);
if (spec.resultLimit != null) {
task = getServiceState(null,
QueryTask.class,
UriUtils.buildUri(hostUri, task.results.nextPageLink));
}
if (task.results.documentLinks.size() == expectedResultCount) {
tasks[0] = task;
return true;
}
log("Expected %d, got %d, Query task: %s", expectedResultCount,
task.results.documentLinks.size(), task);
return false;
});
QueryTask resultTask = tasks[0];
assertTrue(
String.format("Got %d links, expected %d", resultTask.results.documentLinks.size(),
expectedResultCount),
resultTask.results.documentLinks.size() == expectedResultCount);
long end = System.nanoTime() / 1000;
double delta = (end - start) / 1000000.0;
double thpt = documentCount / delta;
log("Document count: %d, Expected match count: %d, Documents / sec: %f",
documentCount, expectedResultCount, thpt);
if (testResults != null) {
String key = spec.query.term.propertyName + " docs/s";
testResults.getReport().all(key, thpt);
}
return resultTask.results;
}
public void validatePermanentServiceDocumentDeletion(String linkPrefix, long count,
boolean failOnMismatch)
throws Throwable {
long start = Utils.getNowMicrosUtc();
while (Utils.getNowMicrosUtc() - start < this.getOperationTimeoutMicros()) {
QueryTask.QuerySpecification q = new QueryTask.QuerySpecification();
q.query = new QueryTask.Query()
.setTermPropertyName(ServiceDocument.FIELD_NAME_SELF_LINK)
.setTermMatchType(MatchType.WILDCARD)
.setTermMatchValue(linkPrefix + UriUtils.URI_WILDCARD_CHAR);
URI u = createQueryTaskService(QueryTask.create(q), false);
QueryTask finishedTaskState = waitForQueryTaskCompletion(q,
(int) count, (int) count, u, false, true);
if (finishedTaskState.results.documentLinks.size() == count) {
return;
}
log("got %d links back, expected %d: %s",
finishedTaskState.results.documentLinks.size(), count,
Utils.toJsonHtml(finishedTaskState));
if (!failOnMismatch) {
return;
}
Thread.sleep(100);
}
if (failOnMismatch) {
throw new TimeoutException();
}
}
public String sendHttpRequest(ServiceClient client, String uri, String requestBody, int count) {
Object[] rspBody = new Object[1];
TestContext ctx = testCreate(count);
Operation op = Operation.createGet(URI.create(uri)).setCompletion(
(o, e) -> {
if (e != null) {
ctx.failIteration(e);
return;
}
rspBody[0] = o.getBodyRaw();
ctx.completeIteration();
});
if (requestBody != null) {
op.setAction(Action.POST).setBody(requestBody);
}
op.setExpiration(Utils.fromNowMicrosUtc(getOperationTimeoutMicros()));
op.setReferer(getReferer());
ServiceClient c = client != null ? client : getClient();
for (int i = 0; i < count; i++) {
c.send(op);
}
ctx.await();
String htmlResponse = (String) rspBody[0];
return htmlResponse;
}
public Operation sendUIHttpRequest(String uri, String requestBody, int count) {
Operation op = Operation.createGet(URI.create(uri));
List<Operation> ops = new ArrayList<>();
for (int i = 0; i < count; i++) {
ops.add(op);
}
List<Operation> responses = this.sender.sendAndWait(ops);
return responses.get(0);
}
public <T extends ServiceDocument> T getServiceState(EnumSet<TestProperty> props, Class<T> type,
URI uri) {
Map<URI, T> r = getServiceState(props, type, new URI[] { uri });
return r.values().iterator().next();
}
public <T extends ServiceDocument> Map<URI, T> getServiceState(EnumSet<TestProperty> props,
Class<T> type,
Collection<URI> uris) {
URI[] array = new URI[uris.size()];
int i = 0;
for (URI u : uris) {
array[i++] = u;
}
return getServiceState(props, type, array);
}
public <T extends TaskService.TaskServiceState> T getServiceStateUsingQueryTask(
Class<T> type, String uri) {
QueryTask.Query q = QueryTask.Query.Builder.create()
.setTerm(ServiceDocument.FIELD_NAME_SELF_LINK, uri)
.build();
QueryTask queryTask = new QueryTask();
queryTask.querySpec = new QueryTask.QuerySpecification();
queryTask.querySpec.query = q;
queryTask.querySpec.options.add(QueryOption.EXPAND_CONTENT);
this.createQueryTaskService(null, queryTask, false, true, queryTask, null);
return Utils.fromJson(queryTask.results.documents.get(uri), type);
}
/**
* Retrieve in parallel, state from N services. This method will block execution until responses
* are received or a failure occurs. It is not optimized for throughput measurements
*
* @param type
* @param uris
*/
public <T extends ServiceDocument> Map<URI, T> getServiceState(EnumSet<TestProperty> props,
Class<T> type, URI... uris) {
if (type == null) {
throw new IllegalArgumentException("type is required");
}
if (uris == null || uris.length == 0) {
throw new IllegalArgumentException("uris are required");
}
List<Operation> ops = new ArrayList<>();
for (URI u : uris) {
Operation get = Operation.createGet(u).setReferer(getReferer());
if (props != null && props.contains(TestProperty.FORCE_REMOTE)) {
get.forceRemote();
}
if (props != null && props.contains(TestProperty.HTTP2)) {
get.setConnectionSharing(true);
}
if (props != null && props.contains(TestProperty.DISABLE_CONTEXT_ID_VALIDATION)) {
get.setContextId(TestProperty.DISABLE_CONTEXT_ID_VALIDATION.toString());
}
ops.add(get);
}
Map<URI, T> results = new HashMap<>();
List<Operation> responses = this.sender.sendAndWait(ops);
for (Operation response : responses) {
T doc = response.getBody(type);
results.put(UriUtils.buildUri(response.getUri(), doc.documentSelfLink), doc);
}
return results;
}
/**
* Retrieve in parallel, state from N services. This method will block execution until responses
* are received or a failure occurs. It is not optimized for throughput measurements
*/
public <T extends ServiceDocument> Map<URI, T> getServiceState(EnumSet<TestProperty> props,
Class<T> type,
List<Service> services) {
URI[] uris = new URI[services.size()];
int i = 0;
for (Service s : services) {
uris[i++] = s.getUri();
}
return this.getServiceState(props, type, uris);
}
public ServiceDocumentQueryResult getFactoryState(URI factoryUri) {
return this.getServiceState(null, ServiceDocumentQueryResult.class, factoryUri);
}
public ServiceDocumentQueryResult getExpandedFactoryState(URI factoryUri) {
factoryUri = UriUtils.buildExpandLinksQueryUri(factoryUri);
return this.getServiceState(null, ServiceDocumentQueryResult.class, factoryUri);
}
public Map<String, ServiceStat> getServiceStats(URI serviceUri) {
ServiceStats stats = this.sender.sendStatsGetAndWait(serviceUri);
return stats.entries;
}
public void doExampleServiceUpdateAndQueryByVersion(URI hostUri, int serviceCount) {
Consumer<Operation> bodySetter = (o) -> {
ExampleServiceState s = new ExampleServiceState();
s.name = UUID.randomUUID().toString();
o.setBody(s);
};
Map<URI, ExampleServiceState> services = doFactoryChildServiceStart(null,
serviceCount,
ExampleServiceState.class, bodySetter,
UriUtils.buildUri(hostUri, ExampleService.FACTORY_LINK));
Map<URI, ExampleServiceState> statesBeforeUpdate = getServiceState(null,
ExampleServiceState.class, services.keySet());
for (ExampleServiceState state : statesBeforeUpdate.values()) {
assertEquals(state.documentVersion, 0);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.POST, 0L,
0L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.POST, null,
0L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.POST, 1L,
null);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.POST, 10L,
null);
}
ExampleServiceState body = new ExampleServiceState();
body.name = UUID.randomUUID().toString();
doServiceUpdates(services.keySet(), Action.PUT, body);
Map<URI, ExampleServiceState> statesAfterPut = getServiceState(null,
ExampleServiceState.class, services.keySet());
for (ExampleServiceState state : statesAfterPut.values()) {
assertEquals(state.documentVersion, 1);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.POST, 0L,
0L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.PUT, 1L,
1L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.PUT, null,
1L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.PUT, 10L,
null);
}
doServiceUpdates(services.keySet(), Action.DELETE, body);
for (ExampleServiceState state : statesAfterPut.values()) {
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.POST, 0L,
0L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.PUT, 1L,
1L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.DELETE, 2L,
2L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.DELETE,
null, 2L);
queryDocumentIndexByVersionAndVerify(hostUri, state.documentSelfLink, Action.DELETE,
10L, null);
}
}
public void doServiceUpdates(Collection<URI> serviceUris, Action action,
ServiceDocument body) {
List<Operation> ops = new ArrayList<>();
for (URI u : serviceUris) {
Operation update = Operation.createPost(u)
.setAction(action)
.setBody(body);
ops.add(update);
}
this.sender.sendAndWait(ops);
}
private void queryDocumentIndexByVersionAndVerify(URI hostUri, String selfLink,
Action expectedAction,
Long version,
Long latestVersion) {
URI localQueryUri = UriUtils.buildDefaultDocumentQueryUri(
hostUri,
selfLink,
false,
true,
ServiceOption.PERSISTENCE);
if (version != null) {
localQueryUri = UriUtils.appendQueryParam(localQueryUri,
ServiceDocument.FIELD_NAME_VERSION,
Long.toString(version));
}
Operation remoteGet = Operation.createGet(localQueryUri);
Operation result = this.sender.sendAndWait(remoteGet);
if (latestVersion == null) {
assertFalse("Document not expected", result.hasBody());
return;
}
ServiceDocument doc = result.getBody(ServiceDocument.class);
int expectedVersion = version == null ? latestVersion.intValue() : version.intValue();
assertEquals("Invalid document version returned", doc.documentVersion, expectedVersion);
String action = doc.documentUpdateAction;
assertEquals("Invalid document update action returned:" + action, expectedAction.name(),
action);
}
public <T> double doPutPerService(List<Service> services)
throws Throwable {
return doPutPerService(EnumSet.noneOf(TestProperty.class), services);
}
public <T> double doPutPerService(EnumSet<TestProperty> properties,
List<Service> services) throws Throwable {
return doPutPerService(computeIterationsFromMemory(properties, services.size()),
properties,
services);
}
public <T> double doPatchPerService(long count,
EnumSet<TestProperty> properties,
List<Service> services) throws Throwable {
return doServiceUpdates(Action.PATCH, count, properties, services);
}
public <T> double doPutPerService(long count, EnumSet<TestProperty> properties,
List<Service> services) throws Throwable {
return doServiceUpdates(Action.PUT, count, properties, services);
}
public double doServiceUpdates(Action action, long count,
EnumSet<TestProperty> properties,
List<Service> services) throws Throwable {
if (properties == null) {
properties = EnumSet.noneOf(TestProperty.class);
}
logMemoryInfo();
StackTraceElement[] e = new Exception().getStackTrace();
String testName = String.format(
"Parent: %s, %s test with properties %s, service caps: %s",
e[1].getMethodName(),
action, properties.toString(), services.get(0).getOptions());
Map<URI, MinimalTestServiceState> statesBeforeUpdate = getServiceState(properties,
MinimalTestServiceState.class, services);
long startTimeMicros = System.nanoTime() / 1000;
TestContext ctx = testCreate(count * services.size());
ctx.setTestName(testName);
ctx.logBefore();
// create a template PUT. Each operation instance is cloned on send, so
// we can re-use across services
Operation updateOp = Operation.createPut(null).setCompletion(ctx.getCompletion());
updateOp.setAction(action);
if (properties.contains(TestProperty.FORCE_REMOTE)) {
updateOp.forceRemote();
}
MinimalTestServiceState body = (MinimalTestServiceState) buildMinimalTestState();
byte[] binaryBody = null;
// put random values in core document properties to verify they are
// ignored
if (!this.isStressTest()) {
body.documentSelfLink = UUID.randomUUID().toString();
body.documentKind = UUID.randomUUID().toString();
} else {
body.stringValue = UUID.randomUUID().toString();
body.id = UUID.randomUUID().toString();
body.responseDelay = 10;
body.documentVersion = 10;
body.documentEpoch = 10L;
body.documentOwner = UUID.randomUUID().toString();
}
if (properties.contains(TestProperty.SET_EXPIRATION)) {
// set expiration to the maintenance interval, which should already be very small
// when the caller sets this test property
body.documentExpirationTimeMicros = Utils.fromNowMicrosUtc(
+this.getMaintenanceIntervalMicros());
}
final int maxByteCount = 256 * 1024;
if (properties.contains(TestProperty.LARGE_PAYLOAD)) {
Random r = new Random();
int byteCount = getClient().getRequestPayloadSizeLimit() / 4;
if (properties.contains(TestProperty.BINARY_PAYLOAD)) {
if (properties.contains(TestProperty.FORCE_FAILURE)) {
byteCount = getClient().getRequestPayloadSizeLimit() * 2;
} else {
// make sure we do not blow memory if max request size is high
byteCount = Math.min(maxByteCount, byteCount);
}
} else {
byteCount = maxByteCount;
}
byte[] data = new byte[byteCount];
r.nextBytes(data);
if (properties.contains(TestProperty.BINARY_PAYLOAD)) {
binaryBody = data;
} else {
body.stringValue = printBase64Binary(data);
}
}
if (properties.contains(TestProperty.HTTP2)) {
updateOp.setConnectionSharing(true);
}
if (properties.contains(TestProperty.BINARY_PAYLOAD)) {
updateOp.setContentType(Operation.MEDIA_TYPE_APPLICATION_OCTET_STREAM);
updateOp.setCompletion((o, eb) -> {
if (eb != null) {
ctx.fail(eb);
return;
}
if (!Operation.MEDIA_TYPE_APPLICATION_OCTET_STREAM.equals(o.getContentType())) {
ctx.fail(new IllegalArgumentException("unexpected content type: "
+ o.getContentType()));
return;
}
ctx.complete();
});
}
boolean isFailureExpected = false;
if (properties.contains(TestProperty.FORCE_FAILURE)
|| properties.contains(TestProperty.EXPECT_FAILURE)) {
toggleNegativeTestMode(true);
isFailureExpected = true;
if (properties.contains(TestProperty.LARGE_PAYLOAD)) {
updateOp.setCompletion((o, ex) -> {
if (ex == null) {
ctx.fail(new IllegalStateException("expected failure"));
} else {
ctx.complete();
}
});
} else {
updateOp.setCompletion((o, ex) -> {
if (ex == null) {
ctx.fail(new IllegalStateException("failure expected"));
return;
}
MinimalTestServiceErrorResponse rsp = o
.getBody(MinimalTestServiceErrorResponse.class);
if (!MinimalTestServiceErrorResponse.KIND.equals(rsp.documentKind)) {
ctx.fail(new IllegalStateException("Response not expected:"
+ Utils.toJson(rsp)));
return;
}
ctx.complete();
});
}
}
int byteCount = Utils.toJson(body).getBytes(Utils.CHARSET).length;
if (properties.contains(TestProperty.BINARY_SERIALIZATION)) {
long c = KryoSerializers.serializeDocument(body, 4096).position();
byteCount = (int) c;
}
log("Bytes per payload %s", byteCount);
boolean isConcurrentSend = properties.contains(TestProperty.CONCURRENT_SEND);
final boolean isFailureExpectedFinal = isFailureExpected;
for (Service s : services) {
if (properties.contains(TestProperty.FORCE_REMOTE)) {
updateOp.setConnectionTag(this.connectionTag);
}
long[] expectedVersion = new long[1];
if (s.hasOption(ServiceOption.STRICT_UPDATE_CHECKING)) {
// we have to serialize requests and properly set version to match expected current
// version
MinimalTestServiceState initialState = statesBeforeUpdate.get(s.getUri());
expectedVersion[0] = isFailureExpected ? Integer.MAX_VALUE
: initialState.documentVersion;
}
URI sUri = s.getUri();
updateOp.setUri(sUri);
for (int i = 0; i < count; i++) {
if (!isFailureExpected) {
body.id = "" + i;
} else if (!properties.contains(TestProperty.LARGE_PAYLOAD)) {
body.id = null;
}
CountDownLatch[] l = new CountDownLatch[1];
if (s.hasOption(ServiceOption.STRICT_UPDATE_CHECKING)) {
// only used for strict update checking, serialized requests
l[0] = new CountDownLatch(1);
// we have to serialize requests and properly set version
body.documentVersion = expectedVersion[0];
updateOp.setCompletion((o, ex) -> {
if (ex == null || isFailureExpectedFinal) {
MinimalTestServiceState rsp = o.getBody(MinimalTestServiceState.class);
expectedVersion[0] = rsp.documentVersion;
ctx.complete();
l[0].countDown();
return;
}
ctx.fail(ex);
l[0].countDown();
});
}
Object b = binaryBody != null ? binaryBody : body;
if (properties.contains(TestProperty.BINARY_SERIALIZATION)) {
// provide hints to runtime on how to serialize the body,
// using binary serialization and a buffer size equal to content length
updateOp.setContentLength(byteCount);
updateOp.setContentType(Operation.MEDIA_TYPE_APPLICATION_KRYO_OCTET_STREAM);
}
if (isConcurrentSend) {
Operation putClone = updateOp.clone();
putClone.setBody(b).setUri(sUri);
run(() -> {
s.sendRequest(putClone);
});
} else {
s.sendRequest(updateOp.setBody(b));
}
if (s.hasOption(ServiceOption.STRICT_UPDATE_CHECKING)) {
// we have to serialize requests and properly set version
if (!isFailureExpected) {
l[0].await();
}
if (this.failure != null) {
throw this.failure;
}
}
}
}
testWait(ctx);
double throughput = ctx.logAfter();
if (isFailureExpected) {
this.toggleNegativeTestMode(false);
return throughput;
}
if (properties.contains(TestProperty.BINARY_PAYLOAD)) {
return throughput;
}
List<URI> getUris = new ArrayList<>();
if (services.get(0).hasOption(ServiceOption.PERSISTENCE)) {
for (Service s : services) {
// bypass the services, which rely on caching, and go straight to the index
URI u = UriUtils.buildDocumentQueryUri(this, s.getSelfLink(), true, false,
ServiceOption.PERSISTENCE);
getUris.add(u);
}
} else {
for (Service s : services) {
getUris.add(s.getUri());
}
}
Map<URI, MinimalTestServiceState> statesAfterUpdate = getServiceState(
properties,
MinimalTestServiceState.class, getUris);
for (MinimalTestServiceState st : statesAfterUpdate.values()) {
URI serviceUri = UriUtils.buildUri(this, st.documentSelfLink);
ServiceDocument beforeSt = statesBeforeUpdate.get(serviceUri);
long expectedVersion = beforeSt.documentVersion + count;
if (st.documentVersion != expectedVersion) {
QueryTestUtils.logVersionInfoForService(this.sender, serviceUri, expectedVersion);
throw new IllegalStateException("got " + st.documentVersion + ", expected "
+ (beforeSt.documentVersion + count));
}
assertTrue(st.documentVersion == beforeSt.documentVersion + count);
assertTrue(st.id != null);
assertTrue(st.documentSelfLink != null
&& st.documentSelfLink.equals(beforeSt.documentSelfLink));
assertTrue(st.documentKind != null
&& st.documentKind.equals(Utils.buildKind(MinimalTestServiceState.class)));
assertTrue(st.documentUpdateTimeMicros > startTimeMicros);
assertTrue(st.documentUpdateAction != null);
assertTrue(st.documentUpdateAction.equals(action.toString()));
}
logMemoryInfo();
return throughput;
}
public void logMemoryInfo() {
log("Memory free:%d, available:%s, total:%s", Runtime.getRuntime().freeMemory(),
Runtime.getRuntime().totalMemory(),
Runtime.getRuntime().maxMemory());
}
public URI getReferer() {
if (this.referer == null) {
this.referer = getUri();
}
return this.referer;
}
public void waitForServiceAvailable(String... links) {
for (String link : links) {
TestContext ctx = testCreate(1);
this.registerForServiceAvailability(ctx.getCompletion(), link);
ctx.await();
}
}
public void waitForReplicatedFactoryServiceAvailable(URI u) {
waitForReplicatedFactoryServiceAvailable(u, ServiceUriPaths.DEFAULT_NODE_SELECTOR);
}
public void waitForReplicatedFactoryServiceAvailable(URI u, String nodeSelectorPath) {
waitFor("replicated available check time out for " + u, () -> {
boolean[] isReady = new boolean[1];
TestContext ctx = testCreate(1);
NodeGroupUtils.checkServiceAvailability((o, e) -> {
if (e != null) {
isReady[0] = false;
ctx.completeIteration();
return;
}
isReady[0] = true;
ctx.completeIteration();
}, this, u, nodeSelectorPath);
ctx.await();
return isReady[0];
});
}
public void waitForServiceAvailable(URI u) {
boolean[] isReady = new boolean[1];
log("Starting /available check on %s", u);
waitFor("available check timeout for " + u, () -> {
TestContext ctx = testCreate(1);
URI available = UriUtils.buildAvailableUri(u);
Operation get = Operation.createGet(available).setCompletion((o, e) -> {
if (e != null) {
// not ready
isReady[0] = false;
ctx.completeIteration();
return;
}
isReady[0] = true;
ctx.completeIteration();
return;
});
send(get);
ctx.await();
if (isReady[0]) {
log("%s /available returned success", get.getUri());
return true;
}
return false;
});
}
public <T extends ServiceDocument> Map<URI, T> doFactoryChildServiceStart(
EnumSet<TestProperty> props,
long c,
Class<T> bodyType,
Consumer<Operation> setInitialStateConsumer,
URI factoryURI) {
Map<URI, T> initialStates = new HashMap<>();
if (props == null) {
props = EnumSet.noneOf(TestProperty.class);
}
log("Sending %d POST requests to %s", c, factoryURI);
List<Operation> ops = new ArrayList<>();
for (int i = 0; i < c; i++) {
Operation createPost = Operation.createPost(factoryURI);
// call callback to set the body
setInitialStateConsumer.accept(createPost);
if (props.contains(TestProperty.FORCE_REMOTE)) {
createPost.forceRemote();
}
ops.add(createPost);
}
List<T> responses = this.sender.sendAndWait(ops, bodyType);
Map<URI, T> docByChildURI = responses.stream().collect(
toMap(doc -> UriUtils.buildUri(factoryURI, doc.documentSelfLink), identity()));
initialStates.putAll(docByChildURI);
log("Done with %d POST requests to %s", c, factoryURI);
return initialStates;
}
public List<Service> doThroughputServiceStart(long c, Class<? extends Service> type,
ServiceDocument initialState,
EnumSet<Service.ServiceOption> options,
EnumSet<Service.ServiceOption> optionsToRemove) throws Throwable {
return doThroughputServiceStart(EnumSet.noneOf(TestProperty.class), c, type, initialState,
options, null);
}
public List<Service> doThroughputServiceStart(
EnumSet<TestProperty> props,
long c, Class<? extends Service> type,
ServiceDocument initialState,
EnumSet<Service.ServiceOption> options,
EnumSet<Service.ServiceOption> optionsToRemove) throws Throwable {
return doThroughputServiceStart(props, c, type, initialState,
options, optionsToRemove, null);
}
public List<Service> doThroughputServiceStart(
EnumSet<TestProperty> props,
long c, Class<? extends Service> type,
ServiceDocument initialState,
EnumSet<Service.ServiceOption> options,
EnumSet<Service.ServiceOption> optionsToRemove,
Long maintIntervalMicros) throws Throwable {
List<Service> services = new ArrayList<>();
TestContext ctx = testCreate((int) c);
for (int i = 0; i < c; i++) {
Service e = type.newInstance();
if (options != null) {
for (Service.ServiceOption cap : options) {
e.toggleOption(cap, true);
}
}
if (optionsToRemove != null) {
for (ServiceOption opt : optionsToRemove) {
e.toggleOption(opt, false);
}
}
Operation post = createServiceStartPost(ctx);
if (initialState != null) {
post.setBody(initialState);
}
if (props != null && props.contains(TestProperty.SET_CONTEXT_ID)) {
post.setContextId(TestProperty.SET_CONTEXT_ID.toString());
}
if (maintIntervalMicros != null) {
e.setMaintenanceIntervalMicros(maintIntervalMicros);
}
startService(post, e);
services.add(e);
}
ctx.await();
logThroughput();
return services;
}
public Service startServiceAndWait(Class<? extends Service> serviceType,
String uriPath)
throws Throwable {
return startServiceAndWait(serviceType.newInstance(), uriPath, null);
}
public Service startServiceAndWait(Service s,
String uriPath,
ServiceDocument body)
throws Throwable {
TestContext ctx = testCreate(1);
URI u = null;
if (uriPath != null) {
u = UriUtils.buildUri(this, uriPath);
}
Operation post = Operation
.createPost(u)
.setBody(body)
.setCompletion(ctx.getCompletion());
startService(post, s);
ctx.await();
return s;
}
public <T extends ServiceDocument> void doServiceRestart(List<Service> services,
Class<T> stateType,
EnumSet<Service.ServiceOption> caps)
throws Throwable {
ServiceDocumentDescription sdd = buildDescription(stateType);
// first collect service state before shutdown so we can compare after
// they restart
Map<URI, T> statesBeforeRestart = getServiceState(null, stateType, services);
List<Service> freshServices = new ArrayList<>();
List<Operation> ops = new ArrayList<>();
for (Service s : services) {
// delete with no body means stop the service
Operation delete = Operation.createDelete(s.getUri());
ops.add(delete);
}
this.sender.sendAndWait(ops);
// restart services
TestContext ctx = testCreate(services.size());
for (Service oldInstance : services) {
Service e = oldInstance.getClass().newInstance();
for (Service.ServiceOption cap : caps) {
e.toggleOption(cap, true);
}
// use the same exact URI so the document index can find the service
// state by self link
startService(
Operation.createPost(oldInstance.getUri()).setCompletion(ctx.getCompletion()),
e);
freshServices.add(e);
}
ctx.await();
services = null;
Map<URI, T> statesAfterRestart = getServiceState(null, stateType, freshServices);
for (Entry<URI, T> e : statesAfterRestart.entrySet()) {
T stateAfter = e.getValue();
if (stateAfter.documentSelfLink == null) {
throw new IllegalStateException("missing selflink");
}
if (stateAfter.documentKind == null) {
throw new IllegalStateException("missing kind");
}
T stateBefore = statesBeforeRestart.get(e.getKey());
if (stateBefore == null) {
throw new IllegalStateException(
"New service has new self link, not in previous service instances");
}
if (!stateBefore.documentKind.equals(stateAfter.documentKind)) {
throw new IllegalStateException("kind mismatch");
}
if (!caps.contains(Service.ServiceOption.PERSISTENCE)) {
continue;
}
if (stateBefore.documentVersion != stateAfter.documentVersion) {
String error = String.format(
"Version mismatch. Before State: %s%n%n After state:%s",
Utils.toJson(stateBefore),
Utils.toJson(stateAfter));
throw new IllegalStateException(error);
}
if (stateBefore.documentUpdateTimeMicros != stateAfter.documentUpdateTimeMicros) {
throw new IllegalStateException("update time mismatch");
}
if (stateBefore.documentVersion == 0) {
throw new IllegalStateException("PUT did not appear to take place before restart");
}
if (!ServiceDocument.equals(sdd, stateBefore, stateAfter)) {
throw new IllegalStateException("content signature mismatch");
}
}
}
private Map<String, NodeState> peerHostIdToNodeState = new ConcurrentHashMap<>();
private Map<URI, URI> peerNodeGroups = new ConcurrentHashMap<>();
private Map<URI, VerificationHost> localPeerHosts = new ConcurrentHashMap<>();
private boolean isRemotePeerTest;
private boolean isSingleton;
public Map<URI, VerificationHost> getInProcessHostMap() {
return new HashMap<>(this.localPeerHosts);
}
public Map<URI, URI> getNodeGroupMap() {
return new HashMap<>(this.peerNodeGroups);
}
public Map<String, NodeState> getNodeStateMap() {
return new HashMap<>(this.peerHostIdToNodeState);
}
public void scheduleSynchronizationIfAutoSyncDisabled(String selectorPath) {
if (this.isPeerSynchronizationEnabled()) {
return;
}
for (VerificationHost peerHost : getInProcessHostMap().values()) {
peerHost.scheduleNodeGroupChangeMaintenance(selectorPath);
ServiceStats selectorStats = getServiceState(null, ServiceStats.class,
UriUtils.buildStatsUri(peerHost, selectorPath));
ServiceStat synchStat = selectorStats.entries
.get(ConsistentHashingNodeSelectorService.STAT_NAME_SYNCHRONIZATION_COUNT);
if (synchStat != null && synchStat.latestValue > 0) {
throw new IllegalStateException("Automatic synchronization was triggered");
}
}
}
public void setUpPeerHosts(int localHostCount) {
CommandLineArgumentParser.parseFromProperties(this);
if (this.peerNodes == null) {
this.setUpLocalPeersHosts(localHostCount, null);
} else {
this.setUpWithRemotePeers(this.peerNodes);
}
}
public void setUpLocalPeersHosts(int localHostCount, Long maintIntervalMillis) {
testStart(localHostCount);
if (maintIntervalMillis == null) {
maintIntervalMillis = this.maintenanceIntervalMillis;
}
final long intervalMicros = TimeUnit.MILLISECONDS.toMicros(maintIntervalMillis);
for (int i = 0; i < localHostCount; i++) {
String location = this.isMultiLocationTest
? ((i < localHostCount / 2) ? LOCATION1 : LOCATION2)
: null;
run(() -> {
try {
this.setUpLocalPeerHost(null, intervalMicros, location);
} catch (Throwable e) {
failIteration(e);
}
});
}
testWait();
}
public Map<URI, URI> getNodeGroupToFactoryMap(String factoryLink) {
Map<URI, URI> nodeGroupToFactoryMap = new HashMap<>();
for (URI nodeGroup : this.peerNodeGroups.values()) {
nodeGroupToFactoryMap.put(nodeGroup,
UriUtils.buildUri(nodeGroup.getScheme(), nodeGroup.getHost(),
nodeGroup.getPort(), factoryLink, null));
}
return nodeGroupToFactoryMap;
}
public VerificationHost setUpLocalPeerHost(Collection<ServiceHost> hosts,
long maintIntervalMicros) throws Throwable {
return setUpLocalPeerHost(0, maintIntervalMicros, hosts);
}
public VerificationHost setUpLocalPeerHost(int port, long maintIntervalMicros,
Collection<ServiceHost> hosts)
throws Throwable {
return setUpLocalPeerHost(port, maintIntervalMicros, hosts, null);
}
public VerificationHost setUpLocalPeerHost(Collection<ServiceHost> hosts,
long maintIntervalMicros, String location) throws Throwable {
return setUpLocalPeerHost(0, maintIntervalMicros, hosts, location);
}
public VerificationHost setUpLocalPeerHost(int port, long maintIntervalMicros,
Collection<ServiceHost> hosts, String location)
throws Throwable {
VerificationHost h = VerificationHost.create(port);
h.setPeerSynchronizationEnabled(this.isPeerSynchronizationEnabled());
h.setAuthorizationEnabled(this.isAuthorizationEnabled());
if (this.getCurrentHttpScheme() == HttpScheme.HTTPS_ONLY) {
// disable HTTP on new peer host
h.setPort(ServiceHost.PORT_VALUE_LISTENER_DISABLED);
// request a random HTTPS port
h.setSecurePort(0);
}
if (this.isAuthorizationEnabled()) {
h.setAuthorizationService(new AuthorizationContextService());
}
try {
VerificationHost.createAndAttachSSLClient(h);
// override with parent cert info.
// Within same node group, all hosts are required to use same cert, private key, and
// passphrase for now.
h.setCertificateFileReference(this.getState().certificateFileReference);
h.setPrivateKeyFileReference(this.getState().privateKeyFileReference);
h.setPrivateKeyPassphrase(this.getState().privateKeyPassphrase);
if (location != null) {
h.setLocation(location);
}
h.start();
h.setMaintenanceIntervalMicros(maintIntervalMicros);
} catch (Throwable e) {
throw new Exception(e);
}
addPeerNode(h);
if (hosts != null) {
hosts.add(h);
}
this.completeIteration();
return h;
}
public void setUpWithRemotePeers(String[] peerNodes) {
this.isRemotePeerTest = true;
this.peerNodeGroups.clear();
for (String remoteNode : peerNodes) {
URI remoteHostBaseURI = URI.create(remoteNode);
if (remoteHostBaseURI.getPort() == 80 || remoteHostBaseURI.getPort() == -1) {
remoteHostBaseURI = UriUtils.buildUri(remoteNode, ServiceHost.DEFAULT_PORT, "",
null);
}
URI remoteNodeGroup = UriUtils.extendUri(remoteHostBaseURI,
ServiceUriPaths.DEFAULT_NODE_GROUP);
this.peerNodeGroups.put(remoteHostBaseURI, remoteNodeGroup);
}
}
public void joinNodesAndVerifyConvergence(int nodeCount) throws Throwable {
joinNodesAndVerifyConvergence(null, nodeCount, nodeCount, null);
}
public boolean isRemotePeerTest() {
return this.isRemotePeerTest;
}
public int getPeerCount() {
return this.peerNodeGroups.size();
}
public URI getPeerHostUri() {
return getPeerServiceUri("");
}
public URI getPeerNodeGroupUri() {
return getPeerServiceUri(ServiceUriPaths.DEFAULT_NODE_GROUP);
}
/**
* Randomly returns one of peer hosts.
*/
public VerificationHost getPeerHost() {
URI hostUri = getPeerServiceUri(null);
if (hostUri != null) {
return this.localPeerHosts.get(hostUri);
}
return null;
}
public URI getPeerServiceUri(String link) {
if (!this.localPeerHosts.isEmpty()) {
List<URI> localPeerList = new ArrayList<>();
for (VerificationHost h : this.localPeerHosts.values()) {
if (h.isStopping() || !h.isStarted()) {
continue;
}
localPeerList.add(h.getUri());
}
return getUriFromList(link, localPeerList);
} else {
List<URI> peerList = new ArrayList<>(this.peerNodeGroups.keySet());
return getUriFromList(link, peerList);
}
}
/**
* Randomly choose one uri from uriList and extend with the link
*/
private URI getUriFromList(String link, List<URI> uriList) {
if (!uriList.isEmpty()) {
Collections.shuffle(uriList, new Random(System.nanoTime()));
URI baseUri = uriList.iterator().next();
return UriUtils.extendUri(baseUri, link);
}
return null;
}
public void createCustomNodeGroupOnPeers(String customGroupName) {
createCustomNodeGroupOnPeers(customGroupName, null);
}
public void createCustomNodeGroupOnPeers(String customGroupName,
Map<URI, NodeState> selfState) {
if (selfState == null) {
selfState = new HashMap<>();
}
// create a custom node group on all peer nodes
List<Operation> ops = new ArrayList<>();
for (URI peerHostBaseUri : getNodeGroupMap().keySet()) {
URI nodeGroupFactoryUri = UriUtils.buildUri(peerHostBaseUri,
ServiceUriPaths.NODE_GROUP_FACTORY);
Operation op = getCreateCustomNodeGroupOperation(customGroupName, nodeGroupFactoryUri,
selfState.get(peerHostBaseUri));
ops.add(op);
}
this.sender.sendAndWait(ops);
}
private Operation getCreateCustomNodeGroupOperation(String customGroupName,
URI nodeGroupFactoryUri,
NodeState selfState) {
NodeGroupState body = new NodeGroupState();
body.documentSelfLink = customGroupName;
if (selfState != null) {
body.nodes.put(selfState.id, selfState);
}
return Operation.createPost(nodeGroupFactoryUri).setBody(body);
}
public void joinNodesAndVerifyConvergence(String customGroupPath, int hostCount,
int memberCount,
Map<URI, EnumSet<NodeOption>> expectedOptionsPerNode)
throws Throwable {
joinNodesAndVerifyConvergence(customGroupPath, hostCount, memberCount,
expectedOptionsPerNode, true);
}
public void joinNodesAndVerifyConvergence(int hostCount, boolean waitForTimeSync)
throws Throwable {
joinNodesAndVerifyConvergence(hostCount, hostCount, waitForTimeSync);
}
public void joinNodesAndVerifyConvergence(int hostCount, int memberCount,
boolean waitForTimeSync) throws Throwable {
joinNodesAndVerifyConvergence(null, hostCount, memberCount, null, waitForTimeSync);
}
public void joinNodesAndVerifyConvergence(String customGroupPath, int hostCount,
int memberCount,
Map<URI, EnumSet<NodeOption>> expectedOptionsPerNode,
boolean waitForTimeSync) throws Throwable {
// invoke op as system user
setAuthorizationContext(getSystemAuthorizationContext());
if (hostCount == 0) {
return;
}
Map<URI, URI> nodeGroupPerHost = new HashMap<>();
if (customGroupPath != null) {
for (Entry<URI, URI> e : this.peerNodeGroups.entrySet()) {
URI ngUri = UriUtils.buildUri(e.getKey(), customGroupPath);
nodeGroupPerHost.put(e.getKey(), ngUri);
}
} else {
nodeGroupPerHost = this.peerNodeGroups;
}
if (this.isRemotePeerTest()) {
memberCount = getPeerCount();
}
if (!isRemotePeerTest() || (isRemotePeerTest() && this.joinNodes)) {
for (URI initialNodeGroupService : this.peerNodeGroups.values()) {
if (customGroupPath != null) {
initialNodeGroupService = UriUtils.buildUri(initialNodeGroupService,
customGroupPath);
}
for (URI nodeGroup : this.peerNodeGroups.values()) {
if (customGroupPath != null) {
nodeGroup = UriUtils.buildUri(nodeGroup, customGroupPath);
}
if (initialNodeGroupService.equals(nodeGroup)) {
continue;
}
testStart(1);
joinNodeGroup(nodeGroup, initialNodeGroupService, memberCount);
testWait();
}
}
}
// for local or remote tests, we still want to wait for convergence
waitForNodeGroupConvergence(nodeGroupPerHost.values(), memberCount, null,
expectedOptionsPerNode, waitForTimeSync);
waitForNodeGroupIsAvailableConvergence(customGroupPath);
//reset auth context
setAuthorizationContext(null);
}
public void joinNodeGroup(URI newNodeGroupService,
URI nodeGroup, Integer quorum) {
if (nodeGroup.equals(newNodeGroupService)) {
return;
}
// to become member of a group of nodes, you send a POST to self
// (the local node group service) with the URI of the remote node
// group you wish to join
JoinPeerRequest joinBody = JoinPeerRequest.create(nodeGroup, quorum);
log("Joining %s through %s", newNodeGroupService, nodeGroup);
// send the request to the node group instance we have picked as the
// "initial" one
send(Operation.createPost(newNodeGroupService)
.setBody(joinBody)
.setCompletion(getCompletion()));
}
public void joinNodeGroup(URI newNodeGroupService, URI nodeGroup) {
joinNodeGroup(newNodeGroupService, nodeGroup, null);
}
public void subscribeForNodeGroupConvergence(URI nodeGroup, int expectedAvailableCount,
CompletionHandler convergedCompletion) {
TestContext ctx = testCreate(1);
Operation subscribeToNodeGroup = Operation.createPost(
UriUtils.buildSubscriptionUri(nodeGroup))
.setCompletion(ctx.getCompletion())
.setReferer(getUri());
startSubscriptionService(subscribeToNodeGroup, (op) -> {
op.complete();
if (op.getAction() != Action.PATCH) {
return;
}
NodeGroupState ngs = op.getBody(NodeGroupState.class);
if (ngs.nodes == null && ngs.nodes.isEmpty()) {
return;
}
int c = 0;
for (NodeState ns : ngs.nodes.values()) {
if (ns.status == NodeStatus.AVAILABLE) {
c++;
}
}
if (c != expectedAvailableCount) {
return;
}
convergedCompletion.handle(op, null);
});
ctx.await();
}
public void waitForNodeGroupIsAvailableConvergence() {
waitForNodeGroupIsAvailableConvergence(ServiceUriPaths.DEFAULT_NODE_GROUP);
}
public void waitForNodeGroupIsAvailableConvergence(String nodeGroupPath) {
waitForNodeGroupIsAvailableConvergence(nodeGroupPath, this.peerNodeGroups.values());
}
public void waitForNodeGroupIsAvailableConvergence(String nodeGroupPath,
Collection<URI> nodeGroupUris) {
if (nodeGroupPath == null) {
nodeGroupPath = ServiceUriPaths.DEFAULT_NODE_GROUP;
}
String finalNodeGroupPath = nodeGroupPath;
waitFor("Node group is not available for convergence", () -> {
boolean isConverged = true;
for (URI nodeGroupUri : nodeGroupUris) {
URI u = UriUtils.buildUri(nodeGroupUri, finalNodeGroupPath);
URI statsUri = UriUtils.buildStatsUri(u);
ServiceStats stats = getServiceState(null, ServiceStats.class, statsUri);
ServiceStat availableStat = stats.entries.get(Service.STAT_NAME_AVAILABLE);
if (availableStat == null || availableStat.latestValue != Service.STAT_VALUE_TRUE) {
log("Service stat available is missing or not 1.0");
isConverged = false;
break;
}
}
return isConverged;
});
}
public void waitForNodeGroupConvergence() {
ArrayList<URI> nodeGroupUris = new ArrayList<>();
nodeGroupUris.add(UriUtils.extendUri(this.getUri(), ServiceUriPaths.DEFAULT_NODE_GROUP));
waitForNodeGroupConvergence(nodeGroupUris, 0, null, new HashMap<>(), false);
}
public void waitForNodeGroupConvergence(int memberCount) {
waitForNodeGroupConvergence(memberCount, null);
}
public void waitForNodeGroupConvergence(int healthyMemberCount, Integer totalMemberCount) {
waitForNodeGroupConvergence(this.peerNodeGroups.values(), healthyMemberCount,
totalMemberCount, true);
}
public void waitForNodeGroupConvergence(Collection<URI> nodeGroupUris, int healthyMemberCount,
Integer totalMemberCount,
boolean waitForTimeSync) {
waitForNodeGroupConvergence(nodeGroupUris, healthyMemberCount, totalMemberCount,
new HashMap<>(), waitForTimeSync);
}
/**
* Check node group convergence.
*
* Due to the implementation of {@link NodeGroupUtils#isNodeGroupAvailable}, quorum needs to
* be set less than the available node counts.
*
* Since {@link TestNodeGroupManager} requires all passing nodes to be in a same nodegroup,
* hosts in in-memory host map({@code this.localPeerHosts}) that do not match with the given
* nodegroup will be skipped for check.
*
* For existing API compatibility, keeping unused variables in signature.
* Only {@code nodeGroupUris} parameter is used.
*
* Sample node group URI: http://127.0.0.1:8000/core/node-groups/default
*
* @see TestNodeGroupManager#waitForConvergence()
*/
public void waitForNodeGroupConvergence(Collection<URI> nodeGroupUris,
int healthyMemberCount,
Integer totalMemberCount,
Map<URI, EnumSet<NodeOption>> expectedOptionsPerNodeGroupUri,
boolean waitForTimeSync) {
Set<String> nodeGroupNames = nodeGroupUris.stream()
.map(URI::getPath)
.map(UriUtils::getLastPathSegment)
.collect(toSet());
if (nodeGroupNames.size() != 1) {
throw new RuntimeException("Multiple nodegroups are not supported. " + nodeGroupNames);
}
String nodeGroupName = nodeGroupNames.iterator().next();
Date exp = getTestExpiration();
Duration timeout = Duration.between(Instant.now(), exp.toInstant());
// Convert "http://127.0.0.1:1234/core/node-groups/default" to "http://127.0.0.1:1234"
Set<URI> baseUris = nodeGroupUris.stream()
.map(uri -> uri.toString().replace(uri.getPath(), ""))
.map(URI::create)
.collect(toSet());
// pick up hosts that match with the base uris of given node group uris
Set<ServiceHost> hosts = getInProcessHostMap().values().stream()
.filter(host -> baseUris.contains(host.getPublicUri()))
.collect(toSet());
// perform "waitForConvergence()"
if (hosts != null && !hosts.isEmpty()) {
TestNodeGroupManager manager = new TestNodeGroupManager(nodeGroupName);
manager.addHosts(hosts);
manager.setTimeout(timeout);
manager.waitForConvergence();
} else {
this.waitFor("Node group did not converge", () -> {
String nodeGroupPath = ServiceUriPaths.NODE_GROUP_FACTORY + "/" + nodeGroupName;
List<Operation> nodeGroupOps = baseUris.stream()
.map(u -> UriUtils.buildUri(u, nodeGroupPath))
.map(Operation::createGet)
.collect(toList());
List<NodeGroupState> nodeGroupStates = getTestRequestSender()
.sendAndWait(nodeGroupOps, NodeGroupState.class);
for (NodeGroupState nodeGroupState : nodeGroupStates) {
TestContext testContext = this.testCreate(1);
// placeholder operation
Operation parentOp = Operation.createGet(null)
.setReferer(this.getUri())
.setCompletion(testContext.getCompletion());
try {
NodeGroupUtils.checkConvergenceFromAnyHost(this, nodeGroupState, parentOp);
testContext.await();
} catch (Exception e) {
return false;
}
}
return true;
});
}
// To be compatible with old behavior, populate peerHostIdToNodeState same way as before
List<Operation> nodeGroupGetOps = nodeGroupUris.stream()
.map(UriUtils::buildExpandLinksQueryUri)
.map(Operation::createGet)
.collect(toList());
List<NodeGroupState> nodeGroupStats = this.sender.sendAndWait(nodeGroupGetOps, NodeGroupState.class);
for (NodeGroupState nodeGroupStat : nodeGroupStats) {
for (NodeState nodeState : nodeGroupStat.nodes.values()) {
if (nodeState.status == NodeStatus.AVAILABLE) {
this.peerHostIdToNodeState.put(nodeState.id, nodeState);
}
}
}
}
/**
* Check replicated factory child services for convergence.
*
* Wait for all hosts until they all have child services
* with the expected URIs and have the same state and replicated
* to desired number of hosts.
*
* For scenarios where a new node is added or restarted, this method
* could be used for convergence of the service.
*
* See also waitForReplicatedFactoryServiceAvailable() which should be used
* in most of the cases when test wants to wait for replicated factory to be
* available. That method just waits for STAT_NAME_AVAILABLE stat on the
* owner of the factory service.
*
*/
public <T extends ServiceDocument> Map<String, T> waitForReplicatedFactoryChildServiceConvergence(
Map<URI, URI> factories,
Map<String, T> serviceStates,
BiPredicate<T, T> stateChecker,
int expectedChildCount, long expectedVersion, long replicationFactor)
throws Throwable, TimeoutException {
// now poll all hosts until they converge: They all have a child service
// with the expected URI and it has the same state
Map<String, T> updatedStatesPerSelfLink = new HashMap<>();
Date expiration = new Date(new Date().getTime()
+ TimeUnit.SECONDS.toMillis(this.getTimeoutSeconds()));
do {
URI node = factories.keySet().iterator().next();
AtomicInteger getFailureCount = new AtomicInteger();
if (expectedChildCount != 0) {
// issue direct GETs to the services, we do not trust the factory
for (String link : serviceStates.keySet()) {
TestContext ctx = this.testCreate(1);
Operation get = Operation.createGet(UriUtils.buildUri(node, link))
.setReferer(this.getReferer())
.setExpiration(
Utils.fromNowMicrosUtc(TimeUnit.SECONDS.toMicros(5)))
.setCompletion(
(o, e) -> {
if (e != null) {
getFailureCount.incrementAndGet();
}
ctx.completeIteration();
});
this.sendRequest(get);
this.testWait(ctx);
}
}
if (getFailureCount.get() > 0) {
this.log("Child services not propagated yet. Failure count: %d",
getFailureCount.get());
Thread.sleep(500);
continue;
}
TestContext testContext = this.testCreate(factories.size());
Map<URI, ServiceDocumentQueryResult> childServicesPerNode = new HashMap<>();
for (URI remoteFactory : factories.values()) {
URI factoryUriWithExpand = UriUtils.extendUriWithQuery(remoteFactory,
UriUtils.URI_PARAM_ODATA_EXPAND,
ServiceDocumentQueryResult.FIELD_NAME_DOCUMENT_LINKS);
if (this.isStressTest()) {
// set an arbitrary, but very high result limit on GET, to avoid query failure
// during long running replication tests
final int resultLimit = 10000000;
factoryUriWithExpand = UriUtils.extendUriWithQuery(remoteFactory,
UriUtils.URI_PARAM_ODATA_EXPAND,
ServiceDocumentQueryResult.FIELD_NAME_DOCUMENT_LINKS,
UriUtils.URI_PARAM_ODATA_TOP,
"" + resultLimit);
}
Operation get = Operation.createGet(factoryUriWithExpand)
.setCompletion(
(o, e) -> {
if (e != null) {
testContext.complete();
return;
}
if (!o.hasBody()) {
testContext.complete();
return;
}
ServiceDocumentQueryResult r = o
.getBody(ServiceDocumentQueryResult.class);
synchronized (childServicesPerNode) {
childServicesPerNode.put(o.getUri(), r);
}
testContext.complete();
});
this.send(get);
}
this.testWait(testContext);
long expectedNodeCountPerLinkMax = factories.size();
long expectedNodeCountPerLinkMin = expectedNodeCountPerLinkMax;
if (replicationFactor != 0) {
// We expect services to end up either on K nodes, or K + 1 nodes,
// if limited replication is enabled. The reason we might end up with services on
// an additional node, is because we elect an owner to synchronize an entire factory,
// using the factory's link, and that might end up on a node not used for any child.
// This will produce children on that node, giving us K+1 replication, which is acceptable
// given K (replication factor) << N (total nodes in group)
expectedNodeCountPerLinkMax = replicationFactor + 1;
expectedNodeCountPerLinkMin = replicationFactor;
}
if (expectedChildCount == 0) {
expectedNodeCountPerLinkMax = 0;
expectedNodeCountPerLinkMin = 0;
}
// build a service link to node map so we can tell on which node each service instance landed
Map<String, Set<URI>> linkToNodeMap = new HashMap<>();
Map<String, String> linkToNodeId = new HashMap<>();
boolean isConverged = true;
for (Entry<URI, ServiceDocumentQueryResult> entry : childServicesPerNode.entrySet()) {
for (String link : entry.getValue().documentLinks) {
if (!serviceStates.containsKey(link)) {
this.log("service %s not expected, node: %s", link, entry.getKey());
isConverged = false;
continue;
}
Set<URI> hostsPerLink = linkToNodeMap.get(link);
if (hostsPerLink == null) {
hostsPerLink = new HashSet<>();
}
hostsPerLink.add(entry.getKey());
linkToNodeMap.put(link, hostsPerLink);
}
}
if (!isConverged) {
Thread.sleep(500);
continue;
}
// each link must exist on N hosts, where N is either the replication factor, or, if not used, all nodes
for (Entry<String, Set<URI>> e : linkToNodeMap.entrySet()) {
if (e.getValue() == null && replicationFactor == 0) {
this.log("Service %s not found on any nodes", e.getKey());
isConverged = false;
continue;
}
if (e.getValue().size() < expectedNodeCountPerLinkMin
|| e.getValue().size() > expectedNodeCountPerLinkMax) {
this.log("Service %s found on %d nodes, expected %d -> %d", e.getKey(), e
.getValue().size(), expectedNodeCountPerLinkMin,
expectedNodeCountPerLinkMax);
isConverged = false;
}
}
if (!isConverged) {
Thread.sleep(500);
continue;
}
if (expectedChildCount == 0) {
// DELETE test, all children removed from all hosts, we are done
return updatedStatesPerSelfLink;
}
// verify /available reports correct results on the factory.
URI factoryUri = factories.values().iterator().next();
Class<?> stateType = serviceStates.values().iterator().next().getClass();
waitForReplicatedFactoryServiceAvailable(factoryUri,
ServiceUriPaths.DEFAULT_NODE_SELECTOR);
// we have the correct number of services on all hosts. Now verify
// the state of each service matches what we expect
isConverged = true;
for (Entry<String, Set<URI>> entry : linkToNodeMap.entrySet()) {
String selfLink = entry.getKey();
int convergedNodeCount = 0;
for (URI nodeUri : entry.getValue()) {
ServiceDocumentQueryResult childLinksAndDocsPerHost = childServicesPerNode
.get(nodeUri);
Object jsonState = childLinksAndDocsPerHost.documents.get(selfLink);
if (jsonState == null && replicationFactor == 0) {
this.log("Service %s not present on host %s", selfLink, entry.getKey());
continue;
}
if (jsonState == null) {
continue;
}
T initialState = serviceStates.get(selfLink);
if (initialState == null) {
continue;
}
@SuppressWarnings("unchecked")
T stateOnNode = (T) Utils.fromJson(jsonState, stateType);
stateOnNode.documentUpdateTimeMicros = 0;
initialState.documentUpdateTimeMicros = 0;
if (!stateChecker.test(initialState, stateOnNode)) {
this.log("111111 State for %s not converged on node %s. Current state: %s, Initial: %s",
selfLink, nodeUri, Utils.toJsonHtml(stateOnNode),
Utils.toJsonHtml(initialState));
break;
}
if (stateOnNode.documentVersion < expectedVersion) {
this.log("Version (%d, expected %d) not converged, state: %s",
stateOnNode.documentVersion,
expectedVersion,
Utils.toJsonHtml(stateOnNode));
break;
}
if (stateOnNode.documentEpoch == null) {
this.log("Epoch is missing, state: %s",
Utils.toJsonHtml(stateOnNode));
break;
}
List<String> nodeIds = this.getInProcessHostMap().values().stream().map(
host -> host.getId()).collect(Collectors.toList());
if (!nodeIds.contains(stateOnNode.documentOwner)) {
this.log("Owner mismatch on peers, Actual Id:%s Expected Id from:%s State: %s",
stateOnNode.documentOwner, String.join(",", nodeIds), Utils.toJsonHtml(stateOnNode));
break;
}
if (linkToNodeId.containsKey(stateOnNode.documentSelfLink)) {
if (!linkToNodeId.get(stateOnNode.documentSelfLink).equals(stateOnNode.documentOwner)) {
this.log("Owner mismatch on peers, Actual Id:%s Expected Id:%s state: %s",
childLinksAndDocsPerHost.documentOwner,
linkToNodeId.get(stateOnNode.documentSelfLink),
Utils.toJsonHtml(stateOnNode));
break;
}
} else {
linkToNodeId.put(stateOnNode.documentSelfLink, stateOnNode.documentOwner);
}
// Do not check exampleState.counter, in this validation loop.
// We can not compare the counter since the replication test sends the updates
// in parallel, meaning some of them will get re-ordered and ignored due to
// version being out of date.
updatedStatesPerSelfLink.put(selfLink, stateOnNode);
convergedNodeCount++;
}
if (convergedNodeCount < expectedNodeCountPerLinkMin
|| convergedNodeCount > expectedNodeCountPerLinkMax) {
isConverged = false;
break;
}
}
if (isConverged) {
return updatedStatesPerSelfLink;
}
Thread.sleep(500);
} while (new Date().before(expiration));
throw new TimeoutException();
}
public int calculateHealthyNodet(NodeGroupState r) {
int healthyNodeCount = 0;
for (NodeState ns : r.nodes.values()) {
if (ns.status == NodeStatus.AVAILABLE) {
healthyNodeCount++;
}
}
return healthyNodeCount;
}
public void getNodeState(URI nodeGroup, Map<URI, NodeGroupState> nodesPerHost) {
getNodeState(nodeGroup, nodesPerHost, null);
}
public void getNodeState(URI nodeGroup, Map<URI, NodeGroupState> nodesPerHost,
TestContext ctx) {
URI u = UriUtils.buildExpandLinksQueryUri(nodeGroup);
Operation get = Operation.createGet(u).setCompletion((o, e) -> {
NodeGroupState ngs = null;
if (e != null) {
// failure is OK, since we might have just stopped a host
log("Host %s failed GET with %s", nodeGroup, e.getMessage());
ngs = new NodeGroupState();
} else {
ngs = o.getBody(NodeGroupState.class);
}
synchronized (nodesPerHost) {
nodesPerHost.put(nodeGroup, ngs);
}
if (ctx == null) {
completeIteration();
} else {
ctx.completeIteration();
}
});
send(get);
}
public void validateNodes(NodeGroupState r, int expectedNodesPerGroup,
Map<URI, EnumSet<NodeOption>> expectedOptionsPerNode) {
int healthyNodes = 0;
NodeState localNode = null;
for (NodeState ns : r.nodes.values()) {
if (ns.status == NodeStatus.AVAILABLE) {
healthyNodes++;
}
assertTrue(ns.documentKind.equals(Utils.buildKind(NodeState.class)));
if (ns.documentSelfLink.endsWith(r.documentOwner)) {
localNode = ns;
}
assertTrue(ns.options != null);
EnumSet<NodeOption> expectedOptions = expectedOptionsPerNode.get(ns.groupReference);
if (expectedOptions == null) {
expectedOptions = NodeState.DEFAULT_OPTIONS;
}
for (NodeOption eo : expectedOptions) {
assertTrue(ns.options.contains(eo));
}
assertTrue(ns.id != null);
assertTrue(ns.groupReference != null);
assertTrue(ns.documentSelfLink.startsWith(ns.groupReference.getPath()));
}
assertTrue(healthyNodes >= expectedNodesPerGroup);
assertTrue(localNode != null);
}
public void doNodeGroupStatsVerification(Map<URI, URI> defaultNodeGroupsPerHost) {
waitFor("peer gossip stats not found", () -> {
List<Operation> ops = new ArrayList<>();
for (URI nodeGroup : defaultNodeGroupsPerHost.values()) {
Operation get = Operation.createGet(UriUtils.extendUri(nodeGroup,
ServiceHost.SERVICE_URI_SUFFIX_STATS));
ops.add(get);
}
int peerCount = defaultNodeGroupsPerHost.size();
List<Operation> results = this.sender.sendAndWait(ops);
for (Operation result : results) {
ServiceStats stats = result.getBody(ServiceStats.class);
if (stats.entries.isEmpty()) {
return false;
}
int gossipPatchStatCount = 0;
for (ServiceStat st : stats.entries.values()) {
if (!st.name
.contains(NodeGroupService.STAT_NAME_PREFIX_GOSSIP_PATCH_DURATION)) {
continue;
}
gossipPatchStatCount++;
if (st.logHistogram == null) {
return false;
}
if (st.timeSeriesStats == null) {
return false;
}
if (st.version < 1) {
return false;
}
}
if (gossipPatchStatCount != peerCount - 1) {
return false;
}
}
return true;
});
}
public void setNodeGroupConfig(NodeGroupConfig config) {
setSystemAuthorizationContext();
List<Operation> ops = new ArrayList<>();
for (URI nodeGroup : getNodeGroupMap().values()) {
NodeGroupState body = new NodeGroupState();
body.config = config;
body.nodes = null;
ops.add(Operation.createPatch(nodeGroup).setBody(body));
}
this.sender.sendAndWait(ops);
resetAuthorizationContext();
}
public void setNodeGroupQuorum(Integer quorum, URI nodeGroup) {
setNodeGroupQuorum(quorum, null, nodeGroup);
}
public void setNodeGroupQuorum(Integer quorum, Integer locationQuorum, URI nodeGroup) {
UpdateQuorumRequest body = UpdateQuorumRequest.create(true);
if (quorum != null) {
body.setMembershipQuorum(quorum);
}
if (locationQuorum != null) {
body.setLocationQuorum(locationQuorum);
}
this.sender.sendAndWait(Operation.createPatch(nodeGroup).setBody(body));
}
public void setNodeGroupQuorum(Integer quorum) throws Throwable {
setNodeGroupQuorum(quorum, (Integer) null);
}
public void setNodeGroupQuorum(Integer quorum, Integer locationQuorum) throws Throwable {
// we can issue the update to any one node and it will update
// everyone in the group
setSystemAuthorizationContext();
for (URI nodeGroup : getNodeGroupMap().values()) {
if (quorum != null) {
log("Changing quorum to %d on group %s", quorum, nodeGroup);
}
if (locationQuorum != null) {
log("Changing location quorum to %d on group %s", locationQuorum, nodeGroup);
}
setNodeGroupQuorum(quorum, locationQuorum, nodeGroup);
// nodes might not be joined, so we need to ask each node to set quorum
}
resetAuthorizationContext();
waitFor("quorum did not converge", () -> {
setSystemAuthorizationContext();
for (URI n : this.peerNodeGroups.values()) {
NodeGroupState s = getServiceState(null, NodeGroupState.class, n);
for (NodeState ns : s.nodes.values()) {
if (!NodeStatus.AVAILABLE.equals(ns.status)) {
continue;
}
if (quorum != ns.membershipQuorum) {
return false;
}
if (locationQuorum != null && !locationQuorum.equals(ns.locationQuorum)) {
return false;
}
}
}
resetAuthorizationContext();
return true;
});
}
public void waitForNodeSelectorQuorumConvergence(String nodeSelectorPath, int quorum) {
waitFor("quorum not updated", () -> {
for (URI peerHostUri : getNodeGroupMap().keySet()) {
URI nodeSelectorUri = UriUtils.buildUri(peerHostUri, nodeSelectorPath);
NodeSelectorState nss = getServiceState(null, NodeSelectorState.class,
nodeSelectorUri);
if (nss.membershipQuorum != quorum) {
return false;
}
}
return true;
});
}
public <T extends ServiceDocument> void validateDocumentPartitioning(
Map<URI, T> provisioningTasks,
Class<T> type) {
Map<String, Map<String, Long>> taskToOwnerCount = new HashMap<>();
for (URI baseHostURI : getNodeGroupMap().keySet()) {
List<URI> documentsPerDcpHost = new ArrayList<>();
for (URI serviceUri : provisioningTasks.keySet()) {
URI u = UriUtils.extendUri(baseHostURI, serviceUri.getPath());
documentsPerDcpHost.add(u);
}
Map<URI, T> tasksOnThisHost = getServiceState(
null,
type, documentsPerDcpHost);
for (T task : tasksOnThisHost.values()) {
Map<String, Long> ownerCount = taskToOwnerCount.get(task.documentSelfLink);
if (ownerCount == null) {
ownerCount = new HashMap<>();
taskToOwnerCount.put(task.documentSelfLink, ownerCount);
}
Long count = ownerCount.get(task.documentOwner);
if (count == null) {
count = 0L;
}
count++;
ownerCount.put(task.documentOwner, count);
}
}
// now verify that each task had a single owner assigned to it
for (Entry<String, Map<String, Long>> e : taskToOwnerCount.entrySet()) {
Map<String, Long> owners = e.getValue();
if (owners.size() > 1) {
throw new IllegalStateException("Multiple owners assigned on task " + e.getKey());
}
}
}
/**
* @return list of full urls of the created example services
*/
public List<URI> createExampleServices(ServiceHost h, long serviceCount, Long expiration) {
return createExampleServices(h, serviceCount, expiration, false, ExampleService.FACTORY_LINK);
}
/**
* @return list of full urls of the created example services
*/
public List<URI> createExampleServices(
ServiceHost h, long serviceCount, Long expiration, boolean skipAvailabilityCheck) {
return createExampleServices(h, serviceCount, expiration, skipAvailabilityCheck, ExampleService.FACTORY_LINK);
}
/**
* @return list of full urls of the created example services
*/
public List<URI> createExampleServices(
ServiceHost h, long serviceCount, Long expiration, boolean skipAvailabilityCheck,
String factoryLink) {
if (!skipAvailabilityCheck) {
waitForServiceAvailable(factoryLink);
}
List<ExampleServiceState> result = createExampleServices(h, serviceCount, expiration, factoryLink);
// returns list of full url
return result.stream()
.map(state -> UriUtils.extendUri(h.getUri(), state.documentSelfLink))
.collect(toList());
}
/**
* @return list of full states of the created example services
*/
public List<ExampleServiceState> createExampleServices(
ServiceHost h, long serviceCount, Long expiration, String factoryLink) {
List<Operation> ops = new ArrayList<>();
for (int i = 0; i < serviceCount; i++) {
ExampleServiceState initState = new ExampleServiceState();
initState.counter = 123L;
if (expiration != null) {
initState.documentExpirationTimeMicros = expiration;
}
initState.name = initState.documentSelfLink = UUID.randomUUID().toString();
Operation post = Operation.createPost(UriUtils.buildUri(h, factoryLink)).setBody(initState);
ops.add(post);
}
return this.sender.sendAndWait(ops, ExampleServiceState.class);
}
public Date getTestExpiration() {
long duration = this.timeoutSeconds + this.testDurationSeconds;
return new Date(new Date().getTime()
+ TimeUnit.SECONDS.toMillis(duration));
}
public boolean isStressTest() {
return this.isStressTest;
}
public void setStressTest(boolean isStressTest) {
this.isStressTest = isStressTest;
if (isStressTest) {
this.timeoutSeconds = 600;
this.setOperationTimeOutMicros(TimeUnit.SECONDS.toMicros(this.timeoutSeconds));
} else {
this.timeoutSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(
ServiceHostState.DEFAULT_OPERATION_TIMEOUT_MICROS);
}
}
public boolean isMultiLocationTest() {
return this.isMultiLocationTest;
}
public void setMultiLocationTest(boolean isMultiLocationTest) {
this.isMultiLocationTest = isMultiLocationTest;
}
public void toggleServiceOptions(URI serviceUri, EnumSet<ServiceOption> optionsToEnable,
EnumSet<ServiceOption> optionsToDisable) {
ServiceConfigUpdateRequest updateBody = ServiceConfigUpdateRequest.create();
updateBody.removeOptions = optionsToDisable;
updateBody.addOptions = optionsToEnable;
URI configUri = UriUtils.buildConfigUri(serviceUri);
this.sender.sendAndWait(Operation.createPatch(configUri).setBody(updateBody));
}
public void setOperationQueueLimit(URI serviceUri, int limit) {
// send a set limit configuration request
ServiceConfigUpdateRequest body = ServiceConfigUpdateRequest.create();
body.operationQueueLimit = limit;
URI configUri = UriUtils.buildConfigUri(serviceUri);
this.sender.sendAndWait(Operation.createPatch(configUri).setBody(body));
// verify new operation limit is set
ServiceConfiguration config = this.sender.sendAndWait(Operation.createGet(configUri),
ServiceConfiguration.class);
assertEquals("Invalid queue limit", body.operationQueueLimit,
(Integer) config.operationQueueLimit);
}
public void toggleNegativeTestMode(boolean enable) {
log("++++++ Negative test mode %s, failure logs expected: %s", enable, enable);
}
public void logNodeProcessLogs(Set<URI> keySet, String logSuffix) {
List<URI> logServices = new ArrayList<>();
for (URI host : keySet) {
logServices.add(UriUtils.extendUri(host, logSuffix));
}
Map<URI, LogServiceState> states = this.getServiceState(null, LogServiceState.class,
logServices);
for (Entry<URI, LogServiceState> entry : states.entrySet()) {
log("Process log for node %s\n\n%s", entry.getKey(),
Utils.toJsonHtml(entry.getValue()));
}
}
public void logNodeManagementState(Set<URI> keySet) {
List<URI> services = new ArrayList<>();
for (URI host : keySet) {
services.add(UriUtils.extendUri(host, ServiceUriPaths.CORE_MANAGEMENT));
}
Map<URI, ServiceHostState> states = this.getServiceState(null, ServiceHostState.class,
services);
for (Entry<URI, ServiceHostState> entry : states.entrySet()) {
log("Management state for node %s\n\n%s", entry.getKey(),
Utils.toJsonHtml(entry.getValue()));
}
}
public void tearDownInProcessPeers() {
for (VerificationHost h : this.localPeerHosts.values()) {
if (h == null) {
continue;
}
stopHost(h);
}
}
public void stopHost(VerificationHost host) {
log("Stopping host %s (%s)", host.getUri(), host.getId());
host.tearDown();
this.peerHostIdToNodeState.remove(host.getId());
this.peerNodeGroups.remove(host.getUri());
this.localPeerHosts.remove(host.getUri());
}
public void stopHostAndPreserveState(ServiceHost host) {
log("Stopping host %s", host.getUri());
// Do not delete the temporary directory with the lucene index. Notice that
// we do not call host.tearDown(), which will delete disk state, we simply
// stop the host and remove it from the peer node tracking tables
host.stop();
this.peerHostIdToNodeState.remove(host.getId());
this.peerNodeGroups.remove(host.getUri());
this.localPeerHosts.remove(host.getUri());
}
public boolean isLongDurationTest() {
return this.testDurationSeconds > 0;
}
public void logServiceStats(URI uri, TestResults testResults) {
ServiceStats serviceStats = logServiceStats(uri);
if (testResults != null) {
testResults.getReport().stats(uri, serviceStats);
}
}
public ServiceStats logServiceStats(URI uri) {
ServiceStats stats = null;
try {
stats = getServiceState(null, ServiceStats.class, UriUtils.buildStatsUri(uri));
if (stats == null || stats.entries == null) {
return null;
}
StringBuilder sb = new StringBuilder();
sb.append(String.format("Stats for %s%n", uri));
sb.append(String.format("\tCount\t\t\tAvg\t\tTotal\t\t\tName%n"));
stats.entries.values().stream()
.sorted((s1, s2) -> s1.name.compareTo(s2.name))
.forEach((s) -> logStat(uri, s, sb));
log(sb.toString());
} catch (Throwable e) {
log("Failure getting stats: %s", e.getMessage());
}
return stats;
}
private void logStat(URI serviceUri, ServiceStat st, StringBuilder sb) {
ServiceStatLogHistogram hist = st.logHistogram;
st.logHistogram = null;
double total = st.accumulatedValue != 0 ? st.accumulatedValue : st.latestValue;
double avg = total / st.version;
sb.append(
String.format("\t%08d\t\t%08.2f\t%010.2f\t%s%n", st.version, avg, total, st.name));
if (hist == null) {
return;
}
}
/**
* Retrieves node group service state from all peers and logs it in JSON format
*/
public void logNodeGroupState() {
List<Operation> ops = new ArrayList<>();
for (URI nodeGroup : getNodeGroupMap().values()) {
ops.add(Operation.createGet(nodeGroup));
}
List<NodeGroupState> stats = this.sender.sendAndWait(ops, NodeGroupState.class);
for (NodeGroupState stat : stats) {
log("%s", Utils.toJsonHtml(stat));
}
}
public void setServiceMaintenanceIntervalMicros(String path, long micros) {
setServiceMaintenanceIntervalMicros(UriUtils.buildUri(this, path), micros);
}
public void setServiceMaintenanceIntervalMicros(URI u, long micros) {
ServiceConfigUpdateRequest updateBody = ServiceConfigUpdateRequest.create();
updateBody.maintenanceIntervalMicros = micros;
URI configUri = UriUtils.extendUri(u, ServiceHost.SERVICE_URI_SUFFIX_CONFIG);
this.sender.sendAndWait(Operation.createPatch(configUri).setBody(updateBody));
}
/**
* Toggles operation tracing on the service host using the management service
*/
public void toggleOperationTracing(URI baseHostURI, boolean enable) {
toggleOperationTracing(baseHostURI, null, enable);
}
/**
* Toggles operation tracing on the service host using the management service
*/
public void toggleOperationTracing(URI baseHostURI, Level level, boolean enable) {
ServiceHostManagementService.ConfigureOperationTracingRequest r = new ServiceHostManagementService.ConfigureOperationTracingRequest();
r.enable = enable ? ServiceHostManagementService.OperationTracingEnable.START
: ServiceHostManagementService.OperationTracingEnable.STOP;
if (level != null) {
r.level = level.toString();
}
r.kind = ServiceHostManagementService.ConfigureOperationTracingRequest.KIND;
this.setSystemAuthorizationContext();
// we convert body to JSON to verify client requests using HTTP client
// with JSON, will work
this.sender.sendAndWait(Operation.createPatch(
UriUtils.extendUri(baseHostURI, ServiceHostManagementService.SELF_LINK))
.setBody(Utils.toJson(r)));
this.resetAuthorizationContext();
}
public CompletionHandler getSuccessOrFailureCompletion() {
return (o, e) -> {
completeIteration();
};
}
public static QueryValidationServiceState buildQueryValidationState() {
QueryValidationServiceState newState = new QueryValidationServiceState();
newState.ignoredStringValue = "should be ignored by index";
newState.exampleValue = new ExampleServiceState();
newState.exampleValue.counter = 10L;
newState.exampleValue.name = "example name";
newState.nestedComplexValue = new NestedType();
newState.nestedComplexValue.id = UUID.randomUUID().toString();
newState.nestedComplexValue.longValue = Long.MIN_VALUE;
newState.listOfExampleValues = new ArrayList<>();
ExampleServiceState exampleItem = new ExampleServiceState();
exampleItem.name = "nested name";
newState.listOfExampleValues.add(exampleItem);
newState.listOfStrings = new ArrayList<>();
for (int i = 0; i < 10; i++) {
newState.listOfStrings.add(UUID.randomUUID().toString());
}
newState.arrayOfExampleValues = new ExampleServiceState[2];
newState.arrayOfExampleValues[0] = new ExampleServiceState();
newState.arrayOfExampleValues[0].name = UUID.randomUUID().toString();
newState.arrayOfStrings = new String[2];
newState.arrayOfStrings[0] = UUID.randomUUID().toString();
newState.arrayOfStrings[1] = UUID.randomUUID().toString();
newState.mapOfStrings = new HashMap<>();
String keyOne = "keyOne";
String keyTwo = "keyTwo";
String valueOne = UUID.randomUUID().toString();
String valueTwo = UUID.randomUUID().toString();
newState.mapOfStrings.put(keyOne, valueOne);
newState.mapOfStrings.put(keyTwo, valueTwo);
newState.mapOfBooleans = new HashMap<>();
newState.mapOfBooleans.put("trueKey", true);
newState.mapOfBooleans.put("falseKey", false);
newState.mapOfBytesArrays = new HashMap<>();
newState.mapOfBytesArrays.put("bytes", new byte[] { 0x01, 0x02 });
newState.mapOfDoubles = new HashMap<>();
newState.mapOfDoubles.put("one", 1.0);
newState.mapOfDoubles.put("minusOne", -1.0);
newState.mapOfEnums = new HashMap<>();
newState.mapOfEnums.put("GET", Service.Action.GET);
newState.mapOfLongs = new HashMap<>();
newState.mapOfLongs.put("one", 1L);
newState.mapOfLongs.put("two", 2L);
newState.mapOfNestedTypes = new HashMap<>();
newState.mapOfNestedTypes.put("nested", newState.nestedComplexValue);
newState.mapOfUris = new HashMap<>();
newState.mapOfUris.put("uri", UriUtils.buildUri("/foo/bar"));
newState.ignoredArrayOfStrings = new String[2];
newState.ignoredArrayOfStrings[0] = UUID.randomUUID().toString();
newState.ignoredArrayOfStrings[1] = UUID.randomUUID().toString();
newState.binaryContent = UUID.randomUUID().toString().getBytes();
return newState;
}
public void updateServiceOptions(Collection<String> selfLinks,
ServiceConfigUpdateRequest cfgBody) {
List<Operation> ops = new ArrayList<>();
for (String link : selfLinks) {
URI bUri = UriUtils.buildUri(getUri(), link,
ServiceHost.SERVICE_URI_SUFFIX_CONFIG);
ops.add(Operation.createPatch(bUri).setBody(cfgBody));
}
this.sender.sendAndWait(ops);
}
public void addPeerNode(VerificationHost h) {
URI localBaseURI = h.getPublicUri();
URI nodeGroup = UriUtils.buildUri(h.getPublicUri(), ServiceUriPaths.DEFAULT_NODE_GROUP);
this.peerNodeGroups.put(localBaseURI, nodeGroup);
this.localPeerHosts.put(localBaseURI, h);
}
public void addPeerNode(URI ngUri) {
URI hostUri = UriUtils.buildUri(ngUri.getScheme(), ngUri.getHost(), ngUri.getPort(), null,
null);
this.peerNodeGroups.put(hostUri, ngUri);
}
public ServiceDocumentDescription buildDescription(Class<? extends ServiceDocument> type) {
EnumSet<ServiceOption> options = EnumSet.noneOf(ServiceOption.class);
return Builder.create().buildDescription(type, options);
}
public void logAllDocuments(Set<URI> baseHostUris) {
QueryTask task = new QueryTask();
task.setDirect(true);
task.querySpec = new QuerySpecification();
task.querySpec.query.setTermPropertyName("documentSelfLink").setTermMatchValue("*");
task.querySpec.query.setTermMatchType(MatchType.WILDCARD);
task.querySpec.options = EnumSet.of(QueryOption.EXPAND_CONTENT);
List<Operation> ops = new ArrayList<>();
for (URI baseHost : baseHostUris) {
Operation queryPost = Operation
.createPost(UriUtils.buildUri(baseHost, ServiceUriPaths.CORE_QUERY_TASKS))
.setBody(task);
ops.add(queryPost);
}
List<QueryTask> queryTasks = this.sender.sendAndWait(ops, QueryTask.class);
for (QueryTask queryTask : queryTasks) {
log(Utils.toJsonHtml(queryTask));
}
}
public void setSystemAuthorizationContext() {
setAuthorizationContext(getSystemAuthorizationContext());
}
public void resetSystemAuthorizationContext() {
super.setAuthorizationContext(null);
}
@Override
public void addPrivilegedService(Class<? extends Service> serviceType) {
// Overriding just for test cases
super.addPrivilegedService(serviceType);
}
@Override
public void setAuthorizationContext(AuthorizationContext context) {
super.setAuthorizationContext(context);
}
public void resetAuthorizationContext() {
super.setAuthorizationContext(null);
}
/**
* Inject user identity into operation context.
*
* @param userServicePath user document link
*/
public AuthorizationContext assumeIdentity(String userServicePath)
throws GeneralSecurityException {
return assumeIdentity(userServicePath, null);
}
/**
* Inject user identity into operation context.
*
* @param userServicePath user document link
* @param properties custom properties in claims
* @throws GeneralSecurityException any generic security exception
*/
public AuthorizationContext assumeIdentity(String userServicePath,
Map<String, String> properties) throws GeneralSecurityException {
Claims.Builder builder = new Claims.Builder();
builder.setSubject(userServicePath);
builder.setProperties(properties);
Claims claims = builder.getResult();
String token = getTokenSigner().sign(claims);
AuthorizationContext.Builder ab = AuthorizationContext.Builder.create();
ab.setClaims(claims);
ab.setToken(token);
// Associate resulting authorization context with this thread
AuthorizationContext authContext = ab.getResult();
setAuthorizationContext(authContext);
return authContext;
}
public void deleteAllChildServices(URI factoryURI) {
deleteOrStopAllChildServices(factoryURI, false, true);
}
public void deleteOrStopAllChildServices(
URI factoryURI, boolean stopOnly, boolean useFullQuorum) {
ServiceDocumentQueryResult res = getFactoryState(factoryURI);
if (res.documentLinks.isEmpty()) {
return;
}
List<Operation> ops = new ArrayList<>();
for (String link : res.documentLinks) {
Operation op = Operation.createDelete(UriUtils.buildUri(factoryURI, link));
if (stopOnly) {
op.addPragmaDirective(Operation.PRAGMA_DIRECTIVE_NO_INDEX_UPDATE);
} else {
if (useFullQuorum) {
op.addRequestHeader(Operation.REPLICATION_QUORUM_HEADER,
Operation.REPLICATION_QUORUM_HEADER_VALUE_ALL);
}
}
ops.add(op);
}
this.sender.sendAndWait(ops);
}
public <T extends ServiceDocument> ServiceDocument verifyPost(Class<T> documentType,
String factoryLink,
T state,
int expectedStatusCode) {
URI uri = UriUtils.buildUri(this, factoryLink);
Operation op = Operation.createPost(uri).setBody(state);
Operation response = this.sender.sendAndWait(op);
String message = String.format("Status code expected: %s, actual: %s", expectedStatusCode,
response.getStatusCode());
assertEquals(message, expectedStatusCode, response.getStatusCode());
return response.getBody(documentType);
}
protected TemporaryFolder getTemporaryFolder() {
return this.temporaryFolder;
}
public void setTemporaryFolder(TemporaryFolder temporaryFolder) {
this.temporaryFolder = temporaryFolder;
}
/**
* Sends an operation and waits for completion. CompletionHandler on passed operation will be cleared.
*/
public void sendAndWaitExpectSuccess(Operation op) {
// to be compatible with old behavior, clear the completion handler
op.setCompletion(null);
this.sender.sendAndWait(op);
}
public void sendAndWaitExpectFailure(Operation op) {
sendAndWaitExpectFailure(op, null);
}
public void sendAndWaitExpectFailure(Operation op, Integer expectedFailureCode) {
// to be compatible with old behavior, clear the completion handler
op.setCompletion(null);
FailureResponse resposne = this.sender.sendAndWaitFailure(op);
if (expectedFailureCode == null) {
return;
}
String msg = "got unexpected status: " + expectedFailureCode;
assertEquals(msg, (int) expectedFailureCode, resposne.op.getStatusCode());
}
/**
* Sends an operation and waits for completion.
*/
public void sendAndWait(Operation op) {
// assume completion is attached, using our getCompletion() or
// getExpectedFailureCompletion()
testStart(1);
send(op);
testWait();
}
/**
* Sends an operation, waits for completion and return the response representation.
*/
public Operation waitForResponse(Operation op) {
final Operation[] result = new Operation[1];
op.nestCompletion((o, e) -> {
result[0] = o;
completeIteration();
});
sendAndWait(op);
return result[0];
}
/**
* Decorates a {@link CompletionHandler} with a try/catch-all
* and fails the current iteration on exception. Allow for calling
* Assert.assert* directly in a handler.
*
* A safe handler will call completeIteration or failIteration exactly once.
*
* @param handler
* @return
*/
public CompletionHandler getSafeHandler(CompletionHandler handler) {
return (o, e) -> {
try {
handler.handle(o, e);
completeIteration();
} catch (Throwable t) {
failIteration(t);
}
};
}
public CompletionHandler getSafeHandler(TestContext ctx, CompletionHandler handler) {
return (o, e) -> {
try {
handler.handle(o, e);
ctx.completeIteration();
} catch (Throwable t) {
ctx.failIteration(t);
}
};
}
/**
* Creates a new service instance of type {@code service} via a {@code HTTP POST} to the service
* factory URI (which is discovered automatically based on {@code service}). It passes {@code
* state} as the body of the {@code POST}.
* <p/>
* See javadoc for <i>handler</i> param for important details on how to properly use this
* method. If your test expects the service instance to be created successfully, you might use:
* <pre>
* String[] taskUri = new String[1];
* CompletionHandler successHandler = getCompletionWithUri(taskUri);
* sendFactoryPost(ExampleTaskService.class, new ExampleTaskServiceState(), successHandler);
* </pre>
*
* @param service the type of service to create
* @param state the body of the {@code POST} to use to create the service instance
* @param handler the completion handler to use when creating the service instance.
* <b>IMPORTANT</b>: This handler must properly call {@code host.failIteration()}
* or {@code host.completeIteration()}.
* @param <T> the state that represents the service instance
*/
public <T extends ServiceDocument> void sendFactoryPost(Class<? extends Service> service,
T state, CompletionHandler handler) {
URI factoryURI = UriUtils.buildFactoryUri(this, service);
log(Level.INFO, "Creating POST for [uri=%s] [body=%s]", factoryURI, state);
Operation createPost = Operation.createPost(factoryURI)
.setBody(state)
.setCompletion(handler);
this.sender.sendAndWait(createPost);
}
/**
* Helper completion handler that:
* <ul>
* <li>Expects valid response to be returned; no exceptions when processing the operation</li>
* <li>Expects a {@code ServiceDocument} to be returned in the response body. The response's
* {@link ServiceDocument#documentSelfLink} will be stored in {@code storeUri[0]} so it can be
* used for test assertions and logic</li>
* </ul>
*
* @param storedLink The {@code documentSelfLink} of the created {@code ServiceDocument} will be
* stored in {@code storedLink[0]} so it can be used for test assertions and
* logic. This must be non-null and its length cannot be zero
* @return a completion handler, handy for using in methods like {@link
* #sendFactoryPost(Class, ServiceDocument, CompletionHandler)}
*/
public CompletionHandler getCompletionWithSelflink(String[] storedLink) {
if (storedLink == null || storedLink.length == 0) {
throw new IllegalArgumentException(
"storeUri must be initialized and have room for at least one item");
}
return (op, ex) -> {
if (ex != null) {
failIteration(ex);
return;
}
ServiceDocument response = op.getBody(ServiceDocument.class);
if (response == null) {
failIteration(new IllegalStateException(
"Expected non-null ServiceDocument in response body"));
return;
}
log(Level.INFO, "Created service instance. [selfLink=%s] [kind=%s]",
response.documentSelfLink, response.documentKind);
storedLink[0] = response.documentSelfLink;
completeIteration();
};
}
/**
* Helper completion handler that:
* <ul>
* <li>Expects an exception when processing the handler; it is a {@code failIteration} if an
* exception is <b>not</b> thrown.</li>
* <li>The exception will be stored in {@code storeException[0]} so it can be used for test
* assertions and logic.</li>
* </ul>
*
* @param storeException the exception that occurred in completion handler will be stored in
* {@code storeException[0]} so it can be used for test assertions and
* logic. This must be non-null and its length cannot be zero.
* @return a completion handler, handy for using in methods like {@link
* #sendFactoryPost(Class, ServiceDocument, CompletionHandler)}
*/
public CompletionHandler getExpectedFailureCompletionReturningThrowable(
Throwable[] storeException) {
if (storeException == null || storeException.length == 0) {
throw new IllegalArgumentException(
"storeException must be initialized and have room for at least one item");
}
return (op, ex) -> {
if (ex == null) {
failIteration(new IllegalStateException("Failure expected"));
}
storeException[0] = ex;
completeIteration();
};
}
/**
* Helper method that waits for a query task to reach the expected stage
*/
public QueryTask waitForQueryTask(URI uri, TaskState.TaskStage expectedStage) {
// If the task's state ever reaches one of these "final" stages, we can stop waiting...
List<TaskState.TaskStage> finalTaskStages = Arrays
.asList(TaskState.TaskStage.CANCELLED, TaskState.TaskStage.FAILED,
TaskState.TaskStage.FINISHED, expectedStage);
String error = String.format("Task did not reach expected state %s", expectedStage);
Object[] r = new Object[1];
final URI finalUri = uri;
waitFor(error, () -> {
QueryTask state = this.getServiceState(null, QueryTask.class, finalUri);
r[0] = state;
if (state.taskInfo != null) {
if (finalTaskStages.contains(state.taskInfo.stage)) {
return true;
}
}
return false;
});
return (QueryTask) r[0];
}
/**
* Helper method that waits for {@code taskUri} to have a {@link TaskState.TaskStage} == {@code
* TaskStage.FINISHED}.
*
* @param type The class type that represent's the task's state
* @param taskUri the URI of the task to wait for
* @param <T> the type that represent's the task's state
* @return the state of the task once's it's {@code FINISHED}
*/
public <T extends TaskService.TaskServiceState> T waitForFinishedTask(Class<T> type,
String taskUri) {
return waitForTask(type, taskUri, TaskState.TaskStage.FINISHED);
}
/**
* Helper method that waits for {@code taskUri} to have a {@link TaskState.TaskStage} == {@code
* TaskStage.FINISHED}.
*
* @param type The class type that represent's the task's state
* @param taskUri the URI of the task to wait for
* @param <T> the type that represent's the task's state
* @return the state of the task once's it's {@code FINISHED}
*/
public <T extends TaskService.TaskServiceState> T waitForFinishedTask(Class<T> type,
URI taskUri) {
return waitForTask(type, taskUri.toString(), TaskState.TaskStage.FINISHED);
}
/**
* Helper method that waits for {@code taskUri} to have a {@link TaskState.TaskStage} == {@code
* TaskStage.FAILED}.
*
* @param type The class type that represent's the task's state
* @param taskUri the URI of the task to wait for
* @param <T> the type that represent's the task's state
* @return the state of the task once's it s {@code FAILED}
*/
public <T extends TaskService.TaskServiceState> T waitForFailedTask(Class<T> type,
String taskUri) {
return waitForTask(type, taskUri, TaskState.TaskStage.FAILED);
}
/**
* Helper method that waits for {@code taskUri} to have a {@link TaskState.TaskStage} == {@code
* expectedStage}.
*
* @param type The class type of that represents the task's state
* @param taskUri the URI of the task to wait for
* @param expectedStage the stage we expect the task to eventually get to
* @param <T> the type that represents the task's state
* @return the state of the task once it's {@link TaskState.TaskStage} == {@code expectedStage}
*/
public <T extends TaskService.TaskServiceState> T waitForTask(Class<T> type, String taskUri,
TaskState.TaskStage expectedStage) {
return waitForTask(type, taskUri, expectedStage, false);
}
/**
* Helper method that waits for {@code taskUri} to have a {@link TaskState.TaskStage} == {@code
* expectedStage}.
*
* @param type The class type of that represents the task's state
* @param taskUri the URI of the task to wait for
* @param expectedStage the stage we expect the task to eventually get to
* @param useQueryTask Uses {@link QueryTask} to retrieve the current stage of the Task
* @param <T> the type that represents the task's state
* @return the state of the task once it's {@link TaskState.TaskStage} == {@code expectedStage}
*/
@SuppressWarnings("unchecked")
public <T extends TaskService.TaskServiceState> T waitForTask(Class<T> type, String taskUri,
TaskState.TaskStage expectedStage, boolean useQueryTask) {
URI uri = UriUtils.buildUri(taskUri);
if (!uri.isAbsolute()) {
uri = UriUtils.buildUri(this, taskUri);
}
List<TaskState.TaskStage> finalTaskStages = Arrays
.asList(TaskState.TaskStage.CANCELLED, TaskState.TaskStage.FAILED,
TaskState.TaskStage.FINISHED);
String error = String.format("Task did not reach expected state %s", expectedStage);
Object[] r = new Object[1];
final URI finalUri = uri;
waitFor(error, () -> {
T state = (useQueryTask)
? this.getServiceStateUsingQueryTask(type, taskUri)
: this.getServiceState(null, type, finalUri);
r[0] = state;
if (state.taskInfo != null) {
if (expectedStage == state.taskInfo.stage) {
return true;
}
if (finalTaskStages.contains(state.taskInfo.stage)) {
fail(String.format(
"Task was expected to reach stage %s but reached a final stage %s",
expectedStage, state.taskInfo.stage));
}
}
return false;
});
return (T) r[0];
}
@FunctionalInterface
public interface WaitHandler {
boolean isReady() throws Throwable;
}
public void waitFor(String timeoutMsg, WaitHandler wh) {
ExceptionTestUtils.executeSafely(() -> {
Date exp = getTestExpiration();
while (new Date().before(exp)) {
if (wh.isReady()) {
return;
}
// sleep for a tenth of the maintenance interval
Thread.sleep(TimeUnit.MICROSECONDS.toMillis(getMaintenanceIntervalMicros()) / 10);
}
throw new TimeoutException(timeoutMsg);
});
}
public void setSingleton(boolean enable) {
this.isSingleton = enable;
}
/*
* Running restart tests in VMs, in over provisioned CI will cause a restart using the same
* index sand box to fail, due to a file system LockHeldException.
* The sleep just reduces the false negative test failure rate, but it can still happen.
* Not much else we can do other adding some weird polling on all the index files.
*
* Returns true if host restarted, false if retry attempts expired or other exceptions where thrown
*/
public static boolean restartStatefulHost(ServiceHost host, boolean failOnIndexDeletion)
throws Throwable {
long exp = Utils.fromNowMicrosUtc(host.getOperationTimeoutMicros());
do {
Thread.sleep(2000);
try {
if (host.isAuthorizationEnabled()) {
host.setAuthenticationService(new AuthorizationContextService());
}
host.start();
return true;
} catch (Throwable e) {
Logger.getAnonymousLogger().warning(String
.format("exception on host restart: %s", e.getMessage()));
try {
host.stop();
} catch (Throwable e1) {
return false;
}
if (e instanceof LockObtainFailedException && !failOnIndexDeletion) {
Logger.getAnonymousLogger()
.warning("Lock held exception on host restart, retrying");
continue;
}
return false;
}
} while (Utils.getSystemNowMicrosUtc() < exp);
return false;
}
public void waitForGC() {
if (!isStressTest()) {
return;
}
for (int k = 0; k < 10; k++) {
Runtime.getRuntime().gc();
Runtime.getRuntime().runFinalization();
}
}
public TestRequestSender getTestRequestSender() {
return this.sender;
}
}
