/* * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://www.apache.org/licenses/LICENSE-2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.amazon.opendistroforelasticsearch.ad.transport.handler; import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.createIndexBlockedState; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import com.amazon.opendistroforelasticsearch.ad.AbstractADTest; import com.amazon.opendistroforelasticsearch.ad.TestHelpers; import com.amazon.opendistroforelasticsearch.ad.common.exception.AnomalyDetectionException; import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyResult; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyResultTests; public class AnomalyResultHandlerTests extends AbstractADTest { private static Settings settings; @Mock private ClusterService clusterService; @Mock private Client client; @Mock private AnomalyDetectionIndices anomalyDetectionIndices; @Mock private IndexNameExpressionResolver indexNameResolver; @BeforeClass public static void setUpBeforeClass() { setUpThreadPool(AnomalyResultTests.class.getSimpleName()); settings = Settings.EMPTY; } @AfterClass public static void tearDownAfterClass() { tearDownThreadPool(); settings = null; } @Override @Before public void setUp() throws Exception { super.setUp(); super.setUpLog4jForJUnit(AnomalyResultHandler.class); MockitoAnnotations.initMocks(this); setWriteBlockAdResultIndex(false); } @Override public void tearDown() throws Exception { super.tearDown(); super.tearDownLog4jForJUnit(); } @Rule public ExpectedException expectedEx = ExpectedException.none(); @Test public void testSavingAdResult() throws IOException { setUpSavingAnomalyResultIndex(false); doAnswer(invocation -> { Object[] args = invocation.getArguments(); assertTrue(String.format("The size of args is %d. Its content is %s", args.length, Arrays.toString(args)), args.length >= 2); IndexRequest request = invocation.getArgument(0); ActionListener<IndexResponse> listener = invocation.getArgument(1); assertTrue(request != null && listener != null); listener.onResponse(mock(IndexResponse.class)); return null; }).when(client).index(any(IndexRequest.class), ArgumentMatchers.<ActionListener<IndexResponse>>any()); AnomalyResultHandler handler = new AnomalyResultHandler( client, settings, clusterService, indexNameResolver, anomalyDetectionIndices, threadPool ); handler.indexAnomalyResult(TestHelpers.randomAnomalyDetectResult()); assertEquals(1, testAppender.countMessage((AnomalyResultHandler.SUCCESS_SAVING_MSG))); } @Test public void testSavingFailureNotRetry() throws InterruptedException, IOException { savingFailureTemplate(false, 1, true); assertEquals(1, testAppender.countMessage((AnomalyResultHandler.FAIL_TO_SAVE_ERR_MSG))); assertTrue(!testAppender.containsMessage(AnomalyResultHandler.SUCCESS_SAVING_MSG)); assertTrue(!testAppender.containsMessage(AnomalyResultHandler.RETRY_SAVING_ERR_MSG)); } @Test public void testSavingFailureRetry() throws InterruptedException, IOException { setWriteBlockAdResultIndex(false); savingFailureTemplate(true, 3, true); assertEquals(2, testAppender.countMessage((AnomalyResultHandler.RETRY_SAVING_ERR_MSG))); assertEquals(1, testAppender.countMessage((AnomalyResultHandler.FAIL_TO_SAVE_ERR_MSG))); assertTrue(!testAppender.containsMessage(AnomalyResultHandler.SUCCESS_SAVING_MSG)); } @Test public void testIndexWriteBlock() { setWriteBlockAdResultIndex(true); AnomalyResultHandler handler = new AnomalyResultHandler( client, settings, clusterService, indexNameResolver, anomalyDetectionIndices, threadPool ); handler.indexAnomalyResult(TestHelpers.randomAnomalyDetectResult()); assertTrue(testAppender.containsMessage(AnomalyResultHandler.CANNOT_SAVE_ERR_MSG)); } @Test public void testAdResultIndexExist() throws IOException { setInitAnomalyResultIndexException(true); AnomalyResultHandler handler = new AnomalyResultHandler( client, settings, clusterService, indexNameResolver, anomalyDetectionIndices, threadPool ); handler.indexAnomalyResult(TestHelpers.randomAnomalyDetectResult()); verify(client, times(1)).index(any(), any()); } @Test public void testAdResultIndexOtherException() throws IOException { expectedEx.expect(AnomalyDetectionException.class); expectedEx.expectMessage("Error in saving anomaly index for ID"); setInitAnomalyResultIndexException(false); AnomalyResultHandler handler = new AnomalyResultHandler( client, settings, clusterService, indexNameResolver, anomalyDetectionIndices, threadPool ); handler.indexAnomalyResult(TestHelpers.randomAnomalyDetectResult()); verify(client, never()).index(any(), any()); } private void setInitAnomalyResultIndexException(boolean indexExistException) throws IOException { Exception e = indexExistException ? mock(ResourceAlreadyExistsException.class) : mock(RuntimeException.class); doAnswer(invocation -> { Object[] args = invocation.getArguments(); assertTrue(String.format("The size of args is %d. Its content is %s", args.length, Arrays.toString(args)), args.length >= 1); ActionListener<CreateIndexResponse> listener = invocation.getArgument(0); assertTrue(listener != null); listener.onFailure(e); return null; }).when(anomalyDetectionIndices).initAnomalyResultIndexDirectly(any()); } private void setWriteBlockAdResultIndex(boolean blocked) { String indexName = randomAlphaOfLength(10); Settings settings = blocked ? Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build() : Settings.EMPTY; ClusterState blockedClusterState = createIndexBlockedState(indexName, settings, AnomalyResult.ANOMALY_RESULT_INDEX); when(clusterService.state()).thenReturn(blockedClusterState); when(indexNameResolver.concreteIndexNames(any(), any(), any())).thenReturn(new String[] { indexName }); } /** * Template to test exponential backoff retry during saving anomaly result. * * @param throwEsRejectedExecutionException whether to throw * EsRejectedExecutionException in the * client::index mock or not * @param latchCount used for coordinating. Equal to * number of expected retries plus 1. * @throws InterruptedException if thread execution is interrupted * @throws IOException if IO failures */ @SuppressWarnings("unchecked") private void savingFailureTemplate(boolean throwEsRejectedExecutionException, int latchCount, boolean adResultIndexExists) throws InterruptedException, IOException { setUpSavingAnomalyResultIndex(adResultIndexExists); final CountDownLatch backoffLatch = new CountDownLatch(latchCount); doAnswer(invocation -> { Object[] args = invocation.getArguments(); assertTrue(String.format("The size of args is %d. Its content is %s", args.length, Arrays.toString(args)), args.length >= 2); IndexRequest request = invocation.getArgument(0); ActionListener<IndexResponse> listener = invocation.getArgument(1); assertTrue(request != null && listener != null); if (throwEsRejectedExecutionException) { listener.onFailure(new EsRejectedExecutionException("")); } else { listener.onFailure(new IllegalArgumentException()); } backoffLatch.countDown(); return null; }).when(client).index(any(IndexRequest.class), ArgumentMatchers.<ActionListener<IndexResponse>>any()); Settings backoffSettings = Settings .builder() .put("opendistro.anomaly_detection.max_retry_for_backoff", 2) .put("opendistro.anomaly_detection.backoff_initial_delay", TimeValue.timeValueMillis(1)) .build(); AnomalyResultHandler handler = new AnomalyResultHandler( client, backoffSettings, clusterService, indexNameResolver, anomalyDetectionIndices, threadPool ); handler.indexAnomalyResult(TestHelpers.randomAnomalyDetectResult()); backoffLatch.await(); } @SuppressWarnings("unchecked") private void setUpSavingAnomalyResultIndex(boolean anomalyResultIndexExists) throws IOException { doAnswer(invocation -> { Object[] args = invocation.getArguments(); assertTrue(String.format("The size of args is %d. Its content is %s", args.length, Arrays.toString(args)), args.length >= 1); ActionListener<CreateIndexResponse> listener = invocation.getArgument(0); assertTrue(listener != null); listener.onResponse(new CreateIndexResponse(true, true, AnomalyResult.ANOMALY_RESULT_INDEX) { }); return null; }).when(anomalyDetectionIndices).initAnomalyResultIndexDirectly(any()); when(anomalyDetectionIndices.doesAnomalyResultIndexExist()).thenReturn(anomalyResultIndexExists); } }