Java Code Examples for org.apache.nifi.events.EventReporter

The following examples show how to use org.apache.nifi.events.EventReporter. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: nifi   Source File: TestFileSystemSwapManager.java    License: Apache License 2.0 6 votes vote down vote up
private FileSystemSwapManager createSwapManager(final FlowFileRepository flowFileRepo) {
    final FileSystemSwapManager swapManager = new FileSystemSwapManager();
    final ResourceClaimManager resourceClaimManager = new NopResourceClaimManager();
    swapManager.initialize(new SwapManagerInitializationContext() {
        @Override
        public ResourceClaimManager getResourceClaimManager() {
            return resourceClaimManager;
        }

        @Override
        public FlowFileRepository getFlowFileRepository() {
            return flowFileRepo;
        }

        @Override
        public EventReporter getEventReporter() {
            return EventReporter.NO_OP;
        }
    });

    return swapManager;
}
 
Example 2
Source Project: nifi   Source File: NodeClusterCoordinatorFactoryBean.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public NodeClusterCoordinator getObject() throws Exception {
    if (nodeClusterCoordinator == null && properties.isNode()) {
        final ClusterCoordinationProtocolSenderListener protocolSenderListener =
                applicationContext.getBean("clusterCoordinationProtocolSenderListener", ClusterCoordinationProtocolSenderListener.class);
        final EventReporter eventReporter = applicationContext.getBean("eventReporter", EventReporter.class);
        final ClusterNodeFirewall clusterFirewall = applicationContext.getBean("clusterFirewall", ClusterNodeFirewall.class);
        final RevisionManager revisionManager = applicationContext.getBean("revisionManager", RevisionManager.class);
        final LeaderElectionManager electionManager = applicationContext.getBean("leaderElectionManager", LeaderElectionManager.class);
        final FlowElection flowElection = applicationContext.getBean("flowElection", FlowElection.class);
        final NodeProtocolSender nodeProtocolSender = applicationContext.getBean("nodeProtocolSender", NodeProtocolSender.class);
        nodeClusterCoordinator = new NodeClusterCoordinator(protocolSenderListener, eventReporter, electionManager, flowElection, clusterFirewall,
                revisionManager, properties, extensionManager, nodeProtocolSender);
    }

    return nodeClusterCoordinator;
}
 
Example 3
Source Project: nifi   Source File: StatelessRemoteInputPort.java    License: Apache License 2.0 6 votes vote down vote up
public StatelessRemoteInputPort(final VersionedRemoteProcessGroup rpg, final VersionedRemoteGroupPort remotePort, final SSLContext sslContext) {
    final String timeout = rpg.getCommunicationsTimeout();
    final long timeoutMillis = FormatUtils.getTimeDuration(timeout, TimeUnit.MILLISECONDS);

    url = rpg.getTargetUris();
    name = remotePort.getName();

    client = new SiteToSiteClient.Builder()
        .portName(remotePort.getName())
        .timeout(timeoutMillis, TimeUnit.MILLISECONDS)
        .transportProtocol(SiteToSiteTransportProtocol.valueOf(rpg.getTransportProtocol()))
        .url(rpg.getTargetUris())
        .useCompression(remotePort.isUseCompression())
        .sslContext(sslContext)
        .eventReporter(EventReporter.NO_OP)
        .build();
}
 
Example 4
Source Project: nifi   Source File: TestSocketClientTransaction.java    License: Apache License 2.0 6 votes vote down vote up
private SocketClientTransaction getClientTransaction(ByteArrayInputStream bis, ByteArrayOutputStream bos, TransferDirection direction) throws IOException {
    PeerDescription description = null;
    String peerUrl = "";
    SocketCommunicationsSession commsSession = mock(SocketCommunicationsSession.class);
    SocketInput socketIn = mock(SocketInput.class);
    SocketOutput socketOut = mock(SocketOutput.class);
    when(commsSession.getInput()).thenReturn(socketIn);
    when(commsSession.getOutput()).thenReturn(socketOut);

    when(socketIn.getInputStream()).thenReturn(bis);
    when(socketOut.getOutputStream()).thenReturn(bos);

    String clusterUrl = "";
    Peer peer = new Peer(description, commsSession, peerUrl, clusterUrl);
    boolean useCompression = false;
    int penaltyMillis = 1000;
    EventReporter eventReporter = null;
    int protocolVersion = 5;
    String destinationId = "destinationId";
    return new SocketClientTransaction(protocolVersion, destinationId, peer, codec, direction, useCompression, penaltyMillis, eventReporter);
}
 
Example 5
public PartitionedWriteAheadEventStore(final RepositoryConfiguration repoConfig, final RecordWriterFactory recordWriterFactory,
    final RecordReaderFactory recordReaderFactory, final EventReporter eventReporter, final EventFileManager fileManager) {
    super(repoConfig, eventReporter);
    this.repoConfig = repoConfig;
    this.eventReporter = eventReporter;
    this.filesToCompress = new LinkedBlockingQueue<>(100);
    final AtomicLong idGenerator = new AtomicLong(0L);
    this.partitions = createPartitions(repoConfig, recordWriterFactory, recordReaderFactory, idGenerator);
    this.fileManager = fileManager;

    // Creates tasks to compress data on rollover
    if (repoConfig.isCompressOnRollover()) {
        compressionExecutor = Executors.newFixedThreadPool(repoConfig.getIndexThreadPoolSize(), new NamedThreadFactory("Compress Provenance Logs"));
    } else {
        compressionExecutor = null;
    }
}
 
Example 6
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
 
Example 7
Source Project: nifi   Source File: SwappablePriorityQueue.java    License: Apache License 2.0 6 votes vote down vote up
public SwappablePriorityQueue(final FlowFileSwapManager swapManager, final int swapThreshold, final EventReporter eventReporter, final FlowFileQueue flowFileQueue,
    final DropFlowFileAction dropAction, final String swapPartitionName) {
    this.swapManager = swapManager;
    this.swapThreshold = swapThreshold;

    this.activeQueue = new PriorityQueue<>(20, new QueuePrioritizer(Collections.emptyList()));
    this.swapQueue = new ArrayList<>();
    this.eventReporter = eventReporter;
    this.flowFileQueue = flowFileQueue;
    this.dropAction = dropAction;
    this.swapPartitionName = swapPartitionName;

    final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
    readLock = new TimedLock(lock.readLock(), flowFileQueue.getIdentifier() + " Read Lock", 100);
    writeLock = new TimedLock(lock.writeLock(), flowFileQueue.getIdentifier() + " Write Lock", 100);
}
 
Example 8
Source Project: nifi   Source File: SiteToSiteRestApiClient.java    License: Apache License 2.0 6 votes vote down vote up
public SiteToSiteRestApiClient(final SSLContext sslContext, final HttpProxy proxy, final EventReporter eventReporter) {
    this.sslContext = sslContext;
    this.proxy = proxy;
    this.eventReporter = eventReporter;

    ttlExtendTaskExecutor = Executors.newScheduledThreadPool(1, new ThreadFactory() {
        private final ThreadFactory defaultFactory = Executors.defaultThreadFactory();

        @Override
        public Thread newThread(final Runnable r) {
            final Thread thread = defaultFactory.newThread(r);
            thread.setName(Thread.currentThread().getName() + " TTLExtend");
            thread.setDaemon(true);
            return thread;
        }
    });
}
 
Example 9
@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }

    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}
 
Example 10
@Override
public NodeClusterCoordinator getObject() throws Exception {
    if (nodeClusterCoordinator == null && properties.isNode()) {
        final ClusterCoordinationProtocolSenderListener protocolSenderListener =
                applicationContext.getBean("clusterCoordinationProtocolSenderListener", ClusterCoordinationProtocolSenderListener.class);
        final EventReporter eventReporter = applicationContext.getBean("eventReporter", EventReporter.class);
        final ClusterNodeFirewall clusterFirewall = applicationContext.getBean("clusterFirewall", ClusterNodeFirewall.class);
        final RevisionManager revisionManager = applicationContext.getBean("revisionManager", RevisionManager.class);
        final LeaderElectionManager electionManager = applicationContext.getBean("leaderElectionManager", LeaderElectionManager.class);
        final FlowElection flowElection = applicationContext.getBean("flowElection", FlowElection.class);
        final NodeProtocolSender nodeProtocolSender = applicationContext.getBean("nodeProtocolSender", NodeProtocolSender.class);
        nodeClusterCoordinator = new NodeClusterCoordinator(protocolSenderListener, eventReporter, electionManager, flowElection, clusterFirewall, revisionManager, properties, nodeProtocolSender);
    }

    return nodeClusterCoordinator;
}
 
Example 11
@Override
public ThreadPoolRequestReplicator getObject() throws Exception {
    if (replicator == null && nifiProperties.isNode()) {
        final EventReporter eventReporter = applicationContext.getBean("eventReporter", EventReporter.class);
        final ClusterCoordinator clusterCoordinator = applicationContext.getBean("clusterCoordinator", ClusterCoordinator.class);
        final RequestCompletionCallback requestCompletionCallback = applicationContext.getBean("clusterCoordinator", RequestCompletionCallback.class);

        final int numThreads = nifiProperties.getClusterNodeProtocolThreads();
        final Client jerseyClient = WebUtils.createClient(new DefaultClientConfig(), SslContextFactory.createSslContext(nifiProperties));
        final String connectionTimeout = nifiProperties.getClusterNodeConnectionTimeout();
        final String readTimeout = nifiProperties.getClusterNodeReadTimeout();

        replicator = new ThreadPoolRequestReplicator(numThreads, jerseyClient, clusterCoordinator,
            connectionTimeout, readTimeout, requestCompletionCallback, eventReporter, nifiProperties);
    }

    return replicator;
}
 
Example 12
@Before
public void setup() throws IOException {
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "src/test/resources/conf/nifi.properties");

    senderListener = Mockito.mock(ClusterCoordinationProtocolSenderListener.class);
    nodeStatuses = Collections.synchronizedList(new ArrayList<>());

    final EventReporter eventReporter = Mockito.mock(EventReporter.class);
    final RevisionManager revisionManager = Mockito.mock(RevisionManager.class);
    Mockito.when(revisionManager.getAllRevisions()).thenReturn(Collections.emptyList());

    coordinator = new NodeClusterCoordinator(senderListener, eventReporter, null, new FirstVoteWinsFlowElection(), null, revisionManager, createProperties(), null) {
        @Override
        void notifyOthersOfNodeStatusChange(NodeConnectionStatus updatedStatus, boolean notifyAllNodes, boolean waitForCoordinator) {
            nodeStatuses.add(updatedStatus);
        }
    };

    final FlowService flowService = Mockito.mock(FlowService.class);
    final StandardDataFlow dataFlow = new StandardDataFlow(new byte[50], new byte[50], new byte[50]);
    Mockito.when(flowService.createDataFlow()).thenReturn(dataFlow);
    coordinator.setFlowService(flowService);
}
 
Example 13
Source Project: nifi   Source File: TestPartitionedWriteAheadEventStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }

    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}
 
Example 14
Source Project: nifi   Source File: TestPartitionedWriteAheadEventStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxEventIdRestored() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
    }

    assertEquals(19, store.getMaxEventId());
    store.close();

    final PartitionedWriteAheadEventStore recoveredStore = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    recoveredStore.initialize();
    assertEquals(19, recoveredStore.getMaxEventId());
}
 
Example 15
Source Project: nifi   Source File: TestPartitionedWriteAheadEventStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
 
Example 16
Source Project: nifi   Source File: TestNodeClusterCoordinator.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTryAgainIfNoFlowServiceSet() throws IOException {
    final ClusterCoordinationProtocolSenderListener senderListener = Mockito.mock(ClusterCoordinationProtocolSenderListener.class);
    final EventReporter eventReporter = Mockito.mock(EventReporter.class);
    final RevisionManager revisionManager = Mockito.mock(RevisionManager.class);
    when(revisionManager.getAllRevisions()).thenReturn(Collections.emptyList());

    final NodeClusterCoordinator coordinator = new NodeClusterCoordinator(senderListener, eventReporter, null, new FirstVoteWinsFlowElection(),
            null, revisionManager, createProperties(), null, stateManagerProvider) {
        @Override
        void notifyOthersOfNodeStatusChange(NodeConnectionStatus updatedStatus, boolean notifyAllNodes, boolean waitForCoordinator) {
        }
    };

    final NodeIdentifier requestedNodeId = createNodeId(6);
    final ConnectionRequest request = new ConnectionRequest(requestedNodeId, new StandardDataFlow(new byte[0], new byte[0], new byte[0], new HashSet<>()));
    final ConnectionRequestMessage requestMsg = new ConnectionRequestMessage();
    requestMsg.setConnectionRequest(request);

    coordinator.setConnected(true);

    final ProtocolMessage protocolResponse = coordinator.handle(requestMsg, Collections.emptySet());
    assertNotNull(protocolResponse);
    assertTrue(protocolResponse instanceof ConnectionResponseMessage);

    final ConnectionResponse response = ((ConnectionResponseMessage) protocolResponse).getConnectionResponse();
    assertNotNull(response);
    assertEquals(5, response.getTryLaterSeconds());
}
 
Example 17
Source Project: nifi   Source File: ClusteredSwapFileIT.java    License: Apache License 2.0 5 votes vote down vote up
protected FlowFileQueue createFlowFileQueue(final String uuid) {
    final ProcessScheduler processScheduler = getFlowController().getProcessScheduler();
    final ResourceClaimManager resourceClaimManager = getFlowController().getResourceClaimManager();
    final FlowFileSwapManager swapManager = getFlowController().createSwapManager();

    final AsyncLoadBalanceClientRegistry clientRegistry = Mockito.mock(AsyncLoadBalanceClientRegistry.class);

    return new SocketLoadBalancedFlowFileQueue(uuid, ConnectionEventListener.NOP_EVENT_LISTENER, processScheduler, getFlowFileRepository(), getProvenanceRepository(),
        getContentRepository(), resourceClaimManager, getClusterCoordinator(), clientRegistry, swapManager, 20000, EventReporter.NO_OP);
}
 
Example 18
Source Project: localization_nifi   Source File: SocketClientTransaction.java    License: Apache License 2.0 5 votes vote down vote up
SocketClientTransaction(final int protocolVersion, final String destinationId, final Peer peer, final FlowFileCodec codec,
        final TransferDirection direction, final boolean useCompression, final int penaltyMillis, final EventReporter eventReporter) throws IOException {
    super(peer, direction, useCompression, codec, eventReporter, protocolVersion, penaltyMillis, destinationId);
    this.dis = new DataInputStream(peer.getCommunicationsSession().getInput().getInputStream());
    this.dos = new DataOutputStream(peer.getCommunicationsSession().getOutput().getOutputStream());

    initialize();
}
 
Example 19
Source Project: localization_nifi   Source File: AbstractTransaction.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractTransaction(final Peer peer, final TransferDirection direction, final boolean useCompression,
                           final FlowFileCodec codec, final EventReporter eventReporter, final int protocolVersion,
                           final int penaltyMillis, final String destinationId) {
    this.peer = peer;
    this.state = TransactionState.TRANSACTION_STARTED;
    this.direction = direction;
    this.compress = useCompression;
    this.codec = codec;
    this.eventReporter = eventReporter;
    this.protocolVersion = protocolVersion;
    this.penaltyMillis = penaltyMillis;
    this.destinationId = destinationId;
}
 
Example 20
Source Project: nifi   Source File: FlowController.java    License: Apache License 2.0 5 votes vote down vote up
public EventReporter createEventReporter() {
    return new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(final Severity severity, final String category, final String message) {
            final Bulletin bulletin = BulletinFactory.createBulletin(category, severity.name(), message);
            bulletinRepository.addBulletin(bulletin);
        }
    };
}
 
Example 21
Source Project: localization_nifi   Source File: EventReportUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static void warn(final Logger logger, final EventReporter eventReporter, final String msg, final Throwable t) {
    logger.warn(msg, t);

    if (eventReporter != null) {
        eventReporter.reportEvent(Severity.WARNING, CATEGORY, msg + ": " + t.toString());
    }
}
 
Example 22
Source Project: nifi   Source File: TestPartitionedWriteAheadEventStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteAfterRecoveringRepo() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    for (int i = 0; i < 4; i++) {
        store.addEvents(Collections.singleton(createEvent()));
    }

    store.close();

    final PartitionedWriteAheadEventStore recoveredStore = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    recoveredStore.initialize();

    List<ProvenanceEventRecord> recoveredEvents = recoveredStore.getEvents(0, 10);
    assertEquals(4, recoveredEvents.size());

    // ensure that we can still write to the store
    for (int i = 0; i < 4; i++) {
        recoveredStore.addEvents(Collections.singleton(createEvent()));
    }

    recoveredEvents = recoveredStore.getEvents(0, 10);
    assertEquals(8, recoveredEvents.size());

    for (int i = 0; i < 8; i++) {
        assertEquals(i, recoveredEvents.get(i).getEventId());
    }
}
 
Example 23
Source Project: localization_nifi   Source File: SiteToSiteRestApiClient.java    License: Apache License 2.0 5 votes vote down vote up
private void startExtendingTtl(final String transactionUrl, final Closeable stream, final CloseableHttpResponse response) {
    if (ttlExtendingFuture != null) {
        // Already started.
        return;
    }

    logger.debug("Starting extending TTL thread...");

    extendingApiClient = new SiteToSiteRestApiClient(sslContext, proxy, EventReporter.NO_OP);
    extendingApiClient.transportProtocolVersionNegotiator = this.transportProtocolVersionNegotiator;
    extendingApiClient.connectTimeoutMillis = this.connectTimeoutMillis;
    extendingApiClient.readTimeoutMillis = this.readTimeoutMillis;
    extendingApiClient.localAddress = this.localAddress;

    final int extendFrequency = serverTransactionTtl / 2;

    ttlExtendingFuture = ttlExtendTaskExecutor.scheduleWithFixedDelay(() -> {
        try {
            extendingApiClient.extendTransaction(transactionUrl);
        } catch (final Exception e) {
            logger.warn("Failed to extend transaction ttl", e);

            try {
                // Without disconnecting, Site-to-Site client keep reading data packet,
                // while server has already rollback.
                this.close();
            } catch (final IOException ec) {
                logger.warn("Failed to close", e);
            }
        }
    }, extendFrequency, extendFrequency, TimeUnit.SECONDS);
}
 
Example 24
Source Project: nifi   Source File: SiteToSiteUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static SiteToSiteClient getClient(PropertyContext reportContext, ComponentLog logger, StateManager stateManager) {
    final SSLContextService sslContextService = reportContext.getProperty(SiteToSiteUtils.SSL_CONTEXT).asControllerService(SSLContextService.class);
    final SSLContext sslContext = sslContextService == null ? null : sslContextService.createSSLContext(SslContextFactory.ClientAuth.REQUIRED);
    final EventReporter eventReporter = (EventReporter) (severity, category, message) -> {
        switch (severity) {
            case WARNING:
                logger.warn(message);
                break;
            case ERROR:
                logger.error(message);
                break;
            default:
                break;
        }
    };
    final String destinationUrl = reportContext.getProperty(SiteToSiteUtils.DESTINATION_URL).evaluateAttributeExpressions().getValue();

    final SiteToSiteTransportProtocol mode = SiteToSiteTransportProtocol.valueOf(reportContext.getProperty(SiteToSiteUtils.TRANSPORT_PROTOCOL).getValue());
    final HttpProxy httpProxy = mode.equals(SiteToSiteTransportProtocol.RAW) || StringUtils.isEmpty(reportContext.getProperty(SiteToSiteUtils.HTTP_PROXY_HOSTNAME).getValue()) ? null
            : new HttpProxy(reportContext.getProperty(SiteToSiteUtils.HTTP_PROXY_HOSTNAME).getValue(), reportContext.getProperty(SiteToSiteUtils.HTTP_PROXY_PORT).asInteger(),
            reportContext.getProperty(SiteToSiteUtils.HTTP_PROXY_USERNAME).getValue(), reportContext.getProperty(SiteToSiteUtils.HTTP_PROXY_PASSWORD).getValue());

    // If no state manager was provided and this context supports retrieving it, do so
    if (stateManager == null && reportContext instanceof ReportingContext) {
        stateManager = ((ReportingContext) reportContext).getStateManager();
    }
    return new SiteToSiteClient.Builder()
            .urls(SiteToSiteRestApiClient.parseClusterUrls(destinationUrl))
            .portName(reportContext.getProperty(SiteToSiteUtils.PORT_NAME).getValue())
            .useCompression(reportContext.getProperty(SiteToSiteUtils.COMPRESS).asBoolean())
            .eventReporter(eventReporter)
            .sslContext(sslContext)
            .timeout(reportContext.getProperty(SiteToSiteUtils.TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS)
            .transportProtocol(mode)
            .httpProxy(httpProxy)
            .stateManager(stateManager)
            .build();
}
 
Example 25
Source Project: localization_nifi   Source File: EventIndexTask.java    License: Apache License 2.0 5 votes vote down vote up
public EventIndexTask(final BlockingQueue<StoredDocument> documentQueue, final RepositoryConfiguration repoConfig, final IndexManager indexManager,
    final IndexDirectoryManager directoryManager, final int maxEventsPerCommit, final EventReporter eventReporter) {
    this.documentQueue = documentQueue;
    this.indexManager = indexManager;
    this.directoryManager = directoryManager;
    this.commitThreshold = maxEventsPerCommit;
    this.eventReporter = eventReporter;
}
 
Example 26
Source Project: localization_nifi   Source File: LuceneEventIndex.java    License: Apache License 2.0 5 votes vote down vote up
public LuceneEventIndex(final RepositoryConfiguration config, final IndexManager indexManager, final int maxEventsPerCommit, final EventReporter eventReporter) {
    this.eventReporter = eventReporter;
    queryExecutor = Executors.newFixedThreadPool(config.getQueryThreadPoolSize(), new NamedThreadFactory("Provenance Query"));
    indexExecutor = Executors.newFixedThreadPool(config.getIndexThreadPoolSize(), new NamedThreadFactory("Index Provenance Events"));
    cacheWarmerExecutor = Executors.newScheduledThreadPool(config.getStorageDirectories().size(), new NamedThreadFactory("Warm Lucene Index", true));
    directoryManager = new IndexDirectoryManager(config);

    // Limit number of indexing threads to 100. When we restore the repository on restart,
    // we have to re-index up to MAX_THREADS * MAX_DOCUMENTS_PER_THREADS events prior to
    // the last event that the index holds. This is done because we could have that many
    // events 'in flight', waiting to be indexed when the last index writer was committed,
    // so even though the index says the largest event ID is 1,000,000 for instance, Event
    // with ID 999,999 may still not have been indexed because another thread was in the
    // process of writing the event to the index.
    final int configuredIndexPoolSize = config.getIndexThreadPoolSize();
    final int numIndexThreads;
    if (configuredIndexPoolSize > MAX_INDEX_THREADS) {
        logger.warn("The Provenance Repository is configured to perform indexing of events using {} threads. This number exceeds the maximum allowable number of threads, which is {}. "
            + "Will proceed using {} threads. This value is limited because the performance of indexing will decrease and startup times will increase when setting this value too high.",
            configuredIndexPoolSize, MAX_INDEX_THREADS, MAX_INDEX_THREADS);
        numIndexThreads = MAX_INDEX_THREADS;
    } else {
        numIndexThreads = configuredIndexPoolSize;
    }

    for (int i = 0; i < numIndexThreads; i++) {
        final EventIndexTask task = new EventIndexTask(documentQueue, config, indexManager, directoryManager, maxEventsPerCommit, eventReporter);
        indexTasks.add(task);
        indexExecutor.submit(task);
    }
    this.config = config;
    this.indexManager = indexManager;
    this.eventConverter = new ConvertEventToLuceneDocument(config.getSearchableFields(), config.getSearchableAttributes());
}
 
Example 27
@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
    final IdentifierLookup idLookup) throws IOException {
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
    };

    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            return RecordReaders.newRecordReader(file, logs, maxChars);
        } finally {
            fileManager.releaseReadLock(file);
        }
    };

    eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager);

    final IndexManager indexManager = new SimpleIndexManager(config);
    eventIndex = new LuceneEventIndex(config, indexManager, eventReporter);

    this.eventReporter = eventReporter;
    this.authorizer = authorizer;
    this.resourceFactory = resourceFactory;

    eventStore.initialize();
    eventIndex.initialize(eventStore);

    try {
        eventStore.reindexLatestEvents(eventIndex);
    } catch (final Exception e) {
        logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest "
            + "events will not be available from the Provenance Repository when a query is issued.", e);
    }
}
 
Example 28
public WriteAheadStorePartition(final File storageDirectory, final String partitionName, final RepositoryConfiguration repoConfig, final RecordWriterFactory recordWriterFactory,
    final RecordReaderFactory recordReaderFactory, final BlockingQueue<File> filesToCompress, final AtomicLong idGenerator, final EventReporter eventReporter) {

    this.partitionName = partitionName;
    this.config = repoConfig;
    this.idGenerator = idGenerator;
    this.partitionDirectory = storageDirectory;
    this.recordWriterFactory = recordWriterFactory;
    this.recordReaderFactory = recordReaderFactory;
    this.filesToCompress = filesToCompress;
}
 
Example 29
Source Project: localization_nifi   Source File: TestLuceneEventIndex.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testUnauthorizedEventsGetPlaceholdersForLineage() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);

    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }

    final NiFiUser user = createUser();

    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 3) {
        final ComputeLineageSubmission submission = index.submitLineageComputation(1L, user, EventAuthorizer.DENY_ALL);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));

        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }

    assertEquals(3, nodes.size());

    for (final LineageNode node : nodes) {
        assertEquals(LineageNodeType.PROVENANCE_EVENT_NODE, node.getNodeType());
        final ProvenanceEventLineageNode eventNode = (ProvenanceEventLineageNode) node;
        assertEquals(ProvenanceEventType.UNKNOWN, eventNode.getEventType());
    }
}
 
Example 30
Source Project: localization_nifi   Source File: TestLuceneEventIndex.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testExpiration() throws InterruptedException, IOException {
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final List<ProvenanceEventRecord> events = new ArrayList<>();
    events.add(createEvent(500000L));
    events.add(createEvent());

    final EventStore eventStore = Mockito.mock(EventStore.class);
    Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {
        @Override
        public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
            final Long eventId = invocation.getArgumentAt(0, Long.class);
            assertEquals(0, eventId.longValue());
            assertEquals(1, invocation.getArgumentAt(1, Integer.class).intValue());
            return Collections.singletonList(events.get(0));
        }
    }).when(eventStore).getEvents(Mockito.anyLong(), Mockito.anyInt());

    index.initialize(eventStore);
    index.addEvent(events.get(0), createStorageSummary(events.get(0).getEventId()));

    // Add the first event to the index and wait for it to be indexed, since indexing is asynchronous.
    List<File> allDirectories = Collections.emptyList();
    while (allDirectories.isEmpty()) {
        allDirectories = index.getDirectoryManager().getDirectories(null, null);
    }

    events.remove(0); // Remove the first event from the store
    index.performMaintenance();
    assertEquals(1, index.getDirectoryManager().getDirectories(null, null).size());
}