org.apache.nifi.events.EventReporter Java Examples

The following examples show how to use org.apache.nifi.events.EventReporter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NodeClusterCoordinatorFactoryBean.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public NodeClusterCoordinator getObject() throws Exception {
    if (nodeClusterCoordinator == null && properties.isNode()) {
        final ClusterCoordinationProtocolSenderListener protocolSenderListener =
                applicationContext.getBean("clusterCoordinationProtocolSenderListener", ClusterCoordinationProtocolSenderListener.class);
        final EventReporter eventReporter = applicationContext.getBean("eventReporter", EventReporter.class);
        final ClusterNodeFirewall clusterFirewall = applicationContext.getBean("clusterFirewall", ClusterNodeFirewall.class);
        final RevisionManager revisionManager = applicationContext.getBean("revisionManager", RevisionManager.class);
        final LeaderElectionManager electionManager = applicationContext.getBean("leaderElectionManager", LeaderElectionManager.class);
        final FlowElection flowElection = applicationContext.getBean("flowElection", FlowElection.class);
        final NodeProtocolSender nodeProtocolSender = applicationContext.getBean("nodeProtocolSender", NodeProtocolSender.class);
        nodeClusterCoordinator = new NodeClusterCoordinator(protocolSenderListener, eventReporter, electionManager, flowElection, clusterFirewall,
                revisionManager, properties, extensionManager, nodeProtocolSender);
    }

    return nodeClusterCoordinator;
}
 
Example #2
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }

    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}
 
Example #3
Source File: TestPartitionedWriteAheadEventStore.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }

    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}
 
Example #4
Source File: SiteToSiteRestApiClient.java    From nifi with Apache License 2.0 6 votes vote down vote up
public SiteToSiteRestApiClient(final SSLContext sslContext, final HttpProxy proxy, final EventReporter eventReporter) {
    this.sslContext = sslContext;
    this.proxy = proxy;
    this.eventReporter = eventReporter;

    ttlExtendTaskExecutor = Executors.newScheduledThreadPool(1, new ThreadFactory() {
        private final ThreadFactory defaultFactory = Executors.defaultThreadFactory();

        @Override
        public Thread newThread(final Runnable r) {
            final Thread thread = defaultFactory.newThread(r);
            thread.setName(Thread.currentThread().getName() + " TTLExtend");
            thread.setDaemon(true);
            return thread;
        }
    });
}
 
Example #5
Source File: StatelessRemoteInputPort.java    From nifi with Apache License 2.0 6 votes vote down vote up
public StatelessRemoteInputPort(final VersionedRemoteProcessGroup rpg, final VersionedRemoteGroupPort remotePort, final SSLContext sslContext) {
    final String timeout = rpg.getCommunicationsTimeout();
    final long timeoutMillis = FormatUtils.getTimeDuration(timeout, TimeUnit.MILLISECONDS);

    url = rpg.getTargetUris();
    name = remotePort.getName();

    client = new SiteToSiteClient.Builder()
        .portName(remotePort.getName())
        .timeout(timeoutMillis, TimeUnit.MILLISECONDS)
        .transportProtocol(SiteToSiteTransportProtocol.valueOf(rpg.getTransportProtocol()))
        .url(rpg.getTargetUris())
        .useCompression(remotePort.isUseCompression())
        .sslContext(sslContext)
        .eventReporter(EventReporter.NO_OP)
        .build();
}
 
Example #6
Source File: TestSocketClientTransaction.java    From nifi with Apache License 2.0 6 votes vote down vote up
private SocketClientTransaction getClientTransaction(ByteArrayInputStream bis, ByteArrayOutputStream bos, TransferDirection direction) throws IOException {
    PeerDescription description = null;
    String peerUrl = "";
    SocketCommunicationsSession commsSession = mock(SocketCommunicationsSession.class);
    SocketInput socketIn = mock(SocketInput.class);
    SocketOutput socketOut = mock(SocketOutput.class);
    when(commsSession.getInput()).thenReturn(socketIn);
    when(commsSession.getOutput()).thenReturn(socketOut);

    when(socketIn.getInputStream()).thenReturn(bis);
    when(socketOut.getOutputStream()).thenReturn(bos);

    String clusterUrl = "";
    Peer peer = new Peer(description, commsSession, peerUrl, clusterUrl);
    boolean useCompression = false;
    int penaltyMillis = 1000;
    EventReporter eventReporter = null;
    int protocolVersion = 5;
    String destinationId = "destinationId";
    return new SocketClientTransaction(protocolVersion, destinationId, peer, codec, direction, useCompression, penaltyMillis, eventReporter);
}
 
Example #7
Source File: NodeClusterCoordinatorFactoryBean.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public NodeClusterCoordinator getObject() throws Exception {
    if (nodeClusterCoordinator == null && properties.isNode()) {
        final ClusterCoordinationProtocolSenderListener protocolSenderListener =
                applicationContext.getBean("clusterCoordinationProtocolSenderListener", ClusterCoordinationProtocolSenderListener.class);
        final EventReporter eventReporter = applicationContext.getBean("eventReporter", EventReporter.class);
        final ClusterNodeFirewall clusterFirewall = applicationContext.getBean("clusterFirewall", ClusterNodeFirewall.class);
        final RevisionManager revisionManager = applicationContext.getBean("revisionManager", RevisionManager.class);
        final LeaderElectionManager electionManager = applicationContext.getBean("leaderElectionManager", LeaderElectionManager.class);
        final FlowElection flowElection = applicationContext.getBean("flowElection", FlowElection.class);
        final NodeProtocolSender nodeProtocolSender = applicationContext.getBean("nodeProtocolSender", NodeProtocolSender.class);
        nodeClusterCoordinator = new NodeClusterCoordinator(protocolSenderListener, eventReporter, electionManager, flowElection, clusterFirewall, revisionManager, properties, nodeProtocolSender);
    }

    return nodeClusterCoordinator;
}
 
Example #8
Source File: ThreadPoolRequestReplicatorFactoryBean.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public ThreadPoolRequestReplicator getObject() throws Exception {
    if (replicator == null && nifiProperties.isNode()) {
        final EventReporter eventReporter = applicationContext.getBean("eventReporter", EventReporter.class);
        final ClusterCoordinator clusterCoordinator = applicationContext.getBean("clusterCoordinator", ClusterCoordinator.class);
        final RequestCompletionCallback requestCompletionCallback = applicationContext.getBean("clusterCoordinator", RequestCompletionCallback.class);

        final int numThreads = nifiProperties.getClusterNodeProtocolThreads();
        final Client jerseyClient = WebUtils.createClient(new DefaultClientConfig(), SslContextFactory.createSslContext(nifiProperties));
        final String connectionTimeout = nifiProperties.getClusterNodeConnectionTimeout();
        final String readTimeout = nifiProperties.getClusterNodeReadTimeout();

        replicator = new ThreadPoolRequestReplicator(numThreads, jerseyClient, clusterCoordinator,
            connectionTimeout, readTimeout, requestCompletionCallback, eventReporter, nifiProperties);
    }

    return replicator;
}
 
Example #9
Source File: SwappablePriorityQueue.java    From nifi with Apache License 2.0 6 votes vote down vote up
public SwappablePriorityQueue(final FlowFileSwapManager swapManager, final int swapThreshold, final EventReporter eventReporter, final FlowFileQueue flowFileQueue,
    final DropFlowFileAction dropAction, final String swapPartitionName) {
    this.swapManager = swapManager;
    this.swapThreshold = swapThreshold;

    this.activeQueue = new PriorityQueue<>(20, new QueuePrioritizer(Collections.emptyList()));
    this.swapQueue = new ArrayList<>();
    this.eventReporter = eventReporter;
    this.flowFileQueue = flowFileQueue;
    this.dropAction = dropAction;
    this.swapPartitionName = swapPartitionName;

    final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
    readLock = new TimedLock(lock.readLock(), flowFileQueue.getIdentifier() + " Read Lock", 100);
    writeLock = new TimedLock(lock.writeLock(), flowFileQueue.getIdentifier() + " Write Lock", 100);
}
 
Example #10
Source File: TestNodeClusterCoordinator.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "src/test/resources/conf/nifi.properties");

    senderListener = Mockito.mock(ClusterCoordinationProtocolSenderListener.class);
    nodeStatuses = Collections.synchronizedList(new ArrayList<>());

    final EventReporter eventReporter = Mockito.mock(EventReporter.class);
    final RevisionManager revisionManager = Mockito.mock(RevisionManager.class);
    Mockito.when(revisionManager.getAllRevisions()).thenReturn(Collections.emptyList());

    coordinator = new NodeClusterCoordinator(senderListener, eventReporter, null, new FirstVoteWinsFlowElection(), null, revisionManager, createProperties(), null) {
        @Override
        void notifyOthersOfNodeStatusChange(NodeConnectionStatus updatedStatus, boolean notifyAllNodes, boolean waitForCoordinator) {
            nodeStatuses.add(updatedStatus);
        }
    };

    final FlowService flowService = Mockito.mock(FlowService.class);
    final StandardDataFlow dataFlow = new StandardDataFlow(new byte[50], new byte[50], new byte[50]);
    Mockito.when(flowService.createDataFlow()).thenReturn(dataFlow);
    coordinator.setFlowService(flowService);
}
 
Example #11
Source File: TestFileSystemSwapManager.java    From nifi with Apache License 2.0 6 votes vote down vote up
private FileSystemSwapManager createSwapManager(final FlowFileRepository flowFileRepo) {
    final FileSystemSwapManager swapManager = new FileSystemSwapManager();
    final ResourceClaimManager resourceClaimManager = new NopResourceClaimManager();
    swapManager.initialize(new SwapManagerInitializationContext() {
        @Override
        public ResourceClaimManager getResourceClaimManager() {
            return resourceClaimManager;
        }

        @Override
        public FlowFileRepository getFlowFileRepository() {
            return flowFileRepo;
        }

        @Override
        public EventReporter getEventReporter() {
            return EventReporter.NO_OP;
        }
    });

    return swapManager;
}
 
Example #12
Source File: TestPartitionedWriteAheadEventStore.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
 
Example #13
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
 
Example #14
Source File: PartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
public PartitionedWriteAheadEventStore(final RepositoryConfiguration repoConfig, final RecordWriterFactory recordWriterFactory,
    final RecordReaderFactory recordReaderFactory, final EventReporter eventReporter, final EventFileManager fileManager) {
    super(repoConfig, eventReporter);
    this.repoConfig = repoConfig;
    this.eventReporter = eventReporter;
    this.filesToCompress = new LinkedBlockingQueue<>(100);
    final AtomicLong idGenerator = new AtomicLong(0L);
    this.partitions = createPartitions(repoConfig, recordWriterFactory, recordReaderFactory, idGenerator);
    this.fileManager = fileManager;

    // Creates tasks to compress data on rollover
    if (repoConfig.isCompressOnRollover()) {
        compressionExecutor = Executors.newFixedThreadPool(repoConfig.getIndexThreadPoolSize(), new NamedThreadFactory("Compress Provenance Logs"));
    } else {
        compressionExecutor = null;
    }
}
 
Example #15
Source File: TestPartitionedWriteAheadEventStore.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxEventIdRestored() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
    }

    assertEquals(19, store.getMaxEventId());
    store.close();

    final PartitionedWriteAheadEventStore recoveredStore = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    recoveredStore.initialize();
    assertEquals(19, recoveredStore.getMaxEventId());
}
 
Example #16
Source File: AbstractSiteToSiteReportingTask.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@OnScheduled
public void setup(final ConfigurationContext context) throws IOException {
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);
    final SSLContext sslContext = sslContextService == null ? null : sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
    final ComponentLog logger = getLogger();
    final EventReporter eventReporter = new EventReporter() {
        @Override
        public void reportEvent(final Severity severity, final String category, final String message) {
            switch (severity) {
                case WARNING:
                    logger.warn(message);
                    break;
                case ERROR:
                    logger.error(message);
                    break;
                default:
                    break;
            }
        }
    };

    final String destinationUrl = context.getProperty(DESTINATION_URL).evaluateAttributeExpressions().getValue();

    siteToSiteClient = new SiteToSiteClient.Builder()
            .url(destinationUrl)
            .portName(context.getProperty(PORT_NAME).getValue())
            .useCompression(context.getProperty(COMPRESS).asBoolean())
            .eventReporter(eventReporter)
            .sslContext(sslContext)
            .timeout(context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS)
            .build();
}
 
Example #17
Source File: StandardPublicPort.java    From nifi with Apache License 2.0 5 votes vote down vote up
public StandardPublicPort(final String id, final String name,
                          final TransferDirection direction, final ConnectableType type, final Authorizer authorizer,
                          final BulletinRepository bulletinRepository, final ProcessScheduler scheduler, final boolean secure,
                          final String yieldPeriod, final List<IdentityMapping> identityMappings) {

    super(id, name, type, scheduler);

    setScheduldingPeriod(MINIMUM_SCHEDULING_NANOS + " nanos");
    this.authorizer = authorizer;
    this.secure = secure;
    this.identityMappings = identityMappings;
    this.bulletinRepository = bulletinRepository;
    this.scheduler = scheduler;
    this.direction = direction;
    setYieldPeriod(yieldPeriod);
    eventReporter = new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(final Severity severity, final String category, final String message) {
            final String groupId = StandardPublicPort.this.getProcessGroup().getIdentifier();
            final String groupName = StandardPublicPort.this.getProcessGroup().getName();
            final String sourceId = StandardPublicPort.this.getIdentifier();
            final String sourceName = StandardPublicPort.this.getName();
            final ComponentType componentType = direction == TransferDirection.RECEIVE ? ComponentType.INPUT_PORT : ComponentType.OUTPUT_PORT;
            bulletinRepository.addBulletin(BulletinFactory.createBulletin(groupId, groupName, sourceId, componentType, sourceName, category, severity.name(), message));
        }
    };

    relationships = direction == TransferDirection.RECEIVE ? Collections.singleton(AbstractPort.PORT_RELATIONSHIP) : Collections.emptySet();
}
 
Example #18
Source File: StandardRootGroupPort.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public StandardRootGroupPort(final String id, final String name, final ProcessGroup processGroup,
        final TransferDirection direction, final ConnectableType type, final Authorizer authorizer,
        final BulletinRepository bulletinRepository, final ProcessScheduler scheduler, final boolean secure,
        final NiFiProperties nifiProperties) {
    super(id, name, processGroup, type, scheduler);

    setScheduldingPeriod(MINIMUM_SCHEDULING_NANOS + " nanos");
    this.authorizer = authorizer;
    this.secure = secure;
    this.identityMappings = IdentityMappingUtil.getIdentityMappings(nifiProperties);
    this.bulletinRepository = bulletinRepository;
    this.scheduler = scheduler;
    setYieldPeriod("100 millis");
    eventReporter = new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(final Severity severity, final String category, final String message) {
            final String groupId = StandardRootGroupPort.this.getProcessGroup().getIdentifier();
            final String sourceId = StandardRootGroupPort.this.getIdentifier();
            final String sourceName = StandardRootGroupPort.this.getName();
            final ComponentType componentType = direction == TransferDirection.RECEIVE ? ComponentType.INPUT_PORT : ComponentType.OUTPUT_PORT;
            bulletinRepository.addBulletin(BulletinFactory.createBulletin(groupId, sourceId, componentType, sourceName, category, severity.name(), message));
        }
    };

    relationships = direction == TransferDirection.RECEIVE ? Collections.singleton(AbstractPort.PORT_RELATIONSHIP) : Collections.<Relationship>emptySet();
}
 
Example #19
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Before
public void printTestName() {
    System.out.println("\n\n\n***********************  " + name.getMethodName() + "  *****************************");

    reportedEvents.clear();
    eventReporter = new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(Severity severity, String category, String message) {
            reportedEvents.add(new ReportedEvent(severity, category, message));
            System.out.println(severity + " : " + category + " : " + message);
        }
    };
}
 
Example #20
Source File: VolatileProvenanceRepository.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
    final IdentifierLookup idLookup) throws IOException {
    if (initialized.getAndSet(true)) {
        return;
    }

    this.authorizer = authorizer;
    this.resourceFactory = resourceFactory;

    scheduledExecService.scheduleWithFixedDelay(new RemoveExpiredQueryResults(), 30L, 30L, TimeUnit.SECONDS);
}
 
Example #21
Source File: SiteToSiteRestApiClient.java    From nifi with Apache License 2.0 5 votes vote down vote up
private void startExtendingTtl(final String transactionUrl, final Closeable stream, final CloseableHttpResponse response) {
    if (ttlExtendingFuture != null) {
        // Already started.
        return;
    }

    logger.debug("Starting extending TTL thread...");

    extendingApiClient = new SiteToSiteRestApiClient(sslContext, proxy, EventReporter.NO_OP);
    extendingApiClient.transportProtocolVersionNegotiator = this.transportProtocolVersionNegotiator;
    extendingApiClient.connectTimeoutMillis = this.connectTimeoutMillis;
    extendingApiClient.readTimeoutMillis = this.readTimeoutMillis;
    extendingApiClient.localAddress = this.localAddress;

    final int extendFrequency = serverTransactionTtl / 2;

    ttlExtendingFuture = ttlExtendTaskExecutor.scheduleWithFixedDelay(() -> {
        try {
            extendingApiClient.extendTransaction(transactionUrl);
        } catch (final Exception e) {
            logger.warn("Failed to extend transaction ttl", e);

            try {
                // Without disconnecting, Site-to-Site client keep reading data packet,
                // while server has already rollback.
                this.close();
            } catch (final IOException ec) {
                logger.warn("Failed to close", e);
            }
        }
    }, extendFrequency, extendFrequency, TimeUnit.SECONDS);
}
 
Example #22
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteAfterRecoveringRepo() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    for (int i = 0; i < 4; i++) {
        store.addEvents(Collections.singleton(createEvent()));
    }

    store.close();

    final PartitionedWriteAheadEventStore recoveredStore = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    recoveredStore.initialize();

    List<ProvenanceEventRecord> recoveredEvents = recoveredStore.getEvents(0, 10);
    assertEquals(4, recoveredEvents.size());

    // ensure that we can still write to the store
    for (int i = 0; i < 4; i++) {
        recoveredStore.addEvents(Collections.singleton(createEvent()));
    }

    recoveredEvents = recoveredStore.getEvents(0, 10);
    assertEquals(8, recoveredEvents.size());

    for (int i = 0; i < 8; i++) {
        assertEquals(i, recoveredEvents.get(i).getEventId());
    }
}
 
Example #23
Source File: SocketClientTransaction.java    From nifi with Apache License 2.0 5 votes vote down vote up
SocketClientTransaction(final int protocolVersion, final String destinationId, final Peer peer, final FlowFileCodec codec,
        final TransferDirection direction, final boolean useCompression, final int penaltyMillis, final EventReporter eventReporter) throws IOException {
    super(peer, direction, useCompression, codec, eventReporter, protocolVersion, penaltyMillis, destinationId);
    this.dis = new DataInputStream(peer.getCommunicationsSession().getInput().getInputStream());
    this.dos = new DataOutputStream(peer.getCommunicationsSession().getOutput().getOutputStream());

    initialize();
}
 
Example #24
Source File: FlowController.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
private static EventReporter createEventReporter(final BulletinRepository bulletinRepository) {
    return new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(final Severity severity, final String category, final String message) {
            final Bulletin bulletin = BulletinFactory.createBulletin(category, severity.name(), message);
            bulletinRepository.addBulletin(bulletin);
        }
    };
}
 
Example #25
Source File: StandardRemoteProcessGroup.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public StandardRemoteProcessGroup(final String id, final String targetUris, final ProcessGroup processGroup,
                                  final FlowController flowController, final SSLContext sslContext, final NiFiProperties nifiProperties) {
    this.nifiProperties = nifiProperties;
    this.id = requireNonNull(id);

    this.targetUris = targetUris;
    this.targetId = null;
    this.processGroup = new AtomicReference<>(processGroup);
    this.sslContext = sslContext;
    this.scheduler = flowController.getProcessScheduler();
    this.authorizationIssue = "Establishing connection to " + targetUris;

    final BulletinRepository bulletinRepository = flowController.getBulletinRepository();
    eventReporter = new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(final Severity severity, final String category, final String message) {
            final String groupId = StandardRemoteProcessGroup.this.getProcessGroup().getIdentifier();
            final String sourceId = StandardRemoteProcessGroup.this.getIdentifier();
            final String sourceName = StandardRemoteProcessGroup.this.getName();
            bulletinRepository.addBulletin(BulletinFactory.createBulletin(groupId, sourceId, ComponentType.REMOTE_PROCESS_GROUP,
                    sourceName, category, severity.name(), message));
        }
    };

    final Runnable checkAuthorizations = new InitializationTask();
    backgroundThreadExecutor = new FlowEngine(1, "Remote Process Group " + id + ": " + targetUris);
    backgroundThreadExecutor.scheduleWithFixedDelay(checkAuthorizations, 5L, 30L, TimeUnit.SECONDS);
}
 
Example #26
Source File: EventIndexTask.java    From nifi with Apache License 2.0 5 votes vote down vote up
public EventIndexTask(final BlockingQueue<StoredDocument> documentQueue, final IndexManager indexManager,
    final IndexDirectoryManager directoryManager, final int maxEventsPerCommit, final EventReporter eventReporter) {
    this.documentQueue = documentQueue;
    this.indexManager = indexManager;
    this.directoryManager = directoryManager;
    this.commitThreshold = maxEventsPerCommit;
    this.eventReporter = eventReporter;
}
 
Example #27
Source File: TestNodeClusterCoordinator.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws IOException {
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "src/test/resources/conf/nifi.properties");

    senderListener = Mockito.mock(ClusterCoordinationProtocolSenderListener.class);
    nodeStatuses = Collections.synchronizedList(new ArrayList<>());
    stateManagerProvider = Mockito.mock(StateManagerProvider.class);

    final StateManager stateManager = Mockito.mock(StateManager.class);
    when(stateManager.getState(any(Scope.class))).thenReturn(new MockStateMap(Collections.emptyMap(), 1));
    when(stateManagerProvider.getStateManager(anyString())).thenReturn(stateManager);


    final EventReporter eventReporter = Mockito.mock(EventReporter.class);
    final RevisionManager revisionManager = Mockito.mock(RevisionManager.class);
    when(revisionManager.getAllRevisions()).thenReturn(Collections.emptyList());

    coordinator = new NodeClusterCoordinator(senderListener, eventReporter, null, new FirstVoteWinsFlowElection(), null, revisionManager, createProperties(), null, stateManagerProvider) {
        @Override
        void notifyOthersOfNodeStatusChange(NodeConnectionStatus updatedStatus, boolean notifyAllNodes, boolean waitForCoordinator) {
            nodeStatuses.add(updatedStatus);
        }
    };

    final FlowService flowService = Mockito.mock(FlowService.class);
    final StandardDataFlow dataFlow = new StandardDataFlow(new byte[50], new byte[50], new byte[50], new HashSet<>());
    when(flowService.createDataFlow()).thenReturn(dataFlow);
    coordinator.setFlowService(flowService);
}
 
Example #28
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final ProvenanceEventRecord event = createEvent();

    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));

    final Query query = new Query(UUID.randomUUID().toString());

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(4000, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #29
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Before
public void printTestName() {
    System.out.println("\n\n\n***********************  " + name.getMethodName() + "  *****************************");

    reportedEvents.clear();
    eventReporter = new EventReporter() {
        private static final long serialVersionUID = 1L;

        @Override
        public void reportEvent(Severity severity, String category, String message) {
            reportedEvents.add(new ReportedEvent(severity, category, message));
            System.out.println(severity + " : " + category + " : " + message);
        }
    };
}
 
Example #30
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testExpiration() throws InterruptedException, IOException {
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final List<ProvenanceEventRecord> events = new ArrayList<>();
    events.add(createEvent(500000L));
    events.add(createEvent());

    final EventStore eventStore = Mockito.mock(EventStore.class);
    Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {
        @Override
        public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
            final Long eventId = invocation.getArgumentAt(0, Long.class);
            assertEquals(0, eventId.longValue());
            assertEquals(1, invocation.getArgumentAt(1, Integer.class).intValue());
            return Collections.singletonList(events.get(0));
        }
    }).when(eventStore).getEvents(Mockito.anyLong(), Mockito.anyInt());

    index.initialize(eventStore);
    index.addEvent(events.get(0), createStorageSummary(events.get(0).getEventId()));

    // Add the first event to the index and wait for it to be indexed, since indexing is asynchronous.
    List<File> allDirectories = Collections.emptyList();
    while (allDirectories.isEmpty()) {
        allDirectories = index.getDirectoryManager().getDirectories(null, null);
    }

    events.remove(0); // Remove the first event from the store
    index.performMaintenance();
    assertEquals(1, index.getDirectoryManager().getDirectories(null, null).size());
}