Java Code Examples for org.apache.nifi.logging.ComponentLog

The following examples show how to use org.apache.nifi.logging.ComponentLog. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: nifi   Source File: TestCSVRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTimeNoCoersionUnexpectedFormat() throws IOException, MalformedRecordException {
    final String text = "time\n01:02:03";

    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("time", RecordFieldType.TIME.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);

    try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
         final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
                 RecordFieldType.DATE.getDefaultFormat(), "HH-MM-SS", RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

        final Record record = reader.nextRecord(false, false);
        assertEquals("01:02:03", (String)record.getValue("time"));
    }
}
 
Example 2
Source Project: localization_nifi   Source File: TransformXml.java    License: Apache License 2.0 6 votes vote down vote up
@OnScheduled
public void onScheduled(final ProcessContext context) {
    final ComponentLog logger = getLogger();
    final Integer cacheSize = context.getProperty(CACHE_SIZE).asInteger();
    final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);

    if (cacheSize > 0) {
        CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
        if (cacheTTL > 0) {
            cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
        }

        cache = cacheBuilder.build(
           new CacheLoader<String, Templates>() {
               public Templates load(String path) throws TransformerConfigurationException {
                   return newTemplates(path);
               }
           });
    } else {
        cache = null;
        logger.warn("Stylesheet cache disabled because cache size is set to 0");
    }
}
 
Example 3
Source Project: nifi   Source File: ConsumerLease.java    License: Apache License 2.0 6 votes vote down vote up
ConsumerLease(
        final long maxWaitMillis,
        final Consumer<byte[], byte[]> kafkaConsumer,
        final byte[] demarcatorBytes,
        final String keyEncoding,
        final String securityProtocol,
        final String bootstrapServers,
        final RecordReaderFactory readerFactory,
        final RecordSetWriterFactory writerFactory,
        final ComponentLog logger,
        final Charset headerCharacterSet,
        final Pattern headerNamePattern) {
    this.maxWaitMillis = maxWaitMillis;
    this.kafkaConsumer = kafkaConsumer;
    this.demarcatorBytes = demarcatorBytes;
    this.keyEncoding = keyEncoding;
    this.securityProtocol = securityProtocol;
    this.bootstrapServers = bootstrapServers;
    this.readerFactory = readerFactory;
    this.writerFactory = writerFactory;
    this.logger = logger;
    this.headerCharacterSet = headerCharacterSet;
    this.headerNamePattern = headerNamePattern;
}
 
Example 4
Source Project: nifi   Source File: AbstractScriptedControllerService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reloads the script located at the given path
 *
 * @param scriptPath the path to the script file to be loaded
 * @return true if the script was loaded successfully; false otherwise
 */
protected boolean reloadScriptFile(final String scriptPath) {
    final Collection<ValidationResult> results = new HashSet<>();

    try (final FileInputStream scriptStream = new FileInputStream(scriptPath)) {
        return reloadScript(IOUtils.toString(scriptStream, Charset.defaultCharset()));

    } catch (final Exception e) {
        final ComponentLog logger = getLogger();
        final String message = "Unable to load script: " + e;

        logger.error(message, e);
        results.add(new ValidationResult.Builder()
                .subject("ScriptValidation")
                .valid(false)
                .explanation("Unable to load script due to " + e)
                .input(scriptPath)
                .build());
    }

    // store the updated validation results
    validationResults.set(results);

    // return whether there was any issues loading the configured script
    return results.isEmpty();
}
 
Example 5
Source Project: nifi   Source File: ReportingTaskWrapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void run() {
    lifecycleState.incrementActiveThreadCount(null);
    try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(extensionManager, taskNode.getReportingTask().getClass(), taskNode.getIdentifier())) {
        taskNode.getReportingTask().onTrigger(taskNode.getReportingContext());
    } catch (final Throwable t) {
        final ComponentLog componentLog = new SimpleProcessLogger(taskNode.getIdentifier(), taskNode.getReportingTask());
        componentLog.error("Error running task {} due to {}", new Object[]{taskNode.getReportingTask(), t.toString()});
        if (componentLog.isDebugEnabled()) {
            componentLog.error("", t);
        }
    } finally {
        try {
            // if the reporting task is no longer scheduled to run and this is the last thread,
            // invoke the OnStopped methods
            if (!lifecycleState.isScheduled() && lifecycleState.getActiveThreadCount() == 1 && lifecycleState.mustCallOnStoppedMethods()) {
                try (final NarCloseable x = NarCloseable.withComponentNarLoader(extensionManager, taskNode.getReportingTask().getClass(), taskNode.getIdentifier())) {
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, taskNode.getReportingTask(), taskNode.getConfigurationContext());
                }
            }
        } finally {
            lifecycleState.decrementActiveThreadCount(null);
        }
    }
}
 
Example 6
Source Project: localization_nifi   Source File: TestKerberosProperties.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testValidatePrincipalAndKeytab() {
    final ComponentLog log = Mockito.mock(ComponentLog.class);
    final Configuration config = new Configuration();

    // no security enabled in config so doesn't matter what principal and keytab are
    List<ValidationResult> results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, null, null, log);
    Assert.assertEquals(0, results.size());

    results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, "principal", null, log);
    Assert.assertEquals(0, results.size());

    results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, "principal", "keytab", log);
    Assert.assertEquals(0, results.size());

    // change the config to have kerberos turned on
    config.set("hadoop.security.authentication", "kerberos");
    config.set("hadoop.security.authorization", "true");

    results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, null, null, log);
    Assert.assertEquals(2, results.size());
}
 
Example 7
Source Project: nifi   Source File: HiveConfigurator.java    License: Apache License 2.0 6 votes vote down vote up
public Collection<ValidationResult> validate(String configFiles, String principal, String keyTab, String password,
                                             AtomicReference<ValidationResources> validationResourceHolder, ComponentLog log) {

    final List<ValidationResult> problems = new ArrayList<>();
    ValidationResources resources = validationResourceHolder.get();

    // if no resources in the holder, or if the holder has different resources loaded,
    // then load the Configuration and set the new resources in the holder
    if (resources == null || !configFiles.equals(resources.getConfigResources())) {
        log.debug("Reloading validation resources");
        resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles));
        validationResourceHolder.set(resources);
    }

    final Configuration hiveConfig = resources.getConfiguration();

    problems.addAll(KerberosProperties.validatePrincipalWithKeytabOrPassword(this.getClass().getSimpleName(), hiveConfig, principal, keyTab, password, log));

    return problems;
}
 
Example 8
Source Project: nifi   Source File: InvokeScriptedProcessor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the valid relationships for this processor as supplied by the
 * script itself.
 *
 * @return a Set of Relationships supported by this processor
 */
@Override
public Set<Relationship> getRelationships() {
    final Set<Relationship> relationships = new HashSet<>();
    final Processor instance = processor.get();
    if (instance != null) {
        try {
            final Set<Relationship> rels = instance.getRelationships();
            if (rels != null && !rels.isEmpty()) {
                relationships.addAll(rels);
            }
        } catch (final Throwable t) {
            final ComponentLog logger = getLogger();
            final String message = "Unable to get relationships from scripted Processor: " + t;

            logger.error(message);
            if (logger.isDebugEnabled()) {
                logger.error(message, t);
            }
        }
    }
    return Collections.unmodifiableSet(relationships);
}
 
Example 9
Source Project: localization_nifi   Source File: ConsumerLease.java    License: Apache License 2.0 6 votes vote down vote up
ConsumerLease(
        final long maxWaitMillis,
        final Consumer<byte[], byte[]> kafkaConsumer,
        final byte[] demarcatorBytes,
        final String keyEncoding,
        final String securityProtocol,
        final String bootstrapServers,
        final ComponentLog logger) {
    this.maxWaitMillis = maxWaitMillis;
    this.kafkaConsumer = kafkaConsumer;
    this.demarcatorBytes = demarcatorBytes;
    this.keyEncoding = keyEncoding;
    this.securityProtocol = securityProtocol;
    this.bootstrapServers = bootstrapServers;
    this.logger = logger;
}
 
Example 10
Source Project: nifi   Source File: ComponentFactory.java    License: Apache License 2.0 6 votes vote down vote up
private Set<URL> getAdditionalClasspathResources(final List<PropertyDescriptor> propertyDescriptors, final String componentId, final Map<String, String> properties,
                                                 final ParameterLookup parameterLookup, final VariableRegistry variableRegistry, final ComponentLog logger) {
    final Set<String> modulePaths = new LinkedHashSet<>();
    for (final PropertyDescriptor descriptor : propertyDescriptors) {
        if (descriptor.isDynamicClasspathModifier()) {
            final String value = properties.get(descriptor.getName());
            if (!StringUtils.isEmpty(value)) {
                final StandardPropertyValue propertyValue = new StandardPropertyValue(value, null, parameterLookup, variableRegistry);
                modulePaths.add(propertyValue.evaluateAttributeExpressions().getValue());
            }
        }
    }

    final Set<URL> additionalUrls = new LinkedHashSet<>();
    try {
        final URL[] urls = ClassLoaderUtils.getURLsForClasspath(modulePaths, null, true);
        if (urls != null) {
            additionalUrls.addAll(Arrays.asList(urls));
        }
    } catch (MalformedURLException mfe) {
        logger.error("Error processing classpath resources for " + componentId + ": " + mfe.getMessage(), mfe);
    }

    return additionalUrls;
}
 
Example 11
Source Project: nifi   Source File: GetJMSQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();

    WrappedMessageConsumer wrappedConsumer = consumerQueue.poll();
    if (wrappedConsumer == null) {
        try {
            wrappedConsumer = JmsFactory.createQueueMessageConsumer(context);
        } catch (JMSException e) {
            logger.error("Failed to connect to JMS Server due to {}", e);
            context.yield();
            return;
        }
    }

    try {
        super.consume(context, session, wrappedConsumer);
    } finally {
        if (!wrappedConsumer.isClosed()) {
            consumerQueue.offer(wrappedConsumer);
        }
    }
}
 
Example 12
Source Project: localization_nifi   Source File: SmtpConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public SmtpConsumer(
        final MessageContext context,
        final ProcessSessionFactory sessionFactory,
        final int port,
        final String host,
        final ComponentLog log,
        final int maxMessageSize
) {
    this.context = context;
    this.sessionFactory = sessionFactory;
    this.port = port;
    if (host == null || host.trim().isEmpty()) {
        this.host = context.getSMTPServer().getHostName();
    } else {
        this.host = host;
    }
    this.log = log;
    this.maxMessageSize = maxMessageSize;
}
 
Example 13
Source Project: nifi   Source File: TestJsonPathRowRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadOneLine() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());

    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-oneline.json"));
         final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {

        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
        assertEquals(expectedFieldNames, fieldNames);

        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
                RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
        assertEquals(expectedTypes, dataTypes);

        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues);

        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues);

        assertNull(reader.nextRecord());
    }
}
 
Example 14
Source Project: nifi   Source File: TestCSVRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDateNoCoersionExpectedFormat() throws IOException, MalformedRecordException {
    final String text = "date\n11/30/1983";

    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);

    try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
         final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
                 "MM/dd/yyyy", RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

        final Record record = reader.nextRecord(false, false);
        final java.sql.Date date = (Date) record.getValue("date");
        final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("gmt"));
        calendar.setTimeInMillis(date.getTime());

        assertEquals(1983, calendar.get(Calendar.YEAR));
        assertEquals(10, calendar.get(Calendar.MONTH));
        assertEquals(30, calendar.get(Calendar.DAY_OF_MONTH));
    }
}
 
Example 15
/**
 * At the moment the only two supported message types are TextMessage and
 * BytesMessage which is sufficient for the type if JMS use cases NiFi is
 * used. The may change to the point where all message types are supported
 * at which point this test will no be longer required.
 */
@Test(expected = IllegalStateException.class)
public void validateFailOnUnsupportedMessageType() throws Exception {
    final String destinationName = "testQueue";
    JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false);

    jmsTemplate.send(destinationName, new MessageCreator() {
        @Override
        public Message createMessage(Session session) throws JMSException {
            return session.createObjectMessage();
        }
    });

    JMSConsumer consumer = new JMSConsumer(jmsTemplate, mock(ComponentLog.class));
    try {
        consumer.consume(destinationName, new ConsumerCallback() {
            @Override
            public void accept(JMSResponse response) {
                // noop
            }
        });
    } finally {
        ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy();
    }
}
 
Example 16
Source Project: localization_nifi   Source File: HiveConfigurator.java    License: Apache License 2.0 6 votes vote down vote up
public Collection<ValidationResult> validate(String configFiles, String principal, String keyTab, AtomicReference<ValidationResources> validationResourceHolder, ComponentLog log) {

        final List<ValidationResult> problems = new ArrayList<>();
        ValidationResources resources = validationResourceHolder.get();

        // if no resources in the holder, or if the holder has different resources loaded,
        // then load the Configuration and set the new resources in the holder
        if (resources == null || !configFiles.equals(resources.getConfigResources())) {
            log.debug("Reloading validation resources");
            resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles));
            validationResourceHolder.set(resources);
        }

        final Configuration hiveConfig = resources.getConfiguration();

        problems.addAll(KerberosProperties.validatePrincipalAndKeytab(this.getClass().getSimpleName(), hiveConfig, principal, keyTab, log));

        return problems;
    }
 
Example 17
Source Project: nifi   Source File: TestXMLRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleRecordWithAttribute2() throws IOException, MalformedRecordException {
    InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
    List<RecordField> fields = getSimpleRecordFields();
    fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType()));
    XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true,
            "ATTR_", "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class));

    Record first = reader.nextRecord();
    assertTrue(Arrays.asList(first.getValues()).containsAll(Arrays.asList("Cleve Butler", 42, "USA")));
    assertEquals("P1", first.getAsString("ATTR_ID"));

    Record second = reader.nextRecord();
    assertTrue(Arrays.asList(second.getValues()).containsAll(Arrays.asList("Ainslie Fletcher", 33, "UK")));
    assertEquals("P2", second.getAsString("ATTR_ID"));

    Record third = reader.nextRecord();
    assertTrue(Arrays.asList(third.getValues()).containsAll(Arrays.asList("Amélie Bonfils", 74, "FR")));
    assertEquals("P3", third.getAsString("ATTR_ID"));

    Record fourth = reader.nextRecord();
    assertTrue(Arrays.asList(fourth.getValues()).containsAll(Arrays.asList("Elenora Scrivens", 16, "USA")));
    assertEquals("P4", fourth.getAsString("ATTR_ID"));
}
 
Example 18
Source Project: nifi   Source File: TestJsonTreeRowRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReadMixedJSON() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());

    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-mixed.json"));
         final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {

        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
        assertEquals(expectedFieldNames, fieldNames);

        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
                RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
        assertEquals(expectedTypes, dataTypes);

        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues);

        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues);

        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {3, "Maria Doe", 4750.89, "123 My Street", "My City", "ME", "11111", "USA"}, thirdRecordValues);

        final Object[] fourthRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {4, "Xi Doe", 4820.09, "321 Your Street", "Your City", "NV", "33333", "USA"}, fourthRecordValues);


        assertNull(reader.nextRecord());
    }
}
 
Example 19
Source Project: nifi   Source File: SocketChannelDispatcher.java    License: Apache License 2.0 5 votes vote down vote up
public SocketChannelDispatcher(final EventFactory<E> eventFactory,
                               final ChannelHandlerFactory<E, AsyncChannelDispatcher> handlerFactory,
                               final BlockingQueue<ByteBuffer> bufferPool,
                               final BlockingQueue<E> events,
                               final ComponentLog logger,
                               final int maxConnections,
                               final SSLContext sslContext,
                               final Charset charset) {
    this(eventFactory, handlerFactory, bufferPool, events, logger, maxConnections, sslContext, SslContextFactory.ClientAuth.REQUIRED, charset);
}
 
Example 20
Source Project: nifi   Source File: ConsumerPool.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a pool of KafkaConsumer objects that will grow up to the maximum
 * indicated threads from the given context. Consumers are lazily
 * initialized. We may elect to not create up to the maximum number of
 * configured consumers if the broker reported lag time for all topics is
 * below a certain threshold.
 *
 * @param maxConcurrentLeases max allowable consumers at once
 * @param demarcator bytes to use as demarcator between messages; null or
 * empty means no demarcator
 * @param kafkaProperties properties to use to initialize kafka consumers
 * @param topics the topics to subscribe to
 * @param maxWaitMillis maximum time to wait for a given lease to acquire
 * data before committing
 * @param keyEncoding the encoding to use for the key of a kafka message if
 * found
 * @param securityProtocol the security protocol used
 * @param bootstrapServers the bootstrap servers
 * @param logger the logger to report any errors/warnings
 */
public ConsumerPool(
        final int maxConcurrentLeases,
        final byte[] demarcator,
        final Map<String, Object> kafkaProperties,
        final List<String> topics,
        final long maxWaitMillis,
        final String keyEncoding,
        final String securityProtocol,
        final String bootstrapServers,
        final ComponentLog logger,
        final boolean honorTransactions,
        final Charset headerCharacterSet,
        final Pattern headerNamePattern) {
    this.pooledLeases = new ArrayBlockingQueue<>(maxConcurrentLeases);
    this.maxWaitMillis = maxWaitMillis;
    this.logger = logger;
    this.demarcatorBytes = demarcator;
    this.keyEncoding = keyEncoding;
    this.securityProtocol = securityProtocol;
    this.bootstrapServers = bootstrapServers;
    this.kafkaProperties = Collections.unmodifiableMap(kafkaProperties);
    this.topics = Collections.unmodifiableList(topics);
    this.topicPattern = null;
    this.readerFactory = null;
    this.writerFactory = null;
    this.honorTransactions = honorTransactions;
    this.headerCharacterSet = headerCharacterSet;
    this.headerNamePattern = headerNamePattern;
}
 
Example 21
Source Project: nifi   Source File: ITConsumeKafka.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void validateGetAllMessages() throws Exception {
    String groupName = "validateGetAllMessages";

    when(mockConsumerPool.obtainConsumer(any())).thenReturn(mockLease);
    when(mockLease.continuePolling()).thenReturn(Boolean.TRUE, Boolean.TRUE, Boolean.FALSE);
    when(mockLease.commit()).thenReturn(Boolean.TRUE);

    ConsumeKafka proc = new ConsumeKafka() {
        @Override
        protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) {
            return mockConsumerPool;
        }
    };
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS, "0.0.0.0:1234");
    runner.setProperty(ConsumeKafka.TOPICS, "foo,bar");
    runner.setProperty(ConsumeKafka.GROUP_ID, groupName);
    runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, ConsumeKafka.OFFSET_EARLIEST);
    runner.run(1, false);

    verify(mockConsumerPool, times(1)).obtainConsumer(any());
    verify(mockLease, times(3)).continuePolling();
    verify(mockLease, times(2)).poll();
    verify(mockLease, times(1)).commit();
    verify(mockLease, times(1)).close();
    verifyNoMoreInteractions(mockConsumerPool);
    verifyNoMoreInteractions(mockLease);
}
 
Example 22
public RELPSocketChannelHandler(final SelectionKey key,
                                final AsyncChannelDispatcher dispatcher,
                                final Charset charset,
                                final EventFactory<E> eventFactory,
                                final BlockingQueue<E> events,
                                final ComponentLog logger) {
    super(key, dispatcher, charset, eventFactory, events, logger);
    this.decoder = new RELPDecoder(charset);
    this.frameHandler = new RELPFrameHandler<>(key, charset, eventFactory, events, dispatcher, logger);
}
 
Example 23
Source Project: nifi   Source File: ITConsumeKafka.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void validateGetAllMessages() throws Exception {
    String groupName = "validateGetAllMessages";

    when(mockConsumerPool.obtainConsumer(any(), any())).thenReturn(mockLease);
    when(mockLease.continuePolling()).thenReturn(Boolean.TRUE, Boolean.TRUE, Boolean.FALSE);
    when(mockLease.commit()).thenReturn(Boolean.TRUE);

    ConsumeKafka_0_10 proc = new ConsumeKafka_0_10() {
        @Override
        protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) {
            return mockConsumerPool;
        }
    };
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS, "0.0.0.0:1234");
    runner.setProperty(ConsumeKafka_0_10.TOPICS, "foo,bar");
    runner.setProperty(ConsumeKafka_0_10.GROUP_ID, groupName);
    runner.setProperty(ConsumeKafka_0_10.AUTO_OFFSET_RESET, ConsumeKafka_0_10.OFFSET_EARLIEST);
    runner.run(1, false);

    verify(mockConsumerPool, times(1)).obtainConsumer(any(), any());
    verify(mockLease, times(3)).continuePolling();
    verify(mockLease, times(2)).poll();
    verify(mockLease, times(1)).commit();
    verify(mockLease, times(1)).close();
    verifyNoMoreInteractions(mockConsumerPool);
    verifyNoMoreInteractions(mockLease);
}
 
Example 24
Source Project: nifi   Source File: TestXMLRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRecordExpectedSimpleFieldFoundAndNoContentFieldConfigured() throws IOException, MalformedRecordException {
    InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
    List<RecordField> fields = getSimpleRecordFields2();
    final DataType recordType = RecordFieldType.RECORD.getRecordDataType(getNestedSchema());
    fields.add(new RecordField("AGE", recordType));
    XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, null,
            "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class));

    Assert.assertArrayEquals(new Object[] {"Cleve Butler", "USA", null}, reader.nextRecord().getValues());
    Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "UK", null}, reader.nextRecord().getValues());
    Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "FR", null}, reader.nextRecord().getValues());
    Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "USA", null}, reader.nextRecord().getValues());
}
 
Example 25
Source Project: nifi   Source File: TestBeatsFrameHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
    this.charset = StandardCharsets.UTF_8;
    this.eventFactory = new BeatsEventFactory();
    this.events = new LinkedBlockingQueue<>();
    this.key = Mockito.mock(SelectionKey.class);
    this.dispatcher = Mockito.mock(AsyncChannelDispatcher.class);
    this.logger = Mockito.mock(ComponentLog.class);

    this.frameHandler = new BeatsFrameHandler<>(key, charset, eventFactory, events, dispatcher, logger);
}
 
Example 26
Source Project: nifi   Source File: WriteParquetResult.java    License: Apache License 2.0 5 votes vote down vote up
public WriteParquetResult(final Schema schema, final OutputStream out, final ParquetConfig parquetConfig, final ComponentLog componentLogger) throws IOException {
    super(out);
    this.schema = schema;
    this.componentLogger = componentLogger;

    final Configuration conf = new Configuration();
    final OutputFile outputFile = new NifiParquetOutputFile(out);

    final AvroParquetWriter.Builder<GenericRecord> writerBuilder =
            AvroParquetWriter.<GenericRecord>builder(outputFile).withSchema(schema);
    applyCommonConfig(writerBuilder, conf, parquetConfig);
    parquetWriter = writerBuilder.build();
}
 
Example 27
Source Project: nifi   Source File: JMSPublisherConsumerIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void validateJmsHeadersAndPropertiesAreTransferredFromFFAttributes() throws Exception {
    final String destinationName = "validateJmsHeadersAndPropertiesAreTransferredFromFFAttributes";
    JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false);

    try {
        JMSPublisher publisher = new JMSPublisher((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class));
        Map<String, String> flowFileAttributes = new HashMap<>();
        flowFileAttributes.put("foo", "foo");
        flowFileAttributes.put("hyphen-property", "value");
        flowFileAttributes.put("fullstop.property", "value");
        flowFileAttributes.put(JmsHeaders.REPLY_TO, "myTopic");
        flowFileAttributes.put(JmsHeaders.DELIVERY_MODE, "1");
        flowFileAttributes.put(JmsHeaders.PRIORITY, "1");
        flowFileAttributes.put(JmsHeaders.EXPIRATION, "never"); // value expected to be integer, make sure non-integer doesn't cause problems
        publisher.publish(destinationName, "hellomq".getBytes(), flowFileAttributes);

        Message receivedMessage = jmsTemplate.receive(destinationName);
        assertTrue(receivedMessage instanceof BytesMessage);
        assertEquals("foo", receivedMessage.getStringProperty("foo"));
        assertTrue(receivedMessage.propertyExists("hyphen-property"));
        assertTrue(receivedMessage.propertyExists("fullstop.property"));
        assertTrue(receivedMessage.getJMSReplyTo() instanceof Topic);
        assertEquals(1, receivedMessage.getJMSDeliveryMode());
        assertEquals(1, receivedMessage.getJMSPriority());
        assertEquals("myTopic", ((Topic) receivedMessage.getJMSReplyTo()).getTopicName());

    } finally {
        ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy();
    }
}
 
Example 28
Source Project: nifi   Source File: ScriptingComponentHelper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Performs common setup operations when the processor is scheduled to run. This method assumes the member
 * variables associated with properties have been filled.
 *
 * @param numberOfScriptEngines number of engines to setup
 */
public void setup(int numberOfScriptEngines, ComponentLog log) {

    if (scriptEngineConfiguratorMap.isEmpty()) {
        ServiceLoader<ScriptEngineConfigurator> configuratorServiceLoader =
                ServiceLoader.load(ScriptEngineConfigurator.class);
        for (ScriptEngineConfigurator configurator : configuratorServiceLoader) {
            scriptEngineConfiguratorMap.put(configurator.getScriptEngineName().toLowerCase(), configurator);
        }
    }
    setupEngines(numberOfScriptEngines, log);
}
 
Example 29
Source Project: nifi   Source File: JMSPublisherConsumerIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void validateConsumeWithCustomHeadersAndProperties() throws Exception {
    final String destinationName = "validateConsumeWithCustomHeadersAndProperties";
    JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false);

    try {
        jmsTemplate.send(destinationName, new MessageCreator() {
            @Override
            public Message createMessage(Session session) throws JMSException {
                TextMessage message = session.createTextMessage("hello from the other side");
                message.setStringProperty("foo", "foo");
                message.setBooleanProperty("bar", false);
                message.setJMSReplyTo(session.createQueue("fooQueue"));
                return message;
            }
        });

        JMSConsumer consumer = new JMSConsumer((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class));
        final AtomicBoolean callbackInvoked = new AtomicBoolean();
        consumer.consume(destinationName, null, false, false, null, "UTF-8", new ConsumerCallback() {
            @Override
            public void accept(JMSResponse response) {
                callbackInvoked.set(true);
                assertEquals("hello from the other side", new String(response.getMessageBody()));
                assertEquals("fooQueue", response.getMessageHeaders().get(JmsHeaders.REPLY_TO));
                assertEquals("foo", response.getMessageProperties().get("foo"));
                assertEquals("false", response.getMessageProperties().get("bar"));
            }
        });
        assertTrue(callbackInvoked.get());

    } finally {
        ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy();
    }
}
 
Example 30
Source Project: nifi   Source File: ParseEvtx.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    ComponentLog logger = getLogger();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    String basename = getBasename(flowFile, logger);
    String granularity = context.getProperty(GRANULARITY).getValue();
    if (FILE.equals(granularity)) {
        // File granularity will emit a FlowFile for each input
        FlowFile original = session.clone(flowFile);
        AtomicReference<Exception> exceptionReference = new AtomicReference<>(null);
        FlowFile updated = session.write(flowFile, (in, out) -> {
            processFileGranularity(session, logger, original, basename, exceptionReference, in, out);
        });
        session.transfer(original, REL_ORIGINAL);
        resultProcessor.process(session, logger, updated, exceptionReference.get(), getName(basename, null, null, XML_EXTENSION));
    } else {
        session.read(flowFile, in -> {
            if (RECORD.equals(granularity)) {
                // Record granularity will emit a FlowFile for every record (event)
                processRecordGranularity(session, logger, flowFile, basename, in);
            } else if (CHUNK.equals(granularity)) {
                // Chunk granularity will emit a FlowFile for each chunk of the file
                processChunkGranularity(session, logger, flowFile, basename, in);
            }
        });
        session.transfer(flowFile, REL_ORIGINAL);
    }
}