org.apache.nifi.logging.ComponentLog Java Examples

@OnScheduled
public void onScheduled(final ProcessContext context) {
    final ComponentLog logger = getLogger();
    final Integer cacheSize = context.getProperty(CACHE_SIZE).asInteger();
    final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);

    if (cacheSize > 0) {
        CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
        if (cacheTTL > 0) {
            cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
        }

        cache = cacheBuilder.build(
           new CacheLoader<String, Templates>() {
               public Templates load(String path) throws TransformerConfigurationException {
                   return newTemplates(path);
               }
           });
    } else {
        cache = null;
        logger.warn("Stylesheet cache disabled because cache size is set to 0");
    }
}
 

@Test
public void testValidatePrincipalAndKeytab() {
    final ComponentLog log = Mockito.mock(ComponentLog.class);
    final Configuration config = new Configuration();

    // no security enabled in config so doesn't matter what principal and keytab are
    List<ValidationResult> results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, null, null, log);
    Assert.assertEquals(0, results.size());

    results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, "principal", null, log);
    Assert.assertEquals(0, results.size());

    results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, "principal", "keytab", log);
    Assert.assertEquals(0, results.size());

    // change the config to have kerberos turned on
    config.set("hadoop.security.authentication", "kerberos");
    config.set("hadoop.security.authorization", "true");

    results = KerberosProperties.validatePrincipalAndKeytab(
            "test", config, null, null, log);
    Assert.assertEquals(2, results.size());
}
 

/**
 * Returns the valid relationships for this processor as supplied by the
 * script itself.
 *
 * @return a Set of Relationships supported by this processor
 */
@Override
public Set<Relationship> getRelationships() {
    final Set<Relationship> relationships = new HashSet<>();
    final Processor instance = processor.get();
    if (instance != null) {
        try {
            final Set<Relationship> rels = instance.getRelationships();
            if (rels != null && !rels.isEmpty()) {
                relationships.addAll(rels);
            }
        } catch (final Throwable t) {
            final ComponentLog logger = getLogger();
            final String message = "Unable to get relationships from scripted Processor: " + t;

            logger.error(message);
            if (logger.isDebugEnabled()) {
                logger.error(message, t);
            }
        }
    }
    return Collections.unmodifiableSet(relationships);
}
 

public Collection<ValidationResult> validate(String configFiles, String principal, String keyTab, String password,
                                             AtomicReference<ValidationResources> validationResourceHolder, ComponentLog log) {

    final List<ValidationResult> problems = new ArrayList<>();
    ValidationResources resources = validationResourceHolder.get();

    // if no resources in the holder, or if the holder has different resources loaded,
    // then load the Configuration and set the new resources in the holder
    if (resources == null || !configFiles.equals(resources.getConfigResources())) {
        log.debug("Reloading validation resources");
        resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles));
        validationResourceHolder.set(resources);
    }

    final Configuration hiveConfig = resources.getConfiguration();

    problems.addAll(KerberosProperties.validatePrincipalWithKeytabOrPassword(this.getClass().getSimpleName(), hiveConfig, principal, keyTab, password, log));

    return problems;
}
 

/**
 * Reloads the script located at the given path
 *
 * @param scriptPath the path to the script file to be loaded
 * @return true if the script was loaded successfully; false otherwise
 */
protected boolean reloadScriptFile(final String scriptPath) {
    final Collection<ValidationResult> results = new HashSet<>();

    try (final FileInputStream scriptStream = new FileInputStream(scriptPath)) {
        return reloadScript(IOUtils.toString(scriptStream, Charset.defaultCharset()));

    } catch (final Exception e) {
        final ComponentLog logger = getLogger();
        final String message = "Unable to load script: " + e;

        logger.error(message, e);
        results.add(new ValidationResult.Builder()
                .subject("ScriptValidation")
                .valid(false)
                .explanation("Unable to load script due to " + e)
                .input(scriptPath)
                .build());
    }

    // store the updated validation results
    validationResults.set(results);

    // return whether there was any issues loading the configured script
    return results.isEmpty();
}
 

ConsumerLease(
        final long maxWaitMillis,
        final Consumer<byte[], byte[]> kafkaConsumer,
        final byte[] demarcatorBytes,
        final String keyEncoding,
        final String securityProtocol,
        final String bootstrapServers,
        final ComponentLog logger) {
    this.maxWaitMillis = maxWaitMillis;
    this.kafkaConsumer = kafkaConsumer;
    this.demarcatorBytes = demarcatorBytes;
    this.keyEncoding = keyEncoding;
    this.securityProtocol = securityProtocol;
    this.bootstrapServers = bootstrapServers;
    this.logger = logger;
}
 

private Set<URL> getAdditionalClasspathResources(final List<PropertyDescriptor> propertyDescriptors, final String componentId, final Map<String, String> properties,
                                                 final ParameterLookup parameterLookup, final VariableRegistry variableRegistry, final ComponentLog logger) {
    final Set<String> modulePaths = new LinkedHashSet<>();
    for (final PropertyDescriptor descriptor : propertyDescriptors) {
        if (descriptor.isDynamicClasspathModifier()) {
            final String value = properties.get(descriptor.getName());
            if (!StringUtils.isEmpty(value)) {
                final StandardPropertyValue propertyValue = new StandardPropertyValue(value, null, parameterLookup, variableRegistry);
                modulePaths.add(propertyValue.evaluateAttributeExpressions().getValue());
            }
        }
    }

    final Set<URL> additionalUrls = new LinkedHashSet<>();
    try {
        final URL[] urls = ClassLoaderUtils.getURLsForClasspath(modulePaths, null, true);
        if (urls != null) {
            additionalUrls.addAll(Arrays.asList(urls));
        }
    } catch (MalformedURLException mfe) {
        logger.error("Error processing classpath resources for " + componentId + ": " + mfe.getMessage(), mfe);
    }

    return additionalUrls;
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();

    WrappedMessageConsumer wrappedConsumer = consumerQueue.poll();
    if (wrappedConsumer == null) {
        try {
            wrappedConsumer = JmsFactory.createQueueMessageConsumer(context);
        } catch (JMSException e) {
            logger.error("Failed to connect to JMS Server due to {}", e);
            context.yield();
            return;
        }
    }

    try {
        super.consume(context, session, wrappedConsumer);
    } finally {
        if (!wrappedConsumer.isClosed()) {
            consumerQueue.offer(wrappedConsumer);
        }
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final Map<String, Pattern> patterns = regexMapRef.get();
    final ComponentLog logger = getLogger();

    final SortedMap<String, String> attributes = getRelevantAttributes(flowFile, patterns);
    if (attributes.size() != patterns.size()) {
        final Set<String> wantedKeys = patterns.keySet();
        final Set<String> foundKeys = attributes.keySet();
        final StringBuilder missingKeys = new StringBuilder();
        for (final String wantedKey : wantedKeys) {
            if (!foundKeys.contains(wantedKey)) {
                missingKeys.append(wantedKey).append(" ");
            }
        }

        logger.error("routing {} to 'failure' because of missing attributes: {}", new Object[]{flowFile, missingKeys.toString()});
        session.transfer(flowFile, REL_FAILURE);
    } else {
        // create single string of attribute key/value pairs to use for group ID hash
        final StringBuilder hashableValue = new StringBuilder();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            hashableValue.append(entry.getKey());
            if (StringUtils.isBlank(entry.getValue())) {
                hashableValue.append("EMPTY");
            } else {
                hashableValue.append(entry.getValue());
            }
        }

        // create group ID
        final String hashValue = DigestUtils.md5Hex(hashableValue.toString());

        logger.info("adding Hash Value {} to attributes for {} and routing to success", new Object[]{hashValue, flowFile});
        flowFile = session.putAttribute(flowFile, context.getProperty(HASH_VALUE_ATTRIBUTE).getValue(), hashValue);
        session.getProvenanceReporter().modifyAttributes(flowFile);
        session.transfer(flowFile, REL_SUCCESS);
    }
}
 

public SmtpConsumer(
        final MessageContext context,
        final ProcessSessionFactory sessionFactory,
        final int port,
        final String host,
        final ComponentLog log,
        final int maxMessageSize
) {
    this.context = context;
    this.sessionFactory = sessionFactory;
    this.port = port;
    if (host == null || host.trim().isEmpty()) {
        this.host = context.getSMTPServer().getHostName();
    } else {
        this.host = host;
    }
    this.log = log;
    this.maxMessageSize = maxMessageSize;
}
 

@Test
public void testReadOneLine() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());

    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-oneline.json"));
         final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {

        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
        assertEquals(expectedFieldNames, fieldNames);

        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
                RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
        assertEquals(expectedTypes, dataTypes);

        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues);

        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues);

        assertNull(reader.nextRecord());
    }
}
 

@Test
public void testDateNoCoersionExpectedFormat() throws IOException, MalformedRecordException {
    final String text = "date\n11/30/1983";

    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);

    try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
         final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
                 "MM/dd/yyyy", RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

        final Record record = reader.nextRecord(false, false);
        final java.sql.Date date = (Date) record.getValue("date");
        final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("gmt"));
        calendar.setTimeInMillis(date.getTime());

        assertEquals(1983, calendar.get(Calendar.YEAR));
        assertEquals(10, calendar.get(Calendar.MONTH));
        assertEquals(30, calendar.get(Calendar.DAY_OF_MONTH));
    }
}
 

ConsumerLease(
        final long maxWaitMillis,
        final Consumer<byte[], byte[]> kafkaConsumer,
        final byte[] demarcatorBytes,
        final String keyEncoding,
        final String securityProtocol,
        final String bootstrapServers,
        final RecordReaderFactory readerFactory,
        final RecordSetWriterFactory writerFactory,
        final ComponentLog logger,
        final Charset headerCharacterSet,
        final Pattern headerNamePattern) {
    this.maxWaitMillis = maxWaitMillis;
    this.kafkaConsumer = kafkaConsumer;
    this.demarcatorBytes = demarcatorBytes;
    this.keyEncoding = keyEncoding;
    this.securityProtocol = securityProtocol;
    this.bootstrapServers = bootstrapServers;
    this.readerFactory = readerFactory;
    this.writerFactory = writerFactory;
    this.logger = logger;
    this.headerCharacterSet = headerCharacterSet;
    this.headerNamePattern = headerNamePattern;
}
 

/**
 * At the moment the only two supported message types are TextMessage and
 * BytesMessage which is sufficient for the type if JMS use cases NiFi is
 * used. The may change to the point where all message types are supported
 * at which point this test will no be longer required.
 */
@Test(expected = IllegalStateException.class)
public void validateFailOnUnsupportedMessageType() throws Exception {
    final String destinationName = "testQueue";
    JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false);

    jmsTemplate.send(destinationName, new MessageCreator() {
        @Override
        public Message createMessage(Session session) throws JMSException {
            return session.createObjectMessage();
        }
    });

    JMSConsumer consumer = new JMSConsumer(jmsTemplate, mock(ComponentLog.class));
    try {
        consumer.consume(destinationName, new ConsumerCallback() {
            @Override
            public void accept(JMSResponse response) {
                // noop
            }
        });
    } finally {
        ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy();
    }
}
 

public Collection<ValidationResult> validate(String configFiles, String principal, String keyTab, AtomicReference<ValidationResources> validationResourceHolder, ComponentLog log) {

        final List<ValidationResult> problems = new ArrayList<>();
        ValidationResources resources = validationResourceHolder.get();

        // if no resources in the holder, or if the holder has different resources loaded,
        // then load the Configuration and set the new resources in the holder
        if (resources == null || !configFiles.equals(resources.getConfigResources())) {
            log.debug("Reloading validation resources");
            resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles));
            validationResourceHolder.set(resources);
        }

        final Configuration hiveConfig = resources.getConfiguration();

        problems.addAll(KerberosProperties.validatePrincipalAndKeytab(this.getClass().getSimpleName(), hiveConfig, principal, keyTab, log));

        return problems;
    }
 

@Override
public synchronized void run() {
    lifecycleState.incrementActiveThreadCount(null);
    try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(extensionManager, taskNode.getReportingTask().getClass(), taskNode.getIdentifier())) {
        taskNode.getReportingTask().onTrigger(taskNode.getReportingContext());
    } catch (final Throwable t) {
        final ComponentLog componentLog = new SimpleProcessLogger(taskNode.getIdentifier(), taskNode.getReportingTask());
        componentLog.error("Error running task {} due to {}", new Object[]{taskNode.getReportingTask(), t.toString()});
        if (componentLog.isDebugEnabled()) {
            componentLog.error("", t);
        }
    } finally {
        try {
            // if the reporting task is no longer scheduled to run and this is the last thread,
            // invoke the OnStopped methods
            if (!lifecycleState.isScheduled() && lifecycleState.getActiveThreadCount() == 1 && lifecycleState.mustCallOnStoppedMethods()) {
                try (final NarCloseable x = NarCloseable.withComponentNarLoader(extensionManager, taskNode.getReportingTask().getClass(), taskNode.getIdentifier())) {
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, taskNode.getReportingTask(), taskNode.getConfigurationContext());
                }
            }
        } finally {
            lifecycleState.decrementActiveThreadCount(null);
        }
    }
}
 

@Test
public void testSimpleRecordWithAttribute2() throws IOException, MalformedRecordException {
    InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
    List<RecordField> fields = getSimpleRecordFields();
    fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType()));
    XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true,
            "ATTR_", "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class));

    Record first = reader.nextRecord();
    assertTrue(Arrays.asList(first.getValues()).containsAll(Arrays.asList("Cleve Butler", 42, "USA")));
    assertEquals("P1", first.getAsString("ATTR_ID"));

    Record second = reader.nextRecord();
    assertTrue(Arrays.asList(second.getValues()).containsAll(Arrays.asList("Ainslie Fletcher", 33, "UK")));
    assertEquals("P2", second.getAsString("ATTR_ID"));

    Record third = reader.nextRecord();
    assertTrue(Arrays.asList(third.getValues()).containsAll(Arrays.asList("Amélie Bonfils", 74, "FR")));
    assertEquals("P3", third.getAsString("ATTR_ID"));

    Record fourth = reader.nextRecord();
    assertTrue(Arrays.asList(fourth.getValues()).containsAll(Arrays.asList("Elenora Scrivens", 16, "USA")));
    assertEquals("P4", fourth.getAsString("ATTR_ID"));
}
 

@Test
public void testTimeNoCoersionUnexpectedFormat() throws IOException, MalformedRecordException {
    final String text = "time\n01:02:03";

    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("time", RecordFieldType.TIME.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);

    try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
         final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
                 RecordFieldType.DATE.getDefaultFormat(), "HH-MM-SS", RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

        final Record record = reader.nextRecord(false, false);
        assertEquals("01:02:03", (String)record.getValue("time"));
    }
}
 

public BeatsSSLSocketChannelHandler(final SelectionKey key,
                                    final AsyncChannelDispatcher dispatcher,
                                    final Charset charset,
                                    final EventFactory<E> eventFactory,
                                    final BlockingQueue<E> events,
                                    final ComponentLog logger) {
    super(key, dispatcher, charset, eventFactory, events, logger);
    this.decoder = new BeatsDecoder(charset, logger);
    this.frameHandler = new BeatsFrameHandler<>(key, charset, eventFactory, events, dispatcher, logger);
}
 

@Before
public void setup() throws InitializationException {
    mockLease = mock(ConsumerLease.class);
    mockConsumerPool = mock(ConsumerPool.class);

    ConsumeKafkaRecord_0_10 proc = new ConsumeKafkaRecord_0_10() {
        @Override
        protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) {
            return mockConsumerPool;
        }
    };

    runner = TestRunners.newTestRunner(proc);
    runner.setProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS, "okeydokey:1234");

    final String readerId = "record-reader";
    final MockRecordParser readerService = new MockRecordParser();
    readerService.addSchemaField("name", RecordFieldType.STRING);
    readerService.addSchemaField("age", RecordFieldType.INT);
    runner.addControllerService(readerId, readerService);
    runner.enableControllerService(readerService);

    final String writerId = "record-writer";
    final RecordSetWriterFactory writerService = new MockRecordWriter("name, age");
    runner.addControllerService(writerId, writerService);
    runner.enableControllerService(writerService);

    runner.setProperty(ConsumeKafkaRecord_0_10.RECORD_READER, readerId);
    runner.setProperty(ConsumeKafkaRecord_0_10.RECORD_WRITER, writerId);
}
 

public WriteJsonResult(final ComponentLog logger, final RecordSchema recordSchema, final SchemaAccessWriter schemaAccess, final OutputStream out, final boolean prettyPrint,
    final NullSuppression nullSuppression, final OutputGrouping outputGrouping, final String dateFormat, final String timeFormat, final String timestampFormat,
    final String mimeType) throws IOException {

    super(out);
    this.logger = logger;
    this.recordSchema = recordSchema;
    this.schemaAccess = schemaAccess;
    this.nullSuppression = nullSuppression;
    this.outputGrouping = outputGrouping;
    this.mimeType = mimeType;

    final DateFormat df = dateFormat == null ? null : DataTypeUtils.getDateFormat(dateFormat);
    final DateFormat tf = timeFormat == null ? null : DataTypeUtils.getDateFormat(timeFormat);
    final DateFormat tsf = timestampFormat == null ? null : DataTypeUtils.getDateFormat(timestampFormat);

    LAZY_DATE_FORMAT = () -> df;
    LAZY_TIME_FORMAT = () -> tf;
    LAZY_TIMESTAMP_FORMAT = () -> tsf;

    final JsonFactory factory = new JsonFactory();
    factory.setCodec(objectMapper);

    this.generator = factory.createJsonGenerator(out);
    if (prettyPrint) {
        generator.useDefaultPrettyPrinter();
    } else if (OutputGrouping.OUTPUT_ONELINE.equals(outputGrouping)) {
        // Use a minimal pretty printer with a newline object separator, will output one JSON object per line
        generator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
    }
}
 

@Override
public ChannelHandler<E, AsyncChannelDispatcher> createHandler(final SelectionKey key,
                                       final AsyncChannelDispatcher dispatcher,
                                       final Charset charset,
                                       final EventFactory<E> eventFactory,
                                       final BlockingQueue<E> events,
                                       final ComponentLog logger) {
    return new StandardSocketChannelHandler<>(key, dispatcher, charset, eventFactory, events, logger);
}
 

@Test
public void testSimpleRecordWithHeader() throws IOException, MalformedRecordException {
    InputStream is = new FileInputStream("src/test/resources/xml/people_with_header_and_comments.xml");
    XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true,
            null, null, dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class));

    Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues());
    Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, reader.nextRecord().getValues());
    Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, reader.nextRecord().getValues());
    Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, reader.nextRecord().getValues());
}
 

@Override
public RecordReader createRecordReader(Map<String, String> variables, InputStream in, long inputLength, ComponentLog logger) throws IOException, SchemaNotFoundException {
    final Iterator<Object[]> itr = records.iterator();

    return new RecordReader() {
        private int recordCount = 0;

        @Override
        public void close() throws IOException {
        }

        @Override
        public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException {
            if (failAfterN >= 0 && recordCount >= failAfterN) {
                throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read");
            }
            recordCount++;

            if (!itr.hasNext()) {
                return null;
            }

            final Object[] values = itr.next();
            final Map<String, Object> valueMap = new HashMap<>();
            int i = 0;
            for (final RecordField field : fields) {
                final String fieldName = field.getFieldName();
                valueMap.put(fieldName, values[i++]);
            }

            return new MapRecord(new SimpleRecordSchema(fields), valueMap);
        }

        @Override
        public RecordSchema getSchema() {
            return new SimpleRecordSchema(fields);
        }
    };
}
 

@Test
public void validateSuccessfulSendAsDelimited() throws Exception {
    InputStream contentStream = new ByteArrayInputStream(
            "Hello Kafka\nHello Kafka\nHello Kafka\nHello Kafka\n".getBytes(StandardCharsets.UTF_8));
    String topicName = "validateSuccessfulSendAsDelimited";

    Properties kafkaProperties = this.buildProducerProperties();
    KafkaPublisher publisher = new KafkaPublisher(kafkaProperties, mock(ComponentLog.class));

    PublishingContext publishingContext = new PublishingContext(contentStream, topicName);
    publishingContext.setDelimiterBytes("\n".getBytes(StandardCharsets.UTF_8));
    KafkaPublisherResult result = publisher.publish(publishingContext);

    assertEquals(3, result.getLastMessageAcked());
    assertEquals(4, result.getMessagesSent());
    contentStream.close();
    publisher.close();

    ConsumerIterator<byte[], byte[]> iter = this.buildConsumer(topicName);
    assertNotNull(iter.next());
    assertNotNull(iter.next());
    assertNotNull(iter.next());
    assertNotNull(iter.next());
    try {
        iter.next();
        fail();
    } catch (ConsumerTimeoutException e) {
        // that's OK since this is the Kafka mechanism to unblock
    }
}
 

private MockScriptedRecordSink initTask() throws InitializationException {

        final MockScriptedRecordSink recordSink = new MockScriptedRecordSink();
        ConfigurationContext context = mock(ConfigurationContext.class);
        StateManager stateManager = new MockStateManager(recordSink);

        final PropertyValue pValue = mock(StandardPropertyValue.class);
        MockRecordWriter writer = new MockRecordWriter(null, false); // No header, don"t quote values
        when(context.getProperty(RecordSinkService.RECORD_WRITER_FACTORY)).thenReturn(pValue);
        when(pValue.asControllerService(RecordSetWriterFactory.class)).thenReturn(writer);


        final ComponentLog logger = mock(ComponentLog.class);
        final ControllerServiceInitializationContext initContext = new MockControllerServiceInitializationContext(writer, UUID.randomUUID().toString(), logger, stateManager);
        recordSink.initialize(initContext);

        // Call something that sets up the ScriptingComponentHelper, so we can mock it
        recordSink.getSupportedPropertyDescriptors();

        when(context.getProperty(recordSink.getScriptingComponentHelper().SCRIPT_ENGINE))
                .thenReturn(new MockPropertyValue("Groovy"));
        when(context.getProperty(ScriptingComponentUtils.SCRIPT_FILE))
                .thenReturn(new MockPropertyValue("src/test/resources/groovy/test_record_sink.groovy"));
        when(context.getProperty(ScriptingComponentUtils.SCRIPT_BODY))
                .thenReturn(new MockPropertyValue(null));
        when(context.getProperty(ScriptingComponentUtils.MODULES))
                .thenReturn(new MockPropertyValue(null));
        try {
            recordSink.onEnabled(context);
        } catch (Exception e) {
            e.printStackTrace();
            fail("onEnabled error: " + e.getMessage());
        }
        return recordSink;
    }
 

@Test
public void testReadMixedJSON() throws IOException, MalformedRecordException {
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());

    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-mixed.json"));
         final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {

        final List<String> fieldNames = schema.getFieldNames();
        final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
        assertEquals(expectedFieldNames, fieldNames);

        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
                RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
        assertEquals(expectedTypes, dataTypes);

        final Object[] firstRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues);

        final Object[] secondRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues);

        final Object[] thirdRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {3, "Maria Doe", 4750.89, "123 My Street", "My City", "ME", "11111", "USA"}, thirdRecordValues);

        final Object[] fourthRecordValues = reader.nextRecord().getValues();
        Assert.assertArrayEquals(new Object[] {4, "Xi Doe", 4820.09, "321 Your Street", "Your City", "NV", "33333", "USA"}, fourthRecordValues);


        assertNull(reader.nextRecord());
    }
}
 

PublisherPool(final Map<String, Object> kafkaProperties, final ComponentLog logger, final int maxMessageSize, final long maxAckWaitMillis,
              final boolean useTransactions, final Supplier<String> transactionalIdSupplier, final Pattern attributeNameRegex, final Charset headerCharacterSet) {
    this.logger = logger;
    this.publisherQueue = new LinkedBlockingQueue<>();
    this.kafkaProperties = kafkaProperties;
    this.maxMessageSize = maxMessageSize;
    this.maxAckWaitMillis = maxAckWaitMillis;
    this.useTransactions = useTransactions;
    this.attributeNameRegex = attributeNameRegex;
    this.headerCharacterSet = headerCharacterSet;
    this.transactionalIdSupplier = transactionalIdSupplier;
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    ComponentLog logger = getLogger();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    String basename = getBasename(flowFile, logger);
    String granularity = context.getProperty(GRANULARITY).getValue();
    if (FILE.equals(granularity)) {
        // File granularity will emit a FlowFile for each input
        FlowFile original = session.clone(flowFile);
        AtomicReference<Exception> exceptionReference = new AtomicReference<>(null);
        FlowFile updated = session.write(flowFile, (in, out) -> {
            processFileGranularity(session, logger, original, basename, exceptionReference, in, out);
        });
        session.transfer(original, REL_ORIGINAL);
        resultProcessor.process(session, logger, updated, exceptionReference.get(), getName(basename, null, null, XML_EXTENSION));
    } else {
        session.read(flowFile, in -> {
            if (RECORD.equals(granularity)) {
                // Record granularity will emit a FlowFile for every record (event)
                processRecordGranularity(session, logger, flowFile, basename, in);
            } else if (CHUNK.equals(granularity)) {
                // Chunk granularity will emit a FlowFile for each chunk of the file
                processChunkGranularity(session, logger, flowFile, basename, in);
            }
        });
        session.transfer(flowFile, REL_ORIGINAL);
    }
}
 

@Override
protected void executeAction(Action action, Map<String, Object> facts) {
    ComponentLog logger = getLogger();
    Map<String, String> attributes = action.getAttributes();
    final String logLevel = attributes.get("logLevel");
    final LogLevel level = getLogLevel(logLevel, LogLevel.valueOf(defaultLogLevel));
    final String eventMessage = StringUtils.isNotEmpty(attributes.get("message")) ? attributes.get("message") : defaultLogMessage;
    final String factsMessage = createFactsLogMessage(facts, eventMessage);
    logger.log(level, factsMessage);
}