org.apache.nifi.processor.ProcessContext Java Examples

The following examples show how to use org.apache.nifi.processor.ProcessContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LookupRecord.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
protected Tuple<Map<String, RecordPath>, RecordPath> getFlowFileContext(final FlowFile flowFile, final ProcessContext context) {
    final Map<String, RecordPath> recordPaths = new HashMap<>();
    for (final PropertyDescriptor prop : context.getProperties().keySet()) {
        if (!prop.isDynamic()) {
            continue;
        }

        final String pathText = context.getProperty(prop).evaluateAttributeExpressions(flowFile).getValue();
        final RecordPath lookupRecordPath = recordPathCache.getCompiled(pathText);
        recordPaths.put(prop.getName(), lookupRecordPath);
    }

    final RecordPath resultRecordPath;
    if (context.getProperty(RESULT_RECORD_PATH).isSet()) {
        final String resultPathText = context.getProperty(RESULT_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
        resultRecordPath = recordPathCache.getCompiled(resultPathText);
    } else {
        resultRecordPath = null;
    }

    return new Tuple<>(recordPaths, resultRecordPath);
}
 
Example #2
Source File: GetTCP.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@OnScheduled
public void onScheduled(final ProcessContext context) throws ProcessException {
    this.receiveBufferSize = context.getProperty(RECEIVE_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    this.originalServerAddressList = context.getProperty(ENDPOINT_LIST).getValue();
    this.endOfMessageByte = ((byte) context.getProperty(END_OF_MESSAGE_BYTE).asInteger().intValue());
    this.connectionAttemptCount = context.getProperty(CONNECTION_ATTEMPT_COUNT).asInteger();
    this.reconnectInterval = context.getProperty(RECONNECT_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS);

    this.clientScheduler = new ScheduledThreadPoolExecutor(originalServerAddressList.split(",").length + 1);
    this.clientScheduler.setKeepAliveTime(10, TimeUnit.SECONDS);
    this.clientScheduler.allowCoreThreadTimeOut(true);

    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        if (descriptor.isDynamic()) {
            this.dynamicAttributes.put(descriptor.getName(), entry.getValue());
        }
    }
}
 
Example #3
Source File: ListFile.java    From nifi with Apache License 2.0 6 votes vote down vote up
@OnScheduled
public void onScheduled(final ProcessContext context) {
    fileFilterRef.set(createFileFilter(context));
    includeFileAttributes = context.getProperty(INCLUDE_FILE_ATTRIBUTES).asBoolean();

    final long maxDiskOperationMillis = context.getProperty(MAX_DISK_OPERATION_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);
    final long maxListingMillis = context.getProperty(MAX_LISTING_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);

    final boolean trackPerformance = context.getProperty(TRACK_PERFORMANCE).asBoolean();
    if (trackPerformance) {
        final int maxEntries = context.getProperty(MAX_TRACKED_FILES).evaluateAttributeExpressions().asInteger();
        performanceTracker = new RollingMetricPerformanceTracker(getLogger(), maxDiskOperationMillis, maxEntries);
    } else {
        performanceTracker = new UntrackedPerformanceTracker(getLogger(), maxDiskOperationMillis);
    }

    final long millisToKeepStats = TimeUnit.MINUTES.toMillis(15);
    final MonitorActiveTasks monitorTask = new MonitorActiveTasks(performanceTracker, getLogger(), maxDiskOperationMillis, maxListingMillis, millisToKeepStats);
    monitoringFuture = monitoringThreadPool.scheduleAtFixedRate(monitorTask, 15, 15, TimeUnit.SECONDS);
}
 
Example #4
Source File: ListenLumberjack.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
protected ChannelDispatcher createDispatcher(final ProcessContext context, final BlockingQueue<LumberjackEvent> events) throws IOException {
    final EventFactory<LumberjackEvent> eventFactory = new LumberjackEventFactory();
    final ChannelHandlerFactory<LumberjackEvent, AsyncChannelDispatcher> handlerFactory = new LumberjackSocketChannelHandlerFactory<>();

    final int maxConnections = context.getProperty(MAX_CONNECTIONS).asInteger();
    final int bufferSize = context.getProperty(RECV_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    final Charset charSet = Charset.forName(context.getProperty(CHARSET).getValue());

    // initialize the buffer pool based on max number of connections and the buffer size
    final BlockingQueue<ByteBuffer> bufferPool = createBufferPool(maxConnections, bufferSize);

    // if an SSLContextService was provided then create an SSLContext to pass down to the dispatcher
    SSLContext sslContext = null;
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
    }

    // if we decide to support SSL then get the context and pass it in here
    return new SocketChannelDispatcher<>(eventFactory, handlerFactory, bufferPool, events,
        getLogger(), maxConnections, sslContext, charSet);
}
 
Example #5
Source File: ParseCEF.java    From nifi with Apache License 2.0 6 votes vote down vote up
@OnScheduled
public void OnScheduled(final ProcessContext context) {

    // Configure jackson mapper before spawning onTriggers
    final SimpleModule module = new SimpleModule()
                                    .addSerializer(MacAddress.class, new MacAddressToStringSerializer());
    mapper.registerModule(module);
    mapper.setDateFormat(this.simpleDateFormat);

    switch (context.getProperty(TIME_REPRESENTATION).getValue()) {
        case LOCAL_TZ:
            // set the mapper TZ to local TZ
            mapper.setTimeZone(TimeZone.getDefault());
            tzId = TimeZone.getDefault().getID();
            break;
        case UTC:
            // set the mapper TZ to local TZ
            mapper.setTimeZone(TimeZone.getTimeZone(UTC));
            tzId = UTC;
            break;
    }

}
 
Example #6
Source File: TransformXml.java    From nifi with Apache License 2.0 6 votes vote down vote up
@OnScheduled
public void onScheduled(final ProcessContext context) {
    final ComponentLog logger = getLogger();
    final Integer cacheSize = context.getProperty(CACHE_SIZE).asInteger();
    final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);

    if (cacheSize > 0) {
        CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
        if (cacheTTL > 0) {
            cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
        }

        cache = cacheBuilder.build(
                new CacheLoader<String, Templates>() {
                    @Override
                    public Templates load(String path) throws TransformerConfigurationException, LookupFailureException {
                        return newTemplates(context, path);
                    }
                });
    } else {
        cache = null;
        logger.info("Stylesheet cache disabled because cache size is set to 0");
    }
}
 
Example #7
Source File: PublishAMQP.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Will construct AMQP message by extracting its body from the incoming
 * {@link FlowFile}. AMQP Properties will be extracted from the
 * {@link FlowFile} and converted to {@link BasicProperties} to be sent
 * along with the message. Upon success the incoming {@link FlowFile} is
 * transferred to 'success' {@link Relationship} and upon failure FlowFile is
 * penalized and transferred to the 'failure' {@link Relationship}
 * <br>
 * NOTE: Attributes extracted from {@link FlowFile} are considered
 * candidates for AMQP properties if their names are prefixed with
 * {@link AMQPUtils#AMQP_PROP_PREFIX} (e.g., amqp$contentType=text/xml)
 *
 */
@Override
protected void rendezvousWithAmqp(ProcessContext context, ProcessSession processSession) throws ProcessException {
    FlowFile flowFile = processSession.get();
    if (flowFile != null) {
        BasicProperties amqpProperties = this.extractAmqpPropertiesFromFlowFile(flowFile);
        String routingKey = context.getProperty(ROUTING_KEY).evaluateAttributeExpressions(flowFile).getValue();
        if (routingKey == null){
            throw new IllegalArgumentException("Failed to determine 'routing key' with provided value '"
                    + context.getProperty(ROUTING_KEY) + "' after evaluating it as expression against incoming FlowFile.");
        }
        String exchange = context.getProperty(EXCHANGE).evaluateAttributeExpressions(flowFile).getValue();

        byte[] messageContent = this.extractMessage(flowFile, processSession);

        try {
            this.targetResource.publish(messageContent, amqpProperties, routingKey, exchange);
            processSession.transfer(flowFile, REL_SUCCESS);
            processSession.getProvenanceReporter().send(flowFile, this.amqpConnection.toString() + "/E:" + exchange + "/RK:" + routingKey);
        } catch (Exception e) {
            processSession.transfer(processSession.penalize(flowFile), REL_FAILURE);
            this.getLogger().error("Failed while sending message to AMQP via " + this.targetResource, e);
            context.yield();
        }
    }
}
 
Example #8
Source File: KafkaProcessorUtils.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Method used to configure the 'sasl.jaas.config' property based on KAFKA-4259<br />
 * https://cwiki.apache.org/confluence/display/KAFKA/KIP-85%3A+Dynamic+JAAS+configuration+for+Kafka+clients<br />
 * <br />
 * It expects something with the following format: <br />
 * <br />
 * &lt;LoginModuleClass&gt; &lt;ControlFlag&gt; *(&lt;OptionName&gt;=&lt;OptionValue&gt;); <br />
 * ControlFlag = required / requisite / sufficient / optional
 *
 * @param mapToPopulate Map of configuration properties
 * @param context Context
 */
private static void setJaasConfig(Map<String, Object> mapToPopulate, ProcessContext context) {
    String keytab = context.getProperty(USER_KEYTAB).evaluateAttributeExpressions().getValue();
    String principal = context.getProperty(USER_PRINCIPAL).evaluateAttributeExpressions().getValue();

    // If the Kerberos Credentials Service is specified, we need to use its configuration, not the explicit properties for principal/keytab.
    // The customValidate method ensures that only one can be set, so we know that the principal & keytab above are null.
    final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
    if (credentialsService != null) {
        principal = credentialsService.getPrincipal();
        keytab = credentialsService.getKeytab();
    }


    String serviceName = context.getProperty(JAAS_SERVICE_NAME).evaluateAttributeExpressions().getValue();
    if(StringUtils.isNotBlank(keytab) && StringUtils.isNotBlank(principal) && StringUtils.isNotBlank(serviceName)) {
        mapToPopulate.put(SaslConfigs.SASL_JAAS_CONFIG, "com.sun.security.auth.module.Krb5LoginModule required "
                + "useTicketCache=false "
                + "renewTicket=true "
                + "serviceName=\"" + serviceName + "\" "
                + "useKeyTab=true "
                + "keyTab=\"" + keytab + "\" "
                + "principal=\"" + principal + "\";");
    }
}
 
Example #9
Source File: AbstractAWSProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
protected AWSCredentials getCredentials(final ProcessContext context) {
    final String accessKey = context.getProperty(ACCESS_KEY).evaluateAttributeExpressions().getValue();
    final String secretKey = context.getProperty(SECRET_KEY).evaluateAttributeExpressions().getValue();

    final String credentialsFile = context.getProperty(CREDENTIALS_FILE).getValue();

    if (credentialsFile != null) {
        try {
            return new PropertiesCredentials(new File(credentialsFile));
        } catch (final IOException ioe) {
            throw new ProcessException("Could not read Credentials File", ioe);
        }
    }

    if (accessKey != null && secretKey != null) {
        return new BasicAWSCredentials(accessKey, secretKey);
    }

    return new AnonymousAWSCredentials();

}
 
Example #10
Source File: AbstractAWSProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
protected void initializeRegionAndEndpoint(ProcessContext context) {
    // if the processor supports REGION, get the configured region.
    if (getSupportedPropertyDescriptors().contains(REGION)) {
        final String region = context.getProperty(REGION).getValue();
        if (region != null) {
            this.region = Region.getRegion(Regions.fromName(region));
            client.setRegion(this.region);
        } else {
            this.region = null;
        }
    }

    // if the endpoint override has been configured, set the endpoint.
    // (per Amazon docs this should only be configured at client creation)
    final String urlstr = StringUtils.trimToEmpty(context.getProperty(ENDPOINT_OVERRIDE).getValue());
    if (!urlstr.isEmpty()) {
        this.client.setEndpoint(urlstr);
    }

}
 
Example #11
Source File: ConsumeWindowsEventLog.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Register subscriber via native call
 *
 * @param context the process context
 */
private String subscribe(ProcessContext context) throws URISyntaxException {
    String channel = context.getProperty(CHANNEL).getValue();
    String query = context.getProperty(QUERY).getValue();

    renderedXMLs = new LinkedBlockingQueue<>(context.getProperty(MAX_EVENT_QUEUE_SIZE).asInteger());
    provenanceUri = new URI("winlog", name, "/" + channel, query, null).toASCIIString();

    evtSubscribeCallback = new EventSubscribeXmlRenderingCallback(getLogger(), s -> {
        try {
            renderedXMLs.put(s);
        } catch (InterruptedException e) {
            throw new IllegalStateException("Got interrupted while waiting to add to queue.", e);
        }
    }, context.getProperty(MAX_BUFFER_SIZE).asInteger(), wEvtApi, kernel32, errorLookup);

    subscriptionHandle = wEvtApi.EvtSubscribe(null, null, channel, query, null, null,
            evtSubscribeCallback, WEvtApi.EvtSubscribeFlags.SUBSCRIBE_TO_FUTURE | WEvtApi.EvtSubscribeFlags.EVT_SUBSCRIBE_STRICT);
    if (!isSubscribed()) {
        return UNABLE_TO_SUBSCRIBE + errorLookup.getLastError();
    }
    return null;
}
 
Example #12
Source File: PutAccumuloRecord.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Adapted from HBASEUtils. Their approach seemed ideal for what our intent is here.
 * @param columnFamily column family from which to extract the visibility or to execute an expression against
 * @param columnQualifier column qualifier from which to extract the visibility or to execute an expression against
 * @param flowFile flow file being written
 * @param context process context
 * @return
 */
public static String produceVisibility(String columnFamily, String columnQualifier, FlowFile flowFile, ProcessContext context) {
    if (org.apache.commons.lang3.StringUtils.isNotEmpty(columnFamily)) {
        return null;
    }
    String lookupKey = String.format("visibility.%s%s%s", columnFamily, !org.apache.commons.lang3.StringUtils.isNotEmpty(columnQualifier) ? "." : "", columnQualifier);
    String fromAttribute = flowFile.getAttribute(lookupKey);

    if (fromAttribute == null && !org.apache.commons.lang3.StringUtils.isBlank(columnQualifier)) {
        String lookupKeyFam = String.format("visibility.%s", columnFamily);
        fromAttribute = flowFile.getAttribute(lookupKeyFam);
    }

    if (fromAttribute != null) {
        return fromAttribute;
    } else {
        PropertyValue descriptor = context.getProperty(lookupKey);
        if (descriptor == null || !descriptor.isSet()) {
            descriptor = context.getProperty(String.format("visibility.%s", columnFamily));
        }

        String retVal = descriptor != null ? descriptor.evaluateAttributeExpressions(flowFile).getValue() : null;

        return retVal;
    }
}
 
Example #13
Source File: ReplaceText.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public RegexReplace(final byte[] buffer, final ProcessContext context) {
    this.buffer = buffer;

    final String regexValue = context.getProperty(SEARCH_VALUE).evaluateAttributeExpressions().getValue();
    numCapturingGroups = Pattern.compile(regexValue).matcher("").groupCount();
    additionalAttrs = new HashMap<>(numCapturingGroups);
}
 
Example #14
Source File: AbstractAMQPProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will builds target resource ({@link AMQPPublisher} or
 * {@link AMQPConsumer}) upon first invocation and will delegate to the
 * implementation of
 * {@link #rendezvousWithAmqp(ProcessContext, ProcessSession)} method for
 * further processing.
 */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    synchronized (this) {
        this.buildTargetResource(context);
    }
    this.rendezvousWithAmqp(context, session);
}
 
Example #15
Source File: PutSplunk.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Send the entire FlowFile as a single message.
 */
private void processSingleMessage(ProcessContext context, ProcessSession session, FlowFile flowFile, ChannelSender sender) {
    // copy the contents of the FlowFile to the ByteArrayOutputStream
    final ByteArrayOutputStream baos = new ByteArrayOutputStream((int)flowFile.getSize() + 1);
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.copy(in, baos);
        }
    });

    // if TCP and we don't end in a new line then add one
    final String protocol = context.getProperty(PROTOCOL).getValue();
    byte[] buf = baos.toByteArray();
    if (protocol.equals(TCP_VALUE.getValue()) && buf[buf.length - 1] != NEW_LINE_CHAR) {
        final byte[] updatedBuf = new byte[buf.length + 1];
        System.arraycopy(buf, 0, updatedBuf, 0, buf.length);
        updatedBuf[updatedBuf.length - 1] = NEW_LINE_CHAR;
        buf = updatedBuf;
    }

    // create a message batch of one message and add to active batches
    final FlowFileMessageBatch messageBatch = new FlowFileMessageBatch(session, flowFile);
    messageBatch.setNumMessages(1);
    activeBatches.add(messageBatch);

    // attempt to send the data and add the appropriate range
    try {
        sender.send(buf);
        messageBatch.addSuccessfulRange(0L, flowFile.getSize());
    } catch (IOException e) {
        messageBatch.addFailedRange(0L, flowFile.getSize(), e);
        context.yield();
    }
}
 
Example #16
Source File: GetAzureEventHub.java    From nifi with Apache License 2.0 5 votes vote down vote up
PartitionReceiver getReceiver(final ProcessContext context, final String partitionId) throws IOException, EventHubException, ExecutionException, InterruptedException {
    PartitionReceiver existingReceiver = partitionToReceiverMap.get(partitionId);
    if (existingReceiver != null) {
        return existingReceiver;
    }

    // we want to avoid allowing multiple threads to create Receivers simultaneously because that could result in
    // having multiple Receivers for the same partition. So if the map does not contain a receiver for this partition,
    // we will enter a synchronized block and check again (because once we enter the synchronized block, we know that no
    // other thread is creating a client). If within the synchronized block, we still do not have an entry in the map,
    // it is up to use to create the receiver, initialize it, and then put it into the map.
    // We do not use the putIfAbsent method in order to do a CAS operation here because we want to also initialize the
    // receiver if and only if it is not present in the map. As a result, we need to initialize the receiver and add it
    // to the map atomically. Hence, the synchronized block.
    synchronized (this) {
        existingReceiver = partitionToReceiverMap.get(partitionId);
        if (existingReceiver != null) {
            return existingReceiver;
        }

        final String consumerGroupName = context.getProperty(CONSUMER_GROUP).getValue();

        final PartitionReceiver receiver = eventHubClient.createReceiver(
                consumerGroupName,
                partitionId,
                EventPosition.fromEnqueuedTime(
                        configuredEnqueueTime == null ? Instant.now() : configuredEnqueueTime)).get();

        receiver.setReceiveTimeout(receiverFetchTimeout == null ? Duration.ofMillis(60000) : receiverFetchTimeout);
        partitionToReceiverMap.put(partitionId, receiver);
        return receiver;

    }
}
 
Example #17
Source File: MergeContent.java    From nifi with Apache License 2.0 5 votes vote down vote up
private byte[] getDelimiterTextContent(final ProcessContext context, final List<FlowFile> flowFiles, final PropertyDescriptor descriptor)
        throws IOException {
    byte[] property = null;
    if (flowFiles != null && flowFiles.size() > 0) {
        final FlowFile flowFile = flowFiles.get(0);
        if (flowFile != null) {
            final String value = context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue();
            if (value != null) {
                property = value.getBytes(StandardCharsets.UTF_8);
            }
        }
    }
    return property;
}
 
Example #18
Source File: AbstractS3Processor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Create CannedAccessControlList if {@link #CANNED_ACL} property specified.
 *
 * @param context ProcessContext
 * @param flowFile FlowFile
 * @return CannedAccessControlList or null if not specified
 */
protected final CannedAccessControlList createCannedACL(final ProcessContext context, final FlowFile flowFile) {
    CannedAccessControlList cannedAcl = null;

    final String cannedAclString = context.getProperty(CANNED_ACL).evaluateAttributeExpressions(flowFile).getValue();
    if (!StringUtils.isEmpty(cannedAclString)) {
        cannedAcl = CannedAccessControlList.valueOf(cannedAclString);
    }

    return cannedAcl;
}
 
Example #19
Source File: AbstractKinesisFirehoseProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Create client using aws credentials provider. This is the preferred way for creating clients
 */
@Override
protected AmazonKinesisFirehoseClient createClient(final ProcessContext context, final AWSCredentialsProvider credentialsProvider, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials provider");

    return new AmazonKinesisFirehoseClient(credentialsProvider, config);
}
 
Example #20
Source File: BaseTransformer.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
InvocationContextProperties(ProcessContext context, FlowFile flowFile) {
    List<PropertyDescriptor> propertyDescriptors = BaseTransformer.this.getSupportedPropertyDescriptors();
    for (PropertyDescriptor propertyDescriptor : propertyDescriptors) {
        if (propertyDescriptor.isExpressionLanguageSupported()) {
            PropertyValue value = context.getProperty(propertyDescriptor)
                    .evaluateAttributeExpressions(flowFile);
            this.propertyInvocationValues.put(propertyDescriptor, value.getValue());
        }
    }
}
 
Example #21
Source File: UpdateAttribute.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
private boolean evaluateRule(final ProcessContext context, final Rule rule, FlowFile flowfile, final Map<String, String> statefulAttributes) {
    // go through each condition
    for (final Condition condition : rule.getConditions()) {

        // fail if any condition is not met
        if (!evaluateCondition(context, condition, flowfile, statefulAttributes)) {
            return false;
        }
    }

    return true;
}
 
Example #22
Source File: ListenBeats.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
protected ChannelDispatcher createDispatcher(final ProcessContext context, final BlockingQueue<BeatsEvent> events) throws IOException {
    final EventFactory<BeatsEvent> eventFactory = new BeatsEventFactory();
    final ChannelHandlerFactory<BeatsEvent, AsyncChannelDispatcher> handlerFactory = new BeatsSocketChannelHandlerFactory<>();

    final int maxConnections = context.getProperty(MAX_CONNECTIONS).asInteger();
    final int bufferSize = context.getProperty(RECV_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    final Charset charSet = Charset.forName(context.getProperty(CHARSET).getValue());

    // initialize the buffer pool based on max number of connections and the buffer size
    final BlockingQueue<ByteBuffer> bufferPool = createBufferPool(maxConnections, bufferSize);

    // if an SSLContextService was provided then create an SSLContext to pass down to the dispatcher
    SSLContext sslContext = null;
    SslContextFactory.ClientAuth clientAuth = null;
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
        sslContext = sslContextService.createSSLContext(SslContextFactory.ClientAuth.valueOf(clientAuthValue));
        clientAuth = SslContextFactory.ClientAuth.valueOf(clientAuthValue);

    }

    // if we decide to support SSL then get the context and pass it in here
    return new SocketChannelDispatcher<>(eventFactory, handlerFactory, bufferPool, events,
        getLogger(), maxConnections, sslContext, clientAuth, charSet);
}
 
Example #23
Source File: PutIgniteCache.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Add failed flow file attributes
 * @param flowFiles all flow files
 * @param failedFlowFiles list of failed flow files
 * @param session process session
 * @param context the process context
 * @return failed flow files with updated attributes
 */
protected List<FlowFile> updateFailedFlowFileAttributes(
        List<FlowFile> flowFiles,
        List<FlowFile> failedFlowFiles, ProcessSession session, ProcessContext context) {

    int flowFileCount = flowFiles.size();
    int flowFileFailed = failedFlowFiles.size();
    List<FlowFile> updatedFailedFlowFiles = new ArrayList<>();

    for (int i = 0; i < flowFileFailed; i++) {
        FlowFile flowFile = failedFlowFiles.get(i);

        Map<String,String> attributes = new HashMap<>();
        attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_ITEM_NUMBER, Integer.toString(i));
        attributes.put(IGNITE_BATCH_FLOW_FILE_TOTAL_COUNT, Integer.toString(flowFileCount));
        attributes.put(IGNITE_BATCH_FLOW_FILE_ITEM_NUMBER, Integer.toString(flowFiles.indexOf(flowFile)));
        attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_COUNT, Integer.toString(flowFileFailed));

        String key = context.getProperty(IGNITE_CACHE_ENTRY_KEY).evaluateAttributeExpressions(flowFile).getValue();

        if (StringUtils.isEmpty(key)) {
            attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_REASON_ATTRIBUTE_KEY,
                    IGNITE_BATCH_FLOW_FILE_FAILED_MISSING_KEY_MESSAGE);
        } else if (flowFile.getSize() == 0) {
            attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_REASON_ATTRIBUTE_KEY,
                    IGNITE_BATCH_FLOW_FILE_FAILED_ZERO_SIZE_MESSAGE);
        } else {
            throw new ProcessException("Unknown reason for failing file: " + flowFile);
        }

        flowFile = session.putAllAttributes(flowFile, attributes);
        updatedFailedFlowFiles.add(flowFile);
    }

    return updatedFailedFlowFiles;

}
 
Example #24
Source File: AttributesToJSON.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final FlowFile original = session.get();
    if (original == null) {
        return;
    }

    final Map<String, String> atrList = buildAttributesMapForFlowFile(original, attributes, attributesToRemove, nullValueForEmptyString, pattern);

    try {
        if (destinationContent) {
            FlowFile conFlowfile = session.write(original, (in, out) -> {
                try (OutputStream outputStream = new BufferedOutputStream(out)) {
                    outputStream.write(objectMapper.writeValueAsBytes(atrList));
                }
            });
            conFlowfile = session.putAttribute(conFlowfile, CoreAttributes.MIME_TYPE.key(), APPLICATION_JSON);
            session.transfer(conFlowfile, REL_SUCCESS);
        } else {
            FlowFile atFlowfile = session.putAttribute(original, JSON_ATTRIBUTE_NAME, objectMapper.writeValueAsString(atrList));
            session.transfer(atFlowfile, REL_SUCCESS);
        }
    } catch (JsonProcessingException e) {
        getLogger().error(e.getMessage());
        session.transfer(original, REL_FAILURE);
    }
}
 
Example #25
Source File: SelectHiveQL.java    From nifi with Apache License 2.0 5 votes vote down vote up
@OnScheduled
public void setup(ProcessContext context) {
    // If the query is not set, then an incoming flow file is needed. Otherwise fail the initialization
    if (!context.getProperty(HIVEQL_SELECT_QUERY).isSet() && !context.hasIncomingConnection()) {
        final String errorString = "Either the Select Query must be specified or there must be an incoming connection "
                + "providing flowfile(s) containing a SQL select query";
        getLogger().error(errorString);
        throw new ProcessException(errorString);
    }
}
 
Example #26
Source File: DeleteGridFS.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile input = session.get();
    if (input == null) {
        return;
    }

    final String deleteQuery = getQuery(context, input);
    final String queryAttribute = context.getProperty(QUERY_ATTRIBUTE).isSet()
            ? context.getProperty(QUERY_ATTRIBUTE).evaluateAttributeExpressions(input).getValue()
            : null;
    GridFSBucket bucket = getBucket(input, context);

    try {
        Document query = Document.parse(deleteQuery);
        MongoCursor cursor = bucket.find(query).iterator();
        if (cursor.hasNext()) {
            GridFSFile file = (GridFSFile)cursor.next();
            bucket.delete(file.getObjectId());

            if (!StringUtils.isEmpty(queryAttribute)) {
                input = session.putAttribute(input, queryAttribute, deleteQuery);
            }

            session.transfer(input, REL_SUCCESS);
        } else {
            getLogger().error(String.format("Query %s did not delete anything in %s", deleteQuery, bucket.getBucketName()));
            session.transfer(input, REL_FAILURE);
        }

        cursor.close();
    } catch (Exception ex) {
        getLogger().error(String.format("Error deleting using query: %s", deleteQuery), ex);
        session.transfer(input, REL_FAILURE);
    }
}
 
Example #27
Source File: TestListenSyslog.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalid() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    final int numMessages = 10;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some TCP messages to the port in the background
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, numMessages, 100, INVALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(50);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_INVALID).size();
        }

        // all messages should be transferred to invalid
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}
 
Example #28
Source File: JmsFactory.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public static Connection createConnection(final ProcessContext context, final String clientId) throws JMSException {
    Objects.requireNonNull(context);
    Objects.requireNonNull(clientId);

    final ConnectionFactory connectionFactory = createConnectionFactory(context);

    final String username = context.getProperty(USERNAME).getValue();
    final String password = context.getProperty(PASSWORD).getValue();
    final Connection connection = (username == null && password == null) ? connectionFactory.createConnection() : connectionFactory.createConnection(username, password);

    connection.setClientID(clientId);
    connection.start();
    return connection;
}
 
Example #29
Source File: BaseTransformer.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 *
 */
protected FlowFile doTransform(ProcessContext context, ProcessSession session, FlowFile flowFile,  InvocationContextProperties contextProperties) {
    AtomicReference<Map<String, String>> attributeRef = new AtomicReference<Map<String, String>>();
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(InputStream in) throws IOException {
            attributeRef.set(transform(in, null, contextProperties));
        }
    });
    if (attributeRef.get() != null) {
        flowFile = session.putAllAttributes(flowFile, attributeRef.get());
    }
    return flowFile;
}
 
Example #30
Source File: PublishGCPubSub.java    From nifi with Apache License 2.0 5 votes vote down vote up
private Publisher.Builder getPublisherBuilder(ProcessContext context) {
    final Long batchSize = context.getProperty(BATCH_SIZE).asLong();

    return Publisher.newBuilder(getTopicName(context))
            .setCredentialsProvider(FixedCredentialsProvider.create(getGoogleCredentials(context)))
            .setBatchingSettings(BatchingSettings.newBuilder()
            .setElementCountThreshold(batchSize)
            .setIsEnabled(true)
            .build());
}