Java Code Examples for org.apache.nifi.processor.Relationship#SELF

The following examples show how to use org.apache.nifi.processor.Relationship#SELF . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void transfer(final FlowFile flowFile, final Relationship relationship) {
    if (relationship == Relationship.SELF) {
        transfer(flowFile);
        return;
    }
    if(!processor.getRelationships().contains(relationship)){
        throw new IllegalArgumentException("this relationship " + relationship.getName() + " is not known");
    }

    validateState(flowFile);
    List<MockFlowFile> list = transferMap.get(relationship);
    if (list == null) {
        list = new ArrayList<>();
        transferMap.put(relationship, list);
    }

    beingProcessed.remove(flowFile.getId());
    list.add((MockFlowFile) flowFile);
}
 
Example 2
Source File: MockProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void transfer(FlowFile flowFile, final Relationship relationship) {
    if (relationship == Relationship.SELF) {
        transfer(flowFile);
        return;
    }
    if(!processor.getRelationships().contains(relationship)){
        throw new IllegalArgumentException("this relationship " + relationship.getName() + " is not known");
    }

    flowFile = validateState(flowFile);
    List<MockFlowFile> list = transferMap.computeIfAbsent(relationship, r -> new ArrayList<>());

    beingProcessed.remove(flowFile.getId());
    list.add((MockFlowFile) flowFile);
    updateLastQueuedDate((MockFlowFile) flowFile);
}
 
Example 3
Source File: StatelessProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void transfer(FlowFile flowFile, final Relationship relationship) {
    if (relationship == Relationship.SELF) {
        transfer(flowFile);
        return;
    }
    if (!processor.getRelationships().contains(relationship)) {
        throw new IllegalArgumentException("this relationship " + relationship.getName() + " is not known");
    }

    flowFile = validateState(flowFile);

    if (outputMap.containsKey(relationship)) {
        Queue<StatelessFlowFile> queue = this.outputMap.get(relationship);
        queue.add((StatelessFlowFile) flowFile);

    }
    beingProcessed.remove(flowFile.getId());
    updateLastQueuedDate((StatelessFlowFile) flowFile);
}
 
Example 4
Source File: StandardProcessSession.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void transfer(final FlowFile flowFile, final Relationship relationship) {
    validateRecordState(flowFile);
    final int numDestinations = context.getConnections(relationship).size();
    final int multiplier = Math.max(1, numDestinations);

    boolean autoTerminated = false;
    boolean selfRelationship = false;
    if (numDestinations == 0 && context.getConnectable().isAutoTerminated(relationship)) {
        // auto terminated.
        autoTerminated = true;
    } else if (numDestinations == 0 && relationship == Relationship.SELF) {
        selfRelationship = true;
    } else if (numDestinations == 0) {
        // the relationship specified is not known in this session/context
        throw new IllegalArgumentException("Relationship '" + relationship.getName() + "' is not known");
    }
    final StandardRepositoryRecord record = records.get(flowFile);
    record.setTransferRelationship(relationship);
    updateLastQueuedDate(record);

    if (autoTerminated) {
        removedCount += multiplier;
        removedBytes += flowFile.getSize();
    } else if (!selfRelationship) {
        flowFilesOut += multiplier;
        contentSizeOut += flowFile.getSize() * multiplier;
    }
}
 
Example 5
Source File: StandardProcessSession.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void transfer(final Collection<FlowFile> flowFiles, final Relationship relationship) {
    validateRecordState(flowFiles);

    boolean autoTerminated = false;
    boolean selfRelationship = false;
    final int numDestinations = context.getConnections(relationship).size();
    if (numDestinations == 0 && context.getConnectable().isAutoTerminated(relationship)) {
        // auto terminated.
        autoTerminated = true;
    } else if (numDestinations == 0 && relationship == Relationship.SELF) {
        selfRelationship = true;
    } else if (numDestinations == 0) {
        // the relationship specified is not known in this session/context
        throw new IllegalArgumentException("Relationship '" + relationship.getName() + "' is not known");
    }

    final int multiplier = Math.max(1, numDestinations);

    long contentSize = 0L;
    for (final FlowFile flowFile : flowFiles) {
        final StandardRepositoryRecord record = records.get(flowFile);
        record.setTransferRelationship(relationship);
        updateLastQueuedDate(record);

        contentSize += flowFile.getSize() * multiplier;
    }

    if (autoTerminated) {
        removedCount += multiplier * flowFiles.size();
        removedBytes += contentSize;
    } else if (!selfRelationship) {
        flowFilesOut += multiplier * flowFiles.size();
        contentSizeOut += multiplier * contentSize;
    }
}
 
Example 6
Source File: MockProcessSession.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void transfer(Collection<FlowFile> flowFiles, final Relationship relationship) {
    if (relationship == Relationship.SELF) {
        transfer(flowFiles);
        return;
    }
    for (final FlowFile flowFile : flowFiles) {
        transfer(flowFile, relationship);
    }
}
 
Example 7
Source File: StatelessProcessSession.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void transfer(Collection<FlowFile> flowFiles, final Relationship relationship) {
    if (relationship == Relationship.SELF) {
        transfer(flowFiles);
        return;
    }
    for (final FlowFile flowFile : flowFiles) {
        transfer(flowFile, relationship);
    }
}
 
Example 8
Source File: StandardProcessSession.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void transfer(FlowFile flowFile, final Relationship relationship) {
    verifyTaskActive();
    flowFile = validateRecordState(flowFile);
    final int numDestinations = context.getConnections(relationship).size();
    final int multiplier = Math.max(1, numDestinations);

    boolean autoTerminated = false;
    boolean selfRelationship = false;
    if (numDestinations == 0 && context.getConnectable().isAutoTerminated(relationship)) {
        // auto terminated.
        autoTerminated = true;
    } else if (numDestinations == 0 && relationship == Relationship.SELF) {
        selfRelationship = true;
    } else if (numDestinations == 0) {
        // the relationship specified is not known in this session/context
        throw new IllegalArgumentException("Relationship '" + relationship.getName() + "' is not known");
    }
    final StandardRepositoryRecord record = getRecord(flowFile);
    record.setTransferRelationship(relationship);
    updateLastQueuedDate(record);

    if (autoTerminated) {
        removedCount += multiplier;
        removedBytes += flowFile.getSize();
    } else if (!selfRelationship) {
        flowFilesOut += multiplier;
        contentSizeOut += flowFile.getSize() * multiplier;
    }
}
 
Example 9
Source File: StandardProcessSession.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void transfer(Collection<FlowFile> flowFiles, final Relationship relationship) {
    verifyTaskActive();
    flowFiles = validateRecordState(flowFiles);

    boolean autoTerminated = false;
    boolean selfRelationship = false;
    final int numDestinations = context.getConnections(relationship).size();
    if (numDestinations == 0 && context.getConnectable().isAutoTerminated(relationship)) {
        // auto terminated.
        autoTerminated = true;
    } else if (numDestinations == 0 && relationship == Relationship.SELF) {
        selfRelationship = true;
    } else if (numDestinations == 0) {
        // the relationship specified is not known in this session/context
        throw new IllegalArgumentException("Relationship '" + relationship.getName() + "' is not known");
    }

    final int multiplier = Math.max(1, numDestinations);

    final long queuedTime = System.currentTimeMillis();
    long contentSize = 0L;
    for (final FlowFile flowFile : flowFiles) {
        final FlowFileRecord flowFileRecord = (FlowFileRecord) flowFile;
        final StandardRepositoryRecord record = getRecord(flowFileRecord);
        record.setTransferRelationship(relationship);
        updateLastQueuedDate(record, queuedTime);

        contentSize += flowFile.getSize();
    }

    if (autoTerminated) {
        removedCount += multiplier * flowFiles.size();
        removedBytes += multiplier * contentSize;
    } else if (!selfRelationship) {
        flowFilesOut += multiplier * flowFiles.size();
        contentSizeOut += multiplier * contentSize;
    }
}
 
Example 10
Source File: PutSQL.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Pulls a batch of FlowFiles from the incoming queues. If no FlowFiles are available, returns <code>null</code>.
 * Otherwise, a List of FlowFiles will be returned.
 *
 * If all FlowFiles pulled are not eligible to be processed, the FlowFiles will be penalized and transferred back
 * to the input queue and an empty List will be returned.
 *
 * Otherwise, if the Support Fragmented Transactions property is true, all FlowFiles that belong to the same
 * transaction will be sorted in the order that they should be evaluated.
 *
 * @param context the process context for determining properties
 * @param session the process session for pulling flowfiles
 * @return a FlowFilePoll containing a List of FlowFiles to process, or <code>null</code> if there are no FlowFiles to process
 */
private FlowFilePoll pollFlowFiles(final ProcessContext context, final ProcessSession session) {
    // Determine which FlowFile Filter to use in order to obtain FlowFiles.
    final boolean useTransactions = context.getProperty(SUPPORT_TRANSACTIONS).asBoolean();
    boolean fragmentedTransaction = false;

    final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
    List<FlowFile> flowFiles;
    if (useTransactions) {
        final TransactionalFlowFileFilter filter = new TransactionalFlowFileFilter();
        flowFiles = session.get(filter);
        fragmentedTransaction = filter.isFragmentedTransaction();
    } else {
        flowFiles = session.get(batchSize);
    }

    if (flowFiles.isEmpty()) {
        return null;
    }

    // If we are supporting fragmented transactions, verify that all FlowFiles are correct
    if (fragmentedTransaction) {
        final Relationship relationship = determineRelationship(flowFiles, context.getProperty(TRANSACTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS));
        if (relationship != null) {
            // if transferring back to self, penalize the FlowFiles.
            if (relationship == Relationship.SELF) {
                // penalize all of the FlowFiles that we are going to route to SELF.
                final ListIterator<FlowFile> itr = flowFiles.listIterator();
                while (itr.hasNext()) {
                    final FlowFile flowFile = itr.next();
                    final FlowFile penalized = session.penalize(flowFile);
                    itr.remove();
                    itr.add(penalized);
                }
            }

            session.transfer(flowFiles, relationship);
            return null;
        }

        // sort by fragment index.
        Collections.sort(flowFiles, new Comparator<FlowFile>() {
            @Override
            public int compare(final FlowFile o1, final FlowFile o2) {
                return Integer.compare(Integer.parseInt(o1.getAttribute(FRAGMENT_INDEX_ATTR)), Integer.parseInt(o2.getAttribute(FRAGMENT_INDEX_ATTR)));
            }
        });
    }

    return new FlowFilePoll(flowFiles, fragmentedTransaction);
}
 
Example 11
Source File: ExceptionHandler.java    From nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Same as {@link #createOnError(ProcessContext, ProcessSession, RoutingResult, Relationship, Relationship)} for FlowFileGroup.
 * @param context process context is used to yield a processor
 * @param session process session is used to penalize FlowFiles
 * @param routingResult input FlowFiles will be routed to a destination relationship in this {@link RoutingResult}
 * @param relFailure specify failure relationship of a processor
 * @param relRetry specify retry relationship of a processor
 * @return composed function
 */
public static <C, I extends PartialFunctions.FlowFileGroup> ExceptionHandler.OnError<C, I> createOnGroupError(
        final ProcessContext context, final ProcessSession session, final RoutingResult routingResult,
        final Relationship relFailure, final Relationship relRetry) {
    return (c, g, r, e) -> {
        final Relationship routeTo;
        switch (r.destination()) {
            case Failure:
                routeTo = relFailure;
                break;
            case Retry:
                routeTo = relRetry;
                break;
            case Self:
                routeTo = Relationship.SELF;
                break;
            default:
                if (e instanceof ProcessException) {
                    throw (ProcessException)e;
                } else {
                    Object inputs = null;
                    if (g != null) {
                        final List<FlowFile> flowFiles = g.getFlowFiles();
                        switch (flowFiles.size()) {
                            case 0:
                                inputs = "[]";
                                break;
                            case 1:
                                inputs = flowFiles.get(0);
                                break;
                            default:
                                inputs = String.format("%d FlowFiles including %s", flowFiles.size(), flowFiles.get(0));
                                break;
                        }
                    }
                    throw new ProcessException(String.format("Failed to process %s due to %s", inputs, e), e);
                }
        }
        for (FlowFile f : g.getFlowFiles()) {
            final FlowFile maybePenalized = penalize(context, session, f, r.penalty());
            routingResult.routeTo(maybePenalized, routeTo);
        }
    };
}