Java Code Examples for org.apache.nifi.flowfile.FlowFilePrioritizer

The following examples show how to use org.apache.nifi.flowfile.FlowFilePrioritizer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: localization_nifi   Source File: RelationshipAuditor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Extracts configured settings from the specified connection only if they have also been specified in the connectionDTO.
 *
 * @param connection connection
 * @param connectionDTO dto
 * @return properties
 */
private Map<String, String> extractConfiguredPropertyValues(Connection connection, ConnectionDTO connectionDTO) {
    Map<String, String> values = new HashMap<>();

    if (connectionDTO.getName() != null) {
        values.put(NAME, connection.getName());
    }
    if (connectionDTO.getFlowFileExpiration() != null) {
        values.put(FLOW_FILE_EXPIRATION, String.valueOf(connection.getFlowFileQueue().getFlowFileExpiration()));
    }
    if (connectionDTO.getBackPressureObjectThreshold() != null) {
        values.put(BACK_PRESSURE_OBJECT_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureObjectThreshold()));
    }
    if (connectionDTO.getBackPressureDataSizeThreshold() != null) {
        values.put(BACK_PRESSURE_DATA_SIZE_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureDataSizeThreshold()));
    }
    if (connectionDTO.getPrioritizers() != null) {
        List<String> prioritizers = new ArrayList<>();
        for (FlowFilePrioritizer prioritizer : connection.getFlowFileQueue().getPriorities()) {
            prioritizers.add(prioritizer.getClass().getCanonicalName());
        }
        values.put(PRIORITIZERS, StringUtils.join(prioritizers, ", "));
    }

    return values;
}
 
Example 2
Source Project: localization_nifi   Source File: StandardFlowFileQueue.java    License: Apache License 2.0 6 votes vote down vote up
public StandardFlowFileQueue(final String identifier, final Connection connection, final FlowFileRepository flowFileRepo, final ProvenanceEventRepository provRepo,
    final ResourceClaimManager resourceClaimManager, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter, final int swapThreshold) {
    activeQueue = new PriorityQueue<>(20, new Prioritizer(new ArrayList<FlowFilePrioritizer>()));
    priorities = new ArrayList<>();
    swapQueue = new ArrayList<>();
    this.eventReporter = eventReporter;
    this.swapManager = swapManager;
    this.flowFileRepository = flowFileRepo;
    this.provRepository = provRepo;
    this.resourceClaimManager = resourceClaimManager;

    this.identifier = identifier;
    this.swapThreshold = swapThreshold;
    this.scheduler = scheduler;
    this.connection = connection;

    readLock = new TimedLock(this.lock.readLock(), identifier + " Read Lock", 100);
    writeLock = new TimedLock(this.lock.writeLock(), identifier + " Write Lock", 100);
}
 
Example 3
@Test
public void testLowestPrioritySwappedOutFirst() {
    final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
    prioritizers.add(new FlowFileSizePrioritizer());
    queue.setPriorities(prioritizers);

    long maxSize = 20000;
    for (int i = 1; i <= 20000; i++) {
        queue.put(new TestFlowFile(maxSize - i));
    }

    assertEquals(1, swapManager.swapOutCalledCount);
    assertEquals(20000, queue.size().getObjectCount());

    assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
    final List<FlowFileRecord> flowFiles = queue.poll(Integer.MAX_VALUE, new HashSet<FlowFileRecord>());
    assertEquals(10000, flowFiles.size());
    for (int i = 0; i < 10000; i++) {
        assertEquals(i, flowFiles.get(i).getSize());
    }
}
 
Example 4
Source Project: nifi   Source File: RelationshipAuditor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Extracts configured settings from the specified connection only if they have also been specified in the connectionDTO.
 *
 * @param connection connection
 * @param connectionDTO dto
 * @return properties
 */
private Map<String, String> extractConfiguredPropertyValues(Connection connection, ConnectionDTO connectionDTO) {
    Map<String, String> values = new HashMap<>();

    if (connectionDTO.getName() != null) {
        values.put(NAME, connection.getName());
    }
    if (connectionDTO.getFlowFileExpiration() != null) {
        values.put(FLOW_FILE_EXPIRATION, String.valueOf(connection.getFlowFileQueue().getFlowFileExpiration()));
    }
    if (connectionDTO.getBackPressureObjectThreshold() != null) {
        values.put(BACK_PRESSURE_OBJECT_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureObjectThreshold()));
    }
    if (connectionDTO.getBackPressureDataSizeThreshold() != null) {
        values.put(BACK_PRESSURE_DATA_SIZE_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureDataSizeThreshold()));
    }
    if (connectionDTO.getPrioritizers() != null) {
        List<String> prioritizers = new ArrayList<>();
        for (FlowFilePrioritizer prioritizer : connection.getFlowFileQueue().getPriorities()) {
            prioritizers.add(prioritizer.getClass().getCanonicalName());
        }
        values.put(PRIORITIZERS, StringUtils.join(prioritizers, ", "));
    }

    return values;
}
 
Example 5
Source Project: nifi   Source File: SocketLoadBalancedFlowFileQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    prioritizers.clear();
    prioritizers.addAll(newPriorities);

    partitionReadLock.lock();
    try {
        for (final QueuePartition partition : queuePartitions) {
            partition.setPriorities(newPriorities);
        }

        rebalancingPartition.setPriorities(newPriorities);
    } finally {
        partitionReadLock.unlock();
    }
}
 
Example 6
Source Project: nifi   Source File: StandardExtensionDiscoveringManager.java    License: Apache License 2.0 6 votes vote down vote up
public StandardExtensionDiscoveringManager() {
    definitionMap.put(Processor.class, new HashSet<>());
    definitionMap.put(FlowFilePrioritizer.class, new HashSet<>());
    definitionMap.put(ReportingTask.class, new HashSet<>());
    definitionMap.put(ControllerService.class, new HashSet<>());
    definitionMap.put(Authorizer.class, new HashSet<>());
    definitionMap.put(UserGroupProvider.class, new HashSet<>());
    definitionMap.put(AccessPolicyProvider.class, new HashSet<>());
    definitionMap.put(LoginIdentityProvider.class, new HashSet<>());
    definitionMap.put(ProvenanceRepository.class, new HashSet<>());
    definitionMap.put(ComponentStatusRepository.class, new HashSet<>());
    definitionMap.put(FlowFileRepository.class, new HashSet<>());
    definitionMap.put(FlowFileSwapManager.class, new HashSet<>());
    definitionMap.put(ContentRepository.class, new HashSet<>());
    definitionMap.put(StateProvider.class, new HashSet<>());
    definitionMap.put(StatusAnalyticsModel.class, new HashSet<>());
}
 
Example 7
Source Project: localization_nifi   Source File: FlowController.java    License: Apache License 2.0 5 votes vote down vote up
public FlowFilePrioritizer createPrioritizer(final String type) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    FlowFilePrioritizer prioritizer;

    final ClassLoader ctxClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final ClassLoader detectedClassLoaderForType = ExtensionManager.getClassLoader(type);
        final Class<?> rawClass;
        if (detectedClassLoaderForType == null) {
            // try to find from the current class loader
            rawClass = Class.forName(type);
        } else {
            // try to find from the registered classloader for that type
            rawClass = Class.forName(type, true, ExtensionManager.getClassLoader(type));
        }

        Thread.currentThread().setContextClassLoader(detectedClassLoaderForType);
        final Class<? extends FlowFilePrioritizer> prioritizerClass = rawClass.asSubclass(FlowFilePrioritizer.class);
        final Object processorObj = prioritizerClass.newInstance();
        prioritizer = prioritizerClass.cast(processorObj);

        return prioritizer;
    } finally {
        if (ctxClassLoader != null) {
            Thread.currentThread().setContextClassLoader(ctxClassLoader);
        }
    }
}
 
Example 8
Source Project: localization_nifi   Source File: StandardFlowFileQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    writeLock.lock();
    try {
        final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new Prioritizer(newPriorities));
        newQueue.addAll(activeQueue);
        activeQueue = newQueue;
        priorities.clear();
        priorities.addAll(newPriorities);
    } finally {
        writeLock.unlock("setPriorities");
    }
}
 
Example 9
Source Project: nifi   Source File: ComponentMockUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static Connection getConnection(
        final String id,
        final String name,
        final Optional<String> versionedId,
        final Collection<Relationship> relationships,
        final List<FlowFilePrioritizer> flowFilePrioritizers,
        final int flowFileExpirationInMs,
        final String backPressureDataSize,
        final long backPressureCount,
        final Connectable source,
        final Connectable destination,
        final boolean isAuthorized) {
    final Connection result = Mockito.mock(Connection.class);
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);

    Mockito.when(flowFileQueue.getPriorities()).thenReturn(flowFilePrioritizers);
    Mockito.when(flowFileQueue.getFlowFileExpiration()).thenReturn(String.valueOf(flowFileExpirationInMs));
    Mockito.when(flowFileQueue.getFlowFileExpiration(TimeUnit.MILLISECONDS)).thenReturn(flowFileExpirationInMs);
    Mockito.when(flowFileQueue.getBackPressureDataSizeThreshold()).thenReturn(backPressureDataSize);
    Mockito.when(flowFileQueue.getBackPressureObjectThreshold()).thenReturn(backPressureCount);

    Mockito.when(result.getIdentifier()).thenReturn(id);
    Mockito.when(result.getName()).thenReturn(name);
    Mockito.when(result.getVersionedComponentId()).thenReturn(versionedId);
    Mockito.when(result.getRelationships()).thenReturn(relationships);
    Mockito.when(result.getFlowFileQueue()).thenReturn(flowFileQueue);
    Mockito.when(result.getSource()).thenReturn(source);
    Mockito.when(result.getDestination()).thenReturn(destination);

    setAuthorized(result, isAuthorized);
    return result;
}
 
Example 10
@Test
public void testSearchBasedOnPriorities() {
    // given
    final ProcessorNode processor1 = getProcessorNode("processor1", "processor1Name", AUTHORIZED);
    final ProcessorNode processor2 = getProcessorNode("processor2", "processor2Name", AUTHORIZED);
    final Connection connection = getConnection("connection", "connectionName", getBasicRelationships(), processor1, processor2, AUTHORIZED);

    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
    prioritizers.add(Mockito.mock(ComponentMockUtil.DummyFlowFilePrioritizer.class));
    Mockito.when(flowFileQueue.getPriorities()).thenReturn(prioritizers);
    Mockito.when(connection.getFlowFileQueue()).thenReturn(flowFileQueue);

    givenRootProcessGroup()
            .withProcessor(processor1)
            .withProcessor(processor2)
            .withConnection(connection);

    // when
    whenExecuteSearch("dummy");

    // then
    Assert.assertEquals(1, results.getConnectionResults().size());
    Assert.assertEquals(1, results.getConnectionResults().get(0).getMatches().size());
    Assert.assertTrue(results.getConnectionResults().get(0).getMatches().get(0)
            .startsWith("Prioritizer: org.apache.nifi.web.controller.ComponentMockUtil$DummyFlowFilePrioritizer$"));
}
 
Example 11
Source Project: nifi   Source File: StandardFlowManager.java    License: Apache License 2.0 5 votes vote down vote up
public FlowFilePrioritizer createPrioritizer(final String type) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    FlowFilePrioritizer prioritizer;

    final ClassLoader ctxClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final List<Bundle> prioritizerBundles = flowController.getExtensionManager().getBundles(type);
        if (prioritizerBundles.size() == 0) {
            throw new IllegalStateException(String.format("The specified class '%s' is not known to this nifi.", type));
        }
        if (prioritizerBundles.size() > 1) {
            throw new IllegalStateException(String.format("Multiple bundles found for the specified class '%s', only one is allowed.", type));
        }

        final Bundle bundle = prioritizerBundles.get(0);
        final ClassLoader detectedClassLoaderForType = bundle.getClassLoader();
        final Class<?> rawClass = Class.forName(type, true, detectedClassLoaderForType);

        Thread.currentThread().setContextClassLoader(detectedClassLoaderForType);
        final Class<? extends FlowFilePrioritizer> prioritizerClass = rawClass.asSubclass(FlowFilePrioritizer.class);
        final Object processorObj = prioritizerClass.newInstance();
        prioritizer = prioritizerClass.cast(processorObj);

        return prioritizer;
    } finally {
        if (ctxClassLoader != null) {
            Thread.currentThread().setContextClassLoader(ctxClassLoader);
        }
    }
}
 
Example 12
Source Project: nifi   Source File: SwappablePriorityQueue.java    License: Apache License 2.0 5 votes vote down vote up
public List<FlowFilePrioritizer> getPriorities() {
    readLock.lock();
    try {
        return Collections.unmodifiableList(priorities);
    } finally {
        readLock.unlock("getPriorities");
    }
}
 
Example 13
Source Project: nifi   Source File: SwappablePriorityQueue.java    License: Apache License 2.0 5 votes vote down vote up
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    writeLock.lock();
    try {
        priorities.clear();
        priorities.addAll(newPriorities);

        final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new QueuePrioritizer(newPriorities));
        newQueue.addAll(activeQueue);
        activeQueue = newQueue;
    } finally {
        writeLock.unlock("setPriorities");
    }
}
 
Example 14
Source Project: nifi   Source File: TestSocketLoadBalancedFlowFileQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPriorities() {
    final FlowFilePrioritizer iValuePrioritizer = new FlowFilePrioritizer() {
        @Override
        public int compare(final FlowFile o1, final FlowFile o2) {
            final int i1 = Integer.parseInt(o1.getAttribute("i"));
            final int i2 = Integer.parseInt(o2.getAttribute("i"));
            return Integer.compare(i1, i2);
        }
    };

    queue.setPriorities(Collections.singletonList(iValuePrioritizer));

    final Map<String, String> attributes = new HashMap<>();

    // Add 100 FlowFiles, each with a descending 'i' value (first has i=99, second has i=98, etc.)
    for (int i = 99; i >= 0; i--) {
        attributes.put("i", String.valueOf(i));
        final MockFlowFileRecord flowFile = new MockFlowFileRecord(new HashMap<>(attributes), 0L);
        queue.put(flowFile);
    }

    for (int i=0; i < 100; i++) {
        final FlowFileRecord polled = queue.poll(Collections.emptySet());
        assertNotNull(polled);
        assertEquals(String.valueOf(i), polled.getAttribute("i"));
    }

    assertNull(queue.poll(Collections.emptySet()));
}
 
Example 15
Source Project: nifi   Source File: TestSocketLoadBalancedFlowFileQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPrioritiesWhenSetBeforeLocalNodeIdDetermined() {
    final FlowFilePrioritizer iValuePrioritizer = new FlowFilePrioritizer() {
        @Override
        public int compare(final FlowFile o1, final FlowFile o2) {
            final int i1 = Integer.parseInt(o1.getAttribute("i"));
            final int i2 = Integer.parseInt(o2.getAttribute("i"));
            return Integer.compare(i1, i2);
        }
    };

    final ProcessScheduler scheduler = mock(ProcessScheduler.class);
    final AsyncLoadBalanceClientRegistry registry = mock(AsyncLoadBalanceClientRegistry.class);
    when(clusterCoordinator.getLocalNodeIdentifier()).thenReturn(null);

    queue = new SocketLoadBalancedFlowFileQueue("unit-test", new NopConnectionEventListener(), scheduler, flowFileRepo, provRepo,
        contentRepo, claimManager, clusterCoordinator, registry, swapManager, 10000, eventReporter);
    queue.setPriorities(Collections.singletonList(iValuePrioritizer));

    when(clusterCoordinator.getLocalNodeIdentifier()).thenReturn(null);
    queue.setNodeIdentifiers(new HashSet<>(nodeIds), true);

    final Map<String, String> attributes = new HashMap<>();

    // Add 100 FlowFiles, each with a descending 'i' value (first has i=99, second has i=98, etc.)
    for (int i = 99; i >= 0; i--) {
        attributes.put("i", String.valueOf(i));
        final MockFlowFileRecord flowFile = new MockFlowFileRecord(new HashMap<>(attributes), 0L);
        queue.put(flowFile);
    }

    for (int i=0; i < 100; i++) {
        final FlowFileRecord polled = queue.poll(Collections.emptySet());
        assertNotNull(polled);
        assertEquals(String.valueOf(i), polled.getAttribute("i"));
    }

    assertNull(queue.poll(Collections.emptySet()));
}
 
Example 16
Source Project: localization_nifi   Source File: DtoFactory.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Creates a ConnectionDTO from the specified Connection.
 *
 * @param connection connection
 * @return dto
 */
public ConnectionDTO createConnectionDto(final Connection connection) {
    if (connection == null) {
        return null;
    }

    final ConnectionDTO dto = new ConnectionDTO();

    dto.setId(connection.getIdentifier());
    dto.setParentGroupId(connection.getProcessGroup().getIdentifier());

    final List<PositionDTO> bendPoints = new ArrayList<>();
    for (final Position bendPoint : connection.getBendPoints()) {
        bendPoints.add(createPositionDto(bendPoint));
    }
    dto.setBends(bendPoints);
    dto.setName(connection.getName());
    dto.setLabelIndex(connection.getLabelIndex());
    dto.setzIndex(connection.getZIndex());
    dto.setSource(createConnectableDto(connection.getSource()));
    dto.setDestination(createConnectableDto(connection.getDestination()));

    dto.setBackPressureObjectThreshold(connection.getFlowFileQueue().getBackPressureObjectThreshold());
    dto.setBackPressureDataSizeThreshold(connection.getFlowFileQueue().getBackPressureDataSizeThreshold());
    dto.setFlowFileExpiration(connection.getFlowFileQueue().getFlowFileExpiration());
    dto.setPrioritizers(new ArrayList<String>());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        dto.getPrioritizers().add(comparator.getClass().getCanonicalName());
    }

    // For ports, we do not want to populate the relationships.
    for (final Relationship selectedRelationship : connection.getRelationships()) {
        if (!Relationship.ANONYMOUS.equals(selectedRelationship)) {
            if (dto.getSelectedRelationships() == null) {
                dto.setSelectedRelationships(new TreeSet<String>(Collator.getInstance(Locale.US)));
            }

            dto.getSelectedRelationships().add(selectedRelationship.getName());
        }
    }

    // For ports, we do not want to populate the relationships.
    for (final Relationship availableRelationship : connection.getSource().getRelationships()) {
        if (!Relationship.ANONYMOUS.equals(availableRelationship)) {
            if (dto.getAvailableRelationships() == null) {
                dto.setAvailableRelationships(new TreeSet<String>(Collator.getInstance(Locale.US)));
            }

            dto.getAvailableRelationships().add(availableRelationship.getName());
        }
    }

    return dto;
}
 
Example 17
Source Project: localization_nifi   Source File: StandardConnectionDAO.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Configures the specified connection using the specified dto.
 */
private void configureConnection(Connection connection, ConnectionDTO connectionDTO) {
    // validate flow file comparators/prioritizers
    List<FlowFilePrioritizer> newPrioritizers = null;
    final List<String> prioritizers = connectionDTO.getPrioritizers();
    if (isNotNull(prioritizers)) {
        final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
        newPrioritizers = new ArrayList<>();
        for (final String className : newPrioritizersClasses) {
            try {
                newPrioritizers.add(flowController.createPrioritizer(className));
            } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
            }
        }
    }

    // update connection queue
    if (isNotNull(connectionDTO.getFlowFileExpiration())) {
        connection.getFlowFileQueue().setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
    }
    if (isNotNull(connectionDTO.getBackPressureObjectThreshold())) {
        connection.getFlowFileQueue().setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
    }
    if (isNotNull(connectionDTO.getBackPressureDataSizeThreshold())) {
        connection.getFlowFileQueue().setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
    }
    if (isNotNull(newPrioritizers)) {
        connection.getFlowFileQueue().setPriorities(newPrioritizers);
    }

    // update the connection state
    if (isNotNull(connectionDTO.getBends())) {
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : connectionDTO.getBends()) {
            if (bend != null) {
                bendPoints.add(new Position(bend.getX(), bend.getY()));
            }
        }
        connection.setBendPoints(bendPoints);
    }
    if (isNotNull(connectionDTO.getName())) {
        connection.setName(connectionDTO.getName());
    }
    if (isNotNull(connectionDTO.getLabelIndex())) {
        connection.setLabelIndex(connectionDTO.getLabelIndex());
    }
    if (isNotNull(connectionDTO.getzIndex())) {
        connection.setZIndex(connectionDTO.getzIndex());
    }
}
 
Example 18
Source Project: localization_nifi   Source File: StandardFlowSerializer.java    License: Apache License 2.0 4 votes vote down vote up
private void addConnection(final Element parentElement, final Connection connection) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("connection");
    parentElement.appendChild(element);
    addTextElement(element, "id", connection.getIdentifier());
    addTextElement(element, "name", connection.getName());

    final Element bendPointsElement = doc.createElement("bendPoints");
    element.appendChild(bendPointsElement);
    for (final Position bendPoint : connection.getBendPoints()) {
        addPosition(bendPointsElement, bendPoint, "bendPoint");
    }

    addTextElement(element, "labelIndex", connection.getLabelIndex());
    addTextElement(element, "zIndex", connection.getZIndex());

    final String sourceId = connection.getSource().getIdentifier();
    final ConnectableType sourceType = connection.getSource().getConnectableType();
    final String sourceGroupId;
    if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
        sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
    } else {
        sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
    }

    final ConnectableType destinationType = connection.getDestination().getConnectableType();
    final String destinationId = connection.getDestination().getIdentifier();
    final String destinationGroupId;
    if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
        destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
    } else {
        destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
    }

    addTextElement(element, "sourceId", sourceId);
    addTextElement(element, "sourceGroupId", sourceGroupId);
    addTextElement(element, "sourceType", sourceType.toString());

    addTextElement(element, "destinationId", destinationId);
    addTextElement(element, "destinationGroupId", destinationGroupId);
    addTextElement(element, "destinationType", destinationType.toString());

    for (final Relationship relationship : connection.getRelationships()) {
        addTextElement(element, "relationship", relationship.getName());
    }

    addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold());
    addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold());

    addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        final String className = comparator.getClass().getCanonicalName();
        addTextElement(element, "queuePrioritizerClass", className);
    }

    parentElement.appendChild(element);
}
 
Example 19
Source Project: localization_nifi   Source File: FlowController.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
public Set<Class> getFlowFileComparatorClasses() {
    return ExtensionManager.getExtensions(FlowFilePrioritizer.class);
}
 
Example 20
Source Project: localization_nifi   Source File: StandardFlowFileQueue.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<FlowFilePrioritizer> getPriorities() {
    return Collections.unmodifiableList(priorities);
}
 
Example 21
Source Project: localization_nifi   Source File: StandardFlowFileQueue.java    License: Apache License 2.0 4 votes vote down vote up
private Prioritizer(final List<FlowFilePrioritizer> priorities) {
    if (null != priorities) {
        prioritizers.addAll(priorities);
    }
}
 
Example 22
Source Project: localization_nifi   Source File: StandardFlowFileQueue.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public int compare(final FlowFileRecord f1, final FlowFileRecord f2) {
    int returnVal = 0;
    final boolean f1Penalized = f1.isPenalized();
    final boolean f2Penalized = f2.isPenalized();

    if (f1Penalized && !f2Penalized) {
        return 1;
    } else if (!f1Penalized && f2Penalized) {
        return -1;
    }

    if (f1Penalized && f2Penalized) {
        if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) {
            return -1;
        } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) {
            return 1;
        }
    }

    if (!prioritizers.isEmpty()) {
        for (final FlowFilePrioritizer prioritizer : prioritizers) {
            returnVal = prioritizer.compare(f1, f2);
            if (returnVal != 0) {
                return returnVal;
            }
        }
    }

    final ContentClaim claim1 = f1.getContentClaim();
    final ContentClaim claim2 = f2.getContentClaim();

    // put the one without a claim first
    if (claim1 == null && claim2 != null) {
        return -1;
    } else if (claim1 != null && claim2 == null) {
        return 1;
    } else if (claim1 != null && claim2 != null) {
        final int claimComparison = claim1.compareTo(claim2);
        if (claimComparison != 0) {
            return claimComparison;
        }

        final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset());
        if (claimOffsetComparison != 0) {
            return claimOffsetComparison;
        }
    }

    return Long.compare(f1.getId(), f2.getId());
}
 
Example 23
Source Project: nifi   Source File: StandardConnectionDAO.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Configures the specified connection using the specified dto.
 */
private void configureConnection(Connection connection, ConnectionDTO connectionDTO) {
    // validate flow file comparators/prioritizers
    List<FlowFilePrioritizer> newPrioritizers = null;
    final List<String> prioritizers = connectionDTO.getPrioritizers();
    if (isNotNull(prioritizers)) {
        final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
        newPrioritizers = new ArrayList<>();
        for (final String className : newPrioritizersClasses) {
            try {
                newPrioritizers.add(flowController.getFlowManager().createPrioritizer(className));
            } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
            }
        }
    }

    // update connection queue
    if (isNotNull(connectionDTO.getFlowFileExpiration())) {
        connection.getFlowFileQueue().setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
    }
    if (isNotNull(connectionDTO.getBackPressureObjectThreshold())) {
        connection.getFlowFileQueue().setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
    }
    if (isNotNull(connectionDTO.getBackPressureDataSizeThreshold())) {
        connection.getFlowFileQueue().setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
    }
    if (isNotNull(newPrioritizers)) {
        connection.getFlowFileQueue().setPriorities(newPrioritizers);
    }

    final String loadBalanceStrategyName = connectionDTO.getLoadBalanceStrategy();
    final String loadBalancePartitionAttribute = connectionDTO.getLoadBalancePartitionAttribute();
    if (isNotNull(loadBalanceStrategyName)) {
        final LoadBalanceStrategy loadBalanceStrategy = LoadBalanceStrategy.valueOf(loadBalanceStrategyName);
        connection.getFlowFileQueue().setLoadBalanceStrategy(loadBalanceStrategy, loadBalancePartitionAttribute);
    }

    final String loadBalanceCompressionName = connectionDTO.getLoadBalanceCompression();
    if (isNotNull(loadBalanceCompressionName)) {
        connection.getFlowFileQueue().setLoadBalanceCompression(LoadBalanceCompression.valueOf(loadBalanceCompressionName));
    }

    // update the connection state
    if (isNotNull(connectionDTO.getBends())) {
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : connectionDTO.getBends()) {
            if (bend != null) {
                bendPoints.add(new Position(bend.getX(), bend.getY()));
            }
        }
        connection.setBendPoints(bendPoints);
    }
    if (isNotNull(connectionDTO.getName())) {
        connection.setName(connectionDTO.getName());
    }
    if (isNotNull(connectionDTO.getLabelIndex())) {
        connection.setLabelIndex(connectionDTO.getLabelIndex());
    }
    if (isNotNull(connectionDTO.getzIndex())) {
        connection.setZIndex(connectionDTO.getzIndex());
    }
}
 
Example 24
Source Project: nifi   Source File: PrioritiesMatcherTest.java    License: Apache License 2.0 4 votes vote down vote up
private List<FlowFilePrioritizer> givenPriorizers() {
    final List<FlowFilePrioritizer> result = new ArrayList<>();
    result.add(new FlowFilePrioritizerOne());
    result.add(new FlowFilePrioritizerTwo());
    return result;
}
 
Example 25
Source Project: nifi   Source File: StandardFlowSerializer.java    License: Apache License 2.0 4 votes vote down vote up
private void addConnection(final Element parentElement, final Connection connection) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("connection");
    parentElement.appendChild(element);
    addTextElement(element, "id", connection.getIdentifier());
    addTextElement(element, "versionedComponentId", connection.getVersionedComponentId());
    addTextElement(element, "name", connection.getName());

    final Element bendPointsElement = doc.createElement("bendPoints");
    element.appendChild(bendPointsElement);
    for (final Position bendPoint : connection.getBendPoints()) {
        addPosition(bendPointsElement, bendPoint, "bendPoint");
    }

    addTextElement(element, "labelIndex", connection.getLabelIndex());
    addTextElement(element, "zIndex", connection.getZIndex());

    final String sourceId = connection.getSource().getIdentifier();
    final ConnectableType sourceType = connection.getSource().getConnectableType();
    final String sourceGroupId;
    if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
        sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
    } else {
        sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
    }

    final ConnectableType destinationType = connection.getDestination().getConnectableType();
    final String destinationId = connection.getDestination().getIdentifier();
    final String destinationGroupId;
    if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
        destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
    } else {
        destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
    }

    addTextElement(element, "sourceId", sourceId);
    addTextElement(element, "sourceGroupId", sourceGroupId);
    addTextElement(element, "sourceType", sourceType.toString());

    addTextElement(element, "destinationId", destinationId);
    addTextElement(element, "destinationGroupId", destinationGroupId);
    addTextElement(element, "destinationType", destinationType.toString());

    for (final Relationship relationship : connection.getRelationships()) {
        addTextElement(element, "relationship", relationship.getName());
    }

    addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold());
    addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold());

    addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        final String className = comparator.getClass().getCanonicalName();
        addTextElement(element, "queuePrioritizerClass", className);
    }

    addTextElement(element, "loadBalanceStrategy", connection.getFlowFileQueue().getLoadBalanceStrategy().name());
    addTextElement(element, "partitioningAttribute", connection.getFlowFileQueue().getPartitioningAttribute());
    addTextElement(element, "loadBalanceCompression", connection.getFlowFileQueue().getLoadBalanceCompression().name());

    parentElement.appendChild(element);
}
 
Example 26
Source Project: nifi   Source File: QueuePrioritizer.java    License: Apache License 2.0 4 votes vote down vote up
public QueuePrioritizer(final List<FlowFilePrioritizer> priorities) {
    if (null != priorities) {
        prioritizers.addAll(priorities);
    }
}
 
Example 27
Source Project: nifi   Source File: QueuePrioritizer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public int compare(final FlowFileRecord f1, final FlowFileRecord f2) {
    int returnVal = 0;
    final boolean f1Penalized = f1.isPenalized();
    final boolean f2Penalized = f2.isPenalized();

    if (f1Penalized && !f2Penalized) {
        return 1;
    } else if (!f1Penalized && f2Penalized) {
        return -1;
    }

    if (f1Penalized && f2Penalized) {
        if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) {
            return -1;
        } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) {
            return 1;
        }
    }

    if (!prioritizers.isEmpty()) {
        for (final FlowFilePrioritizer prioritizer : prioritizers) {
            returnVal = prioritizer.compare(f1, f2);
            if (returnVal != 0) {
                return returnVal;
            }
        }
    }

    final ContentClaim claim1 = f1.getContentClaim();
    final ContentClaim claim2 = f2.getContentClaim();


    // put the one without a claim first
    if (claim1 == null && claim2 != null) {
        return -1;
    } else if (claim1 != null && claim2 == null) {
        return 1;
    } else if (claim1 != null && claim2 != null) {
        final int claimComparison = claim1.compareTo(claim2);
        if (claimComparison != 0) {
            return claimComparison;
        }

        final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset());
        if (claimOffsetComparison != 0) {
            return claimOffsetComparison;
        }
    }

    return Long.compare(f1.getId(), f2.getId());
}
 
Example 28
Source Project: nifi   Source File: SocketLoadBalancedFlowFileQueue.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public synchronized List<FlowFilePrioritizer> getPriorities() {
    return new ArrayList<>(prioritizers);
}
 
Example 29
Source Project: nifi   Source File: StandardRebalancingPartition.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    queue.setPriorities(newPriorities);
}
 
Example 30
Source Project: nifi   Source File: RemoteQueuePartition.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    priorityQueue.setPriorities(newPriorities);
}