org.apache.nifi.flowfile.FlowFilePrioritizer Java Examples

The following examples show how to use org.apache.nifi.flowfile.FlowFilePrioritizer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RelationshipAuditor.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Extracts configured settings from the specified connection only if they have also been specified in the connectionDTO.
 *
 * @param connection connection
 * @param connectionDTO dto
 * @return properties
 */
private Map<String, String> extractConfiguredPropertyValues(Connection connection, ConnectionDTO connectionDTO) {
    Map<String, String> values = new HashMap<>();

    if (connectionDTO.getName() != null) {
        values.put(NAME, connection.getName());
    }
    if (connectionDTO.getFlowFileExpiration() != null) {
        values.put(FLOW_FILE_EXPIRATION, String.valueOf(connection.getFlowFileQueue().getFlowFileExpiration()));
    }
    if (connectionDTO.getBackPressureObjectThreshold() != null) {
        values.put(BACK_PRESSURE_OBJECT_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureObjectThreshold()));
    }
    if (connectionDTO.getBackPressureDataSizeThreshold() != null) {
        values.put(BACK_PRESSURE_DATA_SIZE_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureDataSizeThreshold()));
    }
    if (connectionDTO.getPrioritizers() != null) {
        List<String> prioritizers = new ArrayList<>();
        for (FlowFilePrioritizer prioritizer : connection.getFlowFileQueue().getPriorities()) {
            prioritizers.add(prioritizer.getClass().getCanonicalName());
        }
        values.put(PRIORITIZERS, StringUtils.join(prioritizers, ", "));
    }

    return values;
}
 
Example #2
Source File: StandardExtensionDiscoveringManager.java    From nifi with Apache License 2.0 6 votes vote down vote up
public StandardExtensionDiscoveringManager() {
    definitionMap.put(Processor.class, new HashSet<>());
    definitionMap.put(FlowFilePrioritizer.class, new HashSet<>());
    definitionMap.put(ReportingTask.class, new HashSet<>());
    definitionMap.put(ControllerService.class, new HashSet<>());
    definitionMap.put(Authorizer.class, new HashSet<>());
    definitionMap.put(UserGroupProvider.class, new HashSet<>());
    definitionMap.put(AccessPolicyProvider.class, new HashSet<>());
    definitionMap.put(LoginIdentityProvider.class, new HashSet<>());
    definitionMap.put(ProvenanceRepository.class, new HashSet<>());
    definitionMap.put(ComponentStatusRepository.class, new HashSet<>());
    definitionMap.put(FlowFileRepository.class, new HashSet<>());
    definitionMap.put(FlowFileSwapManager.class, new HashSet<>());
    definitionMap.put(ContentRepository.class, new HashSet<>());
    definitionMap.put(StateProvider.class, new HashSet<>());
    definitionMap.put(StatusAnalyticsModel.class, new HashSet<>());
}
 
Example #3
Source File: SocketLoadBalancedFlowFileQueue.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    prioritizers.clear();
    prioritizers.addAll(newPriorities);

    partitionReadLock.lock();
    try {
        for (final QueuePartition partition : queuePartitions) {
            partition.setPriorities(newPriorities);
        }

        rebalancingPartition.setPriorities(newPriorities);
    } finally {
        partitionReadLock.unlock();
    }
}
 
Example #4
Source File: TestStandardFlowFileQueue.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testLowestPrioritySwappedOutFirst() {
    final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
    prioritizers.add(new FlowFileSizePrioritizer());
    queue.setPriorities(prioritizers);

    long maxSize = 20000;
    for (int i = 1; i <= 20000; i++) {
        queue.put(new TestFlowFile(maxSize - i));
    }

    assertEquals(1, swapManager.swapOutCalledCount);
    assertEquals(20000, queue.size().getObjectCount());

    assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
    final List<FlowFileRecord> flowFiles = queue.poll(Integer.MAX_VALUE, new HashSet<FlowFileRecord>());
    assertEquals(10000, flowFiles.size());
    for (int i = 0; i < 10000; i++) {
        assertEquals(i, flowFiles.get(i).getSize());
    }
}
 
Example #5
Source File: StandardFlowFileQueue.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
public StandardFlowFileQueue(final String identifier, final Connection connection, final FlowFileRepository flowFileRepo, final ProvenanceEventRepository provRepo,
    final ResourceClaimManager resourceClaimManager, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter, final int swapThreshold) {
    activeQueue = new PriorityQueue<>(20, new Prioritizer(new ArrayList<FlowFilePrioritizer>()));
    priorities = new ArrayList<>();
    swapQueue = new ArrayList<>();
    this.eventReporter = eventReporter;
    this.swapManager = swapManager;
    this.flowFileRepository = flowFileRepo;
    this.provRepository = provRepo;
    this.resourceClaimManager = resourceClaimManager;

    this.identifier = identifier;
    this.swapThreshold = swapThreshold;
    this.scheduler = scheduler;
    this.connection = connection;

    readLock = new TimedLock(this.lock.readLock(), identifier + " Read Lock", 100);
    writeLock = new TimedLock(this.lock.writeLock(), identifier + " Write Lock", 100);
}
 
Example #6
Source File: RelationshipAuditor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Extracts configured settings from the specified connection only if they have also been specified in the connectionDTO.
 *
 * @param connection connection
 * @param connectionDTO dto
 * @return properties
 */
private Map<String, String> extractConfiguredPropertyValues(Connection connection, ConnectionDTO connectionDTO) {
    Map<String, String> values = new HashMap<>();

    if (connectionDTO.getName() != null) {
        values.put(NAME, connection.getName());
    }
    if (connectionDTO.getFlowFileExpiration() != null) {
        values.put(FLOW_FILE_EXPIRATION, String.valueOf(connection.getFlowFileQueue().getFlowFileExpiration()));
    }
    if (connectionDTO.getBackPressureObjectThreshold() != null) {
        values.put(BACK_PRESSURE_OBJECT_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureObjectThreshold()));
    }
    if (connectionDTO.getBackPressureDataSizeThreshold() != null) {
        values.put(BACK_PRESSURE_DATA_SIZE_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureDataSizeThreshold()));
    }
    if (connectionDTO.getPrioritizers() != null) {
        List<String> prioritizers = new ArrayList<>();
        for (FlowFilePrioritizer prioritizer : connection.getFlowFileQueue().getPriorities()) {
            prioritizers.add(prioritizer.getClass().getCanonicalName());
        }
        values.put(PRIORITIZERS, StringUtils.join(prioritizers, ", "));
    }

    return values;
}
 
Example #7
Source File: StandardFlowManager.java    From nifi with Apache License 2.0 5 votes vote down vote up
public FlowFilePrioritizer createPrioritizer(final String type) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    FlowFilePrioritizer prioritizer;

    final ClassLoader ctxClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final List<Bundle> prioritizerBundles = flowController.getExtensionManager().getBundles(type);
        if (prioritizerBundles.size() == 0) {
            throw new IllegalStateException(String.format("The specified class '%s' is not known to this nifi.", type));
        }
        if (prioritizerBundles.size() > 1) {
            throw new IllegalStateException(String.format("Multiple bundles found for the specified class '%s', only one is allowed.", type));
        }

        final Bundle bundle = prioritizerBundles.get(0);
        final ClassLoader detectedClassLoaderForType = bundle.getClassLoader();
        final Class<?> rawClass = Class.forName(type, true, detectedClassLoaderForType);

        Thread.currentThread().setContextClassLoader(detectedClassLoaderForType);
        final Class<? extends FlowFilePrioritizer> prioritizerClass = rawClass.asSubclass(FlowFilePrioritizer.class);
        final Object processorObj = prioritizerClass.newInstance();
        prioritizer = prioritizerClass.cast(processorObj);

        return prioritizer;
    } finally {
        if (ctxClassLoader != null) {
            Thread.currentThread().setContextClassLoader(ctxClassLoader);
        }
    }
}
 
Example #8
Source File: ControllerSearchServiceIntegrationTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testSearchBasedOnPriorities() {
    // given
    final ProcessorNode processor1 = getProcessorNode("processor1", "processor1Name", AUTHORIZED);
    final ProcessorNode processor2 = getProcessorNode("processor2", "processor2Name", AUTHORIZED);
    final Connection connection = getConnection("connection", "connectionName", getBasicRelationships(), processor1, processor2, AUTHORIZED);

    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
    prioritizers.add(Mockito.mock(ComponentMockUtil.DummyFlowFilePrioritizer.class));
    Mockito.when(flowFileQueue.getPriorities()).thenReturn(prioritizers);
    Mockito.when(connection.getFlowFileQueue()).thenReturn(flowFileQueue);

    givenRootProcessGroup()
            .withProcessor(processor1)
            .withProcessor(processor2)
            .withConnection(connection);

    // when
    whenExecuteSearch("dummy");

    // then
    Assert.assertEquals(1, results.getConnectionResults().size());
    Assert.assertEquals(1, results.getConnectionResults().get(0).getMatches().size());
    Assert.assertTrue(results.getConnectionResults().get(0).getMatches().get(0)
            .startsWith("Prioritizer: org.apache.nifi.web.controller.ComponentMockUtil$DummyFlowFilePrioritizer$"));
}
 
Example #9
Source File: ComponentMockUtil.java    From nifi with Apache License 2.0 5 votes vote down vote up
public static Connection getConnection(
        final String id,
        final String name,
        final Optional<String> versionedId,
        final Collection<Relationship> relationships,
        final List<FlowFilePrioritizer> flowFilePrioritizers,
        final int flowFileExpirationInMs,
        final String backPressureDataSize,
        final long backPressureCount,
        final Connectable source,
        final Connectable destination,
        final boolean isAuthorized) {
    final Connection result = Mockito.mock(Connection.class);
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);

    Mockito.when(flowFileQueue.getPriorities()).thenReturn(flowFilePrioritizers);
    Mockito.when(flowFileQueue.getFlowFileExpiration()).thenReturn(String.valueOf(flowFileExpirationInMs));
    Mockito.when(flowFileQueue.getFlowFileExpiration(TimeUnit.MILLISECONDS)).thenReturn(flowFileExpirationInMs);
    Mockito.when(flowFileQueue.getBackPressureDataSizeThreshold()).thenReturn(backPressureDataSize);
    Mockito.when(flowFileQueue.getBackPressureObjectThreshold()).thenReturn(backPressureCount);

    Mockito.when(result.getIdentifier()).thenReturn(id);
    Mockito.when(result.getName()).thenReturn(name);
    Mockito.when(result.getVersionedComponentId()).thenReturn(versionedId);
    Mockito.when(result.getRelationships()).thenReturn(relationships);
    Mockito.when(result.getFlowFileQueue()).thenReturn(flowFileQueue);
    Mockito.when(result.getSource()).thenReturn(source);
    Mockito.when(result.getDestination()).thenReturn(destination);

    setAuthorized(result, isAuthorized);
    return result;
}
 
Example #10
Source File: SwappablePriorityQueue.java    From nifi with Apache License 2.0 5 votes vote down vote up
public List<FlowFilePrioritizer> getPriorities() {
    readLock.lock();
    try {
        return Collections.unmodifiableList(priorities);
    } finally {
        readLock.unlock("getPriorities");
    }
}
 
Example #11
Source File: SwappablePriorityQueue.java    From nifi with Apache License 2.0 5 votes vote down vote up
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    writeLock.lock();
    try {
        priorities.clear();
        priorities.addAll(newPriorities);

        final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new QueuePrioritizer(newPriorities));
        newQueue.addAll(activeQueue);
        activeQueue = newQueue;
    } finally {
        writeLock.unlock("setPriorities");
    }
}
 
Example #12
Source File: StandardFlowFileQueue.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    writeLock.lock();
    try {
        final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new Prioritizer(newPriorities));
        newQueue.addAll(activeQueue);
        activeQueue = newQueue;
        priorities.clear();
        priorities.addAll(newPriorities);
    } finally {
        writeLock.unlock("setPriorities");
    }
}
 
Example #13
Source File: TestSocketLoadBalancedFlowFileQueue.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testPriorities() {
    final FlowFilePrioritizer iValuePrioritizer = new FlowFilePrioritizer() {
        @Override
        public int compare(final FlowFile o1, final FlowFile o2) {
            final int i1 = Integer.parseInt(o1.getAttribute("i"));
            final int i2 = Integer.parseInt(o2.getAttribute("i"));
            return Integer.compare(i1, i2);
        }
    };

    queue.setPriorities(Collections.singletonList(iValuePrioritizer));

    final Map<String, String> attributes = new HashMap<>();

    // Add 100 FlowFiles, each with a descending 'i' value (first has i=99, second has i=98, etc.)
    for (int i = 99; i >= 0; i--) {
        attributes.put("i", String.valueOf(i));
        final MockFlowFileRecord flowFile = new MockFlowFileRecord(new HashMap<>(attributes), 0L);
        queue.put(flowFile);
    }

    for (int i=0; i < 100; i++) {
        final FlowFileRecord polled = queue.poll(Collections.emptySet());
        assertNotNull(polled);
        assertEquals(String.valueOf(i), polled.getAttribute("i"));
    }

    assertNull(queue.poll(Collections.emptySet()));
}
 
Example #14
Source File: TestSocketLoadBalancedFlowFileQueue.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testPrioritiesWhenSetBeforeLocalNodeIdDetermined() {
    final FlowFilePrioritizer iValuePrioritizer = new FlowFilePrioritizer() {
        @Override
        public int compare(final FlowFile o1, final FlowFile o2) {
            final int i1 = Integer.parseInt(o1.getAttribute("i"));
            final int i2 = Integer.parseInt(o2.getAttribute("i"));
            return Integer.compare(i1, i2);
        }
    };

    final ProcessScheduler scheduler = mock(ProcessScheduler.class);
    final AsyncLoadBalanceClientRegistry registry = mock(AsyncLoadBalanceClientRegistry.class);
    when(clusterCoordinator.getLocalNodeIdentifier()).thenReturn(null);

    queue = new SocketLoadBalancedFlowFileQueue("unit-test", new NopConnectionEventListener(), scheduler, flowFileRepo, provRepo,
        contentRepo, claimManager, clusterCoordinator, registry, swapManager, 10000, eventReporter);
    queue.setPriorities(Collections.singletonList(iValuePrioritizer));

    when(clusterCoordinator.getLocalNodeIdentifier()).thenReturn(null);
    queue.setNodeIdentifiers(new HashSet<>(nodeIds), true);

    final Map<String, String> attributes = new HashMap<>();

    // Add 100 FlowFiles, each with a descending 'i' value (first has i=99, second has i=98, etc.)
    for (int i = 99; i >= 0; i--) {
        attributes.put("i", String.valueOf(i));
        final MockFlowFileRecord flowFile = new MockFlowFileRecord(new HashMap<>(attributes), 0L);
        queue.put(flowFile);
    }

    for (int i=0; i < 100; i++) {
        final FlowFileRecord polled = queue.poll(Collections.emptySet());
        assertNotNull(polled);
        assertEquals(String.valueOf(i), polled.getAttribute("i"));
    }

    assertNull(queue.poll(Collections.emptySet()));
}
 
Example #15
Source File: FlowController.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public FlowFilePrioritizer createPrioritizer(final String type) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    FlowFilePrioritizer prioritizer;

    final ClassLoader ctxClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final ClassLoader detectedClassLoaderForType = ExtensionManager.getClassLoader(type);
        final Class<?> rawClass;
        if (detectedClassLoaderForType == null) {
            // try to find from the current class loader
            rawClass = Class.forName(type);
        } else {
            // try to find from the registered classloader for that type
            rawClass = Class.forName(type, true, ExtensionManager.getClassLoader(type));
        }

        Thread.currentThread().setContextClassLoader(detectedClassLoaderForType);
        final Class<? extends FlowFilePrioritizer> prioritizerClass = rawClass.asSubclass(FlowFilePrioritizer.class);
        final Object processorObj = prioritizerClass.newInstance();
        prioritizer = prioritizerClass.cast(processorObj);

        return prioritizer;
    } finally {
        if (ctxClassLoader != null) {
            Thread.currentThread().setContextClassLoader(ctxClassLoader);
        }
    }
}
 
Example #16
Source File: StandardFlowFileQueue.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    queue.setPriorities(newPriorities);
}
 
Example #17
Source File: SwappablePriorityQueueLocalPartition.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    priorityQueue.setPriorities(newPriorities);
}
 
Example #18
Source File: RemoteQueuePartition.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    priorityQueue.setPriorities(newPriorities);
}
 
Example #19
Source File: StandardRebalancingPartition.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    queue.setPriorities(newPriorities);
}
 
Example #20
Source File: DtoFactory.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a ConnectionDTO from the specified Connection.
 *
 * @param connection connection
 * @return dto
 */
public ConnectionDTO createConnectionDto(final Connection connection) {
    if (connection == null) {
        return null;
    }

    final ConnectionDTO dto = new ConnectionDTO();

    dto.setId(connection.getIdentifier());
    dto.setParentGroupId(connection.getProcessGroup().getIdentifier());

    final List<PositionDTO> bendPoints = new ArrayList<>();
    for (final Position bendPoint : connection.getBendPoints()) {
        bendPoints.add(createPositionDto(bendPoint));
    }
    dto.setBends(bendPoints);
    dto.setName(connection.getName());
    dto.setLabelIndex(connection.getLabelIndex());
    dto.setzIndex(connection.getZIndex());
    dto.setSource(createConnectableDto(connection.getSource()));
    dto.setDestination(createConnectableDto(connection.getDestination()));

    dto.setBackPressureObjectThreshold(connection.getFlowFileQueue().getBackPressureObjectThreshold());
    dto.setBackPressureDataSizeThreshold(connection.getFlowFileQueue().getBackPressureDataSizeThreshold());
    dto.setFlowFileExpiration(connection.getFlowFileQueue().getFlowFileExpiration());
    dto.setPrioritizers(new ArrayList<String>());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        dto.getPrioritizers().add(comparator.getClass().getCanonicalName());
    }

    // For ports, we do not want to populate the relationships.
    for (final Relationship selectedRelationship : connection.getRelationships()) {
        if (!Relationship.ANONYMOUS.equals(selectedRelationship)) {
            if (dto.getSelectedRelationships() == null) {
                dto.setSelectedRelationships(new TreeSet<String>(Collator.getInstance(Locale.US)));
            }

            dto.getSelectedRelationships().add(selectedRelationship.getName());
        }
    }

    // For ports, we do not want to populate the relationships.
    for (final Relationship availableRelationship : connection.getSource().getRelationships()) {
        if (!Relationship.ANONYMOUS.equals(availableRelationship)) {
            if (dto.getAvailableRelationships() == null) {
                dto.setAvailableRelationships(new TreeSet<String>(Collator.getInstance(Locale.US)));
            }

            dto.getAvailableRelationships().add(availableRelationship.getName());
        }
    }

    return dto;
}
 
Example #21
Source File: SocketLoadBalancedFlowFileQueue.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized List<FlowFilePrioritizer> getPriorities() {
    return new ArrayList<>(prioritizers);
}
 
Example #22
Source File: StandardFlowFileQueue.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public List<FlowFilePrioritizer> getPriorities() {
    return queue.getPriorities();
}
 
Example #23
Source File: QueuePrioritizer.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public int compare(final FlowFileRecord f1, final FlowFileRecord f2) {
    int returnVal = 0;
    final boolean f1Penalized = f1.isPenalized();
    final boolean f2Penalized = f2.isPenalized();

    if (f1Penalized && !f2Penalized) {
        return 1;
    } else if (!f1Penalized && f2Penalized) {
        return -1;
    }

    if (f1Penalized && f2Penalized) {
        if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) {
            return -1;
        } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) {
            return 1;
        }
    }

    if (!prioritizers.isEmpty()) {
        for (final FlowFilePrioritizer prioritizer : prioritizers) {
            returnVal = prioritizer.compare(f1, f2);
            if (returnVal != 0) {
                return returnVal;
            }
        }
    }

    final ContentClaim claim1 = f1.getContentClaim();
    final ContentClaim claim2 = f2.getContentClaim();


    // put the one without a claim first
    if (claim1 == null && claim2 != null) {
        return -1;
    } else if (claim1 != null && claim2 == null) {
        return 1;
    } else if (claim1 != null && claim2 != null) {
        final int claimComparison = claim1.compareTo(claim2);
        if (claimComparison != 0) {
            return claimComparison;
        }

        final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset());
        if (claimOffsetComparison != 0) {
            return claimOffsetComparison;
        }
    }

    return Long.compare(f1.getId(), f2.getId());
}
 
Example #24
Source File: QueuePrioritizer.java    From nifi with Apache License 2.0 4 votes vote down vote up
public QueuePrioritizer(final List<FlowFilePrioritizer> priorities) {
    if (null != priorities) {
        prioritizers.addAll(priorities);
    }
}
 
Example #25
Source File: StandardFlowSerializer.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
private void addConnection(final Element parentElement, final Connection connection) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("connection");
    parentElement.appendChild(element);
    addTextElement(element, "id", connection.getIdentifier());
    addTextElement(element, "name", connection.getName());

    final Element bendPointsElement = doc.createElement("bendPoints");
    element.appendChild(bendPointsElement);
    for (final Position bendPoint : connection.getBendPoints()) {
        addPosition(bendPointsElement, bendPoint, "bendPoint");
    }

    addTextElement(element, "labelIndex", connection.getLabelIndex());
    addTextElement(element, "zIndex", connection.getZIndex());

    final String sourceId = connection.getSource().getIdentifier();
    final ConnectableType sourceType = connection.getSource().getConnectableType();
    final String sourceGroupId;
    if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
        sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
    } else {
        sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
    }

    final ConnectableType destinationType = connection.getDestination().getConnectableType();
    final String destinationId = connection.getDestination().getIdentifier();
    final String destinationGroupId;
    if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
        destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
    } else {
        destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
    }

    addTextElement(element, "sourceId", sourceId);
    addTextElement(element, "sourceGroupId", sourceGroupId);
    addTextElement(element, "sourceType", sourceType.toString());

    addTextElement(element, "destinationId", destinationId);
    addTextElement(element, "destinationGroupId", destinationGroupId);
    addTextElement(element, "destinationType", destinationType.toString());

    for (final Relationship relationship : connection.getRelationships()) {
        addTextElement(element, "relationship", relationship.getName());
    }

    addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold());
    addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold());

    addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        final String className = comparator.getClass().getCanonicalName();
        addTextElement(element, "queuePrioritizerClass", className);
    }

    parentElement.appendChild(element);
}
 
Example #26
Source File: StandardFlowSerializer.java    From nifi with Apache License 2.0 4 votes vote down vote up
private void addConnection(final Element parentElement, final Connection connection) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("connection");
    parentElement.appendChild(element);
    addTextElement(element, "id", connection.getIdentifier());
    addTextElement(element, "versionedComponentId", connection.getVersionedComponentId());
    addTextElement(element, "name", connection.getName());

    final Element bendPointsElement = doc.createElement("bendPoints");
    element.appendChild(bendPointsElement);
    for (final Position bendPoint : connection.getBendPoints()) {
        addPosition(bendPointsElement, bendPoint, "bendPoint");
    }

    addTextElement(element, "labelIndex", connection.getLabelIndex());
    addTextElement(element, "zIndex", connection.getZIndex());

    final String sourceId = connection.getSource().getIdentifier();
    final ConnectableType sourceType = connection.getSource().getConnectableType();
    final String sourceGroupId;
    if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
        sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
    } else {
        sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
    }

    final ConnectableType destinationType = connection.getDestination().getConnectableType();
    final String destinationId = connection.getDestination().getIdentifier();
    final String destinationGroupId;
    if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
        destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
    } else {
        destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
    }

    addTextElement(element, "sourceId", sourceId);
    addTextElement(element, "sourceGroupId", sourceGroupId);
    addTextElement(element, "sourceType", sourceType.toString());

    addTextElement(element, "destinationId", destinationId);
    addTextElement(element, "destinationGroupId", destinationGroupId);
    addTextElement(element, "destinationType", destinationType.toString());

    for (final Relationship relationship : connection.getRelationships()) {
        addTextElement(element, "relationship", relationship.getName());
    }

    addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold());
    addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold());

    addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        final String className = comparator.getClass().getCanonicalName();
        addTextElement(element, "queuePrioritizerClass", className);
    }

    addTextElement(element, "loadBalanceStrategy", connection.getFlowFileQueue().getLoadBalanceStrategy().name());
    addTextElement(element, "partitioningAttribute", connection.getFlowFileQueue().getPartitioningAttribute());
    addTextElement(element, "loadBalanceCompression", connection.getFlowFileQueue().getLoadBalanceCompression().name());

    parentElement.appendChild(element);
}
 
Example #27
Source File: PrioritiesMatcherTest.java    From nifi with Apache License 2.0 4 votes vote down vote up
private List<FlowFilePrioritizer> givenPriorizers() {
    final List<FlowFilePrioritizer> result = new ArrayList<>();
    result.add(new FlowFilePrioritizerOne());
    result.add(new FlowFilePrioritizerTwo());
    return result;
}
 
Example #28
Source File: StandardConnectionDAO.java    From nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Configures the specified connection using the specified dto.
 */
private void configureConnection(Connection connection, ConnectionDTO connectionDTO) {
    // validate flow file comparators/prioritizers
    List<FlowFilePrioritizer> newPrioritizers = null;
    final List<String> prioritizers = connectionDTO.getPrioritizers();
    if (isNotNull(prioritizers)) {
        final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
        newPrioritizers = new ArrayList<>();
        for (final String className : newPrioritizersClasses) {
            try {
                newPrioritizers.add(flowController.getFlowManager().createPrioritizer(className));
            } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
            }
        }
    }

    // update connection queue
    if (isNotNull(connectionDTO.getFlowFileExpiration())) {
        connection.getFlowFileQueue().setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
    }
    if (isNotNull(connectionDTO.getBackPressureObjectThreshold())) {
        connection.getFlowFileQueue().setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
    }
    if (isNotNull(connectionDTO.getBackPressureDataSizeThreshold())) {
        connection.getFlowFileQueue().setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
    }
    if (isNotNull(newPrioritizers)) {
        connection.getFlowFileQueue().setPriorities(newPrioritizers);
    }

    final String loadBalanceStrategyName = connectionDTO.getLoadBalanceStrategy();
    final String loadBalancePartitionAttribute = connectionDTO.getLoadBalancePartitionAttribute();
    if (isNotNull(loadBalanceStrategyName)) {
        final LoadBalanceStrategy loadBalanceStrategy = LoadBalanceStrategy.valueOf(loadBalanceStrategyName);
        connection.getFlowFileQueue().setLoadBalanceStrategy(loadBalanceStrategy, loadBalancePartitionAttribute);
    }

    final String loadBalanceCompressionName = connectionDTO.getLoadBalanceCompression();
    if (isNotNull(loadBalanceCompressionName)) {
        connection.getFlowFileQueue().setLoadBalanceCompression(LoadBalanceCompression.valueOf(loadBalanceCompressionName));
    }

    // update the connection state
    if (isNotNull(connectionDTO.getBends())) {
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : connectionDTO.getBends()) {
            if (bend != null) {
                bendPoints.add(new Position(bend.getX(), bend.getY()));
            }
        }
        connection.setBendPoints(bendPoints);
    }
    if (isNotNull(connectionDTO.getName())) {
        connection.setName(connectionDTO.getName());
    }
    if (isNotNull(connectionDTO.getLabelIndex())) {
        connection.setLabelIndex(connectionDTO.getLabelIndex());
    }
    if (isNotNull(connectionDTO.getzIndex())) {
        connection.setZIndex(connectionDTO.getzIndex());
    }
}
 
Example #29
Source File: StandardFlowFileQueue.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
public int compare(final FlowFileRecord f1, final FlowFileRecord f2) {
    int returnVal = 0;
    final boolean f1Penalized = f1.isPenalized();
    final boolean f2Penalized = f2.isPenalized();

    if (f1Penalized && !f2Penalized) {
        return 1;
    } else if (!f1Penalized && f2Penalized) {
        return -1;
    }

    if (f1Penalized && f2Penalized) {
        if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) {
            return -1;
        } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) {
            return 1;
        }
    }

    if (!prioritizers.isEmpty()) {
        for (final FlowFilePrioritizer prioritizer : prioritizers) {
            returnVal = prioritizer.compare(f1, f2);
            if (returnVal != 0) {
                return returnVal;
            }
        }
    }

    final ContentClaim claim1 = f1.getContentClaim();
    final ContentClaim claim2 = f2.getContentClaim();

    // put the one without a claim first
    if (claim1 == null && claim2 != null) {
        return -1;
    } else if (claim1 != null && claim2 == null) {
        return 1;
    } else if (claim1 != null && claim2 != null) {
        final int claimComparison = claim1.compareTo(claim2);
        if (claimComparison != 0) {
            return claimComparison;
        }

        final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset());
        if (claimOffsetComparison != 0) {
            return claimOffsetComparison;
        }
    }

    return Long.compare(f1.getId(), f2.getId());
}
 
Example #30
Source File: StandardFlowFileQueue.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
private Prioritizer(final List<FlowFilePrioritizer> priorities) {
    if (null != priorities) {
        prioritizers.addAll(priorities);
    }
}