org.apache.nifi.web.api.dto.ConnectableDTO Java Examples

/**
 * Determines the type of component the specified connectable is.
 */
private Component determineConnectableType(ConnectableDTO connectable) {
    Component component = Component.Controller;

    final String connectableType = connectable.getType();
    if (ConnectableType.PROCESSOR.name().equals(connectableType)) {
        component = Component.Processor;
    } else if (ConnectableType.INPUT_PORT.name().equals(connectableType)) {
        component = Component.InputPort;
    } else if (ConnectableType.OUTPUT_PORT.name().equals(connectableType)) {
        component = Component.OutputPort;
    } else if (ConnectableType.FUNNEL.name().equals(connectableType)) {
        component = Component.Funnel;
    } else {
        component = Component.RemoteProcessGroup;
    }

    return component;
}
 

public ConnectionEntity createConnection(final ConnectableDTO source, final ConnectableDTO destination, final Set<String> relationships) throws NiFiClientException, IOException {
    final String groupId = "OUTPUT_PORT".equals(source.getType()) ? destination.getGroupId() : source.getGroupId();

    final ConnectionDTO connectionDto = new ConnectionDTO();
    connectionDto.setSelectedRelationships(relationships);
    connectionDto.setDestination(destination);
    connectionDto.setSource(source);
    connectionDto.setParentGroupId(groupId);

    final ConnectionEntity connectionEntity = new ConnectionEntity();
    connectionEntity.setComponent(connectionDto);

    connectionEntity.setDestinationGroupId(destination.getGroupId());
    connectionEntity.setDestinationId(destination.getId());
    connectionEntity.setDestinationType(destination.getType());

    connectionEntity.setSourceGroupId(source.getGroupId());
    connectionEntity.setSourceId(source.getId());
    connectionEntity.setSourceType(source.getType());

    connectionEntity.setRevision(createNewRevision());

    return nifiClient.getConnectionClient().createConnection(groupId, connectionEntity);
}
 

/**
 * Determines the type of component the specified connectable is.
 */
private Component determineConnectableType(ConnectableDTO connectable) {
    Component component = Component.Controller;

    final String connectableType = connectable.getType();
    if (ConnectableType.PROCESSOR.name().equals(connectableType)) {
        component = Component.Processor;
    } else if (ConnectableType.INPUT_PORT.name().equals(connectableType)) {
        component = Component.InputPort;
    } else if (ConnectableType.OUTPUT_PORT.name().equals(connectableType)) {
        component = Component.OutputPort;
    } else if (ConnectableType.FUNNEL.name().equals(connectableType)) {
        component = Component.Funnel;
    } else {
        component = Component.RemoteProcessGroup;
    }

    return component;
}
 

public ConnectableDTO createConnectableDTO(final PortEntity port) {
    final ConnectableDTO dto = new ConnectableDTO();
    dto.setGroupId(port.getComponent().getParentGroupId());
    dto.setId(port.getId());
    dto.setName(port.getComponent().getName());
    dto.setRunning("RUNNING".equalsIgnoreCase(port.getComponent().getState()));
    dto.setType(port.getPortType());

    return dto;
}
 

public ConnectableDTO createConnectableDTO(final ProcessorEntity processor) {
    final ConnectableDTO dto = new ConnectableDTO();
    dto.setGroupId(processor.getComponent().getParentGroupId());
    dto.setId(processor.getId());
    dto.setName(processor.getComponent().getName());
    dto.setRunning("RUNNING".equalsIgnoreCase(processor.getComponent().getState()));
    dto.setType("PROCESSOR");

    return dto;
}
 

private void assertEquals(ConnectableDTO expected, ConnectableDTO actual) {
    if (expected == null && actual == null) {
        return;
    }

    Assert.assertEquals(expected.getGroupId(), actual.getGroupId());
    Assert.assertEquals(expected.getId(), actual.getId());
    Assert.assertEquals(expected.getName(), actual.getName());
    Assert.assertEquals(expected.getType(), actual.getType());
}
 

/**
 * Remove unnecessary fields in connectables prior to saving.
 *
 * @param connectable connectable
 */
private static void scrubConnectable(final ConnectableDTO connectable) {
    if (connectable != null) {
        connectable.setComments(null);
        connectable.setExists(null);
        connectable.setRunning(null);
        connectable.setTransmitting(null);
        connectable.setName(null);
    }
}
 

@Before
public void setup() {
    ConnectableDTO source = new ConnectableDTO();
    source.setId(testSourceId);

    ConnectableDTO destination = new ConnectableDTO();
    destination.setId(testDestinationId);

    dto = new ConnectionDTO();
    dto.setId(testId);
    dto.setName(testName);
    dto.setSource(source);
    dto.setSelectedRelationships(Arrays.asList(testSelectedRelationship).stream().collect(Collectors.toSet()));
    dto.setDestination(destination);
    dto.setBackPressureObjectThreshold(testMaxWorkQueueSize);
    dto.setBackPressureDataSizeThreshold(testMaxWorkQueueDataSize);
    dto.setFlowFileExpiration(testFlowfileExpiration);
    dto.setPrioritizers(Arrays.asList(testQueuePrioritizerClass));

    map = new HashMap<>();
    map.put(CommonPropertyKeys.ID_KEY, testId);
    map.put(CommonPropertyKeys.NAME_KEY, testName);
    map.put(ConnectionSchema.SOURCE_ID_KEY, testSourceId);
    map.put(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY, new ArrayList<>(Arrays.asList(testSelectedRelationship)));
    map.put(ConnectionSchema.DESTINATION_ID_KEY, testDestinationId);
    map.put(ConnectionSchema.MAX_WORK_QUEUE_SIZE_KEY, testMaxWorkQueueSize);
    map.put(ConnectionSchema.MAX_WORK_QUEUE_DATA_SIZE_KEY, testMaxWorkQueueDataSize);
    map.put(ConnectionSchema.FLOWFILE_EXPIRATION__KEY, testFlowfileExpiration);
    map.put(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, testQueuePrioritizerClass);
}
 

private static void setName(Map<String, String> connectableNameMap, ConnectableDTO connectableDTO, Map<String, String> nameOverrides) {
    if (connectableDTO != null) {
        final String name = connectableNameMap.get(connectableDTO.getId());
        if (name != null) {
            connectableDTO.setName(Optional.ofNullable(nameOverrides.get(connectableDTO.getId())).orElse(name));
        }
    }
}
 

private static String determineValueForConnectable(ConnectableDTO connectable, Map<String, String> idOverrideMap) {
    String connectionName = "";
    if (connectable != null) {
        connectionName = connectable.getName();
        // If no name is specified, determine the appropriate id to use, preferring any overrides specified
        if (StringUtils.isBlank(connectionName)) {
            connectionName = idOverrideMap.containsKey(connectable.getId()) ? idOverrideMap.get(connectable.getId()) : connectable.getId();
        }
    }
    return connectionName;
}
 

private void assertEquals(ConnectableDTO expected, ConnectableDTO actual) {
    if (expected == null && actual == null) {
        return;
    }

    Assert.assertEquals(expected.getGroupId(), actual.getGroupId());
    Assert.assertEquals(expected.getId(), actual.getId());
    Assert.assertEquals(expected.getName(), actual.getName());
    Assert.assertEquals(expected.getType(), actual.getType());
}
 

/**
 * Remove unnecessary fields in connectables prior to saving.
 *
 * @param connectable connectable
 */
private static void scrubConnectable(final ConnectableDTO connectable) {
    if (connectable != null) {
        connectable.setComments(null);
        connectable.setExists(null);
        connectable.setRunning(null);
        connectable.setTransmitting(null);
        connectable.setName(null);
    }
}
 

@Test
public void testNoSourceId() {
    dto.setSource(new ConnectableDTO());
    map.remove(ConnectionSchema.SOURCE_ID_KEY);
    assertDtoAndMapConstructorAreSame(1);
}
 

@Test
public void testNoDestinationName() {
    dto.setDestination(new ConnectableDTO());
    map.remove(ConnectionSchema.DESTINATION_ID_KEY);
    assertDtoAndMapConstructorAreSame(1);
}
 

/**
 * Validates the proposed processor configuration.
 */
private List<String> validateProposedConfiguration(final String groupId, final ConnectionDTO connectionDTO) {
    List<String> validationErrors = new ArrayList<>();

    if (isNotNull(connectionDTO.getBackPressureObjectThreshold()) && connectionDTO.getBackPressureObjectThreshold() < 0) {
        validationErrors.add("Max queue size must be a non-negative integer");
    }
    if (isNotNull(connectionDTO.getFlowFileExpiration())) {
        Matcher expirationMatcher = FormatUtils.TIME_DURATION_PATTERN.matcher(connectionDTO.getFlowFileExpiration());
        if (!expirationMatcher.matches()) {
            validationErrors.add("Flow file expiration is not a valid time duration (ie 30 sec, 5 min)");
        }
    }
    if (isNotNull(connectionDTO.getLabelIndex())) {
        if (connectionDTO.getLabelIndex() < 0) {
            validationErrors.add("The label index must be positive.");
        }
    }

    // validation is required when connecting to a remote process group since each node in a
    // cluster may or may not be authorized
    final ConnectableDTO proposedDestination = connectionDTO.getDestination();
    if (proposedDestination != null && ConnectableType.REMOTE_INPUT_PORT.name().equals(proposedDestination.getType())) {
        // the group id must be specified
        if (proposedDestination.getGroupId() == null) {
            validationErrors.add("When the destination is a remote input port its group id is required.");
            return validationErrors;
        }

        // attempt to location the proprosed destination
        final ProcessGroup destinationParentGroup = locateProcessGroup(flowController, groupId);
        final RemoteProcessGroup remoteProcessGroup = destinationParentGroup.getRemoteProcessGroup(proposedDestination.getGroupId());
        if (remoteProcessGroup == null) {
            validationErrors.add("Unable to find the specified remote process group.");
            return validationErrors;
        }

        // ensure the new destination was found
        final RemoteGroupPort remoteInputPort = remoteProcessGroup.getInputPort(proposedDestination.getId());
        if (remoteInputPort == null) {
            validationErrors.add("Unable to find the specified destination.");
            return validationErrors;
        }
    }

    return validationErrors;
}
 

private ConnectionEntity createConnection(final String name) throws Exception {
    String url = helper.getBaseUrl() + "/process-groups/root/connections";

    // get two processors
    final ProcessorEntity one = ITProcessorAccessControl.createProcessor(helper, "one");
    final ProcessorEntity two = ITProcessorAccessControl.createProcessor(helper, "two");

    // create the source connectable
    ConnectableDTO source = new ConnectableDTO();
    source.setId(one.getId());
    source.setGroupId(one.getComponent().getParentGroupId());
    source.setType(ConnectableType.PROCESSOR.name());

    // create the target connectable
    ConnectableDTO target = new ConnectableDTO();
    target.setId(two.getId());
    target.setGroupId(two.getComponent().getParentGroupId());
    target.setType(ConnectableType.PROCESSOR.name());

    // create the relationships
    Set<String> relationships = new HashSet<>();
    relationships.add("success");

    // create the connection
    ConnectionDTO connection = new ConnectionDTO();
    connection.setName(name);
    connection.setSource(source);
    connection.setDestination(target);
    connection.setSelectedRelationships(relationships);

    // create the revision
    final RevisionDTO revision = new RevisionDTO();
    revision.setClientId(READ_WRITE_CLIENT_ID);
    revision.setVersion(0L);

    // create the entity body
    ConnectionEntity entity = new ConnectionEntity();
    entity.setRevision(revision);
    entity.setComponent(connection);

    // perform the request
    Response response = helper.getReadWriteUser().testPost(url, entity);

    // ensure the request is successful
    assertEquals(201, response.getStatus());

    // get the entity body
    entity = response.readEntity(ConnectionEntity.class);

    // verify creation
    connection = entity.getComponent();
    assertEquals(name, connection.getName());

    // get the connection
    return entity;
}
 

public static ConnectionDTO getConnection(final Element element) {
    final ConnectionDTO dto = new ConnectionDTO();
    dto.setId(getString(element, "id"));
    dto.setName(getString(element, "name"));
    dto.setLabelIndex(getOptionalInt(element, "labelIndex"));
    dto.setzIndex(getOptionalLong(element, "zIndex"));
    dto.setVersionedComponentId(getString(element, "versionedComponentId"));

    final List<PositionDTO> bends = new ArrayList<>();
    final Element bendPointsElement = DomUtils.getChild(element, "bendPoints");
    if (bendPointsElement != null) {
        for (final Element bendPointElement : getChildrenByTagName(bendPointsElement, "bendPoint")) {
            final PositionDTO bend = getPosition(bendPointElement);
            bends.add(bend);
        }
    }
    dto.setBends(bends);

    final ConnectableDTO sourceConnectable = new ConnectableDTO();
    dto.setSource(sourceConnectable);
    sourceConnectable.setId(getString(element, "sourceId"));
    sourceConnectable.setGroupId(getString(element, "sourceGroupId"));
    sourceConnectable.setType(getString(element, "sourceType"));

    final ConnectableDTO destConnectable = new ConnectableDTO();
    dto.setDestination(destConnectable);
    destConnectable.setId(getString(element, "destinationId"));
    destConnectable.setGroupId(getString(element, "destinationGroupId"));
    destConnectable.setType(getString(element, "destinationType"));

    final Set<String> relationships = new HashSet<>();
    final List<Element> relationshipNodeList = getChildrenByTagName(element, "relationship");
    for (final Element relationshipElem : relationshipNodeList) {
        relationships.add(relationshipElem.getTextContent());
    }
    dto.setSelectedRelationships(relationships);

    dto.setBackPressureObjectThreshold(getLong(element, "maxWorkQueueSize"));

    final String maxDataSize = getString(element, "maxWorkQueueDataSize");
    if (maxDataSize != null && !maxDataSize.trim().isEmpty()) {
        dto.setBackPressureDataSizeThreshold(maxDataSize);
    }

    String expiration = getString(element, "flowFileExpiration");
    if (expiration == null) {
        expiration = "0 sec";
    }
    dto.setFlowFileExpiration(expiration);

    final List<String> prioritizerClasses = new ArrayList<>();
    final List<Element> prioritizerNodeList = getChildrenByTagName(element, "queuePrioritizerClass");
    for (final Element prioritizerElement : prioritizerNodeList) {
        prioritizerClasses.add(prioritizerElement.getTextContent().trim());
    }
    dto.setPrioritizers(prioritizerClasses);

    dto.setLoadBalanceStrategy(getString(element, "loadBalanceStrategy"));
    dto.setLoadBalancePartitionAttribute(getString(element, "partitioningAttribute"));
    dto.setLoadBalanceCompression(getString(element, "loadBalanceCompression"));

    return dto;
}
 

protected void testRPGBackToSelf(final SiteToSiteTransportProtocol protocol, final String portName) throws NiFiClientException, IOException, InterruptedException {
    final NiFiClientUtil util = getClientUtil();

    // Create a flow that is InputPort -> CountEvents
    final PortEntity port = util.createRemoteInputPort("root", portName);
    final ProcessorEntity count = getClientUtil().createProcessor("CountEvents");
    util.setAutoTerminatedRelationships(count, "success");

    // Create a flow that is GenerateFlowFile -> RPG, connected to the input port
    final ProcessorEntity generateFlowFile = getClientUtil().createProcessor("GenerateFlowFile");
    RemoteProcessGroupEntity rpg = getClientUtil().createRPG("root", protocol);

    util.updateProcessorProperties(generateFlowFile, Collections.singletonMap("File Size", "1 KB"));
    util.updateProcessorProperties(generateFlowFile, Collections.singletonMap("Batch Size", "3"));
    util.updateProcessorSchedulingPeriod(generateFlowFile, "10 min");

    final String rpgId = rpg.getId();

    // Wait for the port to become available. We have to check for the specific port ID because otherwise,
    // the RPG may have an old Port ID cached, since we are running an HTTP-based test and a RAW-based test
    waitFor(() -> {
        try {
            final RemoteProcessGroupEntity entity = getNifiClient().getRemoteProcessGroupClient().getRemoteProcessGroup(rpgId);
            final Set<RemoteProcessGroupPortDTO> ports = entity.getComponent().getContents().getInputPorts();
            if (ports.isEmpty()) {
                return false;
            }

            for (final RemoteProcessGroupPortDTO dto : ports) {
                if (dto.getTargetId().equals(port.getId())) {
                    return true;
                }
            }

            return false;
        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail("Could not retrieve RPG with ID " + rpgId);
            return false;
        }
    });

    rpg = getNifiClient().getRemoteProcessGroupClient().getRemoteProcessGroup(rpg.getId());
    final String rpgPortId = rpg.getComponent().getContents().getInputPorts().stream()
        .filter(dto -> dto.getTargetId().equals(port.getId()))
        .findFirst() // find the port with the desired ID
        .get() // get the Port
        .getId(); // get the Port's ID

    final ConnectableDTO destination = new ConnectableDTO();
    destination.setId(rpgPortId);
    destination.setGroupId(rpg.getId());
    destination.setType("REMOTE_INPUT_PORT");

    final ConnectionEntity generateToRPG = getClientUtil().createConnection(util.createConnectableDTO(generateFlowFile), destination, "success");
    final ConnectionEntity portToCount = getClientUtil().createConnection(util.createConnectableDTO(port), util.createConnectableDTO(count), "");

    getNifiClient().getInputPortClient().startInputPort(port);
    getNifiClient().getProcessorClient().startProcessor(generateFlowFile);
    getNifiClient().getRemoteProcessGroupClient().startTransmitting(rpg);

    waitFor(() -> util.getQueueSize(generateToRPG.getId()).getObjectCount() == 0);
    waitFor(() -> util.getQueueSize(portToCount.getId()).getObjectCount() == 3 * getNumberOfNodes());
}
 

public ConnectionEntity createConnection(final ConnectableDTO source, final ConnectableDTO destination, final String relationship) throws NiFiClientException, IOException {
    return createConnection(source, destination, Collections.singleton(relationship));
}
 

public static ConnectionDTO getConnection(final Element element) {
    final ConnectionDTO dto = new ConnectionDTO();
    dto.setId(getString(element, "id"));
    dto.setName(getString(element, "name"));
    dto.setLabelIndex(getOptionalInt(element, "labelIndex"));
    dto.setzIndex(getOptionalLong(element, "zIndex"));

    final List<PositionDTO> bends = new ArrayList<>();
    final Element bendPointsElement = DomUtils.getChild(element, "bendPoints");
    if (bendPointsElement != null) {
        for (final Element bendPointElement : getChildrenByTagName(bendPointsElement, "bendPoint")) {
            final PositionDTO bend = getPosition(bendPointElement);
            bends.add(bend);
        }
    }
    dto.setBends(bends);

    final ConnectableDTO sourceConnectable = new ConnectableDTO();
    dto.setSource(sourceConnectable);
    sourceConnectable.setId(getString(element, "sourceId"));
    sourceConnectable.setGroupId(getString(element, "sourceGroupId"));
    sourceConnectable.setType(getString(element, "sourceType"));

    final ConnectableDTO destConnectable = new ConnectableDTO();
    dto.setDestination(destConnectable);
    destConnectable.setId(getString(element, "destinationId"));
    destConnectable.setGroupId(getString(element, "destinationGroupId"));
    destConnectable.setType(getString(element, "destinationType"));

    final Set<String> relationships = new HashSet<>();
    final List<Element> relationshipNodeList = getChildrenByTagName(element, "relationship");
    for (final Element relationshipElem : relationshipNodeList) {
        relationships.add(relationshipElem.getTextContent());
    }
    dto.setSelectedRelationships(relationships);

    dto.setBackPressureObjectThreshold(getLong(element, "maxWorkQueueSize"));

    final String maxDataSize = getString(element, "maxWorkQueueDataSize");
    if (maxDataSize != null && !maxDataSize.trim().isEmpty()) {
        dto.setBackPressureDataSizeThreshold(maxDataSize);
    }

    String expiration = getString(element, "flowFileExpiration");
    if (expiration == null) {
        expiration = "0 sec";
    }
    dto.setFlowFileExpiration(expiration);

    final List<String> prioritizerClasses = new ArrayList<>();
    final List<Element> prioritizerNodeList = getChildrenByTagName(element, "queuePrioritizerClass");
    for (final Element prioritizerElement : prioritizerNodeList) {
        prioritizerClasses.add(prioritizerElement.getTextContent().trim());
    }
    dto.setPrioritizers(prioritizerClasses);

    return dto;
}
 

private ConnectionEntity createConnection(final String name) throws Exception {
    String url = helper.getBaseUrl() + "/process-groups/root/connections";

    // get two processors
    final ProcessorEntity one = ITProcessorAccessControl.createProcessor(helper, "one");
    final ProcessorEntity two = ITProcessorAccessControl.createProcessor(helper, "two");

    // create the source connectable
    ConnectableDTO source = new ConnectableDTO();
    source.setId(one.getId());
    source.setType(ConnectableType.PROCESSOR.name());

    // create the target connectable
    ConnectableDTO target = new ConnectableDTO();
    target.setId(two.getId());
    target.setType(ConnectableType.PROCESSOR.name());

    // create the relationships
    Set<String> relationships = new HashSet<>();
    relationships.add("success");

    // create the connection
    ConnectionDTO connection = new ConnectionDTO();
    connection.setName(name);
    connection.setSource(source);
    connection.setDestination(target);
    connection.setSelectedRelationships(relationships);

    // create the revision
    final RevisionDTO revision = new RevisionDTO();
    revision.setClientId(READ_WRITE_CLIENT_ID);
    revision.setVersion(0L);

    // create the entity body
    ConnectionEntity entity = new ConnectionEntity();
    entity.setRevision(revision);
    entity.setComponent(connection);

    // perform the request
    ClientResponse response = helper.getReadWriteUser().testPost(url, entity);

    // ensure the request is successful
    assertEquals(201, response.getStatus());

    // get the entity body
    entity = response.getEntity(ConnectionEntity.class);

    // verify creation
    connection = entity.getComponent();
    assertEquals(name, connection.getName());

    // get the connection
    return entity;
}
 

/**
 * Validates the proposed processor configuration.
 */
private List<String> validateProposedConfiguration(final String groupId, final ConnectionDTO connectionDTO) {
    List<String> validationErrors = new ArrayList<>();

    if (isNotNull(connectionDTO.getBackPressureObjectThreshold()) && connectionDTO.getBackPressureObjectThreshold() < 0) {
        validationErrors.add("Max queue size must be a non-negative integer");
    }
    if (isNotNull(connectionDTO.getFlowFileExpiration())) {
        Matcher expirationMatcher = FormatUtils.TIME_DURATION_PATTERN.matcher(connectionDTO.getFlowFileExpiration());
        if (!expirationMatcher.matches()) {
            validationErrors.add("Flow file expiration is not a valid time duration (ie 30 sec, 5 min)");
        }
    }
    if (isNotNull(connectionDTO.getLabelIndex())) {
        if (connectionDTO.getLabelIndex() < 0) {
            validationErrors.add("The label index must be positive.");
        }
    }

    // validation is required when connecting to a remote process group since each node in a
    // cluster may or may not be authorized
    final ConnectableDTO proposedDestination = connectionDTO.getDestination();
    if (proposedDestination != null && ConnectableType.REMOTE_INPUT_PORT.name().equals(proposedDestination.getType())) {
        // the group id must be specified
        if (proposedDestination.getGroupId() == null) {
            validationErrors.add("When the destination is a remote input port its group id is required.");
            return validationErrors;
        }

        // attempt to location the proprosed destination
        final ProcessGroup destinationParentGroup = locateProcessGroup(flowController, groupId);
        final RemoteProcessGroup remoteProcessGroup = destinationParentGroup.getRemoteProcessGroup(proposedDestination.getGroupId());
        if (remoteProcessGroup == null) {
            validationErrors.add("Unable to find the specified remote process group.");
            return validationErrors;
        }

        // ensure the new destination was found
        final RemoteGroupPort remoteInputPort = remoteProcessGroup.getInputPort(proposedDestination.getId());
        if (remoteInputPort == null) {
            validationErrors.add("Unable to find the specified destination.");
            return validationErrors;
        }
    }

    return validationErrors;
}