Java Code Examples for org.apache.nifi.provenance.ProvenanceEventRecord#getComponentId()

The following examples show how to use org.apache.nifi.provenance.ProvenanceEventRecord#getComponentId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PutHiveStreaming.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    if (event.getTransitUri() == null) {
        return null;
    }

    final URI uri = parseUri(event.getTransitUri());
    final String namespace = context.getNamespaceResolver().fromHostNames(uri.getHost());
    final Set<Tuple<String, String>> outputTables = parseTableNames(null, event.getAttribute(ATTR_OUTPUT_TABLES));
    if (outputTables.isEmpty()) {
        return null;
    }

    final DataSetRefs refs = new DataSetRefs(event.getComponentId());
    outputTables.forEach(tableName -> {
        final Referenceable ref = createTableRef(namespace, tableName);
        refs.addOutput(ref);
    });
    return refs;
}
 
Example 2
Source File: NiFiRootGroupPort.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    if (!ProvenanceEventType.SEND.equals(event.getEventType())
            && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) {
        return null;
    }

    final boolean isInputPort = event.getComponentType().equals("Input Port");
    final String type = isInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT;
    final String rootPortId = event.getComponentId();

    final S2SPort s2SPort = analyzeS2SPort(event, context.getNamespaceResolver());

    // Find connections connecting to/from the remote port.
    final List<ConnectionStatus> connections = isInputPort
            ? context.findConnectionFrom(rootPortId)
            : context.findConnectionTo(rootPortId);
    if (connections == null || connections.isEmpty()) {
        logger.warn("Connection was not found: {}", new Object[]{event});
        return null;
    }

    // The name of the port can be retrieved from any connection, use the first one.
    final ConnectionStatus connection = connections.get(0);
    final Referenceable ref = new Referenceable(type);
    ref.set(ATTR_NAME, isInputPort ? connection.getSourceName() : connection.getDestinationName());
    ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.namespace, rootPortId));

    return singleDataSetRef(event.getComponentId(), event.getEventType(), ref);
}
 
Example 3
Source File: Create.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    // Check if this component is a processor that generates data.
    final String componentId = event.getComponentId();
    final List<ConnectionStatus> incomingConnections = context.findConnectionTo(componentId);
    if (incomingConnections != null && !incomingConnections.isEmpty()) {
        return null;
    }

    return super.analyze(context, event);
}
 
Example 4
Source File: UnknownOutput.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    final String componentId = event.getComponentId();
    final DataSetRefs refs = new DataSetRefs(componentId);
    final Referenceable ref = createDataSetRef(context, event);
    refs.addOutput(ref);

    return refs;
}
 
Example 5
Source File: UnknownInput.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    final String componentId = event.getComponentId();
    final DataSetRefs refs = new DataSetRefs(componentId);
    final Referenceable ref = createDataSetRef(context, event);
    refs.addInput(ref);

    return refs;
}
 
Example 6
Source File: NiFiRemotePort.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    if (!ProvenanceEventType.SEND.equals(event.getEventType())
            && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) {
        return null;
    }

    final boolean isRemoteInputPort = event.getComponentType().equals("Remote Input Port");
    final String type = isRemoteInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT;

    final S2SPort s2SPort = analyzeS2SPort(event, context.getNamespaceResolver());

    // Find connections that connects to/from the remote port.
    final String componentId = event.getComponentId();
    final List<ConnectionStatus> connections = isRemoteInputPort
            ? context.findConnectionTo(componentId)
            : context.findConnectionFrom(componentId);
    if (connections == null || connections.isEmpty()) {
        logger.warn("Connection was not found: {}", new Object[]{event});
        return null;
    }

    // The name of remote port can be retrieved from any connection, use the first one.
    final ConnectionStatus connection = connections.get(0);
    final Referenceable ref = new Referenceable(type);
    ref.set(ATTR_NAME, isRemoteInputPort ? connection.getDestinationName() : connection.getSourceName());
    ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.namespace, s2SPort.targetPortId));

    return singleDataSetRef(event.getComponentId(), event.getEventType(), ref);
}
 
Example 7
Source File: LatestEventsPerProcessorQuery.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void update(final ProvenanceEventRecord event, final StorageSummary storageSummary) {
    final String componentId = event.getComponentId();
    final RingBuffer<Long> ringBuffer = latestRecords.computeIfAbsent(componentId, id -> new RingBuffer<>(1000));
    ringBuffer.add(storageSummary.getEventId());
}
 
Example 8
Source File: LatestEventsPerProcessorQuery.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void update(final ProvenanceEventRecord event, final StorageSummary storageSummary) {
    final String componentId = event.getComponentId();
    final RingBuffer<Long> ringBuffer = latestRecords.computeIfAbsent(componentId, id -> new RingBuffer<>(1000));
    ringBuffer.add(storageSummary.getEventId());
}
 
Example 9
Source File: NiFiRootGroupPort.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
protected String getRawProtocolPortId(ProvenanceEventRecord event) {
    return event.getComponentId();
}
 
Example 10
Source File: Hive2JDBC.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    // Replace the colon so that the schema in the URI can be parsed correctly.
    final String transitUri = event.getTransitUri();
    if (transitUri == null) {
        return null;
    }

    final Matcher uriMatcher = URI_PATTERN.matcher(transitUri);
    if (!uriMatcher.matches()) {
        logger.warn("Unexpected transit URI: {}", new Object[]{transitUri});
        return null;
    }

    final String namespace = context.getNamespaceResolver().fromHostNames(splitHostNames(uriMatcher.group(1)));
    String connectedDatabaseName = null;
    if (uriMatcher.groupCount() > 1) {
        // Try to find connected database name from connection parameters.
        final String[] connectionParams = uriMatcher.group(2).split(";");
        connectedDatabaseName = connectionParams[0];
    }

    if (StringUtils.isEmpty(connectedDatabaseName)) {
        // If not found, then use "default".
        connectedDatabaseName = "default";
    }

    final Set<Tuple<String, String>> inputTables = parseTableNames(connectedDatabaseName, event.getAttribute(ATTR_INPUT_TABLES));
    final Set<Tuple<String, String>> outputTables = parseTableNames(connectedDatabaseName, event.getAttribute(ATTR_OUTPUT_TABLES));

    if (inputTables.isEmpty() && outputTables.isEmpty()) {
        // If input/output tables are unknown, create database level lineage.
        // Handle case insensitivity of database and table names in Hive: send names uniformly in lower case
        return getDatabaseRef(event.getComponentId(), event.getEventType(), namespace, connectedDatabaseName.toLowerCase());
    }

    final DataSetRefs refs = new DataSetRefs(event.getComponentId());
    addRefs(refs, true, namespace, inputTables);
    addRefs(refs, false, namespace, outputTables);
    return refs;
}