Java Code Examples for org.apache.nifi.util.Tuple

The following examples show how to use org.apache.nifi.util.Tuple. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: localization_nifi   Source File: FetchFileTransfer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Close connections that are idle or optionally close all connections.
 * Connections are considered "idle" if they have not been used in 10 seconds.
 *
 * @param closeNonIdleConnections if <code>true</code> will close all connection; if <code>false</code> will close only idle connections
 */
private void closeConnections(final boolean closeNonIdleConnections) {
    for (final Map.Entry<Tuple<String, Integer>, BlockingQueue<FileTransferIdleWrapper>> entry : fileTransferMap.entrySet()) {
        final BlockingQueue<FileTransferIdleWrapper> wrapperQueue = entry.getValue();

        final List<FileTransferIdleWrapper> putBack = new ArrayList<>();
        FileTransferIdleWrapper wrapper;
        while ((wrapper = wrapperQueue.poll()) != null) {
            final long lastUsed = wrapper.getLastUsed();
            final long nanosSinceLastUse = System.nanoTime() - lastUsed;
            if (!closeNonIdleConnections && TimeUnit.NANOSECONDS.toMillis(nanosSinceLastUse) < IDLE_CONNECTION_MILLIS) {
                putBack.add(wrapper);
            } else {
                try {
                    wrapper.getFileTransfer().close();
                } catch (final IOException ioe) {
                    getLogger().warn("Failed to close Idle Connection due to {}", new Object[] {ioe}, ioe);
                }
            }
        }

        for (final FileTransferIdleWrapper toPutBack : putBack) {
            wrapperQueue.offer(toPutBack);
        }
    }
}
 
Example 2
Source Project: localization_nifi   Source File: TransformXml.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ValidationResult validate(final String subject, final String input, final ValidationContext validationContext) {
    final Tuple<String, ValidationResult> lastResult = this.cachedResult;
    if (lastResult != null && lastResult.getKey().equals(input)) {
        return lastResult.getValue();
    } else {
        String error = null;
        final File stylesheet = new File(input);
        final TransformerFactory tFactory = new net.sf.saxon.TransformerFactoryImpl();
        final StreamSource styleSource = new StreamSource(stylesheet);

        try {
            tFactory.newTransformer(styleSource);
        } catch (final Exception e) {
            error = e.toString();
        }

        this.cachedResult = new Tuple<>(input, new ValidationResult.Builder()
                .input(input)
                .subject(subject)
                .valid(error == null)
                .explanation(error)
                .build());
        return this.cachedResult.getValue();
    }
}
 
Example 3
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 6 votes vote down vote up
public Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> getModelMap( String predictionType, Double score,
                                                                            Double targetPrediction, Double variablePrediction){
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = new HashMap<>();
    StatusAnalyticsModel model = Mockito.mock(StatusAnalyticsModel.class);
    StatusMetricExtractFunction extractFunction = Mockito.mock(StatusMetricExtractFunction.class);
    Tuple<StatusAnalyticsModel,StatusMetricExtractFunction> modelTuple = new Tuple<>(model,extractFunction);
    modelMap.put(predictionType,modelTuple);
    Map<String,Double> scores = new HashMap<>();
    scores.put("rSquared",score);

    Double[][] features = new Double[1][1];
    Double[] target = new Double[1];

    when(extractFunction.extractMetric(anyString(),any(StatusHistory.class))).then(new Answer<Tuple<Stream<Double[]>,Stream<Double>>>() {
        @Override
        public Tuple<Stream<Double[]>, Stream<Double>> answer(InvocationOnMock invocationOnMock) throws Throwable {
            return new Tuple<>(Stream.of(features), Stream.of(target));
        }
    });

    when(model.getScores()).thenReturn(scores);
    when(model.predict(any(Double[].class))).thenReturn(targetPrediction);
    when(model.predictVariable(anyInt(),any(),any())).thenReturn(variablePrediction);
    return modelMap;

}
 
Example 4
Source Project: nifi   Source File: EventFileManager.java    License: Apache License 2.0 6 votes vote down vote up
public void releaseWriteLock(final File file) {
    final String key = getMapKey(file);

    boolean updated = false;
    while (!updated) {
        final Tuple<ReadWriteLock, Integer> tuple = lockMap.get(key);
        if (tuple == null) {
            throw new IllegalMonitorStateException("Lock is not owned");
        }

        // If this is the only reference to the lock, remove it from the map and then unlock.
        if (tuple.getValue() <= 1) {
            updated = lockMap.remove(key, tuple);
            if (updated) {
                tuple.getKey().writeLock().unlock();
            }
        } else {
            final Tuple<ReadWriteLock, Integer> updatedTuple = new Tuple<>(tuple.getKey(), tuple.getValue() - 1);
            updated = lockMap.replace(key, tuple, updatedTuple);
            if (updated) {
                tuple.getKey().writeLock().unlock();
            }
        }
    }
}
 
Example 5
Source Project: nifi   Source File: TestLuceneEventIndex.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTimeRange() {
    final long now = System.currentTimeMillis();

    final List<File> indexFiles = new ArrayList<>();
    indexFiles.add(new File("index-1000"));
    indexFiles.add(new File("lucene-8-index-3000"));
    indexFiles.add(new File("index-4000"));
    indexFiles.add(new File("index-5000"));
    indexFiles.add(new File("lucene-8-index-6000"));
    indexFiles.add(new File("index-7000"));

    assertEquals(new Tuple<>(1000L, 3000L), LuceneEventIndex.getTimeRange(new File("index-1000"), indexFiles));

    assertEquals(new Tuple<>(3000L, 4000L), LuceneEventIndex.getTimeRange(new File("lucene-8-index-3000"), indexFiles));
    assertEquals(new Tuple<>(4000L, 5000L), LuceneEventIndex.getTimeRange(new File("index-4000"), indexFiles));
    assertEquals(new Tuple<>(5000L, 6000L), LuceneEventIndex.getTimeRange(new File("index-5000"), indexFiles));
    assertEquals(new Tuple<>(6000L, 7000L), LuceneEventIndex.getTimeRange(new File("lucene-8-index-6000"), indexFiles));

    assertEquals(7000L, LuceneEventIndex.getTimeRange(new File("index-7000"), indexFiles).getKey().longValue());
    assertTrue(LuceneEventIndex.getTimeRange(new File("index-7000"), indexFiles).getValue() >= now);

}
 
Example 6
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetScores() {
    Date now = new Date();
    Long tomorrowMillis = DateUtils.addDays(now,1).toInstant().toEpochMilli();
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> bytesModelMap = getModelMap("queuedBytes",.9,10000000.0,tomorrowMillis.doubleValue());
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> countModelMap = getModelMap("queuedCount",.9,50.0,tomorrowMillis.doubleValue());
    countModelMap.putAll(bytesModelMap);
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(countModelMap);
    connectionStatusAnalytics.loadPredictions(repositoryStatusReport);
    Map<String,Long> scores = connectionStatusAnalytics.getPredictions();
    assertNotNull(scores);
    assertFalse(scores.isEmpty());
    assertTrue(scores.get("nextIntervalPercentageUseCount").equals(50L));
    assertTrue(scores.get("nextIntervalBytes").equals(10000000L));
    assertTrue(scores.get("timeToBytesBackpressureMillis") > 0);
    assertTrue(scores.get("nextIntervalCount").equals(50L));
    assertTrue(scores.get("nextIntervalPercentageUseBytes").equals(10L));
    assertTrue(scores.get("intervalTimeMillis").equals(180000L));
    assertTrue(scores.get("timeToCountBackpressureMillis") > 0);
}
 
Example 7
Source Project: nifi   Source File: PutHiveStreaming.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    if (event.getTransitUri() == null) {
        return null;
    }

    final URI uri = parseUri(event.getTransitUri());
    final String namespace = context.getNamespaceResolver().fromHostNames(uri.getHost());
    final Set<Tuple<String, String>> outputTables = parseTableNames(null, event.getAttribute(ATTR_OUTPUT_TABLES));
    if (outputTables.isEmpty()) {
        return null;
    }

    final DataSetRefs refs = new DataSetRefs(event.getComponentId());
    outputTables.forEach(tableName -> {
        final Referenceable ref = createTableRef(namespace, tableName);
        refs.addOutput(ref);
    });
    return refs;
}
 
Example 8
Source Project: nifi   Source File: NiFiAtlasClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the flow components of type {@code componentType} from Atlas server.
 * Deleted components will be filtered out before calling Atlas.
 * Atlas object ids will be initialized with all the attributes (guid, type, unique attributes) in order to be able
 * to match ids retrieved from Atlas (having guid) and ids created by the reporting task (not having guid yet).
 *
 * @param componentType Atlas type of the flow component (nifi_flow_path, nifi_queue, nifi_input_port, nifi_output_port)
 * @param referredEntities referred entities of the flow entity (returned when the flow fetched) containing the basic data (id, status) of the flow components
 * @return flow component entities mapped to their object ids
 */
private Map<AtlasObjectId, AtlasEntity> fetchFlowComponents(String componentType, Map<String, AtlasEntity> referredEntities) {
    return referredEntities.values().stream()
            .filter(referredEntity -> referredEntity.getTypeName().equals(componentType))
            .filter(referredEntity -> referredEntity.getStatus() == AtlasEntity.Status.ACTIVE)
            .map(referredEntity -> {
                final Map<String, Object> uniqueAttributes = Collections.singletonMap(ATTR_QUALIFIED_NAME, referredEntity.getAttribute(ATTR_QUALIFIED_NAME));
                final AtlasObjectId id = new AtlasObjectId(referredEntity.getGuid(), componentType, uniqueAttributes);
                try {
                    final AtlasEntity.AtlasEntityWithExtInfo fetchedEntityExt = searchEntityDef(id);
                    return new Tuple<>(id, fetchedEntityExt.getEntity());
                } catch (AtlasServiceException e) {
                    logger.warn("Failed to search entity by id {}, due to {}", id, e);
                    return null;
                }
            })
            .filter(Objects::nonNull)
            .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue));
}
 
Example 9
Source Project: nifi   Source File: NiFiFlow.java    License: Apache License 2.0 6 votes vote down vote up
public Tuple<AtlasObjectId, AtlasEntity> getOrCreateQueue(String destinationComponentId) {
    final String qualifiedName = toQualifiedName(destinationComponentId);
    final Optional<AtlasObjectId> existingQueueId = findIdByQualifiedName(queues.keySet(), qualifiedName);

    if (existingQueueId.isPresent()) {
        final AtlasEntity entity = queues.get(existingQueueId.get());
        stillExistingEntityGuids.add(entity.getGuid());
        return new Tuple<>(existingQueueId.get(), entity);
    } else {
        final AtlasObjectId queueId = new AtlasObjectId(TYPE_NIFI_QUEUE, ATTR_QUALIFIED_NAME, qualifiedName);
        final AtlasEntity queue = new AtlasEntity(TYPE_NIFI_QUEUE);
        queue.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId());
        queue.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName);
        queue.setAttribute(ATTR_NAME, "queue");
        queue.setAttribute(ATTR_DESCRIPTION, "Input queue for " + destinationComponentId);
        queues.put(queueId, queue);
        return new Tuple<>(queueId, queue);
    }
}
 
Example 10
Source Project: nifi   Source File: NiFiFlow.java    License: Apache License 2.0 6 votes vote down vote up
public Map<EntityChangeType, List<AtlasEntity>> getChangedFlowPathEntities() {
    // Convert NiFiFlowPath to AtlasEntity.
    final HashMap<EntityChangeType, List<AtlasEntity>> changedPaths = flowPaths.values().stream()
            .map(path -> {
                final EntityChangeType changeType = getFlowPathChangeType(path);
                switch (changeType) {
                    case CREATED:
                    case UPDATED:
                    case AS_IS:
                        return toAtlasEntity(changeType, path);
                    default:
                        return new Tuple<>(changeType, path.getExEntity());
                }
            }).collect(Collectors.groupingBy(Tuple::getKey, HashMap::new, Collectors.mapping(Tuple::getValue, Collectors.toList())));

    updateAudit.add("CREATED NiFiFlowPath=" + changedPaths.get(EntityChangeType.CREATED));
    updateAudit.add("UPDATED NiFiFlowPath=" + changedPaths.get(EntityChangeType.UPDATED));
    updateAudit.add("DELETED NiFiFlowPath=" + changedPaths.get(EntityChangeType.DELETED));
    return changedPaths;
}
 
Example 11
Source Project: nifi   Source File: TestPutKudu.java    License: Apache License 2.0 6 votes vote down vote up
private LinkedList<OperationResponse> queueInsert(MockPutKudu putKudu, KuduSession session, boolean sync, ResultCode... results) throws Exception {
    LinkedList<OperationResponse> responses = new LinkedList<>();
    for (ResultCode result : results) {
        boolean ok = result == OK;
        Tuple<Insert, OperationResponse> tuple = insert(ok);
        putKudu.queue(tuple.getKey());

        if (result == EXCEPTION) {
            when(session.apply(tuple.getKey())).thenThrow(mock(KuduException.class));
            // Stop processing the rest of the records on the first exception
            break;
        } else {
            responses.add(tuple.getValue());
            if (sync) {
                when(session.apply(tuple.getKey())).thenReturn(ok ? null : tuple.getValue());

                // In AUTO_FLUSH_SYNC mode, PutKudu immediately knows when an operation has failed.
                // In that case, it does not process the rest of the records in the FlowFile.
                if (result == FAIL) break;
            }
        }
    }
    return responses;
}
 
Example 12
Source Project: nifi   Source File: LookupRecord.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected Tuple<Map<String, RecordPath>, RecordPath> getFlowFileContext(final FlowFile flowFile, final ProcessContext context) {
    final Map<String, RecordPath> recordPaths = new HashMap<>();
    for (final PropertyDescriptor prop : context.getProperties().keySet()) {
        if (!prop.isDynamic()) {
            continue;
        }

        final String pathText = context.getProperty(prop).evaluateAttributeExpressions(flowFile).getValue();
        final RecordPath lookupRecordPath = recordPathCache.getCompiled(pathText);
        recordPaths.put(prop.getName(), lookupRecordPath);
    }

    final RecordPath resultRecordPath;
    if (context.getProperty(RESULT_RECORD_PATH).isSet()) {
        final String resultPathText = context.getProperty(RESULT_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
        resultRecordPath = recordPathCache.getCompiled(resultPathText);
    } else {
        resultRecordPath = null;
    }

    return new Tuple<>(recordPaths, resultRecordPath);
}
 
Example 13
Source Project: nifi   Source File: ListFile.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void purgeTimingInfo(final long cutoff) {
    logger.debug("Purging any entries from Performance Tracker that is older than {}", new Object[] {new Date(cutoff)});
    final Iterator<Map.Entry<Tuple<String, String>, TimingInfo>> itr = directoryToTimingInfo.entrySet().iterator();

    int purgedCount = 0;
    long earliestTimestamp = System.currentTimeMillis();
    while (itr.hasNext()) {
        final Map.Entry<Tuple<String, String>, TimingInfo> entry = itr.next();
        final TimingInfo timingInfo = entry.getValue();
        final long creationTime = timingInfo.getCreationTimestamp();

        if (creationTime < cutoff) {
            itr.remove();
            purgedCount++;

            directoryCanonicalization.remove(entry.getKey().getKey());
        } else {
            earliestTimestamp = Math.min(earliestTimestamp, creationTime);
        }
    }

    this.earliestTimestamp = earliestTimestamp;
    logger.debug("Purged {} entries from Performance Tracker; now holding {} entries", new Object[] {purgedCount, directoryToTimingInfo.size()});
}
 
Example 14
Source Project: nifi   Source File: AbstractRouteRecord.java    License: Apache License 2.0 6 votes vote down vote up
private void writeRecord(final Record record, final Relationship relationship, final Map<Relationship, Tuple<FlowFile, RecordSetWriter>> writers, final ProcessSession session,
                         final FlowFile original, final Map<String, String> originalAttributes, final RecordSetWriterFactory writerFactory) throws IOException, SchemaNotFoundException {
    final RecordSetWriter recordSetWriter;
    Tuple<FlowFile, RecordSetWriter> tuple = writers.get(relationship);

    if (tuple == null) {
        final FlowFile outFlowFile = session.create(original);
        final OutputStream out = session.write(outFlowFile);

        final RecordSchema recordWriteSchema = writerFactory.getSchema(originalAttributes, record.getSchema());
        recordSetWriter = writerFactory.createWriter(getLogger(), recordWriteSchema, out, outFlowFile);
        recordSetWriter.beginRecordSet();

        tuple = new Tuple<>(outFlowFile, recordSetWriter);
        writers.put(relationship, tuple);
    } else {
        recordSetWriter = tuple.getValue();
    }

    recordSetWriter.write(record);
}
 
Example 15
Source Project: nifi   Source File: FetchFileTransfer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Close connections that are idle or optionally close all connections.
 * Connections are considered "idle" if they have not been used in 10 seconds.
 *
 * @param closeNonIdleConnections if <code>true</code> will close all connection; if <code>false</code> will close only idle connections
 */
private void closeConnections(final boolean closeNonIdleConnections) {
    for (final Map.Entry<Tuple<String, Integer>, BlockingQueue<FileTransferIdleWrapper>> entry : fileTransferMap.entrySet()) {
        final BlockingQueue<FileTransferIdleWrapper> wrapperQueue = entry.getValue();

        final List<FileTransferIdleWrapper> putBack = new ArrayList<>();
        FileTransferIdleWrapper wrapper;
        while ((wrapper = wrapperQueue.poll()) != null) {
            final long lastUsed = wrapper.getLastUsed();
            final long nanosSinceLastUse = System.nanoTime() - lastUsed;
            if (!closeNonIdleConnections && TimeUnit.NANOSECONDS.toMillis(nanosSinceLastUse) < IDLE_CONNECTION_MILLIS) {
                putBack.add(wrapper);
            } else {
                try {
                    wrapper.getFileTransfer().close();
                } catch (final IOException ioe) {
                    getLogger().warn("Failed to close Idle Connection due to {}", new Object[] {ioe}, ioe);
                }
            }
        }

        for (final FileTransferIdleWrapper toPutBack : putBack) {
            wrapperQueue.offer(toPutBack);
        }
    }
}
 
Example 16
Source Project: nifi   Source File: TransformXml.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ValidationResult validate(final String subject, final String input, final ValidationContext validationContext) {
    final Tuple<String, ValidationResult> lastResult = this.cachedResult;
    if (lastResult != null && lastResult.getKey().equals(input)) {
        return lastResult.getValue();
    } else {
        String error = null;
        final File stylesheet = new File(input);
        final TransformerFactory tFactory = new net.sf.saxon.TransformerFactoryImpl();
        final StreamSource styleSource = new StreamSource(stylesheet);

        try {
            tFactory.newTransformer(styleSource);
        } catch (final Exception e) {
            error = e.toString();
        }

        this.cachedResult = new Tuple<>(input, new ValidationResult.Builder()
                .input(input)
                .subject(subject)
                .valid(error == null)
                .explanation(error)
                .build());
        return this.cachedResult.getValue();
    }
}
 
Example 17
Source Project: nifi   Source File: ProcessGroupStatusEnumerator.java    License: Apache License 2.0 6 votes vote down vote up
private Tuple<ProcessGroupStatus, String> getNextProcessGroupStatus() {
    Tuple<Iterator<ProcessGroupStatus>, String> top = iteratorBreadcrumb.peek();
    if (top == null) {
        return null;
    }
    Iterator<ProcessGroupStatus> i = top.getKey();
    String parentId = top.getValue();

    if (i.hasNext()) {
        ProcessGroupStatus nextPG = i.next();
        iteratorBreadcrumb.push(new Tuple<>(nextPG.getProcessGroupStatus().iterator(), nextPG.getId()));
        return new Tuple<>(nextPG, parentId);
    } else {
        // No more child PGs, remove it from the breadcrumb trail and try again
        Tuple<Iterator<ProcessGroupStatus>, String> pop = iteratorBreadcrumb.pop();
        return getNextProcessGroupStatus();
    }
}
 
Example 18
Source Project: nifi   Source File: TestMetricsEventReportingTask.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConnectionStatusTable() throws IOException, InitializationException {
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(QueryMetricsUtil.QUERY, "select connectionId, predictedQueuedCount, predictedTimeToBytesBackpressureMillis from CONNECTION_STATUS_PREDICTIONS");
    reportingTask = initTask(properties);
    reportingTask.onTrigger(context);
    List<Map<String,Object>> metricsList = actionHandler.getRows();
    List<Tuple<String, Action>> defaultLogActions = actionHandler.getDefaultActionsByType("LOG");
    List<Tuple<String, Action>> defaultAlertActions = actionHandler.getDefaultActionsByType("ALERT");
    List<PropertyContext> propertyContexts = actionHandler.getPropertyContexts();
    assertFalse(metricsList.isEmpty());
    assertEquals(2,defaultLogActions.size());
    assertEquals(2,defaultAlertActions.size());
    assertEquals(4,propertyContexts.size());

}
 
Example 19
Source Project: nifi   Source File: FlowRegistryUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static Set<ConfigurableComponent> getRestrictedComponents(final VersionedProcessGroup group, final NiFiServiceFacade serviceFacade) {
    final Set<ConfigurableComponent> restrictedComponents = new HashSet<>();

    final Set<Tuple<String, BundleCoordinate>> componentTypes = new HashSet<>();
    populateComponentTypes(group, componentTypes);

    for (final Tuple<String, BundleCoordinate> tuple : componentTypes) {
        final ConfigurableComponent component = serviceFacade.getTempComponent(tuple.getKey(), tuple.getValue());
        if (component == null) {
            throw new NiFiCoreException("Could not create an instance of component " + tuple.getKey() + " using bundle coordinates " + tuple.getValue());
        }

        final boolean isRestricted = component.getClass().isAnnotationPresent(Restricted.class);
        if (isRestricted) {
            restrictedComponents.add(component);
        }
    }

    return restrictedComponents;
}
 
Example 20
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCannotPredictTimeToBytesNaN() {
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = getModelMap("queuedBytes",.9,Double.NaN,Double.NaN);
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(modelMap);
    Long countTime = connectionStatusAnalytics.getTimeToBytesBackpressureMillis(connection, flowFileEvent);
    assertNotNull(countTime);
    assert (countTime == -1);
}
 
Example 21
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidModelLowScore() {
    Date now = new Date();
    Long tomorrowMillis = DateUtils.addDays(now,1).toInstant().toEpochMilli();
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = getModelMap("queuedCount",.5,100.0,tomorrowMillis.doubleValue());
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(modelMap);
    Long countTime = connectionStatusAnalytics.getTimeToCountBackpressureMillis(connection, flowFileEvent);
    assertNotNull(countTime);
    assert (countTime == -1);
}
 
Example 22
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMissingModelGetTimeToCount() {
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = getModelMap("fakeModel",Double.POSITIVE_INFINITY,Double.POSITIVE_INFINITY,Double.POSITIVE_INFINITY);
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(modelMap);
    try {
        connectionStatusAnalytics.getTimeToCountBackpressureMillis(connection, flowFileEvent);
        fail();
    }catch(IllegalArgumentException iae){
        assertTrue(true);
    }
}
 
Example 23
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetNextIntervalBytes() {
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = getModelMap("queuedBytes",.9,1.0,1.0);
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(modelMap);
    Long nextIntervalBytes = connectionStatusAnalytics.getNextIntervalBytes(flowFileEvent);
    assertNotNull(nextIntervalBytes);
    assert (nextIntervalBytes > 0);
}
 
Example 24
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidModelNaNScore() {
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = getModelMap("queuedCount",Double.NaN,Double.NaN,Double.NaN);
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(modelMap);
    Long countTime = connectionStatusAnalytics.getTimeToCountBackpressureMillis(connection, flowFileEvent);
    assertNotNull(countTime);
    assert (countTime == -1);
}
 
Example 25
Source Project: nifi   Source File: TestConnectionStatusAnalytics.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetNextIntervalCount() {
    Map<String, Tuple<StatusAnalyticsModel, StatusMetricExtractFunction>> modelMap = getModelMap("queuedCount",.9,1.0,1.0);
    ConnectionStatusAnalytics connectionStatusAnalytics = getConnectionStatusAnalytics(modelMap);
    Long nextIntervalBytes = connectionStatusAnalytics.getNextIntervalCount(flowFileEvent);
    assertNotNull(nextIntervalBytes);
    assert (nextIntervalBytes > 0);
}
 
Example 26
Source Project: nifi   Source File: StandardControllerServiceNode.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void addReference(final ComponentNode referencingComponent, final PropertyDescriptor propertyDescriptor) {
    writeLock.lock();
    try {
        referencingComponents.add(new Tuple<>(referencingComponent, propertyDescriptor));
    } finally {
        writeLock.unlock();
    }
}
 
Example 27
@Override
public <K, V> V getAndPutIfAbsent(final K key, final V value, final Serializer<K> keySerializer, final Serializer<V> valueSerializer, final Deserializer<V> valueDeserializer) throws IOException {
    return withConnection(redisConnection -> {
        final Tuple<byte[],byte[]> kv = serialize(key, value, keySerializer, valueSerializer);
        do {
            // start a watch on the key and retrieve the current value
            redisConnection.watch(kv.getKey());
            final byte[] existingValue = redisConnection.get(kv.getKey());

            // start a transaction and perform the put-if-absent
            redisConnection.multi();
            redisConnection.setNX(kv.getKey(), kv.getValue());

            // Set the TTL only if the key doesn't exist already
            if (ttl != -1L && existingValue == null) {
                redisConnection.expire(kv.getKey(), ttl);
            }

            // execute the transaction
            final List<Object> results = redisConnection.exec();

            // if the results list was empty, then the transaction failed (i.e. key was modified after we started watching), so keep looping to retry
            // if the results list has results, then the transaction succeeded and it should have the result of the setNX operation
            if (results.size() > 0) {
                final Object firstResult = results.get(0);
                if (firstResult instanceof Boolean) {
                    final Boolean absent = (Boolean) firstResult;
                    return absent ? null : valueDeserializer.deserialize(existingValue);
                } else {
                    // this shouldn't really happen, but just in case there is a non-boolean result then bounce out of the loop
                    throw new IOException("Unexpected result from Redis transaction: Expected Boolean result, but got "
                            + firstResult.getClass().getName() + " with value " + firstResult.toString());
                }
            }
        } while (isEnabled());

        return null;
    });
}
 
Example 28
@Override
public <K, V> void put(final K key, final V value, final Serializer<K> keySerializer, final Serializer<V> valueSerializer) throws IOException {
    withConnection(redisConnection -> {
        final Tuple<byte[],byte[]> kv = serialize(key, value, keySerializer, valueSerializer);
        redisConnection.set(kv.getKey(), kv.getValue(), Expiration.seconds(ttl), SetOption.upsert());
        return null;
    });
}
 
Example 29
Source Project: nifi   Source File: HortonworksSchemaRegistry.java    License: Apache License 2.0 5 votes vote down vote up
private SchemaVersionInfo getSchemaVersionInfo(final SchemaRegistryClient client, final SchemaVersionKey key) throws org.apache.nifi.schema.access.SchemaNotFoundException {
    try {
        // Try to fetch the SchemaVersionInfo from the cache.
        final Tuple<SchemaVersionInfo, Long> timestampedVersionInfo = schemaVersionByKeyCache.get(key);

        // Determine if the timestampedVersionInfo is expired
        boolean fetch = false;
        if (timestampedVersionInfo == null) {
            fetch = true;
        } else {
            final long minTimestamp = System.nanoTime() - versionInfoCacheNanos;
            fetch = timestampedVersionInfo.getValue() < minTimestamp;
        }

        // If not expired, use what we got from the cache
        if (!fetch) {
            return timestampedVersionInfo.getKey();
        }

        // schema version info was expired or not found in cache. Fetch from schema registry
        final SchemaVersionInfo versionInfo = client.getSchemaVersionInfo(key);
        if (versionInfo == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with name '" + key.getSchemaName() + "' and version " + key.getVersion());
        }

        // Store new version in cache.
        final Tuple<SchemaVersionInfo, Long> tuple = new Tuple<>(versionInfo, System.nanoTime());
        schemaVersionByKeyCache.put(key, tuple);
        return versionInfo;
    } catch (final SchemaNotFoundException e) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException(e);
    }
}
 
Example 30
Source Project: nifi   Source File: AbstractHiveAnalyzer.java    License: Apache License 2.0 5 votes vote down vote up
protected Referenceable createTableRef(String namespace, Tuple<String, String> tableName) {
    final Referenceable ref = new Referenceable(TYPE_TABLE);
    ref.set(ATTR_NAME, tableName.getValue());
    ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(namespace, toTableNameStr(tableName)));
    ref.set(ATTR_DB, createDatabaseRef(namespace, tableName.getKey()));
    return ref;
}