Java Code Examples for org.apache.nifi.flowfile.FlowFile#getAttributes()
The following examples show how to use
org.apache.nifi.flowfile.FlowFile#getAttributes() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PutS3Object.java From nifi with Apache License 2.0 | 6 votes |
private List<Tag> getObjectTags(ProcessContext context, FlowFile flowFile) { final String prefix = context.getProperty(OBJECT_TAGS_PREFIX).evaluateAttributeExpressions(flowFile).getValue(); final List<Tag> objectTags = new ArrayList<>(); final Map<String, String> attributesMap = flowFile.getAttributes(); attributesMap.entrySet().stream().sequential() .filter(attribute -> attribute.getKey().startsWith(prefix)) .forEach(attribute -> { String tagKey = attribute.getKey(); String tagValue = attribute.getValue(); if (context.getProperty(REMOVE_TAG_PREFIX).asBoolean()) { tagKey = tagKey.replace(prefix, ""); } objectTags.add(new Tag(tagKey, tagValue)); }); return objectTags; }
Example 2
Source File: AttributesToJSON.java From localization_nifi with Apache License 2.0 | 6 votes |
/** * Builds the Map of attributes that should be included in the JSON that is emitted from this process. * * @return * Map of values that are feed to a Jackson ObjectMapper */ protected Map<String, String> buildAttributesMapForFlowFile(FlowFile ff, Set<String> attributes, Set<String> attributesToRemove, boolean nullValForEmptyString) { Map<String, String> result; //If list of attributes specified get only those attributes. Otherwise write them all if (attributes != null) { result = new HashMap<>(attributes.size()); for (String attribute : attributes) { String val = ff.getAttribute(attribute); if (val != null || nullValForEmptyString) { result.put(attribute, val); } else { result.put(attribute, ""); } } } else { Map<String, String> ffAttributes = ff.getAttributes(); result = new HashMap<>(ffAttributes.size()); for (Map.Entry<String, String> e : ffAttributes.entrySet()) { if (!attributesToRemove.contains(e.getKey())) { result.put(e.getKey(), e.getValue()); } } } return result; }
Example 3
Source File: MockProcessSession.java From nifi with Apache License 2.0 | 5 votes |
/** * Returns the attributes that are common to every flow file given. The key * and value must match exactly. * * @param flowFileList a list of flow files * * @return the common attributes */ private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) { final Map<String, String> result = new HashMap<>(); // trivial cases if (flowFileList == null || flowFileList.isEmpty()) { return result; } else if (flowFileList.size() == 1) { result.putAll(flowFileList.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFileList) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } return result; }
Example 4
Source File: StandardProcessSession.java From nifi with Apache License 2.0 | 5 votes |
/** * Returns the attributes that are common to every FlowFile given. The key * and value must match exactly. * * @param flowFileList a list of FlowFiles * * @return the common attributes */ private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) { final Map<String, String> result = new HashMap<>(); // trivial cases if (flowFileList == null || flowFileList.isEmpty()) { return result; } else if (flowFileList.size() == 1) { result.putAll(flowFileList.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFileList) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } return result; }
Example 5
Source File: StatelessProcessSession.java From nifi with Apache License 2.0 | 5 votes |
/** * Returns the attributes that are common to every flow file given. The key * and value must match exactly. * * @param flowFileList a list of flow files * * @return the common attributes */ private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) { final Map<String, String> result = new HashMap<>(); // trivial cases if (flowFileList == null || flowFileList.isEmpty()) { return result; } else if (flowFileList.size() == 1) { result.putAll(flowFileList.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFileList) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } return result; }
Example 6
Source File: KeepCommonAttributeStrategy.java From nifi with Apache License 2.0 | 5 votes |
@Override public Map<String, String> getMergedAttributes(final List<FlowFile> flowFiles) { final Map<String, String> result = new HashMap<>(); //trivial cases if (flowFiles == null || flowFiles.isEmpty()) { return result; } else if (flowFiles.size() == 1) { result.putAll(flowFiles.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFiles.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFiles) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } // Never copy the UUID from the parents - which could happen if we don't remove it and there is only 1 parent. result.remove(CoreAttributes.UUID.key()); return result; }
Example 7
Source File: ExecuteSQLRecord.java From nifi with Apache License 2.0 | 5 votes |
@Override protected SqlWriter configureSqlWriter(ProcessSession session, ProcessContext context, FlowFile fileToProcess) { final Integer maxRowsPerFlowFile = context.getProperty(MAX_ROWS_PER_FLOW_FILE).evaluateAttributeExpressions().asInteger(); final boolean convertNamesForAvro = context.getProperty(NORMALIZE_NAMES).asBoolean(); final Boolean useAvroLogicalTypes = context.getProperty(USE_AVRO_LOGICAL_TYPES).asBoolean(); final JdbcCommon.AvroConversionOptions options = JdbcCommon.AvroConversionOptions.builder() .convertNames(convertNamesForAvro) .useLogicalTypes(useAvroLogicalTypes) .build(); final RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class); return new RecordSqlWriter(recordSetWriterFactory, options, maxRowsPerFlowFile, fileToProcess == null ? Collections.emptyMap() : fileToProcess.getAttributes()); }
Example 8
Source File: AbstractAzureDataLakeStorageProcessor.java From nifi with Apache License 2.0 | 5 votes |
public static DataLakeServiceClient getStorageClient(PropertyContext context, FlowFile flowFile) { final Map<String, String> attributes = flowFile != null ? flowFile.getAttributes() : Collections.emptyMap(); final String accountName = context.getProperty(ACCOUNT_NAME).evaluateAttributeExpressions(attributes).getValue(); final String accountKey = context.getProperty(ACCOUNT_KEY).evaluateAttributeExpressions(attributes).getValue(); final String sasToken = context.getProperty(SAS_TOKEN).evaluateAttributeExpressions(attributes).getValue(); final String endpointSuffix = context.getProperty(ENDPOINT_SUFFIX).evaluateAttributeExpressions(attributes).getValue(); final String endpoint = String.format("https://%s.%s", accountName,endpointSuffix); final boolean useManagedIdentity = context.getProperty(USE_MANAGED_IDENTITY).asBoolean(); DataLakeServiceClient storageClient; if (StringUtils.isNotBlank(accountKey)) { final StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey); storageClient = new DataLakeServiceClientBuilder().endpoint(endpoint).credential(credential) .buildClient(); } else if (StringUtils.isNotBlank(sasToken)) { storageClient = new DataLakeServiceClientBuilder().endpoint(endpoint).sasToken(sasToken) .buildClient(); } else if(useManagedIdentity){ final ManagedIdentityCredential misCrendential = new ManagedIdentityCredentialBuilder() .build(); storageClient = new DataLakeServiceClientBuilder() .endpoint(endpoint) .credential(misCrendential) .buildClient(); } else { throw new IllegalArgumentException(String.format("Either '%s' or '%s' must be defined.", ACCOUNT_KEY.getDisplayName(), SAS_TOKEN.getDisplayName())); } return storageClient; }
Example 9
Source File: AzureStorageUtils.java From nifi with Apache License 2.0 | 5 votes |
public static AzureStorageCredentialsDetails getStorageCredentialsDetails(PropertyContext context, FlowFile flowFile) { final Map<String, String> attributes = flowFile != null ? flowFile.getAttributes() : Collections.emptyMap(); final AzureStorageCredentialsService storageCredentialsService = context.getProperty(STORAGE_CREDENTIALS_SERVICE).asControllerService(AzureStorageCredentialsService.class); if (storageCredentialsService != null) { return storageCredentialsService.getStorageCredentialsDetails(attributes); } else { return createStorageCredentialsDetails(context, attributes); } }
Example 10
Source File: MockFlowFile.java From localization_nifi with Apache License 2.0 | 5 votes |
public MockFlowFile(final long id, final FlowFile toCopy) { this.creationTime = System.nanoTime(); this.id = id; entryDate = System.currentTimeMillis(); final Map<String, String> attributesToCopy = toCopy.getAttributes(); String filename = attributesToCopy.get(CoreAttributes.FILENAME.key()); if (filename == null) { filename = String.valueOf(System.nanoTime()) + ".mockFlowFile"; } attributes.put(CoreAttributes.FILENAME.key(), filename); String path = attributesToCopy.get(CoreAttributes.PATH.key()); if (path == null) { path = "target"; } attributes.put(CoreAttributes.PATH.key(), path); String uuid = attributesToCopy.get(CoreAttributes.UUID.key()); if (uuid == null) { uuid = UUID.randomUUID().toString(); } attributes.put(CoreAttributes.UUID.key(), uuid); attributes.putAll(toCopy.getAttributes()); final byte[] dataToCopy = ((MockFlowFile) toCopy).data; this.data = new byte[dataToCopy.length]; System.arraycopy(dataToCopy, 0, this.data, 0, dataToCopy.length); }
Example 11
Source File: MockFlowFile.java From nifi with Apache License 2.0 | 5 votes |
public MockFlowFile(final long id, final FlowFile toCopy) { this.creationTime = System.nanoTime(); this.id = id; entryDate = System.currentTimeMillis(); final Map<String, String> attributesToCopy = toCopy.getAttributes(); String filename = attributesToCopy.get(CoreAttributes.FILENAME.key()); if (filename == null) { filename = String.valueOf(System.nanoTime()) + ".mockFlowFile"; } attributes.put(CoreAttributes.FILENAME.key(), filename); String path = attributesToCopy.get(CoreAttributes.PATH.key()); if (path == null) { path = "target"; } attributes.put(CoreAttributes.PATH.key(), path); String uuid = attributesToCopy.get(CoreAttributes.UUID.key()); if (uuid == null) { uuid = UUID.randomUUID().toString(); } attributes.put(CoreAttributes.UUID.key(), uuid); attributes.putAll(toCopy.getAttributes()); final byte[] dataToCopy = ((MockFlowFile) toCopy).data; this.data = new byte[dataToCopy.length]; System.arraycopy(dataToCopy, 0, this.data, 0, dataToCopy.length); this.penalized = toCopy.isPenalized(); }
Example 12
Source File: StandardProcessSession.java From localization_nifi with Apache License 2.0 | 5 votes |
/** * Returns the attributes that are common to every FlowFile given. The key * and value must match exactly. * * @param flowFileList a list of FlowFiles * * @return the common attributes */ private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) { final Map<String, String> result = new HashMap<>(); // trivial cases if (flowFileList == null || flowFileList.isEmpty()) { return result; } else if (flowFileList.size() == 1) { result.putAll(flowFileList.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFileList) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } return result; }
Example 13
Source File: MergeContent.java From localization_nifi with Apache License 2.0 | 5 votes |
@Override public Map<String, String> getMergedAttributes(final List<FlowFile> flowFiles) { final Map<String, String> result = new HashMap<>(); //trivial cases if (flowFiles == null || flowFiles.isEmpty()) { return result; } else if (flowFiles.size() == 1) { result.putAll(flowFiles.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFiles.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFiles) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } // Never copy the UUID from the parents - which could happen if we don't remove it and there is only 1 parent. result.remove(CoreAttributes.UUID.key()); return result; }
Example 14
Source File: MockProcessSession.java From localization_nifi with Apache License 2.0 | 5 votes |
/** * Returns the attributes that are common to every flow file given. The key * and value must match exactly. * * @param flowFileList a list of flow files * * @return the common attributes */ private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) { final Map<String, String> result = new HashMap<>(); // trivial cases if (flowFileList == null || flowFileList.isEmpty()) { return result; } else if (flowFileList.size() == 1) { result.putAll(flowFileList.iterator().next().getAttributes()); } /* * Start with the first attribute map and only put an entry to the * resultant map if it is common to every map. */ final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes(); outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) { final String key = mapEntry.getKey(); final String value = mapEntry.getValue(); for (final FlowFile flowFile : flowFileList) { final Map<String, String> currMap = flowFile.getAttributes(); final String curVal = currMap.get(key); if (curVal == null || !curVal.equals(value)) { continue outer; } } result.put(key, value); } return result; }
Example 15
Source File: NLPProcessor.java From nifi-nlp-processor with Apache License 2.0 | 4 votes |
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { FlowFile flowFile = session.get(); if (flowFile == null) { flowFile = session.create(); } try { flowFile.getAttributes(); service = new OpenNLPService(); String sentence = flowFile.getAttribute(ATTRIBUTE_INPUT_NAME); String sentence2 = context.getProperty(ATTRIBUTE_INPUT_NAME).evaluateAttributeExpressions(flowFile) .getValue(); if (sentence == null) { sentence = sentence2; } try { // if they pass in a sentence do that instead of flowfile if (sentence == null) { final AtomicReference<String> contentsRef = new AtomicReference<>(null); session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream input) throws IOException { final String contents = IOUtils.toString(input, "UTF-8"); contentsRef.set(contents); } }); // use this as our text if (contentsRef.get() != null) { sentence = contentsRef.get(); } } List<PersonName> people = service.getPeople( context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(), sentence); int count = 1; for (PersonName personName : people) { flowFile = session.putAttribute(flowFile, "nlp_name_" + count, personName.getName()); count++; } List<String> dates = service.getDates( context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(), sentence); count = 1; for (String aDate : dates) { flowFile = session.putAttribute(flowFile, "nlp_date_" + count, aDate); count++; } List<Location> locations = service.getLocations( context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(), sentence); count = 1; for (Location location : locations) { flowFile = session.putAttribute(flowFile, "nlp_location_" + count, location.getLocation()); count++; } List<Organization> orgs = service.getOrganizations( context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(), sentence); count = 1; for (Organization org : orgs) { flowFile = session.putAttribute(flowFile, "nlp_org_" + count, org.getOrganization()); count++; } } catch (Exception e) { throw new ProcessException(e); } session.transfer(flowFile, REL_SUCCESS); session.commit(); } catch (final Throwable t) { getLogger().error("Unable to process NLP Processor file " + t.getLocalizedMessage()); getLogger().error("{} failed to process due to {}; rolling back session", new Object[] { this, t }); throw t; } }
Example 16
Source File: PublishAMQP.java From localization_nifi with Apache License 2.0 | 4 votes |
/** * Extracts AMQP properties from the {@link FlowFile} attributes. Attributes * extracted from {@link FlowFile} are considered candidates for AMQP * properties if their names are prefixed with * {@link AMQPUtils#AMQP_PROP_PREFIX} (e.g., amqp$contentType=text/xml). * * Some fields require a specific format and are validated: * * {@link AMQPUtils#validateAMQPHeaderProperty} * {@link AMQPUtils#validateAMQPDeliveryModeProperty} * {@link AMQPUtils#validateAMQPPriorityProperty} * {@link AMQPUtils#validateAMQPTimestampProperty} */ private BasicProperties extractAmqpPropertiesFromFlowFile(FlowFile flowFile) { Map<String, String> attributes = flowFile.getAttributes(); AMQP.BasicProperties.Builder builder = new AMQP.BasicProperties.Builder(); for (Entry<String, String> attributeEntry : attributes.entrySet()) { if (attributeEntry.getKey().startsWith(AMQPUtils.AMQP_PROP_PREFIX)) { String amqpPropName = attributeEntry.getKey(); String amqpPropValue = attributeEntry.getValue(); AMQPUtils.PropertyNames propertyNames = AMQPUtils.PropertyNames.fromValue(amqpPropName); if (propertyNames != null) { switch (propertyNames){ case CONTENT_TYPE: builder.contentType(amqpPropValue); break; case CONTENT_ENCODING: builder.contentEncoding(amqpPropValue); break; case HEADERS: builder.headers(AMQPUtils.validateAMQPHeaderProperty(amqpPropValue)); break; case DELIVERY_MODE: builder.deliveryMode(AMQPUtils.validateAMQPDeliveryModeProperty(amqpPropValue)); break; case PRIORITY: builder.priority(AMQPUtils.validateAMQPPriorityProperty(amqpPropValue)); break; case CORRELATION_ID: builder.correlationId(amqpPropValue); break; case REPLY_TO: builder.replyTo(amqpPropValue); break; case EXPIRATION: builder.expiration(amqpPropValue); break; case MESSAGE_ID: builder.messageId(amqpPropValue); break; case TIMESTAMP: builder.timestamp(AMQPUtils.validateAMQPTimestampProperty(amqpPropValue)); break; case TYPE: builder.type(amqpPropValue); break; case USER_ID: builder.userId(amqpPropValue); break; case APP_ID: builder.appId(amqpPropValue); break; case CLUSTER_ID: builder.clusterId(amqpPropValue); break; } } else { getLogger().warn("Unrecognised AMQP property '" + amqpPropName + "', will ignore."); } } } return builder.build(); }
Example 17
Source File: PutJMS.java From nifi with Apache License 2.0 | 4 votes |
/** * Iterates through all of the flow file's metadata and for any metadata key that starts with <code>jms.</code>, the value for the corresponding key is written to the JMS message as a property. * The name of this property is equal to the key of the flow file's metadata minus the <code>jms.</code>. For example, if the flowFile has a metadata entry: * <br /><br /> * <code>jms.count</code> = <code>8</code> * <br /><br /> * then the JMS message will have a String property added to it with the property name <code>count</code> and value <code>8</code>. * * If the flow file also has a metadata key with the name <code>jms.count.type</code>, then the value of that metadata entry will determine the JMS property type to use for the value. For example, * if the flow file has the following properties: * <br /><br /> * <code>jms.count</code> = <code>8</code><br /> * <code>jms.count.type</code> = <code>integer</code> * <br /><br /> * Then <code>message</code> will have an INTEGER property added with the value 8. * <br /><br/> * If the type is not valid for the given value (e.g., <code>jms.count.type</code> = <code>integer</code> and <code>jms.count</code> = <code>hello</code>, then this JMS property will not be added * to <code>message</code>. * * @param flowFile The flow file whose metadata should be examined for JMS properties. * @param message The JMS message to which we want to add properties. * @throws JMSException ex */ private void copyAttributesToJmsProps(final FlowFile flowFile, final Message message) throws JMSException { final ComponentLog logger = getLogger(); final Map<String, String> attributes = flowFile.getAttributes(); for (final Entry<String, String> entry : attributes.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); if (key.toLowerCase().startsWith(ATTRIBUTE_PREFIX.toLowerCase()) && !key.toLowerCase().endsWith(ATTRIBUTE_TYPE_SUFFIX.toLowerCase())) { final String jmsPropName = key.substring(ATTRIBUTE_PREFIX.length()); final String type = attributes.get(key + ATTRIBUTE_TYPE_SUFFIX); try { if (type == null || type.equalsIgnoreCase(PROP_TYPE_STRING)) { message.setStringProperty(jmsPropName, value); } else if (type.equalsIgnoreCase(PROP_TYPE_INTEGER)) { message.setIntProperty(jmsPropName, Integer.parseInt(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_BOOLEAN)) { message.setBooleanProperty(jmsPropName, Boolean.parseBoolean(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_SHORT)) { message.setShortProperty(jmsPropName, Short.parseShort(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_LONG)) { message.setLongProperty(jmsPropName, Long.parseLong(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_BYTE)) { message.setByteProperty(jmsPropName, Byte.parseByte(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_DOUBLE)) { message.setDoubleProperty(jmsPropName, Double.parseDouble(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_FLOAT)) { message.setFloatProperty(jmsPropName, Float.parseFloat(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_OBJECT)) { message.setObjectProperty(jmsPropName, value); } else { logger.warn("Attribute key '{}' for {} has value '{}', but expected one of: integer, string, object, byte, double, float, long, short, boolean; not adding this property", new Object[]{key, flowFile, value}); } } catch (NumberFormatException e) { logger.warn("Attribute key '{}' for {} has value '{}', but attribute key '{}' has value '{}'. Not adding this JMS property", new Object[]{key, flowFile, value, key + ATTRIBUTE_TYPE_SUFFIX, PROP_TYPE_INTEGER}); } } } }
Example 18
Source File: WriteAheadRepositoryRecordSerde.java From localization_nifi with Apache License 2.0 | 4 votes |
public void serializeEdit(final RepositoryRecord previousRecordState, final RepositoryRecord record, final DataOutputStream out, final boolean forceAttributesWritten) throws IOException { if (record.isMarkedForAbort()) { logger.warn("Repository Record {} is marked to be aborted; it will be persisted in the FlowFileRepository as a DELETE record", record); out.write(ACTION_DELETE); out.writeLong(getRecordIdentifier(record)); serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out); return; } final UpdateType updateType = getUpdateType(record); if (updateType.equals(UpdateType.DELETE)) { out.write(ACTION_DELETE); out.writeLong(getRecordIdentifier(record)); serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out); return; } // If there's a Destination Connection, that's the one that we want to associated with this record. // However, on restart, we will restore the FlowFile and set this connection to its "originalConnection". // If we then serialize the FlowFile again before it's transferred, it's important to allow this to happen, // so we use the originalConnection instead FlowFileQueue associatedQueue = record.getDestination(); if (associatedQueue == null) { associatedQueue = record.getOriginalQueue(); } if (updateType.equals(UpdateType.SWAP_OUT)) { out.write(ACTION_SWAPPED_OUT); out.writeLong(getRecordIdentifier(record)); out.writeUTF(associatedQueue.getIdentifier()); out.writeUTF(getLocation(record)); return; } final FlowFile flowFile = record.getCurrent(); final ContentClaim claim = record.getCurrentClaim(); switch (updateType) { case UPDATE: out.write(ACTION_UPDATE); break; case CREATE: out.write(ACTION_CREATE); break; case SWAP_IN: out.write(ACTION_SWAPPED_IN); break; default: throw new AssertionError(); } out.writeLong(getRecordIdentifier(record)); out.writeLong(flowFile.getEntryDate()); out.writeLong(flowFile.getLineageStartDate()); out.writeLong(flowFile.getLineageStartIndex()); final Long queueDate = flowFile.getLastQueueDate(); out.writeLong(queueDate == null ? System.currentTimeMillis() : queueDate); out.writeLong(flowFile.getQueueDateIndex()); out.writeLong(flowFile.getSize()); if (associatedQueue == null) { logger.warn("{} Repository Record {} has no Connection associated with it; it will be destroyed on restart", new Object[] {this, record}); writeString("", out); } else { writeString(associatedQueue.getIdentifier(), out); } serializeContentClaim(claim, record.getCurrentClaimOffset(), out); if (forceAttributesWritten || record.isAttributesChanged() || updateType == UpdateType.CREATE || updateType == UpdateType.SWAP_IN) { out.write(1); // indicate attributes changed final Map<String, String> attributes = flowFile.getAttributes(); out.writeInt(attributes.size()); for (final Map.Entry<String, String> entry : attributes.entrySet()) { writeString(entry.getKey(), out); writeString(entry.getValue(), out); } } else { out.write(0); // indicate attributes did not change } if (updateType == UpdateType.SWAP_IN) { out.writeUTF(record.getSwapLocation()); } }
Example 19
Source File: GetMongoRecord.java From nifi with Apache License 2.0 | 4 votes |
@Override public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { FlowFile input = null; if (context.hasIncomingConnection()) { input = session.get(); if (input == null && context.hasNonLoopConnection()) { return; } } final String database = context.getProperty(DATABASE_NAME).evaluateAttributeExpressions(input).getValue(); final String collection = context.getProperty(COLLECTION_NAME).evaluateAttributeExpressions(input).getValue(); final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(input).getValue(); final Document query = getQuery(context, session, input); MongoCollection mongoCollection = clientService.getDatabase(database).getCollection(collection); FindIterable<Document> find = mongoCollection.find(query); if (context.getProperty(SORT).isSet()) { find = find.sort(Document.parse(context.getProperty(SORT).evaluateAttributeExpressions(input).getValue())); } if (context.getProperty(PROJECTION).isSet()) { find = find.projection(Document.parse(context.getProperty(PROJECTION).evaluateAttributeExpressions(input).getValue())); } if (context.getProperty(LIMIT).isSet()) { find = find.limit(context.getProperty(LIMIT).evaluateAttributeExpressions(input).asInteger()); } MongoCursor<Document> cursor = find.iterator(); FlowFile output = input != null ? session.create(input) : session.create(); final FlowFile inputPtr = input; try { final Map<String, String> attributes = getAttributes(context, input, query, mongoCollection); try (OutputStream out = session.write(output)) { Map<String, String> attrs = inputPtr != null ? inputPtr.getAttributes() : new HashMap<String, String>(){{ put("schema.name", schemaName); }}; RecordSchema schema = writerFactory.getSchema(attrs, null); RecordSetWriter writer = writerFactory.createWriter(getLogger(), schema, out, attrs); long count = 0L; writer.beginRecordSet(); while (cursor.hasNext()) { Document next = cursor.next(); if (next.get("_id") instanceof ObjectId) { next.put("_id", next.get("_id").toString()); } Record record = new MapRecord(schema, next); writer.write(record); count++; } writer.finishRecordSet(); writer.close(); out.close(); attributes.put("record.count", String.valueOf(count)); } catch (SchemaNotFoundException e) { throw new RuntimeException(e); } output = session.putAllAttributes(output, attributes); session.getProvenanceReporter().fetch(output, getURI(context)); session.transfer(output, REL_SUCCESS); if (input != null) { session.transfer(input, REL_ORIGINAL); } } catch (Exception ex) { ex.printStackTrace(); getLogger().error("Error writing record set from Mongo query.", ex); session.remove(output); if (input != null) { session.transfer(input, REL_FAILURE); } } }
Example 20
Source File: PutJMS.java From localization_nifi with Apache License 2.0 | 4 votes |
/** * Iterates through all of the flow file's metadata and for any metadata key that starts with <code>jms.</code>, the value for the corresponding key is written to the JMS message as a property. * The name of this property is equal to the key of the flow file's metadata minus the <code>jms.</code>. For example, if the flowFile has a metadata entry: * <br /><br /> * <code>jms.count</code> = <code>8</code> * <br /><br /> * then the JMS message will have a String property added to it with the property name <code>count</code> and value <code>8</code>. * * If the flow file also has a metadata key with the name <code>jms.count.type</code>, then the value of that metadata entry will determine the JMS property type to use for the value. For example, * if the flow file has the following properties: * <br /><br /> * <code>jms.count</code> = <code>8</code><br /> * <code>jms.count.type</code> = <code>integer</code> * <br /><br /> * Then <code>message</code> will have an INTEGER property added with the value 8. * <br /><br/> * If the type is not valid for the given value (e.g., <code>jms.count.type</code> = <code>integer</code> and <code>jms.count</code> = <code>hello</code>, then this JMS property will not be added * to <code>message</code>. * * @param flowFile The flow file whose metadata should be examined for JMS properties. * @param message The JMS message to which we want to add properties. * @throws JMSException ex */ private void copyAttributesToJmsProps(final FlowFile flowFile, final Message message) throws JMSException { final ComponentLog logger = getLogger(); final Map<String, String> attributes = flowFile.getAttributes(); for (final Entry<String, String> entry : attributes.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); if (key.toLowerCase().startsWith(ATTRIBUTE_PREFIX.toLowerCase()) && !key.toLowerCase().endsWith(ATTRIBUTE_TYPE_SUFFIX.toLowerCase())) { final String jmsPropName = key.substring(ATTRIBUTE_PREFIX.length()); final String type = attributes.get(key + ATTRIBUTE_TYPE_SUFFIX); try { if (type == null || type.equalsIgnoreCase(PROP_TYPE_STRING)) { message.setStringProperty(jmsPropName, value); } else if (type.equalsIgnoreCase(PROP_TYPE_INTEGER)) { message.setIntProperty(jmsPropName, Integer.parseInt(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_BOOLEAN)) { message.setBooleanProperty(jmsPropName, Boolean.parseBoolean(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_SHORT)) { message.setShortProperty(jmsPropName, Short.parseShort(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_LONG)) { message.setLongProperty(jmsPropName, Long.parseLong(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_BYTE)) { message.setByteProperty(jmsPropName, Byte.parseByte(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_DOUBLE)) { message.setDoubleProperty(jmsPropName, Double.parseDouble(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_FLOAT)) { message.setFloatProperty(jmsPropName, Float.parseFloat(value)); } else if (type.equalsIgnoreCase(PROP_TYPE_OBJECT)) { message.setObjectProperty(jmsPropName, value); } else { logger.warn("Attribute key '{}' for {} has value '{}', but expected one of: integer, string, object, byte, double, float, long, short, boolean; not adding this property", new Object[]{key, flowFile, value}); } } catch (NumberFormatException e) { logger.warn("Attribute key '{}' for {} has value '{}', but attribute key '{}' has value '{}'. Not adding this JMS property", new Object[]{key, flowFile, value, key + ATTRIBUTE_TYPE_SUFFIX, PROP_TYPE_INTEGER}); } } } }