Java Code Examples for org.apache.nifi.flowfile.FlowFile#getAttributes()

private List<Tag> getObjectTags(ProcessContext context, FlowFile flowFile) {
    final String prefix = context.getProperty(OBJECT_TAGS_PREFIX).evaluateAttributeExpressions(flowFile).getValue();
    final List<Tag> objectTags = new ArrayList<>();
    final Map<String, String> attributesMap = flowFile.getAttributes();

    attributesMap.entrySet().stream().sequential()
            .filter(attribute -> attribute.getKey().startsWith(prefix))
            .forEach(attribute -> {
                String tagKey = attribute.getKey();
                String tagValue = attribute.getValue();

                if (context.getProperty(REMOVE_TAG_PREFIX).asBoolean()) {
                    tagKey = tagKey.replace(prefix, "");
                }
                objectTags.add(new Tag(tagKey, tagValue));
            });

    return objectTags;
}
 

/**
 * Builds the Map of attributes that should be included in the JSON that is emitted from this process.
 *
 * @return
 *  Map of values that are feed to a Jackson ObjectMapper
 */
protected Map<String, String> buildAttributesMapForFlowFile(FlowFile ff, Set<String> attributes, Set<String> attributesToRemove, boolean nullValForEmptyString) {
    Map<String, String> result;
    //If list of attributes specified get only those attributes. Otherwise write them all
    if (attributes != null) {
        result = new HashMap<>(attributes.size());
        for (String attribute : attributes) {
            String val = ff.getAttribute(attribute);
            if (val != null || nullValForEmptyString) {
                result.put(attribute, val);
            } else {
                result.put(attribute, "");
            }
        }
    } else {
        Map<String, String> ffAttributes = ff.getAttributes();
        result = new HashMap<>(ffAttributes.size());
        for (Map.Entry<String, String> e : ffAttributes.entrySet()) {
            if (!attributesToRemove.contains(e.getKey())) {
                result.put(e.getKey(), e.getValue());
            }
        }
    }
    return result;
}
 

/**
 * Returns the attributes that are common to every flow file given. The key
 * and value must match exactly.
 *
 * @param flowFileList a list of flow files
 *
 * @return the common attributes
 */
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
    final Map<String, String> result = new HashMap<>();
    // trivial cases
    if (flowFileList == null || flowFileList.isEmpty()) {
        return result;
    } else if (flowFileList.size() == 1) {
        result.putAll(flowFileList.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();

    outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();
        for (final FlowFile flowFile : flowFileList) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    return result;
}
 

/**
 * Returns the attributes that are common to every FlowFile given. The key
 * and value must match exactly.
 *
 * @param flowFileList a list of FlowFiles
 *
 * @return the common attributes
 */
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
    final Map<String, String> result = new HashMap<>();
    // trivial cases
    if (flowFileList == null || flowFileList.isEmpty()) {
        return result;
    } else if (flowFileList.size() == 1) {
        result.putAll(flowFileList.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();

    outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();
        for (final FlowFile flowFile : flowFileList) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    return result;
}
 

/**
 * Returns the attributes that are common to every flow file given. The key
 * and value must match exactly.
 *
 * @param flowFileList a list of flow files
 *
 * @return the common attributes
 */
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
    final Map<String, String> result = new HashMap<>();
    // trivial cases
    if (flowFileList == null || flowFileList.isEmpty()) {
        return result;
    } else if (flowFileList.size() == 1) {
        result.putAll(flowFileList.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();

    outer:
    for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();
        for (final FlowFile flowFile : flowFileList) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    return result;
}
 

@Override
public Map<String, String> getMergedAttributes(final List<FlowFile> flowFiles) {
    final Map<String, String> result = new HashMap<>();

    //trivial cases
    if (flowFiles == null || flowFiles.isEmpty()) {
        return result;
    } else if (flowFiles.size() == 1) {
        result.putAll(flowFiles.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFiles.iterator().next().getAttributes();

    outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();

        for (final FlowFile flowFile : flowFiles) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    // Never copy the UUID from the parents - which could happen if we don't remove it and there is only 1 parent.
    result.remove(CoreAttributes.UUID.key());
    return result;
}
 

@Override
protected SqlWriter configureSqlWriter(ProcessSession session, ProcessContext context, FlowFile fileToProcess) {
    final Integer maxRowsPerFlowFile = context.getProperty(MAX_ROWS_PER_FLOW_FILE).evaluateAttributeExpressions().asInteger();
    final boolean convertNamesForAvro = context.getProperty(NORMALIZE_NAMES).asBoolean();
    final Boolean useAvroLogicalTypes = context.getProperty(USE_AVRO_LOGICAL_TYPES).asBoolean();
    final JdbcCommon.AvroConversionOptions options = JdbcCommon.AvroConversionOptions.builder()
            .convertNames(convertNamesForAvro)
            .useLogicalTypes(useAvroLogicalTypes)
            .build();
    final RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);

    return new RecordSqlWriter(recordSetWriterFactory, options, maxRowsPerFlowFile, fileToProcess == null ? Collections.emptyMap() : fileToProcess.getAttributes());
}
 

public static DataLakeServiceClient getStorageClient(PropertyContext context, FlowFile flowFile) {
    final Map<String, String> attributes = flowFile != null ? flowFile.getAttributes() : Collections.emptyMap();
    final String accountName = context.getProperty(ACCOUNT_NAME).evaluateAttributeExpressions(attributes).getValue();
    final String accountKey = context.getProperty(ACCOUNT_KEY).evaluateAttributeExpressions(attributes).getValue();
    final String sasToken = context.getProperty(SAS_TOKEN).evaluateAttributeExpressions(attributes).getValue();
    final String endpointSuffix = context.getProperty(ENDPOINT_SUFFIX).evaluateAttributeExpressions(attributes).getValue();
    final String endpoint = String.format("https://%s.%s", accountName,endpointSuffix);
    final boolean useManagedIdentity = context.getProperty(USE_MANAGED_IDENTITY).asBoolean();
    DataLakeServiceClient storageClient;
    if (StringUtils.isNotBlank(accountKey)) {
        final StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName,
                accountKey);
        storageClient = new DataLakeServiceClientBuilder().endpoint(endpoint).credential(credential)
                .buildClient();
    } else if (StringUtils.isNotBlank(sasToken)) {
        storageClient = new DataLakeServiceClientBuilder().endpoint(endpoint).sasToken(sasToken)
                .buildClient();
    } else if(useManagedIdentity){
        final ManagedIdentityCredential misCrendential = new ManagedIdentityCredentialBuilder()
                                                            .build();
        storageClient = new  DataLakeServiceClientBuilder()
                                .endpoint(endpoint)
                                .credential(misCrendential)
                                .buildClient();
    } else {
        throw new IllegalArgumentException(String.format("Either '%s' or '%s' must be defined.",
                ACCOUNT_KEY.getDisplayName(), SAS_TOKEN.getDisplayName()));
    }
    return storageClient;
}
 

public static AzureStorageCredentialsDetails getStorageCredentialsDetails(PropertyContext context, FlowFile flowFile) {
    final Map<String, String> attributes = flowFile != null ? flowFile.getAttributes() : Collections.emptyMap();

    final AzureStorageCredentialsService storageCredentialsService = context.getProperty(STORAGE_CREDENTIALS_SERVICE).asControllerService(AzureStorageCredentialsService.class);

    if (storageCredentialsService != null) {
        return storageCredentialsService.getStorageCredentialsDetails(attributes);
    } else {
        return createStorageCredentialsDetails(context, attributes);
    }
}
 

public MockFlowFile(final long id, final FlowFile toCopy) {
    this.creationTime = System.nanoTime();
    this.id = id;
    entryDate = System.currentTimeMillis();

    final Map<String, String> attributesToCopy = toCopy.getAttributes();
    String filename = attributesToCopy.get(CoreAttributes.FILENAME.key());
    if (filename == null) {
        filename = String.valueOf(System.nanoTime()) + ".mockFlowFile";
    }
    attributes.put(CoreAttributes.FILENAME.key(), filename);

    String path = attributesToCopy.get(CoreAttributes.PATH.key());
    if (path == null) {
        path = "target";
    }
    attributes.put(CoreAttributes.PATH.key(), path);

    String uuid = attributesToCopy.get(CoreAttributes.UUID.key());
    if (uuid == null) {
        uuid = UUID.randomUUID().toString();
    }
    attributes.put(CoreAttributes.UUID.key(), uuid);

    attributes.putAll(toCopy.getAttributes());
    final byte[] dataToCopy = ((MockFlowFile) toCopy).data;
    this.data = new byte[dataToCopy.length];
    System.arraycopy(dataToCopy, 0, this.data, 0, dataToCopy.length);
}
 

public MockFlowFile(final long id, final FlowFile toCopy) {
    this.creationTime = System.nanoTime();
    this.id = id;
    entryDate = System.currentTimeMillis();

    final Map<String, String> attributesToCopy = toCopy.getAttributes();
    String filename = attributesToCopy.get(CoreAttributes.FILENAME.key());
    if (filename == null) {
        filename = String.valueOf(System.nanoTime()) + ".mockFlowFile";
    }
    attributes.put(CoreAttributes.FILENAME.key(), filename);

    String path = attributesToCopy.get(CoreAttributes.PATH.key());
    if (path == null) {
        path = "target";
    }
    attributes.put(CoreAttributes.PATH.key(), path);

    String uuid = attributesToCopy.get(CoreAttributes.UUID.key());
    if (uuid == null) {
        uuid = UUID.randomUUID().toString();
    }
    attributes.put(CoreAttributes.UUID.key(), uuid);

    attributes.putAll(toCopy.getAttributes());
    final byte[] dataToCopy = ((MockFlowFile) toCopy).data;
    this.data = new byte[dataToCopy.length];
    System.arraycopy(dataToCopy, 0, this.data, 0, dataToCopy.length);

    this.penalized = toCopy.isPenalized();
}
 

/**
 * Returns the attributes that are common to every FlowFile given. The key
 * and value must match exactly.
 *
 * @param flowFileList a list of FlowFiles
 *
 * @return the common attributes
 */
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
    final Map<String, String> result = new HashMap<>();
    // trivial cases
    if (flowFileList == null || flowFileList.isEmpty()) {
        return result;
    } else if (flowFileList.size() == 1) {
        result.putAll(flowFileList.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();

    outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();
        for (final FlowFile flowFile : flowFileList) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    return result;
}
 

@Override
public Map<String, String> getMergedAttributes(final List<FlowFile> flowFiles) {
    final Map<String, String> result = new HashMap<>();

    //trivial cases
    if (flowFiles == null || flowFiles.isEmpty()) {
        return result;
    } else if (flowFiles.size() == 1) {
        result.putAll(flowFiles.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFiles.iterator().next().getAttributes();

    outer:
    for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();

        for (final FlowFile flowFile : flowFiles) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    // Never copy the UUID from the parents - which could happen if we don't remove it and there is only 1 parent.
    result.remove(CoreAttributes.UUID.key());
    return result;
}
 

/**
 * Returns the attributes that are common to every flow file given. The key
 * and value must match exactly.
 *
 * @param flowFileList a list of flow files
 *
 * @return the common attributes
 */
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
    final Map<String, String> result = new HashMap<>();
    // trivial cases
    if (flowFileList == null || flowFileList.isEmpty()) {
        return result;
    } else if (flowFileList.size() == 1) {
        result.putAll(flowFileList.iterator().next().getAttributes());
    }

    /*
     * Start with the first attribute map and only put an entry to the
     * resultant map if it is common to every map.
     */
    final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();

    outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
        final String key = mapEntry.getKey();
        final String value = mapEntry.getValue();
        for (final FlowFile flowFile : flowFileList) {
            final Map<String, String> currMap = flowFile.getAttributes();
            final String curVal = currMap.get(key);
            if (curVal == null || !curVal.equals(value)) {
                continue outer;
            }
        }
        result.put(key, value);
    }

    return result;
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
	FlowFile flowFile = session.get();
	if (flowFile == null) {
		flowFile = session.create();
	}
	try {
		flowFile.getAttributes();

		service = new OpenNLPService();

		String sentence = flowFile.getAttribute(ATTRIBUTE_INPUT_NAME);
		String sentence2 = context.getProperty(ATTRIBUTE_INPUT_NAME).evaluateAttributeExpressions(flowFile)
				.getValue();

		if (sentence == null) {
			sentence = sentence2;
		}

		try {
			// if they pass in a sentence do that instead of flowfile
			if (sentence == null) {
				final AtomicReference<String> contentsRef = new AtomicReference<>(null);

				session.read(flowFile, new InputStreamCallback() {
					@Override
					public void process(final InputStream input) throws IOException {
						final String contents = IOUtils.toString(input, "UTF-8");
						contentsRef.set(contents);
					}
				});

				// use this as our text
				if (contentsRef.get() != null) {
					sentence = contentsRef.get();
				}
			}

			List<PersonName> people = service.getPeople(
					context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(),
					sentence);

			int count = 1;
			for (PersonName personName : people) {
				flowFile = session.putAttribute(flowFile, "nlp_name_" + count, personName.getName());
				count++;
			}

			List<String> dates = service.getDates(
					context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(),
					sentence);

			count = 1;
			for (String aDate : dates) {
				flowFile = session.putAttribute(flowFile, "nlp_date_" + count, aDate);
				count++;
			}

			List<Location> locations = service.getLocations(
					context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(),
					sentence);

			count = 1;
			for (Location location : locations) {
				flowFile = session.putAttribute(flowFile, "nlp_location_" + count, location.getLocation());
				count++;
			}

			List<Organization> orgs = service.getOrganizations(
					context.getProperty(EXTRA_RESOURCE).evaluateAttributeExpressions(flowFile).getValue(),
					sentence);

			count = 1;
			for (Organization org : orgs) {
				flowFile = session.putAttribute(flowFile, "nlp_org_" + count, org.getOrganization());
				count++;
			}

		} catch (Exception e) {
			throw new ProcessException(e);
		}

		session.transfer(flowFile, REL_SUCCESS);
		session.commit();
	} catch (final Throwable t) {
		getLogger().error("Unable to process NLP Processor file " + t.getLocalizedMessage());
		getLogger().error("{} failed to process due to {}; rolling back session", new Object[] { this, t });
		throw t;
	}
}
 

/**
 * Extracts AMQP properties from the {@link FlowFile} attributes. Attributes
 * extracted from {@link FlowFile} are considered candidates for AMQP
 * properties if their names are prefixed with
 * {@link AMQPUtils#AMQP_PROP_PREFIX} (e.g., amqp$contentType=text/xml).
 *
 * Some fields require a specific format and are validated:
 *
 * {@link AMQPUtils#validateAMQPHeaderProperty}
 * {@link AMQPUtils#validateAMQPDeliveryModeProperty}
 * {@link AMQPUtils#validateAMQPPriorityProperty}
 * {@link AMQPUtils#validateAMQPTimestampProperty}
 */
private BasicProperties extractAmqpPropertiesFromFlowFile(FlowFile flowFile) {
    Map<String, String> attributes = flowFile.getAttributes();
    AMQP.BasicProperties.Builder builder = new AMQP.BasicProperties.Builder();
    for (Entry<String, String> attributeEntry : attributes.entrySet()) {
        if (attributeEntry.getKey().startsWith(AMQPUtils.AMQP_PROP_PREFIX)) {
            String amqpPropName = attributeEntry.getKey();
            String amqpPropValue = attributeEntry.getValue();

            AMQPUtils.PropertyNames propertyNames = AMQPUtils.PropertyNames.fromValue(amqpPropName);

            if (propertyNames != null) {
                switch (propertyNames){
                    case CONTENT_TYPE:
                        builder.contentType(amqpPropValue);
                        break;
                    case CONTENT_ENCODING:
                        builder.contentEncoding(amqpPropValue);
                        break;
                    case HEADERS:
                        builder.headers(AMQPUtils.validateAMQPHeaderProperty(amqpPropValue));
                        break;
                    case DELIVERY_MODE:
                        builder.deliveryMode(AMQPUtils.validateAMQPDeliveryModeProperty(amqpPropValue));
                        break;
                    case PRIORITY:
                        builder.priority(AMQPUtils.validateAMQPPriorityProperty(amqpPropValue));
                        break;
                    case CORRELATION_ID:
                        builder.correlationId(amqpPropValue);
                        break;
                    case REPLY_TO:
                        builder.replyTo(amqpPropValue);
                        break;
                    case EXPIRATION:
                        builder.expiration(amqpPropValue);
                        break;
                    case MESSAGE_ID:
                        builder.messageId(amqpPropValue);
                        break;
                    case TIMESTAMP:
                        builder.timestamp(AMQPUtils.validateAMQPTimestampProperty(amqpPropValue));
                        break;
                    case TYPE:
                        builder.type(amqpPropValue);
                        break;
                    case USER_ID:
                        builder.userId(amqpPropValue);
                        break;
                    case APP_ID:
                        builder.appId(amqpPropValue);
                        break;
                    case CLUSTER_ID:
                        builder.clusterId(amqpPropValue);
                        break;
                }

            } else {
                getLogger().warn("Unrecognised AMQP property '" + amqpPropName + "', will ignore.");
            }
        }
    }
    return builder.build();
}
 

/**
 * Iterates through all of the flow file's metadata and for any metadata key that starts with <code>jms.</code>, the value for the corresponding key is written to the JMS message as a property.
 * The name of this property is equal to the key of the flow file's metadata minus the <code>jms.</code>. For example, if the flowFile has a metadata entry:
 * <br /><br />
 * <code>jms.count</code> = <code>8</code>
 * <br /><br />
 * then the JMS message will have a String property added to it with the property name <code>count</code> and value <code>8</code>.
 *
 * If the flow file also has a metadata key with the name <code>jms.count.type</code>, then the value of that metadata entry will determine the JMS property type to use for the value. For example,
 * if the flow file has the following properties:
 * <br /><br />
 * <code>jms.count</code> = <code>8</code><br />
 * <code>jms.count.type</code> = <code>integer</code>
 * <br /><br />
 * Then <code>message</code> will have an INTEGER property added with the value 8.
 * <br /><br/>
 * If the type is not valid for the given value (e.g., <code>jms.count.type</code> = <code>integer</code> and <code>jms.count</code> = <code>hello</code>, then this JMS property will not be added
 * to <code>message</code>.
 *
 * @param flowFile The flow file whose metadata should be examined for JMS properties.
 * @param message The JMS message to which we want to add properties.
 * @throws JMSException ex
 */
private void copyAttributesToJmsProps(final FlowFile flowFile, final Message message) throws JMSException {
    final ComponentLog logger = getLogger();

    final Map<String, String> attributes = flowFile.getAttributes();
    for (final Entry<String, String> entry : attributes.entrySet()) {
        final String key = entry.getKey();
        final String value = entry.getValue();

        if (key.toLowerCase().startsWith(ATTRIBUTE_PREFIX.toLowerCase()) && !key.toLowerCase().endsWith(ATTRIBUTE_TYPE_SUFFIX.toLowerCase())) {

            final String jmsPropName = key.substring(ATTRIBUTE_PREFIX.length());
            final String type = attributes.get(key + ATTRIBUTE_TYPE_SUFFIX);

            try {
                if (type == null || type.equalsIgnoreCase(PROP_TYPE_STRING)) {
                    message.setStringProperty(jmsPropName, value);
                } else if (type.equalsIgnoreCase(PROP_TYPE_INTEGER)) {
                    message.setIntProperty(jmsPropName, Integer.parseInt(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_BOOLEAN)) {
                    message.setBooleanProperty(jmsPropName, Boolean.parseBoolean(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_SHORT)) {
                    message.setShortProperty(jmsPropName, Short.parseShort(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_LONG)) {
                    message.setLongProperty(jmsPropName, Long.parseLong(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_BYTE)) {
                    message.setByteProperty(jmsPropName, Byte.parseByte(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_DOUBLE)) {
                    message.setDoubleProperty(jmsPropName, Double.parseDouble(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_FLOAT)) {
                    message.setFloatProperty(jmsPropName, Float.parseFloat(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_OBJECT)) {
                    message.setObjectProperty(jmsPropName, value);
                } else {
                    logger.warn("Attribute key '{}' for {} has value '{}', but expected one of: integer, string, object, byte, double, float, long, short, boolean; not adding this property",
                            new Object[]{key, flowFile, value});
                }
            } catch (NumberFormatException e) {
                logger.warn("Attribute key '{}' for {} has value '{}', but attribute key '{}' has value '{}'. Not adding this JMS property",
                        new Object[]{key, flowFile, value, key + ATTRIBUTE_TYPE_SUFFIX, PROP_TYPE_INTEGER});
            }
        }
    }
}
 

public void serializeEdit(final RepositoryRecord previousRecordState, final RepositoryRecord record, final DataOutputStream out, final boolean forceAttributesWritten) throws IOException {
    if (record.isMarkedForAbort()) {
        logger.warn("Repository Record {} is marked to be aborted; it will be persisted in the FlowFileRepository as a DELETE record", record);
        out.write(ACTION_DELETE);
        out.writeLong(getRecordIdentifier(record));
        serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out);
        return;
    }

    final UpdateType updateType = getUpdateType(record);

    if (updateType.equals(UpdateType.DELETE)) {
        out.write(ACTION_DELETE);
        out.writeLong(getRecordIdentifier(record));
        serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out);
        return;
    }

    // If there's a Destination Connection, that's the one that we want to associated with this record.
    // However, on restart, we will restore the FlowFile and set this connection to its "originalConnection".
    // If we then serialize the FlowFile again before it's transferred, it's important to allow this to happen,
    // so we use the originalConnection instead
    FlowFileQueue associatedQueue = record.getDestination();
    if (associatedQueue == null) {
        associatedQueue = record.getOriginalQueue();
    }

    if (updateType.equals(UpdateType.SWAP_OUT)) {
        out.write(ACTION_SWAPPED_OUT);
        out.writeLong(getRecordIdentifier(record));
        out.writeUTF(associatedQueue.getIdentifier());
        out.writeUTF(getLocation(record));
        return;
    }

    final FlowFile flowFile = record.getCurrent();
    final ContentClaim claim = record.getCurrentClaim();

    switch (updateType) {
        case UPDATE:
            out.write(ACTION_UPDATE);
            break;
        case CREATE:
            out.write(ACTION_CREATE);
            break;
        case SWAP_IN:
            out.write(ACTION_SWAPPED_IN);
            break;
        default:
            throw new AssertionError();
    }

    out.writeLong(getRecordIdentifier(record));
    out.writeLong(flowFile.getEntryDate());
    out.writeLong(flowFile.getLineageStartDate());
    out.writeLong(flowFile.getLineageStartIndex());

    final Long queueDate = flowFile.getLastQueueDate();
    out.writeLong(queueDate == null ? System.currentTimeMillis() : queueDate);
    out.writeLong(flowFile.getQueueDateIndex());
    out.writeLong(flowFile.getSize());

    if (associatedQueue == null) {
        logger.warn("{} Repository Record {} has no Connection associated with it; it will be destroyed on restart",
            new Object[] {this, record});
        writeString("", out);
    } else {
        writeString(associatedQueue.getIdentifier(), out);
    }

    serializeContentClaim(claim, record.getCurrentClaimOffset(), out);

    if (forceAttributesWritten || record.isAttributesChanged() || updateType == UpdateType.CREATE || updateType == UpdateType.SWAP_IN) {
        out.write(1);   // indicate attributes changed
        final Map<String, String> attributes = flowFile.getAttributes();
        out.writeInt(attributes.size());
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            writeString(entry.getKey(), out);
            writeString(entry.getValue(), out);
        }
    } else {
        out.write(0);   // indicate attributes did not change
    }

    if (updateType == UpdateType.SWAP_IN) {
        out.writeUTF(record.getSwapLocation());
    }
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile input = null;

    if (context.hasIncomingConnection()) {
        input = session.get();
        if (input == null && context.hasNonLoopConnection()) {
            return;
        }
    }

    final String database = context.getProperty(DATABASE_NAME).evaluateAttributeExpressions(input).getValue();
    final String collection = context.getProperty(COLLECTION_NAME).evaluateAttributeExpressions(input).getValue();
    final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(input).getValue();
    final Document query = getQuery(context, session, input);

    MongoCollection mongoCollection = clientService.getDatabase(database).getCollection(collection);

    FindIterable<Document> find = mongoCollection.find(query);
    if (context.getProperty(SORT).isSet()) {
        find = find.sort(Document.parse(context.getProperty(SORT).evaluateAttributeExpressions(input).getValue()));
    }
    if (context.getProperty(PROJECTION).isSet()) {
        find = find.projection(Document.parse(context.getProperty(PROJECTION).evaluateAttributeExpressions(input).getValue()));
    }
    if (context.getProperty(LIMIT).isSet()) {
        find = find.limit(context.getProperty(LIMIT).evaluateAttributeExpressions(input).asInteger());
    }

    MongoCursor<Document> cursor = find.iterator();

    FlowFile output = input != null ? session.create(input) : session.create();
    final FlowFile inputPtr = input;
    try {
        final Map<String, String> attributes = getAttributes(context, input, query, mongoCollection);
        try (OutputStream out = session.write(output)) {
            Map<String, String> attrs = inputPtr != null ? inputPtr.getAttributes() : new HashMap<String, String>(){{
                put("schema.name", schemaName);
            }};
            RecordSchema schema = writerFactory.getSchema(attrs, null);
            RecordSetWriter writer = writerFactory.createWriter(getLogger(), schema, out, attrs);
            long count = 0L;
            writer.beginRecordSet();
            while (cursor.hasNext()) {
                Document next = cursor.next();
                if (next.get("_id") instanceof ObjectId) {
                    next.put("_id", next.get("_id").toString());
                }
                Record record = new MapRecord(schema, next);
                writer.write(record);
                count++;
            }
            writer.finishRecordSet();
            writer.close();
            out.close();
            attributes.put("record.count", String.valueOf(count));
        } catch (SchemaNotFoundException e) {
            throw new RuntimeException(e);
        }


        output = session.putAllAttributes(output, attributes);

        session.getProvenanceReporter().fetch(output, getURI(context));
        session.transfer(output, REL_SUCCESS);
        if (input != null) {
            session.transfer(input, REL_ORIGINAL);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        getLogger().error("Error writing record set from Mongo query.", ex);
        session.remove(output);
        if (input != null) {
            session.transfer(input, REL_FAILURE);
        }
    }
}
 

/**
 * Iterates through all of the flow file's metadata and for any metadata key that starts with <code>jms.</code>, the value for the corresponding key is written to the JMS message as a property.
 * The name of this property is equal to the key of the flow file's metadata minus the <code>jms.</code>. For example, if the flowFile has a metadata entry:
 * <br /><br />
 * <code>jms.count</code> = <code>8</code>
 * <br /><br />
 * then the JMS message will have a String property added to it with the property name <code>count</code> and value <code>8</code>.
 *
 * If the flow file also has a metadata key with the name <code>jms.count.type</code>, then the value of that metadata entry will determine the JMS property type to use for the value. For example,
 * if the flow file has the following properties:
 * <br /><br />
 * <code>jms.count</code> = <code>8</code><br />
 * <code>jms.count.type</code> = <code>integer</code>
 * <br /><br />
 * Then <code>message</code> will have an INTEGER property added with the value 8.
 * <br /><br/>
 * If the type is not valid for the given value (e.g., <code>jms.count.type</code> = <code>integer</code> and <code>jms.count</code> = <code>hello</code>, then this JMS property will not be added
 * to <code>message</code>.
 *
 * @param flowFile The flow file whose metadata should be examined for JMS properties.
 * @param message The JMS message to which we want to add properties.
 * @throws JMSException ex
 */
private void copyAttributesToJmsProps(final FlowFile flowFile, final Message message) throws JMSException {
    final ComponentLog logger = getLogger();

    final Map<String, String> attributes = flowFile.getAttributes();
    for (final Entry<String, String> entry : attributes.entrySet()) {
        final String key = entry.getKey();
        final String value = entry.getValue();

        if (key.toLowerCase().startsWith(ATTRIBUTE_PREFIX.toLowerCase()) && !key.toLowerCase().endsWith(ATTRIBUTE_TYPE_SUFFIX.toLowerCase())) {

            final String jmsPropName = key.substring(ATTRIBUTE_PREFIX.length());
            final String type = attributes.get(key + ATTRIBUTE_TYPE_SUFFIX);

            try {
                if (type == null || type.equalsIgnoreCase(PROP_TYPE_STRING)) {
                    message.setStringProperty(jmsPropName, value);
                } else if (type.equalsIgnoreCase(PROP_TYPE_INTEGER)) {
                    message.setIntProperty(jmsPropName, Integer.parseInt(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_BOOLEAN)) {
                    message.setBooleanProperty(jmsPropName, Boolean.parseBoolean(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_SHORT)) {
                    message.setShortProperty(jmsPropName, Short.parseShort(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_LONG)) {
                    message.setLongProperty(jmsPropName, Long.parseLong(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_BYTE)) {
                    message.setByteProperty(jmsPropName, Byte.parseByte(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_DOUBLE)) {
                    message.setDoubleProperty(jmsPropName, Double.parseDouble(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_FLOAT)) {
                    message.setFloatProperty(jmsPropName, Float.parseFloat(value));
                } else if (type.equalsIgnoreCase(PROP_TYPE_OBJECT)) {
                    message.setObjectProperty(jmsPropName, value);
                } else {
                    logger.warn("Attribute key '{}' for {} has value '{}', but expected one of: integer, string, object, byte, double, float, long, short, boolean; not adding this property",
                            new Object[]{key, flowFile, value});
                }
            } catch (NumberFormatException e) {
                logger.warn("Attribute key '{}' for {} has value '{}', but attribute key '{}' has value '{}'. Not adding this JMS property",
                        new Object[]{key, flowFile, value, key + ATTRIBUTE_TYPE_SUFFIX, PROP_TYPE_INTEGER});
            }
        }
    }
}