org.apache.nifi.util.FlowFileUnpackagerV3 Java Examples

The following examples show how to use org.apache.nifi.util.FlowFileUnpackagerV3. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestPostHTTP.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testSendAsFlowFile() throws Exception {
    setup(null);
    runner.setProperty(PostHTTP.URL, server.getUrl());
    runner.setProperty(PostHTTP.SEND_AS_FLOWFILE, "true");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
    Assert.assertNull(receivedAttrs.get("Content-Length"));
}
 
Example #2
Source File: TestPostHTTP.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testSendAsFlowFile() throws Exception {
    setup(null);
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.URL, server.getUrl());
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.SEND_AS_FLOWFILE, "true");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
    Assert.assertNull(receivedAttrs.get("Content-Length"));
}
 
Example #3
Source File: TestPostHTTP.java    From nifi with Apache License 2.0 5 votes vote down vote up
private void checkBatch(TestServer server, CaptureServlet servlet, Set<String> actualContent, int expectedCount) throws Exception {
    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();
    Set<String> actualFFContent = new HashSet<>();
    Set<String> actualPostContent = new HashSet<>();

    runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS, expectedCount);

    // confirm that all FlowFiles transferred to 'success' have the same URL
    // also accumulate content to verify later
    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS);
    for (int i = 0; i < expectedCount; i++) {
        MockFlowFile mff = successFlowFiles.get(i);
        mff.assertAttributeEquals("url", server.getUrl());
        String content = new String(mff.toByteArray());
        actualFFContent.add(content);
    }

    // confirm that all FlowFiles POSTed to server have the same URL
    // also accumulate content to verify later
    try (ByteArrayInputStream bais = new ByteArrayInputStream(servlet.getLastPost());
        ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
        for (int i = 0; i < expectedCount; i++) {
            Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
            String receivedContent = new String(baos.toByteArray());
            actualPostContent.add(receivedContent);
            assertEquals(server.getUrl(), receivedAttrs.get("url"));
            assertTrue(unpacker.hasMoreData() || i == (expectedCount - 1));
            baos.reset();
        }
    }

    // confirm that the transferred and POSTed content match
    assertEquals(actualFFContent, actualPostContent);

    // accumulate actial content
    actualContent.addAll(actualPostContent);
    runner.clearTransferState();
}
 
Example #4
Source File: UnpackContent.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        for (PackageFormat format: PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[]{flowFile, mimeType});
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }

    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch (packagingFormat) {
    case TAR_FORMAT:
    case X_TAR_FORMAT:
        unpacker = tarUnpacker;
        addFragmentAttrs = true;
        break;
    case ZIP_FORMAT:
        unpacker = zipUnpacker;
        addFragmentAttrs = true;
        break;
    case FLOWFILE_STREAM_FORMAT_V2:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_STREAM_FORMAT_V3:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_TAR_FORMAT:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
        addFragmentAttrs = false;
        break;
    case AUTO_DETECT_FORMAT:
    default:
        // The format of the unpacker should be known before initialization
        throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }

    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[]{flowFile, unpacked});
    } catch (final ProcessException | InvalidPathException e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[]{flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}
 
Example #5
Source File: TestPostHTTP.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testSendAsFlowFileSecure() throws Exception {
    final Map<String, String> sslProps = new HashMap<>();
    sslProps.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/localhost-ks.jks");
    sslProps.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "localtest");
    sslProps.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
    sslProps.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");
    sslProps.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "localtest");
    sslProps.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
    sslProps.put(TestServer.NEED_CLIENT_AUTH, "true");
    setup(sslProps);

    final SSLContextService sslContextService = new StandardSSLContextService();
    runner.addControllerService("ssl-context", sslContextService);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, "src/test/resources/localhost-ts.jks");
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, "localtest");
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, "JKS");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE, "src/test/resources/localhost-ks.jks");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_PASSWORD, "localtest");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_TYPE, "JKS");
    runner.enableControllerService(sslContextService);

    runner.setProperty(PostHTTP.URL, server.getSecureUrl());
    runner.setProperty(PostHTTP.SEND_AS_FLOWFILE, "true");
    runner.setProperty(PostHTTP.SSL_CONTEXT_SERVICE, "ssl-context");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
}
 
Example #6
Source File: UnpackContent.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        for (PackageFormat format: PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[]{flowFile, mimeType});
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }

    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch (packagingFormat) {
    case TAR_FORMAT:
    case X_TAR_FORMAT:
        unpacker = tarUnpacker;
        addFragmentAttrs = true;
        break;
    case ZIP_FORMAT:
        unpacker = zipUnpacker;
        addFragmentAttrs = true;
        break;
    case FLOWFILE_STREAM_FORMAT_V2:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_STREAM_FORMAT_V3:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_TAR_FORMAT:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
        addFragmentAttrs = false;
        break;
    case AUTO_DETECT_FORMAT:
    default:
        // The format of the unpacker should be known before initialization
        throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }

    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[]{flowFile, unpacked});
    } catch (final Exception e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[]{flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}
 
Example #7
Source File: TestPostHTTP.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testSendAsFlowFileSecure() throws Exception {
    final Map<String, String> sslProps = new HashMap<>();
    sslProps.put(StandardSSLContextService.KEYSTORE.getName(), KEYSTORE_PATH);
    sslProps.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    sslProps.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), JKS_TYPE);
    sslProps.put(StandardSSLContextService.TRUSTSTORE.getName(), TRUSTSTORE_PATH);
    sslProps.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    sslProps.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), JKS_TYPE);
    sslProps.put(TestServer.NEED_CLIENT_AUTH, "true");
    setup(sslProps);

    final SSLContextService sslContextService = new StandardSSLContextService();
    runner.addControllerService("ssl-context", sslContextService);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, TRUSTSTORE_PATH);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, JKS_TYPE);
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE, KEYSTORE_PATH);
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_PASSWORD, KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_TYPE, JKS_TYPE);
    runner.enableControllerService(sslContextService);

    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.URL, server.getSecureUrl());
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.SEND_AS_FLOWFILE, "true");
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.SSL_CONTEXT_SERVICE, "ssl-context");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
}