org.apache.nifi.processor.exception.FlowFileAccessException Java Examples

The following examples show how to use org.apache.nifi.processor.exception.FlowFileAccessException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlowFileAccessInputStream.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public int read() throws IOException {
    try {
        final int byteRead = super.read();
        if (byteRead < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed++;
        }

        return byteRead;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 
Example #2
Source File: MockProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public MockFlowFile importFrom(final Path path, final boolean keepSourceFile, final FlowFile flowFile) {
    validateState(flowFile);
    if (path == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;
    MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        Files.copy(path, baos);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }

    newFlowFile.setData(baos.toByteArray());
    newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
    return newFlowFile;
}
 
Example #3
Source File: MockProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public MockFlowFile importFrom(final InputStream in, final FlowFile flowFile) {
    validateState(flowFile);
    if (in == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;

    final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);
    try {
        final byte[] data = readFully(in);
        newFlowFile.setData(data);
        return newFlowFile;
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #4
Source File: MockProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(final FlowFile flowFile, final OutputStream out) {
    validateState(flowFile);
    if (flowFile == null || out == null) {
        throw new IllegalArgumentException("arguments cannot be null");
    }

    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    try {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #5
Source File: MockProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(final FlowFile flowFile, final Path path, final boolean append) {
    validateState(flowFile);
    if (flowFile == null || path == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;

    try (final OutputStream out = Files.newOutputStream(path, mode)) {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #6
Source File: StandardProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
private void resetWriteClaims(final boolean suppressExceptions) {
    for (final ByteCountingOutputStream out : appendableStreams.values()) {
        try {
            try {
                out.flush();
            } finally {
                out.close();
            }
        } catch (final IOException e) {
            if (!suppressExceptions) {
                throw new FlowFileAccessException("Unable to flush the output of FlowFile to the Content Repository");
            }
        }
    }
    appendableStreams.clear();
}
 
Example #7
Source File: FlowFileAccessInputStream.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
    try {
        final int count = super.read(b, off, len);
        if (count < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed += count;
        }

        return count;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 
Example #8
Source File: StandardProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(FlowFile source, final Path destination, final boolean append) {
    verifyTaskActive();
    source = validateRecordState(source);
    final StandardRepositoryRecord record = getRecord(source);
    try {
        ensureNotAppending(record.getCurrentClaim());

        claimCache.flush(record.getCurrentClaim());
        final long copyCount = context.getContentRepository().exportTo(record.getCurrentClaim(), destination, append, record.getCurrentClaimOffset(), source.getSize());
        bytesRead += copyCount;
        bytesWritten += copyCount;
    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final Throwable t) {
        throw new FlowFileAccessException("Failed to export " + source + " to " + destination + " due to " + t.toString(), t);
    }
}
 
Example #9
Source File: FlowFileAccessInputStream.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public int read() throws IOException {
    try {
        final int byteRead = super.read();
        if (byteRead < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed++;
        }

        return byteRead;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 
Example #10
Source File: StandardProcessSession.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(final FlowFile source, final Path destination, final boolean append) {
    validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);
    try {
        ensureNotAppending(record.getCurrentClaim());

        claimCache.flush(record.getCurrentClaim());
        final long copyCount = context.getContentRepository().exportTo(record.getCurrentClaim(), destination, append, record.getCurrentClaimOffset(), source.getSize());
        bytesRead += copyCount;
        bytesWritten += copyCount;
    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final Throwable t) {
        throw new FlowFileAccessException("Failed to export " + source + " to " + destination + " due to " + t.toString(), t);
    }
}
 
Example #11
Source File: FlowFileAccessInputStream.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
    try {
        final int count = super.read(b, off, len);
        if (count < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed += count;
        }

        return count;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 
Example #12
Source File: FlowFileAccessInputStream.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public int read(final byte[] b) throws IOException {
    try {
        final int count = super.read(b);
        if (count < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed += count;
        }

        return count;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 
Example #13
Source File: StandardProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
private void resetWriteClaims(final boolean suppressExceptions) {
    for (final ByteCountingOutputStream out : appendableStreams.values()) {
        try {
            try {
                out.flush();
            } finally {
                out.close();
            }
        } catch (final IOException e) {
            if (!suppressExceptions) {
                throw new FlowFileAccessException("Unable to flush the output of FlowFile to the Content Repository");
            }
        }
    }
    appendableStreams.clear();
}
 
Example #14
Source File: StatelessProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public StatelessFlowFile importFrom(final Path path, final boolean keepSourceFile, FlowFile flowFile) {
    flowFile = validateState(flowFile);
    if (path == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof StatelessFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    if (!keepSourceFile) {
        throw new IllegalArgumentException("Not going to delete the file...");
    }
    StatelessFlowFile newFlowFile = new StatelessFlowFile((StatelessFlowFile) flowFile, this.materializeContent);
    try {
        newFlowFile.setData(Files.newInputStream(path));
    } catch (IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
    currentVersions.put(newFlowFile.getId(), newFlowFile);

    newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
    return newFlowFile;
}
 
Example #15
Source File: StatelessProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(FlowFile flowFile, final Path path, final boolean append) {
    flowFile = validateState(flowFile);
    if (flowFile == null || path == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof StatelessFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    StatelessFlowFile statelessFlowFile = (StatelessFlowFile) flowFile;

    final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;

    try (final OutputStream out = Files.newOutputStream(path, mode)) {
        if (statelessFlowFile.materializeContent)
            statelessFlowFile.materializeData();
        copyTo(statelessFlowFile.getDataStream(), out);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #16
Source File: StatelessProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(FlowFile flowFile, final OutputStream out) {
    flowFile = validateState(flowFile);
    if (flowFile == null || out == null) {
        throw new IllegalArgumentException("arguments cannot be null");
    }

    if (!(flowFile instanceof StatelessFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    try {
        copyTo(((StatelessFlowFile) flowFile).getDataStream(), out);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #17
Source File: MockProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public MockFlowFile importFrom(final Path path, final boolean keepSourceFile, FlowFile flowFile) {
    flowFile = validateState(flowFile);
    if (path == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;
    MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        Files.copy(path, baos);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }

    newFlowFile.setData(baos.toByteArray());
    newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
    return newFlowFile;
}
 
Example #18
Source File: MockProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public MockFlowFile importFrom(final InputStream in, FlowFile flowFile) {
    flowFile = validateState(flowFile);
    if (in == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;

    final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);
    try {
        final byte[] data = readFully(in);
        newFlowFile.setData(data);
        return newFlowFile;
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #19
Source File: MockProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(FlowFile flowFile, final Path path, final boolean append) {
    flowFile = validateState(flowFile);
    if (flowFile == null || path == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;

    try (final OutputStream out = Files.newOutputStream(path, mode)) {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #20
Source File: MockProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
private void closeStreams(final Map<FlowFile, ? extends Closeable> streamMap, final boolean enforceClosed) {
    final Map<FlowFile, ? extends Closeable> openStreamCopy = new HashMap<>(streamMap); // avoid ConcurrentModificationException by creating a copy of the List
    for (final Map.Entry<FlowFile, ? extends Closeable> entry : openStreamCopy.entrySet()) {
        final FlowFile flowFile = entry.getKey();
        final Closeable openStream = entry.getValue();

        try {
            openStream.close();
        } catch (IOException e) {
            throw new FlowFileAccessException("Failed to close stream for " + flowFile, e);
        }

        if (enforceClosed) {
            throw new FlowFileHandlingException("Cannot commit session because the following streams were created via "
                + "calls to ProcessSession.read(FlowFile) or ProcessSession.write(FlowFile) and never closed: " + streamMap);
        }
    }
}
 
Example #21
Source File: MockProcessSession.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void exportTo(FlowFile flowFile, final OutputStream out) {
    flowFile = validateState(flowFile);
    if (flowFile == null || out == null) {
        throw new IllegalArgumentException("arguments cannot be null");
    }

    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    try {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 
Example #22
Source File: FlowFileAccessInputStream.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void reset() throws IOException {
    try {
        super.reset();
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not reset stream from " + flowFile, ioe);
    }
}
 
Example #23
Source File: FlowFileAccessInputStream.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public long skip(final long n) throws IOException {
    try {
        final long count = super.skip(n);
        bytesConsumed += count;
        return count;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not skip data in " + flowFile, ioe);
    }
}
 
Example #24
Source File: TestFetchS3Object.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testFlowFileAccessExceptionGoesToFailure() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    AmazonS3Exception amazonException = new AmazonS3Exception("testing");
    Mockito.doThrow(new FlowFileAccessException("testing nested", amazonException)).when(mockS3Client).getObject(Mockito.any());

    runner.run(1);

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1);
}
 
Example #25
Source File: StatelessFlowFile.java    From nifi with Apache License 2.0 5 votes vote down vote up
public StatelessFlowFile(final StatelessFlowFile toCopy, boolean materializeContent) {
    this(materializeContent);
    this.id = toCopy.id;

    attributes.putAll(toCopy.getAttributes());
    this.penalized = toCopy.isPenalized();

    try {
        this.setData(toCopy.getDataArray());
    } catch (IOException e) {
        throw new FlowFileAccessException("Exception creating FlowFile", e);
    }
}
 
Example #26
Source File: TestListenSyslog.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testErrorQueue() throws IOException {
    final List<ListenSyslog.RawSyslogEvent> msgs = new ArrayList<>();
    msgs.add(new ListenSyslog.RawSyslogEvent(VALID_MESSAGE.getBytes(), "sender-01"));
    msgs.add(new ListenSyslog.RawSyslogEvent(VALID_MESSAGE.getBytes(), "sender-01"));

    // Add message that will throw a FlowFileAccessException the first time that we attempt to read
    // the contents but will succeed the second time.
    final AtomicInteger getMessageAttempts = new AtomicInteger(0);
    msgs.add(new ListenSyslog.RawSyslogEvent(VALID_MESSAGE.getBytes(), "sender-01") {
        @Override
        public byte[] getData() {
            final int attempts = getMessageAttempts.incrementAndGet();
            if (attempts == 1) {
                throw new FlowFileAccessException("Unit test failure");
            } else {
                return VALID_MESSAGE.getBytes();
            }
        }
    });

    final CannedMessageProcessor proc = new CannedMessageProcessor(msgs);
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.MAX_BATCH_SIZE, "5");
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.UDP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");
    runner.setProperty(ListenSyslog.PARSE_MESSAGES, "false");

    runner.run();
    assertEquals(1, proc.getErrorQueueSize());
    runner.assertAllFlowFilesTransferred(ListenSyslog.REL_SUCCESS, 1);
    runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0).assertContentEquals(VALID_MESSAGE + "\n" + VALID_MESSAGE);

    // running again should pull from the error queue
    runner.clearTransferState();
    runner.run();
    runner.assertAllFlowFilesTransferred(ListenSyslog.REL_SUCCESS, 1);
    runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0).assertContentEquals(VALID_MESSAGE);
}
 
Example #27
Source File: FlowFileAccessOutputStream.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void write(final int b) throws IOException {
    try {
        out.write(b);
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not write to " + flowFile, ioe);
    }
}
 
Example #28
Source File: FlowFileAccessOutputStream.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void write(final byte[] b) throws IOException {
    try {
        out.write(b);
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not write to " + flowFile, ioe);
    }
}
 
Example #29
Source File: FlowFileAccessOutputStream.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
    try {
        out.flush();
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not flush OutputStream for " + flowFile, ioe);
    }
}
 
Example #30
Source File: TestStandardProcessSession.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testHandlingOfMultipleFlowFilesWithSameId() {
    // Add two FlowFiles with the same ID
    for (int i=0; i < 2; i++) {
        final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder()
            .id(1000L)
            .addAttribute("uuid", "12345678-1234-1234-1234-123456789012")
            .entryDate(System.currentTimeMillis())
            .size(0L)
            .build();

        flowFileQueue.put(flowFileRecord);
    }

    final Relationship relationship = new Relationship.Builder().name("A").build();

    FlowFile ff1 = session.get();
    assertNotNull(ff1);

    session.transfer(ff1, relationship);

    try {
        session.get();
        Assert.fail("Should not have been able to poll second FlowFile with same ID");
    } catch (final FlowFileAccessException e) {
        // Expected
    }
}