Java Code Examples for org.apache.nifi.stream.io.StreamUtils

The following examples show how to use org.apache.nifi.stream.io.StreamUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: nifi   Source File: NifiSeekableInputStream.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void seek(long newPos) throws IOException {
    final long currentPos = getPos();
    if (newPos == currentPos) {
        return;
    }

    if (newPos < currentPos) {
        // seeking backwards so first reset back to beginning of the stream then seek
        input.reset();
        input.mark(Integer.MAX_VALUE);
    }

    // must call getPos() again in case reset was called above
    StreamUtils.skip(input, newPos - getPos());
}
 
Example 2
Source Project: localization_nifi   Source File: SiteToSiteRestApiClient.java    License: Apache License 2.0 6 votes vote down vote up
private TransactionResultEntity readResponse(final InputStream inputStream) throws IOException {
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();

    StreamUtils.copy(inputStream, bos);
    String responseMessage = null;

    try {
        responseMessage = new String(bos.toByteArray(), "UTF-8");
        logger.debug("readResponse responseMessage={}", responseMessage);

        final ObjectMapper mapper = new ObjectMapper();
        return mapper.readValue(responseMessage, TransactionResultEntity.class);
    } catch (JsonParseException | JsonMappingException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("Failed to parse JSON.", e);
        }

        final TransactionResultEntity entity = new TransactionResultEntity();
        entity.setResponseCode(ResponseCode.ABORT.getCode());
        entity.setMessage(responseMessage);
        return entity;
    }
}
 
Example 3
Source Project: localization_nifi   Source File: CaptureServlet.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Capture all the headers for reference.  Intentionally choosing to not special handling for headers with multiple values for clarity
    final Enumeration<String> headerNames = request.getHeaderNames();
    lastPostHeaders = new HashMap<>();
    while (headerNames.hasMoreElements()) {
        final String nextHeader = headerNames.nextElement();
        lastPostHeaders.put(nextHeader, request.getHeader(nextHeader));
    }

    try {
        StreamUtils.copy(request.getInputStream(), baos);
        this.lastPost = baos.toByteArray();
    } finally {
        FileUtils.closeQuietly(baos);
    }
    response.setStatus(Status.OK.getStatusCode());
}
 
Example 4
Source Project: nifi   Source File: FileSystemRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public long exportTo(final ContentClaim claim, final OutputStream destination, final long offset, final long length) throws IOException {
    if (offset < 0) {
        throw new IllegalArgumentException("offset cannot be negative");
    }
    final long claimSize = size(claim);
    if (offset > claimSize) {
        throw new IllegalArgumentException("offset of " + offset + " exceeds claim size of " + claimSize);
    }
    if (offset == 0 && length == claimSize) {
        return exportTo(claim, destination);
    }
    try (final InputStream in = read(claim)) {
        StreamUtils.skip(in, offset);
        final byte[] buffer = new byte[8192];
        int len;
        long copied = 0L;
        while ((len = in.read(buffer, 0, (int) Math.min(length - copied, buffer.length))) > 0) {
            destination.write(buffer, 0, len);
            copied += len;
        }
        return copied;
    }
}
 
Example 5
Source Project: nifi   Source File: EncryptedSchemaRepositoryRecordSerde.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the deserialized and decrypted {@link RepositoryRecord} from the input stream.
 *
 * @param in      stream to read from
 * @param version the version of the SerDe that was used to serialize the
 *                record
 * @return the deserialized record
 * @throws IOException if there is a problem reading from the stream
 */
@Override
public SerializedRepositoryRecord deserializeRecord(final DataInputStream in, final int version) throws IOException {
    // Read the expected length of the encrypted record (including the encryption metadata)
    int encryptedRecordLength = in.readInt();
    if (encryptedRecordLength == -1) {
        return null;
    }

    // Read the encrypted record bytes
    byte[] cipherBytes = new byte[encryptedRecordLength];
    StreamUtils.fillBuffer(in, cipherBytes);
    logger.debug("Read {} bytes (encrypted, including length) from actual input stream", encryptedRecordLength + 4);

    // Decrypt the byte[]
    DataInputStream wrappedInputStream = decryptToStream(cipherBytes);

    // Deserialize the plain bytes using the delegate serde
    final SerializedRepositoryRecord deserializedRecord = wrappedSerDe.deserializeRecord(wrappedInputStream, version);
    logger.debug("Deserialized flowfile record {} from temp stream", getRecordIdentifier(deserializedRecord));
    return deserializedRecord;
}
 
Example 6
Source Project: nifi   Source File: EncryptedSchemaRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected Optional<StandardProvenanceEventRecord> readToEvent(final long eventId, final DataInputStream dis, final int serializationVersion) throws IOException {
    verifySerializationVersion(serializationVersion);

    while (isData(dis)) {
        final long startOffset = getBytesConsumed();
        final long id = dis.readInt() + getFirstEventId();
        final int recordLength = dis.readInt();

        if (id >= eventId) {
            final StandardProvenanceEventRecord event = readRecord(dis, id, startOffset, recordLength);
            return Optional.ofNullable(event);
        } else {
            // This is not the record we want. Skip over it instead of deserializing it.
            StreamUtils.skip(dis, recordLength);
        }
    }

    return Optional.empty();
}
 
Example 7
Source Project: nifi   Source File: TestContentRepositoryFlowFileAccess.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInputStreamFromContentRepo() throws IOException {
    final ContentRepository contentRepo = mock(ContentRepository.class);

    final ResourceClaimManager claimManager = new StandardResourceClaimManager();
    final ResourceClaim resourceClaim = new StandardResourceClaim(claimManager, "container", "section", "id", false);
    final ContentClaim contentClaim = new StandardContentClaim(resourceClaim, 5L);

    final FlowFileRecord flowFile = mock(FlowFileRecord.class);
    when(flowFile.getContentClaim()).thenReturn(contentClaim);
    when(flowFile.getSize()).thenReturn(5L);

    final InputStream inputStream = new ByteArrayInputStream("hello".getBytes());
    when(contentRepo.read(contentClaim)).thenReturn(inputStream);

    final ContentRepositoryFlowFileAccess flowAccess = new ContentRepositoryFlowFileAccess(contentRepo);

    final InputStream repoStream = flowAccess.read(flowFile);
    verify(contentRepo, times(1)).read(contentClaim);

    final byte[] buffer = new byte[5];
    StreamUtils.fillBuffer(repoStream, buffer);
    assertEquals(-1, repoStream.read());
    assertArrayEquals("hello".getBytes(), buffer);
}
 
Example 8
Source Project: localization_nifi   Source File: CaptureServlet.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Capture all the headers for reference.  Intentionally choosing to not special handling for headers with multiple values for clarity
    final Enumeration<String> headerNames = request.getHeaderNames();
    lastPostHeaders = new HashMap<>();
    while (headerNames.hasMoreElements()) {
        final String nextHeader = headerNames.nextElement();
        lastPostHeaders.put(nextHeader, request.getHeader(nextHeader));
    }

    try {
        StreamUtils.copy(request.getInputStream(), baos);
        this.lastPost = baos.toByteArray();
    } finally {
        FileUtils.closeQuietly(baos);
    }
    response.setStatus(Status.OK.getStatusCode());
}
 
Example 9
Source Project: nifi   Source File: CipherUtility.java    License: Apache License 2.0 6 votes vote down vote up
public static byte[] readBytesFromInputStream(InputStream in, String label, int limit, byte[] delimiter) throws IOException, ProcessException {
    if (in == null) {
        throw new IllegalArgumentException("Cannot read " + label + " from null InputStream");
    }

    // If the value is not detected within the first n bytes, throw an exception
    in.mark(limit);

    // The first n bytes of the input stream contain the value up to the custom delimiter
    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    byte[] stoppedBy = StreamUtils.copyExclusive(in, bytesOut, limit + delimiter.length, delimiter);

    if (stoppedBy != null) {
        byte[] bytes = bytesOut.toByteArray();
        return bytes;
    }

    // If no delimiter was found, reset the cursor
    in.reset();
    return null;
}
 
Example 10
Source Project: nifi   Source File: TestFileSystemRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadClaimThenWriteThenReadMore() throws IOException {
    final ContentClaim claim = repository.create(false);

    final OutputStream out = repository.write(claim);
    out.write("hello".getBytes());
    out.flush();

    final InputStream in = repository.read(claim);
    final byte[] buffer = new byte[5];
    StreamUtils.fillBuffer(in, buffer);

    assertEquals("hello", new String(buffer));

    out.write("good-bye".getBytes());
    out.close();

    final byte[] buffer2 = new byte[8];
    StreamUtils.fillBuffer(in, buffer2);
    assertEquals("good-bye", new String(buffer2));
}
 
Example 11
Source Project: localization_nifi   Source File: PutHBaseCell.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected PutFlowFile createPut(final ProcessSession session, final ProcessContext context, final FlowFile flowFile) {
    final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
    final String row = context.getProperty(ROW_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String columnFamily = context.getProperty(COLUMN_FAMILY).evaluateAttributeExpressions(flowFile).getValue();
    final String columnQualifier = context.getProperty(COLUMN_QUALIFIER).evaluateAttributeExpressions(flowFile).getValue();

    final byte[] buffer = new byte[(int) flowFile.getSize()];
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer);
        }
    });


    final Collection<PutColumn> columns = Collections.singletonList(new PutColumn(columnFamily.getBytes(StandardCharsets.UTF_8),
                                                                        columnQualifier.getBytes(StandardCharsets.UTF_8), buffer));
    byte[] rowKeyBytes = getRow(row,context.getProperty(ROW_ID_ENCODING_STRATEGY).getValue());


    return new PutFlowFile(tableName,rowKeyBytes , columns, flowFile);
}
 
Example 12
Source Project: localization_nifi   Source File: SnippetManager.java    License: Apache License 2.0 6 votes vote down vote up
public static List<StandardSnippet> parseBytes(final byte[] bytes) {
    final List<StandardSnippet> snippets = new ArrayList<>();

    try (final InputStream rawIn = new ByteArrayInputStream(bytes);
            final DataInputStream in = new DataInputStream(rawIn)) {
        final int length = in.readInt();
        final byte[] buffer = new byte[length];
        StreamUtils.fillBuffer(in, buffer, true);
        final StandardSnippet snippet = StandardSnippetDeserializer.deserialize(new ByteArrayInputStream(buffer));
        snippets.add(snippet);
    } catch (final IOException e) {
        throw new RuntimeException("Failed to parse bytes", e);  // should never happen because of streams being used
    }

    return snippets;
}
 
Example 13
Source Project: localization_nifi   Source File: FileSystemRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ContentClaim clone(final ContentClaim original, final boolean lossTolerant) throws IOException {
    if (original == null) {
        return null;
    }

    final ContentClaim newClaim = create(lossTolerant);
    try (final InputStream in = read(original);
            final OutputStream out = write(newClaim)) {
        StreamUtils.copy(in, out);
    } catch (final IOException ioe) {
        decrementClaimantCount(newClaim);
        remove(newClaim);
        throw ioe;
    }
    return newClaim;
}
 
Example 14
Source Project: localization_nifi   Source File: FileSystemRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public InputStream read(final ContentClaim claim) throws IOException {
    if (claim == null) {
        return new ByteArrayInputStream(new byte[0]);
    }
    final Path path = getPath(claim, true);
    final FileInputStream fis = new FileInputStream(path.toFile());
    if (claim.getOffset() > 0L) {
        try {
            StreamUtils.skip(fis, claim.getOffset());
        } catch (IOException ioe) {
            IOUtils.closeQuietly(fis);
            throw ioe;
        }

    }

    // see javadocs for claim.getLength() as to why we do this.
    if (claim.getLength() >= 0) {
        return new LimitedInputStream(fis, claim.getLength());
    } else {
        return fis;
    }
}
 
Example 15
Source Project: nifi   Source File: TestStandardProcessSession.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadFromInputStream() throws IOException {
    FlowFile flowFile = session.create();
    flowFile = session.write(flowFile, new OutputStreamCallback() {
        @Override
        public void process(final OutputStream out) throws IOException {
            out.write("hello, world".getBytes());
        }
    });

    try (InputStream in = session.read(flowFile)) {
        final byte[] buffer = new byte[12];
        StreamUtils.fillBuffer(in, buffer);
        assertEquals("hello, world", new String(buffer));
    }

    session.remove(flowFile);
    session.commit();
}
 
Example 16
Source Project: nifi   Source File: OpenSSLPKCS5CipherProvider.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the salt provided as part of the cipher stream, or throws an exception if one cannot be detected.
 *
 * @param in the cipher InputStream
 * @return the salt
 */
@Override
public byte[] readSalt(InputStream in) throws IOException {
    if (in == null) {
        throw new IllegalArgumentException("Cannot read salt from null InputStream");
    }

    // The header and salt format is "Salted__salt x8b" in ASCII
    byte[] salt = new byte[DEFAULT_SALT_LENGTH];

    // Try to read the header and salt from the input
    byte[] header = new byte[OPENSSL_EVP_HEADER_SIZE];

    // Mark the stream in case there is no salt
    in.mark(OPENSSL_EVP_HEADER_SIZE + 1);
    StreamUtils.fillBuffer(in, header);

    final byte[] headerMarkerBytes = OPENSSL_EVP_HEADER_MARKER.getBytes(StandardCharsets.US_ASCII);

    if (!Arrays.equals(headerMarkerBytes, header)) {
        // No salt present
        salt = new byte[0];
        // Reset the stream because we skipped 8 bytes of cipher text
        in.reset();
    }

    StreamUtils.fillBuffer(in, salt);
    return salt;
}
 
Example 17
Source Project: localization_nifi   Source File: TestSiteToSiteClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Ignore("For local testing only; not really a unit test but a manual test")
public void testReceive() throws IOException {
    System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.remote", "DEBUG");

    final SiteToSiteClient client = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("cba")
            .requestBatchCount(10)
            .build();

    try {
        for (int i = 0; i < 1000; i++) {
            final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
            Assert.assertNotNull(transaction);

            DataPacket packet;
            while (true) {
                packet = transaction.receive();
                if (packet == null) {
                    break;
                }

                final InputStream in = packet.getData();
                final long size = packet.getSize();
                final byte[] buff = new byte[(int) size];

                StreamUtils.fillBuffer(in, buff);
            }

            transaction.confirm();
            transaction.complete();
        }
    } finally {
        client.close();
    }
}
 
Example 18
Source Project: nifi   Source File: TestContentClaimInputStream.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRereadWithOffset() throws IOException {
    final ContentClaimInputStream in = new ContentClaimInputStream(repo, contentClaim, 3L);

    final byte[] buff = new byte[2];

    final int invocations = 10;
    for (int i=0; i < invocations; i++) {
        in.mark(5);

        StreamUtils.fillBuffer(in, buff, true);

        final String contentRead = new String(buff);
        assertEquals("lo", contentRead);

        assertEquals(2 * (i+1), in.getBytesConsumed());
        assertEquals(5, in.getCurrentOffset());
        assertEquals(-1, in.read());

        in.reset();
    }

    Mockito.verify(repo, Mockito.times(invocations + 1)).read(contentClaim); // Will call reset() 'invocations' times plus the initial read
    Mockito.verifyNoMoreInteractions(repo);

    // Ensure that underlying stream is closed
    in.close();
    assertTrue(closed.get());
}
 
Example 19
Source Project: nifi   Source File: FlowFileEvent.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public byte[] getBody() {
  synchronized (bodyLock) {
    if (!bodyLoaded) {
      if (flowFile.getSize() > Integer.MAX_VALUE) {
        throw new RuntimeException("Can't get body of Event because the backing FlowFile is too large (" + flowFile.getSize() + " bytes)");
      }

      final ByteArrayOutputStream baos = new ByteArrayOutputStream((int) flowFile.getSize());
      session.read(flowFile, new InputStreamCallback() {

        @Override
        public void process(InputStream in) throws IOException {
          try (BufferedInputStream input = new BufferedInputStream(in)) {
            StreamUtils.copy(input, baos);
          }
          baos.close();
        }
      });

      body = baos.toByteArray();
      bodyLoaded = true;
    }
  }

  return body;
}
 
Example 20
Source Project: nifi   Source File: VolatileContentRepository.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long exportTo(ContentClaim claim, OutputStream destination, long offset, long length) throws IOException {
    final InputStream in = read(claim);
    try {
        StreamUtils.skip(in, offset);
        StreamUtils.copy(in, destination, length);
    } finally {
        IOUtils.closeQuietly(in);
    }
    return length;
}
 
Example 21
Source Project: nifi   Source File: TestStandardProcessSession.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOpenMultipleInputStreamsToFlowFile() throws IOException {
    final ContentClaim claim = contentRepo.create(false);
    try (final OutputStream out = contentRepo.write(claim)) {
        out.write("hello, world".getBytes());
    }

    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder()
            .contentClaim(claim)
            .addAttribute("uuid", "12345678-1234-1234-1234-123456789012")
            .entryDate(System.currentTimeMillis())
            .size(12L)
            .build();
    flowFileQueue.put(flowFileRecord);

    final FlowFile flowFile = session.get();
    InputStream in = session.read(flowFile);
    final byte[] buffer = new byte[12];
    StreamUtils.fillBuffer(in, buffer);
    assertEquals("hello, world", new String(buffer));

    InputStream in2 = session.read(flowFile);
    StreamUtils.fillBuffer(in2, buffer);
    assertEquals("hello, world", new String(buffer));

    in.close();
    in2.close();
    session.remove(flowFile);
    session.commit();
}
 
Example 22
Source Project: nifi   Source File: ByteArraySchemaRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void readHeader(final DataInputStream in, final int serializationVersion) throws IOException {
    verifySerializationVersion(serializationVersion);
    final int schemaLength = in.readInt();
    final byte[] buffer = new byte[schemaLength];
    StreamUtils.fillBuffer(in, buffer);

    try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) {
        schema = RecordSchema.readFrom(bais);
    }

    recordReader = SchemaRecordReader.fromSchema(schema);
}
 
Example 23
Source Project: nifi   Source File: PutCassandraQL.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Determines the CQL statement that should be executed for the given FlowFile
 *
 * @param session  the session that can be used to access the given FlowFile
 * @param flowFile the FlowFile whose CQL statement should be executed
 * @return the CQL that is associated with the given FlowFile
 */

private String getCQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) {
    // Read the CQL from the FlowFile's content
    final byte[] buffer = new byte[(int) flowFile.getSize()];
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer);
        }
    });

    // Create the PreparedStatement string to use for this FlowFile.
    return new String(buffer, charset);
}
 
Example 24
@Override
@SuppressWarnings("unchecked")
protected synchronized void readHeader(final DataInputStream in, final int serializationVersion) throws IOException {
    verifySerializationVersion(serializationVersion);
    final int eventSchemaLength = in.readInt();
    final byte[] buffer = new byte[eventSchemaLength];
    StreamUtils.fillBuffer(in, buffer);

    try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) {
        schema = RecordSchema.readFrom(bais);
    }

    recordReader = SchemaRecordReader.fromSchema(schema);

    final int headerSchemaLength = in.readInt();
    final byte[] headerSchemaBuffer = new byte[headerSchemaLength];
    StreamUtils.fillBuffer(in, headerSchemaBuffer);

    final RecordSchema headerSchema;
    try (final ByteArrayInputStream bais = new ByteArrayInputStream(headerSchemaBuffer)) {
        headerSchema = RecordSchema.readFrom(bais);
    }

    final SchemaRecordReader headerReader = SchemaRecordReader.fromSchema(headerSchema);
    final Record headerRecord = headerReader.readRecord(in);
    componentIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_IDS);
    componentTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_TYPES);
    queueIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.QUEUE_IDS);
    eventTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.EVENT_TYPES);
    firstEventId = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.FIRST_EVENT_ID);
    systemTimeOffset = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.TIMESTAMP_OFFSET);
}
 
Example 25
@Override
protected void readHeader(final DataInputStream in, final int serializationVersion) throws IOException {
    verifySerializationVersion(serializationVersion);
    final int schemaLength = in.readInt();
    final byte[] buffer = new byte[schemaLength];
    StreamUtils.fillBuffer(in, buffer);

    try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) {
        schema = RecordSchema.readFrom(bais);
    }

    recordReader = SchemaRecordReader.fromSchema(schema);
}
 
Example 26
Source Project: nifi   Source File: TestJmsConsumer.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMap2FlowFileTextMessage() throws Exception {

    TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
    TextMessage textMessage = new ActiveMQTextMessage();

    String payload = "Hello world!";
    textMessage.setText(payload);

    ProcessContext context = runner.getProcessContext();
    ProcessSession session = runner.getProcessSessionFactory().createSession();
    ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());

    JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, textMessage, true, pic.getLogger());

    assertEquals("TextMessage content length should equal to FlowFile content size", payload.length(), summary.getLastFlowFile().getSize());

    final byte[] buffer = new byte[payload.length()];
    runner.clearTransferState();

    session.read(summary.getLastFlowFile(), new InputStreamCallback() {
        @Override
        public void process(InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer, false);
        }
    });

    String contentString = new String(buffer, "UTF-8");
    assertEquals("", payload, contentString);
}
 
Example 27
Source Project: nifi   Source File: PutSplunk.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Send the entire FlowFile as a single message.
 */
private void processSingleMessage(ProcessContext context, ProcessSession session, FlowFile flowFile, ChannelSender sender) {
    // copy the contents of the FlowFile to the ByteArrayOutputStream
    final ByteArrayOutputStream baos = new ByteArrayOutputStream((int)flowFile.getSize() + 1);
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.copy(in, baos);
        }
    });

    // if TCP and we don't end in a new line then add one
    final String protocol = context.getProperty(PROTOCOL).getValue();
    byte[] buf = baos.toByteArray();
    if (protocol.equals(TCP_VALUE.getValue()) && buf[buf.length - 1] != NEW_LINE_CHAR) {
        final byte[] updatedBuf = new byte[buf.length + 1];
        System.arraycopy(buf, 0, updatedBuf, 0, buf.length);
        updatedBuf[updatedBuf.length - 1] = NEW_LINE_CHAR;
        buf = updatedBuf;
    }

    // create a message batch of one message and add to active batches
    final FlowFileMessageBatch messageBatch = new FlowFileMessageBatch(session, flowFile);
    messageBatch.setNumMessages(1);
    activeBatches.add(messageBatch);

    // attempt to send the data and add the appropriate range
    try {
        sender.send(buf);
        messageBatch.addSuccessfulRange(0L, flowFile.getSize());
    } catch (IOException e) {
        messageBatch.addFailedRange(0L, flowFile.getSize(), e);
        context.yield();
    }
}
 
Example 28
Source Project: nifi   Source File: NiFiLegacyCipherProvider.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the salt provided as part of the cipher stream, or throws an exception if one cannot be detected.
 * This method is only implemented by {@link NiFiLegacyCipherProvider} because the legacy salt generation was dependent on the cipher block size.
 *
 * @param encryptionMethod the encryption method
 * @param in the cipher InputStream
 * @return the salt
 */
public byte[] readSalt(EncryptionMethod encryptionMethod, InputStream in) throws IOException {
    if (in == null) {
        throw new IllegalArgumentException("Cannot read salt from null InputStream");
    }

    // The first 8-16 bytes (depending on the cipher blocksize) of the input stream are the salt
    final int saltLength = calculateSaltLength(encryptionMethod);
    if (in.available() < saltLength) {
        throw new ProcessException("The cipher stream is too small to contain the salt");
    }
    byte[] salt = new byte[saltLength];
    StreamUtils.fillBuffer(in, salt);
    return salt;
}
 
Example 29
Source Project: localization_nifi   Source File: PutCassandraQL.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Determines the CQL statement that should be executed for the given FlowFile
 *
 * @param session  the session that can be used to access the given FlowFile
 * @param flowFile the FlowFile whose CQL statement should be executed
 * @return the CQL that is associated with the given FlowFile
 */

private String getCQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) {
    // Read the CQL from the FlowFile's content
    final byte[] buffer = new byte[(int) flowFile.getSize()];
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer);
        }
    });

    // Create the PreparedStatement string to use for this FlowFile.
    return new String(buffer, charset);
}
 
Example 30
Source Project: localization_nifi   Source File: PutUDP.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Helper method to read the FlowFile content stream into a byte array.
 *
 * @param session
 *            - the current process session.
 * @param flowFile
 *            - the FlowFile to read the content from.
 *
 * @return byte array representation of the FlowFile content.
 */
protected byte[] readContent(final ProcessSession session, final FlowFile flowFile) {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream((int) flowFile.getSize() + 1);
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.copy(in, baos);
        }
    });

    return baos.toByteArray();
}