com.streamsets.pipeline.api.FileRef Java Examples

The following examples show how to use com.streamsets.pipeline.api.FileRef. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StringTypeSupport.java    From datacollector-api with Apache License 2.0 6 votes vote down vote up
@Override
public String convert(Object value) {
  // Shortcut if the input is already string
  if(value instanceof String) {
    return (String)value;
  }

  if(value instanceof Map || value instanceof List || value instanceof byte[]|| value instanceof FileRef) {
    throw new IllegalArgumentException(Utils.format(Errors.API_18.getMessage()));
  }
  // ZoneDatetime.toString() does not use a standard format which can be parsed.
  if (value instanceof ZonedDateTime) {
    return Utils.format((ZonedDateTime) value);
  }

  return value.toString();
}
 
Example #2
Source File: WholeFileTransformerProcessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up
/**
 * Return the Avro file input stream
 * @param record the {@link com.streamsets.pipeline.api.Record} whole file record
 */
private InputStream getAvroInputStream(Record record) throws StageException {
  try {
    FileRef fileRef = record.get(FileRefUtil.FILE_REF_FIELD_PATH).getValueAsFileRef();

    // get avro reader
    final boolean includeChecksumInTheEvents = false;

    InputStream is = FileRefUtil.getReadableStream(
        getContext(),
        fileRef,
        InputStream.class,
        includeChecksumInTheEvents,
        null,
        null
    );

    return is;
  } catch (IOException ex) {
    throw new TransformerStageCheckedException(Errors.CONVERT_07, ex.toString(), ex);
  }
}
 
Example #3
Source File: TestWholeFileDataParser.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testParse() throws Exception {
  DataParserFactory factory = new DataParserFactoryBuilder(context, DataParserFormat.WHOLE_FILE)
      .setMaxDataLen(1000)
      .build();
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  try (DataParser parser = factory.getParser(
      "id",
      metadata,
      FileRefTestUtil.getLocalFileRef(testDir, false, null, null)
  )) {
    Assert.assertEquals("0", parser.getOffset());
    Record record = parser.parse();
    Assert.assertNotNull(record);
    Assert.assertEquals("-1", parser.getOffset());
    Assert.assertTrue(record.has("/fileInfo"));
    Assert.assertTrue(record.has("/fileRef"));
    FileRef fileRef = record.get("/fileRef").getValueAsFileRef();
    Assert.assertTrue(fileRef instanceof LocalFileRef);
    InputStream is = fileRef.createInputStream(context, InputStream.class);
    byte[] b = new byte[FileRefTestUtil.TEXT.getBytes().length];
    int bytesRead = is.read(b);
    Assert.assertEquals(FileRefTestUtil.TEXT.getBytes().length, bytesRead);
    Assert.assertArrayEquals(FileRefTestUtil.TEXT.getBytes(StandardCharsets.UTF_8), b);
  }
}
 
Example #4
Source File: TestJsonDataGenerator.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testFileRefTypeError() throws Exception {
  File testDir = new File("target", UUID.randomUUID().toString());
  testDir.mkdirs();
  try {
    FileRefTestUtil.writePredefinedTextToFile(testDir);
    Stage.Context context = ContextInfoCreator.createTargetContext("i", false, OnRecordError.TO_ERROR);
    Record record = context.createRecord("id");
    Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
    FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, false, null, null);
    record.set(FileRefUtil.getWholeFileRecordRootField(fileRef, metadata));
    DataGenerator gen = new JsonCharDataGenerator(getContext(), new StringWriter(), Mode.MULTIPLE_OBJECTS);
    gen.write(record);
    Assert.fail("Json should not process FileRef field");
  } catch (DataGeneratorException e) {
    Assert.assertEquals(Errors.JSON_GENERATOR_01, e.getErrorCode());
  } finally {
    testDir.delete();
  }
}
 
Example #5
Source File: TestRateLimitingWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testNIOReadWithHeapByteBuffer() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = Mockito.spy(fileRef.createInputStream(context, ReadableByteChannel.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    ByteBuffer b = ByteBuffer.allocate(10);
    int bytesRead;
    int freeSpaceInBuffer = b.remaining();
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, freeSpaceInBuffer, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
      b.compact();
      freeSpaceInBuffer = b.remaining();
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 
Example #6
Source File: TestRateLimitingWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testNIOReadWithDirectBuffer() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = Mockito.spy(fileRef.createInputStream(context, ReadableByteChannel.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    ByteBuffer b = ByteBuffer.allocateDirect(10);
    int bytesRead;
    int freeSpaceInBuffer = b.remaining();
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, freeSpaceInBuffer, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
      b.compact();
      freeSpaceInBuffer = b.remaining();
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 
Example #7
Source File: TestRateLimitingWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testIOReadsMixed() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    int bytesRead;
    while ((bytesRead= FileRefTestUtil.randomReadMethodsWithInputStream(is)) > 0) {
      remainingFileSize -= bytesRead;
      Assert.assertTrue(isRateLimiterAcquired.get());
      Assert.assertEquals(remainingFileSize, getRemainingStreamSize(is));
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 
Example #8
Source File: TestRateLimitingWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testIOReadParameterized2() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b, 0, b.length)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, b.length, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 
Example #9
Source File: TestRateLimitingWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testIOReadParameterized1() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, b.length, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 
Example #10
Source File: TestRateLimitingWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testIOReadParameterLess() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    while(is.read() != -1) {
      remainingFileSize--;
      checkState(is, remainingFileSize, 1, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
  }
}
 
Example #11
Source File: FileRefTestUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
public static FileRef getLocalFileRefWithCustomFile(
    File file,
    boolean createMetrics,
    String checksum,
    HashingUtil.HashType checksumAlgorithm
) {
  AbstractSpoolerFileRef.Builder builder = new LocalFileRef.Builder()
      .filePath(file.getAbsolutePath())
      //To force multiple reads from the file.
      .bufferSize(TEXT.getBytes().length / 2)
      .totalSizeInBytes(file.length());
  if (checksum != null) {
    builder.verifyChecksum(true)
        .checksum(checksum)
        .checksumAlgorithm(checksumAlgorithm);
  }
  if (createMetrics) {
    builder.createMetrics(true);
  } else {
    builder.createMetrics(false);
  }
  return builder.build();
}
 
Example #12
Source File: TestMetricsEnabledWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testNIOReadWithHeapByteBuffer() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = fileRef.createInputStream(context, ReadableByteChannel.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    ByteBuffer b = ByteBuffer.allocate(10);
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
      b.compact();
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}
 
Example #13
Source File: TestMetricsEnabledWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testNIOReadWithDirectBuffer() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = fileRef.createInputStream(context, ReadableByteChannel.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    ByteBuffer b = ByteBuffer.allocateDirect(10);
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
      b.compact();
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}
 
Example #14
Source File: TestMetricsEnabledWrapperStream.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testIOReadParameterized() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = fileRef.createInputStream(context, InputStream.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b, 0, b.length)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}
 
Example #15
Source File: TestFileRefTypeSupport.java    From datacollector-api with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertFromFilRefInvalid() {
  FileRef fileRef = new TestFileRef.ByteArrayRef("This is file ref Field".getBytes());
  testConvertFromFilRefInvalid(new IntegerTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new LongTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new FloatTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new DoubleTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new DecimalTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ByteTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new CharTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new BooleanTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ByteArrayTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new StringTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ListMapTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ListTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new MapTypeSupport(), fileRef);
}
 
Example #16
Source File: TestHDFSTargetWholeFile.java    From datacollector with Apache License 2.0 6 votes vote down vote up
static Record getFileRefRecordForFile(Path filePath) throws Exception {
  Record fileRefRecord = RecordCreator.create();
  FileRef fileRef =
      new LocalFileRef.Builder()
          .bufferSize(1024)
          .createMetrics(false)
          .verifyChecksum(false)
          .filePath(filePath.toAbsolutePath().toString())
          .build();
  Map<String, Field> fieldMap = new HashMap<>();

  Map<String, Object> metadata = new HashMap<>(Files.readAttributes(filePath, "posix:*"));
  metadata.put("filename", filePath.getFileName());
  metadata.put("file", filePath.toString());
  metadata.put("dir", filePath.getParent().toString());
  metadata.put("permissions", "777");

  fieldMap.put(FileRefUtil.FILE_REF_FIELD_NAME, Field.create(Field.Type.FILE_REF, fileRef));
  fieldMap.put(FileRefUtil.FILE_INFO_FIELD_NAME, FileRefUtil.createFieldForMetadata(metadata));
  fileRefRecord.set(Field.create(fieldMap));

  return fileRefRecord;
}
 
Example #17
Source File: FileRefUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static  <T extends AutoCloseable> T getReadableStream(
    ProtoConfigurableEntity.Context context,
    FileRef fileRef,
    Class<T> streamClass,
    boolean includeChecksumInTheEvents,
    ChecksumAlgorithm checksumAlgorithm,
    StreamCloseEventHandler<?> streamCloseEventHandler
) throws IOException {
  T stream = fileRef.createInputStream(context, streamClass);
  if (includeChecksumInTheEvents) {
    Utils.checkArgument(
        FileRefStreamCloseEventHandler.class.isAssignableFrom(streamCloseEventHandler.getClass()),
        "Stream Close Event handler should be of type " + FileRefStreamCloseEventHandler.class.getCanonicalName()
    );
    stream = (T) new ChecksumCalculatingWrapperStream(stream, checksumAlgorithm.getHashType(), streamCloseEventHandler);
  }
  return stream;
}
 
Example #18
Source File: WrapperDataParserFactory.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
public DataParser getParser(
    String id,
    Map<String, Object> metadata,
    FileRef fileRef
) throws DataParserException {
  return new WrapperDataParser(factory.getParser(id, metadata, fileRef));
}
 
Example #19
Source File: SampleProcessor.java    From tutorials with Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  LOG.info("Input record: {}", record);

  FileRef fileRef = record.get("/fileRef").getValueAsFileRef();
  Metadata metadata;
  try {
    metadata = ImageMetadataReader.readMetadata(fileRef.createInputStream(getContext(), InputStream.class));
  } catch (ImageProcessingException | IOException e) {
    String filename = record.get("/fileInfo/filename").getValueAsString();
    LOG.info("Exception getting metadata from {}", filename, e);
    throw new OnRecordErrorException(record, Errors.SAMPLE_02, e);
  }

  for (Directory directory : metadata.getDirectories()) {
    LinkedHashMap<String, Field> listMap = new LinkedHashMap<>();

    for (Tag tag : directory.getTags()) {
      listMap.put(tag.getTagName(), Field.create(tag.getDescription()));
    }

    if (directory.hasErrors()) {
      for (String error : directory.getErrors()) {
        LOG.info("ERROR: {}", error);
      }
    }

    record.set("/" + directory.getName(), Field.createListMap(listMap));
  }

  LOG.info("Output record: {}", record);

  batchMaker.addRecord(record);
}
 
Example #20
Source File: WholeFileDataParser.java    From datacollector with Apache License 2.0 5 votes vote down vote up
WholeFileDataParser(
    ProtoConfigurableEntity.Context context,
    String id,
    Map<String, Object> metadata,
    FileRef fileRef
)  {
  this.id = id;
  this.context = context;
  this.metadata = metadata;
  this.fileRef = fileRef;
}
 
Example #21
Source File: WholeFileDataParserFactory.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
public DataParser getParser(
    String id,
    Map<String, Object> metadata,
    FileRef fileRef
) throws DataParserException {
  Utils.checkNotNull(fileRef, FileRefUtil.FILE_REF_FIELD_NAME);
  validateMetadata(metadata);
  return new WholeFileDataParser(
      getSettings().getContext(),
      id,
      metadata,
      fileRef
  );
}
 
Example #22
Source File: TestFileRefTypeSupport.java    From datacollector-api with Apache License 2.0 5 votes vote down vote up
private void testConvertFromFilRefInvalid(TypeSupport targetTypeSupport, FileRef fileRef) {
  try {
    new FileRefTypeSupport().convert(fileRef, targetTypeSupport);
    Assert.fail("Convert File Ref Object to " + targetTypeSupport + " should fail.");
  } catch (IllegalArgumentException e) {
    Assert.assertEquals(Utils.format(Errors.API_24.getMessage(), targetTypeSupport), e.getMessage());
  }
}
 
Example #23
Source File: TestWholeFileDataGenerator.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private Record createRecord(String checksum, HashingUtil.HashType checksumAlgorithm) throws Exception {
  Record record = context.createRecord("id");
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, checksum, checksumAlgorithm);
  record.set(FileRefUtil.getWholeFileRecordRootField(fileRef, metadata));
  return record;
}
 
Example #24
Source File: TestWholeFileDataParserFactory.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private void getParserWithInvalidMetadata(
    FileRef fileRef,
    Map<String, Object> metadata,
    DataParserFactory dataParserFactory,
    ErrorCode errorCode
) {
  try {
    dataParserFactory.getParser("id", metadata, fileRef);
    Assert.fail("getParser should fail for missing mandatory metadata");
  } catch (DataParserException e) {
    Assert.assertEquals(errorCode, e.getErrorCode());
  }
}
 
Example #25
Source File: TestWholeFileDataParserFactory.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testWholeFileDataParserFactoryMissingMetadata() throws Exception {
  DataParserFactory factory = new DataParserFactoryBuilder(context, DataParserFormat.WHOLE_FILE)
      .setMaxDataLen(1000)
      .build();
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, false, null, null);
  Set<String> keys = new HashSet<>(FileRefUtil.MANDATORY_METADATA_INFO);
  for (String key :keys) {
    Map<String, Object> metadataCopy = new HashMap<>(metadata);
    metadataCopy.remove(key);
    getParserWithInvalidMetadata(fileRef, metadataCopy, factory, Errors.WHOLE_FILE_PARSER_ERROR_0);
  }
}
 
Example #26
Source File: FileRefTypeSupport.java    From datacollector-api with Apache License 2.0 5 votes vote down vote up
@Override
public FileRef convert(Object value) {
  if (value instanceof FileRef) {
    return (FileRef)value;
  }
  throw new IllegalArgumentException(Utils.format(Errors.API_23.getMessage(), value.getClass().getName()));
}
 
Example #27
Source File: SpoolDirUtil.java    From datacollector with Apache License 2.0 5 votes vote down vote up
public static DataParser getParser(
    WrappedFileSystem fs,
    WrappedFile file,
    DataFormat dataFormat,
    DataParserFactory parserFactory,
    String offset,
    int wholeFileMaxObjectLen,
    ELEval rateLimitElEval,
    ELVars rateLimitElVars,
    String rateLimit
) throws DataParserException, ELEvalException, IOException {
  DataParser parser;

  switch (dataFormat) {
    case WHOLE_FILE:
      FileRef fileRef = fs.getFileRefBuilder()
          .filePath(file.getAbsolutePath())
          .bufferSize(wholeFileMaxObjectLen)
          .rateLimit(FileRefUtil.evaluateAndGetRateLimit(rateLimitElEval, rateLimitElVars, rateLimit))
          .createMetrics(true)
          .totalSizeInBytes(file.getSize())
          .build();
      parser = parserFactory.getParser(file.getFileName(), file.getFileMetadata(), fileRef);
      break;
    default:
      parser = parserFactory.getParser(file.getFileName(), file.getInputStream(), offset);
  }

  return parser;
}
 
Example #28
Source File: DataParserServiceImpl.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
public DataParser getParser(String id, Map<String, Object> metadata, FileRef fileRef) throws DataParserException {
  try {
    return new DataParserWrapper(dataFormatConfig.getParserFactory().getParser(id, metadata, fileRef));
  } catch (com.streamsets.pipeline.lib.parser.DataParserException e) {
    throw ShimUtil.convert(e);
  }
}
 
Example #29
Source File: ServiceRuntime.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override // From DataFormatParserService
public DataParser getParser(String id, Map<String, Object> metadata, FileRef fileRef) throws DataParserException {
  ClassLoader cl = serviceBean.getDefinition().getStageClassLoader();

  return LambdaUtil.privilegedWithClassLoader(
    cl,
    DataParserException.class,
    () -> new DataParserServiceWrapper(cl, ((DataFormatParserService)serviceBean.getService()).getParser(id, metadata, fileRef))
  );
}
 
Example #30
Source File: TestRemoteUploadTarget.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private Record createRecord(File file) throws Exception {
  Record record = context.createRecord("id");
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadataWithCustomFile(file);
  FileRef fileRef = FileRefTestUtil.getLocalFileRefWithCustomFile(file, true, null, null);
  record.set(FileRefUtil.getWholeFileRecordRootField(fileRef, metadata));
  return record;
}