com.streamsets.pipeline.api.Field Java Examples

The following examples show how to use com.streamsets.pipeline.api.Field. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDelimitedDataGenerator.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testGeneratorReplaceNewLines() throws Exception {
  StringWriter writer = new StringWriter();
  DataGenerator gen = new DelimitedCharDataGenerator(writer, CsvMode.CSV.getFormat(), CsvHeader.NO_HEADER, "h", "d", " ");
  Record record = RecordCreator.create();
  List<Field> list = new ArrayList<>();
  Map<String,Field> map = new HashMap<>();
  map.put("h", Field.create("A"));
  map.put("d", Field.create("a\n\r"));
  list.add(Field.create(map));
  map.put("h", Field.create("B"));
  map.put("d", Field.create("b"));
  list.add(Field.create(map));
  map.put("h", Field.create("C"));
  map.put("d", Field.create((String)null));
  list.add(Field.create(map));
  record.set(Field.create(list));
  gen.write(record);
  gen.close();
  Assert.assertEquals("\"a  \",b,\r\n", writer.toString());
}
 
Example #2
Source File: TestDelimitedDataGenerator.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testGeneratorNoHeader() throws Exception {
  StringWriter writer = new StringWriter();
  DataGenerator gen = new DelimitedCharDataGenerator(writer, CsvMode.CSV.getFormat(), CsvHeader.NO_HEADER, "h", "d", null);
  Record record = RecordCreator.create();
  List<Field> list = new ArrayList<>();
  Map<String,Field> map = new HashMap<>();
  map.put("h", Field.create("A"));
  map.put("d", Field.create("a\n\r"));
  list.add(Field.create(map));
  map.put("h", Field.create("B"));
  map.put("d", Field.create("b"));
  list.add(Field.create(map));
  record.set(Field.create(list));
  gen.write(record);
  gen.close();
  Assert.assertEquals("\"a\n\r\",b\r\n", writer.toString());
}
 
Example #3
Source File: Matchers.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Override
public boolean matches(Object item) {
  if (item instanceof Field) {
    Field field = (Field) item;
    if (field.getType() == type) {
      final Object valueFromField = getValueFromField(field);
      if (value == null) {
        return valueFromField == null;
      } else if (value.getClass().isArray()) {
        // arrays are a pain...
        if (byte.class.equals(value.getClass().getComponentType())) {
          return Arrays.equals((byte[]) value, (byte[]) valueFromField);
        } else {
          throw new IllegalStateException(
              "Incomparable array type: " + value.getClass().getComponentType().toString()
          );
        }
      } else {
        return value.equals(valueFromField);
      }
    }
  }
  return false;
}
 
Example #4
Source File: RecordConverter.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private List<Record> toRecords(Table table,
                               EventHeader eventHeader,
                               DeleteRowsEventData eventData,
                               SourceOffset offset) {
  List<Record> res = new ArrayList<>(eventData.getRows().size());
  for (Serializable[] row : eventData.getRows()) {
    Record record = recordFactory.create(offset.format());
    Map<String, Field> fields = createHeader(table, eventHeader, offset);
    fields.put(TYPE_FIELD, create("DELETE"));
    record.getHeader().setAttribute(
        OperationType.SDC_OPERATION_TYPE,
        String.valueOf(OperationType.DELETE_CODE)
    );
    List<ColumnValue> columnValues = zipColumnsValues(eventData.getIncludedColumns(), table, row);
    Map<String, Field> data = toMap(columnValues);
    fields.put(OLD_DATA_FIELD, create(data));

    record.set(create(fields));
    res.add(record);
  }
  return res;
}
 
Example #5
Source File: TestSyslogTarget.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteSingleRecord() throws Exception {
  SyslogDTarget config = getDefaultConfig();
  Target target = config.createTarget();
  TargetRunner runner = new TargetRunner.Builder(SyslogDTarget.class, target)
      .setOnRecordError(OnRecordError.TO_ERROR)
      .addService(DataFormatGeneratorService.class, new SdkJsonDataFormatGeneratorService())
      .build();

  runner.runInit();

  Record record = RecordCreator.create();
  Map<String, Field> fields = new HashMap<>();
  fields.put("text", Field.create("myTestMessage"));
  fields.put("hostname", Field.create("localhost"));
  fields.put("application", Field.create("myTestApp"));
  fields.put("facility", Field.create("1"));
  fields.put("severity", Field.create("1"));
  record.set(Field.create(fields));

  runner.runWrite(Arrays.asList(record));

  runner.runDestroy();
}
 
Example #6
Source File: StartJobSupplier.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private void generateField(Map<String, Object> jobStatus) {
  String status = jobStatus.containsKey("status") ? (String) jobStatus.get("status") : null;
  String statusColor = jobStatus.containsKey("color") ? (String) jobStatus.get("color") : null;
  String errorMessage = jobStatus.containsKey("errorMessage") ? (String) jobStatus.get("errorMessage") : null;
  boolean success = ControlHubApiUtil.determineJobSuccess(status, statusColor);
  LinkedHashMap<String, Field> startOutput = new LinkedHashMap<>();
  startOutput.put(Constants.JOB_ID_FIELD, Field.create(jobIdConfig.jobId));
  startOutput.put(Constants.STARTED_SUCCESSFULLY_FIELD, Field.create(true));
  if (!conf.runInBackground) {
    startOutput.put(Constants.FINISHED_SUCCESSFULLY_FIELD, Field.create(success));
    MetricRegistryJson jobMetrics = ControlHubApiUtil.getJobMetrics(
        clientBuilder,
        conf.baseUrl,
        jobIdConfig.jobId,
        userAuthToken
    );
    startOutput.put(Constants.JOB_METRICS_FIELD, CommonUtil.getMetricsField(jobMetrics));
  }
  startOutput.put(Constants.JOB_STATUS_FIELD, Field.create(status));
  startOutput.put(Constants.JOB_STATUS_COLOR_FIELD, Field.create(statusColor));
  startOutput.put(Constants.ERROR_MESSAGE_FIELD, Field.create(errorMessage));
  responseField = Field.createListMap(startOutput);
}
 
Example #7
Source File: BasicIT.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private static List<Record> createTransactionRecords(int noOfRecords) {
  List<Record> records = new ArrayList<>();
  long currentTime = (System.currentTimeMillis() / 1000) * 1000;
  for (int i = 0; i < noOfRecords; i++) {
    Record record = RecordCreator.create();
    LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
    fields.put("unique_int", Field.create(i + 1));
    fields.put("t_date", Field.create(Field.Type.LONG, currentTime));
    fields.put("random_string", Field.create(UUID.randomUUID().toString()));
    record.set(Field.createListMap(fields));
    records.add(record);
    //making sure time is at least off by a second.
    currentTime = currentTime + 1000;
  }
  return records;
}
 
Example #8
Source File: FieldHasherProcessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private Set<String> validateAndExtractFieldsToHash(
    Record record,
    Set<String> fieldsDontExist,
    Set<String> fieldsWithListOrMapType,
    Set<String> fieldsWithNull,
    Collection<String> matchingFieldsPath
) {
  Set<String> validFieldsToHashForThisConfig = new HashSet<String>();
  for (String matchingFieldPath : matchingFieldsPath) {
    if (record.has(matchingFieldPath)) {
      Field field = record.get(matchingFieldPath);
      if (UNSUPPORTED_FIELD_TYPES.contains(field.getType())) {
        fieldsWithListOrMapType.add(matchingFieldPath);
      } else if (field.getValue() == null) {
        fieldsWithNull.add(matchingFieldPath);
      } else {
        validFieldsToHashForThisConfig.add(matchingFieldPath);
      }
    } else {
      fieldsDontExist.add(matchingFieldPath);
    }
  }
  return validFieldsToHashForThisConfig;
}
 
Example #9
Source File: SchedulerJob.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(JobExecutionContext jobExecutionContext) {
  try {
    SchedulerContext schedulerContext = jobExecutionContext.getScheduler().getContext();
    PushSource.Context pushSourceContext = (PushSource.Context) schedulerContext.get(
        SchedulerPushSource.PUSH_SOURCE_CONTEXT
    );
    if (pushSourceContext != null) {
      BatchContext batchContext = pushSourceContext.startBatch();
      Record record = pushSourceContext.createRecord("cronRecord");
      LinkedHashMap<String, Field> linkedHashMap = new LinkedHashMap<>();
      linkedHashMap.put("timestamp", Field.createDatetime(new Date()));
      record.set(Field.createListMap(linkedHashMap));
      batchContext.getBatchMaker().addRecord(record);
      pushSourceContext.processBatch(batchContext);
    }
  } catch (SchedulerException ex) {
    LOG.error(ex.getMessage(), ex);
  }
}
 
Example #10
Source File: TestAvroTypeUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateListPrimitiveField() throws Exception {
  String schema = "{\"type\": \"array\", \"items\": \"string\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, Arrays.asList("Hari", "Kiran"), false);

  Assert.assertEquals(Field.Type.LIST, field.getType());
  List<Field> valueAsList = field.getValueAsList();
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(0).getType());
  Assert.assertEquals("Hari", valueAsList.get(0).getValueAsString());
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(1).getType());
  Assert.assertEquals("Kiran", valueAsList.get(1).getValueAsString());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof List<?>);

  List<String> listString = (List<String>) avroObject;
  Assert.assertEquals("Hari", listString.get(0));
  Assert.assertEquals("Kiran", listString.get(1));

  //Check invalid type - String to List
  makeBadType(Field.create("notList"), record, avroSchema);
}
 
Example #11
Source File: TestAvroTypeUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateDoubleField() throws Exception {
  String schema = "{\"name\": \"name\", \"type\": \"double\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, 345823746923.863423, false);
  Assert.assertEquals(Field.Type.DOUBLE, field.getType());
  Assert.assertTrue(345823746923.863423 == field.getValueAsDouble());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof Double);
  Assert.assertTrue(345823746923.863423 == (Double) avroObject);

  //Check invalid type - String to Double
  makeBadType(Field.create("notDouble"), record, avroSchema);
}
 
Example #12
Source File: TestWholeFileTransformerProcessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testNotWholeFileFormatRecord() throws StageException {
  Processor wholeFileTransofrmer  = new TestWholeFileTransformerProcessorBuilder()
      .tempDir("${file:parentPath(record:attribute('file'))}/.parquet")
      .build();

  ProcessorRunner runner = new ProcessorRunner.Builder(WholeFileTransformerDProcessor.class, wholeFileTransofrmer)
      .addOutputLane("a")
      .setOnRecordError(OnRecordError.TO_ERROR)
      .build();

  Record record = RecordCreator.create();
  record.set(Field.create("test"));

  try {
    runner.runInit();
    StageRunner.Output output = runner.runProcess(Arrays.asList(record));
    Assert.assertEquals(0, output.getRecords().get("a").size());

    List<Record> errorRecords = runner.getErrorRecords();
    Assert.assertEquals(1, errorRecords.size());
    Assert.assertEquals(Errors.CONVERT_01.getCode(), errorRecords.get(0).getHeader().getErrorCode());
  } finally {
    runner.runDestroy();
  }
}
 
Example #13
Source File: TestTimeNewEL.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testExtractStringFromDate() throws Exception {

  Record.Header header = Mockito.mock(Record.Header.class);
  Record record = Mockito.mock(Record.class);
  Mockito.when(record.getHeader()).thenReturn(header);
  long val = System.currentTimeMillis();
  Date d = new Date(val);
  Mockito.when(record.get(Mockito.eq("/theDate"))).thenReturn(Field.createDate(d));
  Mockito.when(record.get(Mockito.eq("/epochMS"))).thenReturn(Field.create(val));

  String format  = "G yyyy-MM-dd HH:mm:ss.SSS a zzzzzzz";
  Mockito.when(record.get(Mockito.eq("/format"))).thenReturn(Field.create(format));

  RecordEL.setRecordInContext(variables, record);

  SimpleDateFormat sdf = new SimpleDateFormat(format);

  String ans = sdf.format(d);

  Assert.assertEquals(ans, eval.eval(variables,
      "${time:extractStringFromDate(time:millisecondsToDateTime(record:value('/epochMS')), record:value('/format'))}", String.class));

  Assert.assertEquals(ans, eval.eval(variables,
      "${time:extractStringFromDate(record:value('/theDate'), record:value('/format'))}", String.class));
}
 
Example #14
Source File: Matchers.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Override
public boolean matches(Object item) {
  if (item instanceof Field) {
    Field field = (Field) item;
    if (field.getType().isOneOf(Field.Type.MAP, Field.Type.LIST_MAP)) {
      final Map<String, Field> childFields = field.getValueAsMap();
      if (!childFields.containsKey(nestedFieldName)) {
        return false;
      }
      final VT value = valueAccessor.getValue(childFields.get(nestedFieldName));
      if (expectedValue == null) {
        return value == null;
      } else {
        return expectedValue.equals(value);
      }
    }
  }
  return false;
}
 
Example #15
Source File: JdbcUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
public LinkedHashMap<String, Field> resultSetToFields(
    ResultSet rs,
    CommonSourceConfigBean commonSourceBean,
    ErrorRecordHandler errorRecordHandler,
    UnknownTypeAction unknownTypeAction,
    Set<String> recordHeader,
    DatabaseVendor vendor
) throws SQLException, StageException {
  return resultSetToFields(
      rs,
      commonSourceBean.maxClobSize,
      commonSourceBean.maxBlobSize,
      Collections.emptyMap(),
      errorRecordHandler,
      unknownTypeAction,
      recordHeader,
      commonSourceBean.convertTimestampToString,
      vendor
  );
}
 
Example #16
Source File: TestJdbcMetadata.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testInitialRecordStructure() throws Exception {
  JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString);

  ProcessorRunner processorRunner = getProcessorRunner(processor);

  processorRunner.runInit();

  List<Record> singleRecord = ImmutableList.of(makeRecord(fieldMap1));
  StageRunner.Output output = processorRunner.runProcess(singleRecord);
  Assert.assertEquals(1, output.getRecords().get("lane").size());

  Record record = output.getRecords().get("lane").get(0);
  for (Map.Entry<String, Field>entry : record.get().getValueAsListMap().entrySet()) {
    Assert.assertEquals(fieldMap1.get(entry.getKey()).getRight(), entry.getValue().getValue());
  }

  try (ResultSet metaDataColumns = jdbcUtil.getColumnMetadata(connection, null, tableName)) {
    while (metaDataColumns.next()) {
      Pair<Field.Type, Object> typeAndValue = fieldMap1.get(metaDataColumns.getString(JdbcSchemaReader.COLUMN_NAME).toLowerCase());
      Assert.assertEquals(typeAndValue.getLeft(), typeMap.get(metaDataColumns.getInt(JdbcSchemaReader.DATA_TYPE)));
    }
  }
}
 
Example #17
Source File: TestSdcRecordDataGenerator.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test(expected = IOException.class)
public void testWriteAfterClose() throws Exception {
  ByteArrayOutputStream writer = new ByteArrayOutputStream();
  DataGenerator gen = new SdcRecordDataGenerator(
      getContextExtensions().createRecordWriter(writer),
      getContextExtensions()
  );
  Record record = RecordCreator.create();
  record.set(Field.create("Hello"));
  gen.close();
  gen.write(record);
}
 
Example #18
Source File: TestFieldTypeConverterProcessorFields.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testStringToByte() throws StageException {
  FieldTypeConverterConfig fieldTypeConverterConfig =
      new FieldTypeConverterConfig();
  fieldTypeConverterConfig.fields = ImmutableList.of("/beginner", "/intermediate", "/null");
  fieldTypeConverterConfig.targetType = Field.Type.BYTE;
  fieldTypeConverterConfig.dataLocale = "en";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(fieldTypeConverterConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("beginner", Field.create("1"));
    map.put("intermediate", Field.create("126"));
    map.put("null", Field.create(Field.Type.STRING, null));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 3);
    Assert.assertTrue(result.containsKey("beginner"));
    Assert.assertEquals((byte)1, result.get("beginner").getValue());
    Assert.assertTrue(result.containsKey("intermediate"));
    Assert.assertEquals((byte)126, result.get("intermediate").getValue());
    Assert.assertTrue(result.containsKey("null"));
    Assert.assertEquals(null, result.get("null").getValue());
  } finally {
    runner.runDestroy();
  }
}
 
Example #19
Source File: DriftRuleEL.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
protected Set<String> getValue(Field field) {
  Set<String> names;
  switch (field.getType()) {
    case MAP:
    case LIST_MAP:
      names = new HashSet<>(field.getValueAsMap().keySet());
      break;
    default:
      names = null;
      break;
  }
  return names;
}
 
Example #20
Source File: Matchers.java    From datacollector with Apache License 2.0 5 votes vote down vote up
public static Matcher<Field> fieldWithValue(final double value) {
  return new FieldMatcher(Field.Type.DOUBLE, value) {
    @Override
    protected Object getValueFromField(Field field) {
      return field.getValueAsDouble();
    }
  };
}
 
Example #21
Source File: MapRDBCDCSource.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private void setId(Record record, Value id) throws StageException {
  if(id.getType() == Value.Type.STRING) {
    record.set("/_id", Field.create(id.getString()));
  } else if(id.getType() == Value.Type.BINARY) {
    record.set("/_id", Field.create(id.getBinary().array()));
  } else {
    throw new OnRecordErrorException(record, MaprDBCDCErrors.MAPRDB_04, id.getType().name());
  }
}
 
Example #22
Source File: TestXMLFlatteningProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testNonMap() throws Exception {
  Record expected1 = RecordCreator.create();
  expected1.set(Field.create(new ArrayList<Field>()));
  processor = new XMLFlatteningProcessor(ORIGINAL, true, false, "", "", "", "", false, false);
  ProcessorRunner runner = new ProcessorRunner.Builder(dProcessorClass, processor)
      .addOutputLane("xml").setOnRecordError(OnRecordError.STOP_PIPELINE).build();
  try {
    runner.runInit();
    runner.runProcess(ImmutableList.of(expected1));
    Assert.fail();
  } catch (StageException ex) {
    Assert.assertEquals(CMN_0100, ex.getErrorCode());
  }
}
 
Example #23
Source File: StreamingXmlParser.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void addContent(Map<String, Object> contents, String name, Field field) throws
    XMLStreamException,
    ObjectLengthException {
  throwIfOverMaxObjectLength();
  List<Field> list = (List<Field>) contents.get(name);
  if (list == null) {
    list = new ArrayList<>();
    contents.put(name, list);
  }
  list.add(field);
}
 
Example #24
Source File: TestExpressionProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testIsEmptyMapNullable() throws Exception {

  ExpressionProcessorConfig expressionProcessorConfig;
  ProcessorRunner runner;

  Map<String, Field> map = new LinkedHashMap<>();
  map.put("a", Field.create(123));
  Record record = RecordCreator.create("s", "s:1");
  record.set(Field.create(map));

  expressionProcessorConfig = new ExpressionProcessorConfig();
  expressionProcessorConfig.expression = "${isEmptyMap(NULL)}";
  expressionProcessorConfig.fieldToSet = "/setme";

  runner = new ProcessorRunner.Builder(ExpressionDProcessor.class)
          .addConfiguration("expressionProcessorConfigs", ImmutableList.of(expressionProcessorConfig))
          .addOutputLane("a").setOnRecordError(OnRecordError.TO_ERROR).build();
  runner.runInit();

  try {
    runner.runProcess(ImmutableList.of(record));

    Assert.assertEquals(0, runner.getErrorRecords().size());
  } finally {
    runner.runDestroy();
  }
}
 
Example #25
Source File: CollectdParser.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private void startNewRecord() {
  Record record = context.createRecord(fields.get(PART_TYPES.get(HOST)).getValueAsString() + "::" + recordId++);

  record.set(Field.create(fields));
  records.add(record);

  fields = new HashMap<>(fields);
}
 
Example #26
Source File: TestAggregationEL.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testMin() throws Exception {
  final ELEvaluator eval = new ELEvaluator("testMin", elDefinitionExtractor, AggregationEL.class);

  final ELVariables variables = makeTestVariables();

  final Field result1 = eval.eval(variables, "${min(intFields)}", Field.class);
  assertThat(result1, Matchers.fieldWithValue(1l));

  final Field result2 = eval.eval(variables, "${min(doubleFields)}", Field.class);
  assertThat(result2, Matchers.fieldWithValue(0.5d));

  final Field result3 = eval.eval(variables, "${min(decimalFields)}", Field.class);
  assertThat(result3, Matchers.fieldWithValue(new BigDecimal("-8.05")));
}
 
Example #27
Source File: TestFieldFlattenerEntireRecordProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testFlattenEntireRecordDifferentSeparator() throws Exception {
  FieldFlattenerConfig config = new FieldFlattenerConfig();
  config.nameSeparator = "/";
  config.flattenType = FlattenType.ENTIRE_RECORD;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFlattenerDProcessor.class, new FieldFlattenerProcessor(config))
    .addOutputLane("a").build();
  runner.runInit();

  Record record = RecordCreator.create();
  record.set(Field.create(Field.Type.MAP, ImmutableMap.builder()
    .put("top-level", Field.create(Field.Type.STRING, "top-level"))
    .put("map", Field.create(Field.Type.MAP, ImmutableMap.builder()
      .put("map1", Field.create(Field.Type.STRING, "map1"))
      .put("map2", Field.create(Field.Type.STRING, "map2"))
      .build()))
    .put("list", Field.create(Field.Type.LIST, ImmutableList.of(
      Field.create(Field.Type.STRING, "list1"),
      Field.create(Field.Type.STRING, "list2"))))
    .build()
  ));

  StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
  assertEquals(1, output.getRecords().get("a").size());

  Field field = output.getRecords().get("a").get(0).get();
  assertEquals(Field.Type.MAP, field.getType());

  Map<String, Field> result = field.getValueAsMap();
  assertEquals(5, result.size());
  assertTrue(result.containsKey("top-level"));
  assertTrue(result.containsKey("map/map1"));
  assertTrue(result.containsKey("map/map2"));
  assertTrue(result.containsKey("list/0"));
  assertTrue(result.containsKey("list/1"));

  runner.runDestroy();
}
 
Example #28
Source File: StartPipelineSource.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
public String produce(String s, int i, BatchMaker batchMaker) throws StageException {
  List<CompletableFuture<Field>> startPipelineFutures = new ArrayList<>();
  Executor executor = Executors.newCachedThreadPool();

  for (PipelineIdConfig pipelineIdConfig: conf.pipelineIdConfigList) {
    StartPipelineSupplier startPipelineSupplier = startPipelineCommon.getStartPipelineSupplier(
        pipelineIdConfig,
        null
    );
    CompletableFuture<Field> future = CompletableFuture.supplyAsync(startPipelineSupplier, executor);
    startPipelineFutures.add(future);
  }

  try {
    LinkedHashMap<String, Field> outputField = startPipelineCommon.startPipelineInParallel(startPipelineFutures);
    Record outputRecord = CommonUtil.createOrchestratorTaskRecord(
        null,
        getContext(),
        conf.taskName,
        outputField
    );
    batchMaker.addRecord(outputRecord);
  } catch (Exception ex) {
    LOG.error(ex.toString(), ex);
    errorRecordHandler.onError(StartPipelineErrors.START_PIPELINE_04, ex.toString(), ex);
  }

  return null;
}
 
Example #29
Source File: TestFieldDeepCopy.java    From datacollector-api with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testClone() {
  ListTypeSupport ts = new ListTypeSupport();
  List<Field> list = new ArrayList<>();
  list.add(Field.create("a"));
  List<Field> cList = (List) ts.clone(list);
  Assert.assertEquals(list, cList);
  Assert.assertNotSame(list, cList);
  Assert.assertNotSame(list.get(0), cList.get(0));
  List<Field> listX = new ArrayList<>();
  listX.add(Field.create(list));

  Map<String, Field> map = new HashMap<>();
  map.put("list", Field.create(listX));
  MapTypeSupport ms = new MapTypeSupport();
  Map<String, Field> cMap = (Map) ms.clone(map);
  Assert.assertEquals(map, cMap);
  Assert.assertNotSame(map, cMap);
  Assert.assertEquals(map.get("list"), cMap.get("list"));
  Assert.assertNotSame(map.get("list"), cMap.get("list"));
  Assert.assertEquals(map.get("list").getValueAsList().get(0), cMap.get("list").getValueAsList().get(0));
  Assert.assertNotSame(map.get("list").getValueAsList().get(0), cMap.get("list").getValueAsList().get(0));

  LinkedHashMap<String, Field> listMap = new LinkedHashMap<>();
  listMap.put("list", Field.create(listX));
  ListMapTypeSupport listMapTypeSupport = new ListMapTypeSupport();
  LinkedHashMap<String, Field> listMapClone = (LinkedHashMap) listMapTypeSupport.clone(listMap);
  Assert.assertEquals(listMap, listMapClone);
  Assert.assertNotSame(listMap, listMapClone);
  Assert.assertEquals(map.get("list"), listMapClone.get("list"));
  Assert.assertNotSame(map.get("list"), listMapClone.get("list"));
  Assert.assertEquals(map.get("list").getValueAsList().get(0), listMapClone.get("list").getValueAsList().get(0));
  Assert.assertNotSame(map.get("list").getValueAsList().get(0), listMapClone.get("list").getValueAsList().get(0));
}
 
Example #30
Source File: TestFieldValueReplacer.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testReplaceNullLongFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/longField");
  stringFieldReplacement.newValue = "523456789345";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("longField", Field.create(Field.Type.LONG, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("longField"));
    Assert.assertEquals(523456789345L, result.get("longField").getValue());
  } finally {
    runner.runDestroy();
  }
}