Java Code Examples for com.streamsets.pipeline.api.Field

The following examples show how to use com.streamsets.pipeline.api.Field. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: datacollector   Source File: StartJobSupplier.java    License: Apache License 2.0 6 votes vote down vote up
private void generateField(Map<String, Object> jobStatus) {
  String status = jobStatus.containsKey("status") ? (String) jobStatus.get("status") : null;
  String statusColor = jobStatus.containsKey("color") ? (String) jobStatus.get("color") : null;
  String errorMessage = jobStatus.containsKey("errorMessage") ? (String) jobStatus.get("errorMessage") : null;
  boolean success = ControlHubApiUtil.determineJobSuccess(status, statusColor);
  LinkedHashMap<String, Field> startOutput = new LinkedHashMap<>();
  startOutput.put(Constants.JOB_ID_FIELD, Field.create(jobIdConfig.jobId));
  startOutput.put(Constants.STARTED_SUCCESSFULLY_FIELD, Field.create(true));
  if (!conf.runInBackground) {
    startOutput.put(Constants.FINISHED_SUCCESSFULLY_FIELD, Field.create(success));
    MetricRegistryJson jobMetrics = ControlHubApiUtil.getJobMetrics(
        clientBuilder,
        conf.baseUrl,
        jobIdConfig.jobId,
        userAuthToken
    );
    startOutput.put(Constants.JOB_METRICS_FIELD, CommonUtil.getMetricsField(jobMetrics));
  }
  startOutput.put(Constants.JOB_STATUS_FIELD, Field.create(status));
  startOutput.put(Constants.JOB_STATUS_COLOR_FIELD, Field.create(statusColor));
  startOutput.put(Constants.ERROR_MESSAGE_FIELD, Field.create(errorMessage));
  responseField = Field.createListMap(startOutput);
}
 
Example 2
Source Project: datacollector   Source File: TestSyslogTarget.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteSingleRecord() throws Exception {
  SyslogDTarget config = getDefaultConfig();
  Target target = config.createTarget();
  TargetRunner runner = new TargetRunner.Builder(SyslogDTarget.class, target)
      .setOnRecordError(OnRecordError.TO_ERROR)
      .addService(DataFormatGeneratorService.class, new SdkJsonDataFormatGeneratorService())
      .build();

  runner.runInit();

  Record record = RecordCreator.create();
  Map<String, Field> fields = new HashMap<>();
  fields.put("text", Field.create("myTestMessage"));
  fields.put("hostname", Field.create("localhost"));
  fields.put("application", Field.create("myTestApp"));
  fields.put("facility", Field.create("1"));
  fields.put("severity", Field.create("1"));
  record.set(Field.create(fields));

  runner.runWrite(Arrays.asList(record));

  runner.runDestroy();
}
 
Example 3
Source Project: datacollector   Source File: TestDelimitedDataGenerator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGeneratorNoHeader() throws Exception {
  StringWriter writer = new StringWriter();
  DataGenerator gen = new DelimitedCharDataGenerator(writer, CsvMode.CSV.getFormat(), CsvHeader.NO_HEADER, "h", "d", null);
  Record record = RecordCreator.create();
  List<Field> list = new ArrayList<>();
  Map<String,Field> map = new HashMap<>();
  map.put("h", Field.create("A"));
  map.put("d", Field.create("a\n\r"));
  list.add(Field.create(map));
  map.put("h", Field.create("B"));
  map.put("d", Field.create("b"));
  list.add(Field.create(map));
  record.set(Field.create(list));
  gen.write(record);
  gen.close();
  Assert.assertEquals("\"a\n\r\",b\r\n", writer.toString());
}
 
Example 4
Source Project: datacollector   Source File: Matchers.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean matches(Object item) {
  if (item instanceof Field) {
    Field field = (Field) item;
    if (field.getType() == type) {
      final Object valueFromField = getValueFromField(field);
      if (value == null) {
        return valueFromField == null;
      } else if (value.getClass().isArray()) {
        // arrays are a pain...
        if (byte.class.equals(value.getClass().getComponentType())) {
          return Arrays.equals((byte[]) value, (byte[]) valueFromField);
        } else {
          throw new IllegalStateException(
              "Incomparable array type: " + value.getClass().getComponentType().toString()
          );
        }
      } else {
        return value.equals(valueFromField);
      }
    }
  }
  return false;
}
 
Example 5
Source Project: datacollector   Source File: FieldHasherProcessor.java    License: Apache License 2.0 6 votes vote down vote up
private Set<String> validateAndExtractFieldsToHash(
    Record record,
    Set<String> fieldsDontExist,
    Set<String> fieldsWithListOrMapType,
    Set<String> fieldsWithNull,
    Collection<String> matchingFieldsPath
) {
  Set<String> validFieldsToHashForThisConfig = new HashSet<String>();
  for (String matchingFieldPath : matchingFieldsPath) {
    if (record.has(matchingFieldPath)) {
      Field field = record.get(matchingFieldPath);
      if (UNSUPPORTED_FIELD_TYPES.contains(field.getType())) {
        fieldsWithListOrMapType.add(matchingFieldPath);
      } else if (field.getValue() == null) {
        fieldsWithNull.add(matchingFieldPath);
      } else {
        validFieldsToHashForThisConfig.add(matchingFieldPath);
      }
    } else {
      fieldsDontExist.add(matchingFieldPath);
    }
  }
  return validFieldsToHashForThisConfig;
}
 
Example 6
Source Project: datacollector   Source File: SchedulerJob.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void execute(JobExecutionContext jobExecutionContext) {
  try {
    SchedulerContext schedulerContext = jobExecutionContext.getScheduler().getContext();
    PushSource.Context pushSourceContext = (PushSource.Context) schedulerContext.get(
        SchedulerPushSource.PUSH_SOURCE_CONTEXT
    );
    if (pushSourceContext != null) {
      BatchContext batchContext = pushSourceContext.startBatch();
      Record record = pushSourceContext.createRecord("cronRecord");
      LinkedHashMap<String, Field> linkedHashMap = new LinkedHashMap<>();
      linkedHashMap.put("timestamp", Field.createDatetime(new Date()));
      record.set(Field.createListMap(linkedHashMap));
      batchContext.getBatchMaker().addRecord(record);
      pushSourceContext.processBatch(batchContext);
    }
  } catch (SchedulerException ex) {
    LOG.error(ex.getMessage(), ex);
  }
}
 
Example 7
@Test
public void testNotWholeFileFormatRecord() throws StageException {
  Processor wholeFileTransofrmer  = new TestWholeFileTransformerProcessorBuilder()
      .tempDir("${file:parentPath(record:attribute('file'))}/.parquet")
      .build();

  ProcessorRunner runner = new ProcessorRunner.Builder(WholeFileTransformerDProcessor.class, wholeFileTransofrmer)
      .addOutputLane("a")
      .setOnRecordError(OnRecordError.TO_ERROR)
      .build();

  Record record = RecordCreator.create();
  record.set(Field.create("test"));

  try {
    runner.runInit();
    StageRunner.Output output = runner.runProcess(Arrays.asList(record));
    Assert.assertEquals(0, output.getRecords().get("a").size());

    List<Record> errorRecords = runner.getErrorRecords();
    Assert.assertEquals(1, errorRecords.size());
    Assert.assertEquals(Errors.CONVERT_01.getCode(), errorRecords.get(0).getHeader().getErrorCode());
  } finally {
    runner.runDestroy();
  }
}
 
Example 8
Source Project: datacollector   Source File: TestTimeNewEL.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExtractStringFromDate() throws Exception {

  Record.Header header = Mockito.mock(Record.Header.class);
  Record record = Mockito.mock(Record.class);
  Mockito.when(record.getHeader()).thenReturn(header);
  long val = System.currentTimeMillis();
  Date d = new Date(val);
  Mockito.when(record.get(Mockito.eq("/theDate"))).thenReturn(Field.createDate(d));
  Mockito.when(record.get(Mockito.eq("/epochMS"))).thenReturn(Field.create(val));

  String format  = "G yyyy-MM-dd HH:mm:ss.SSS a zzzzzzz";
  Mockito.when(record.get(Mockito.eq("/format"))).thenReturn(Field.create(format));

  RecordEL.setRecordInContext(variables, record);

  SimpleDateFormat sdf = new SimpleDateFormat(format);

  String ans = sdf.format(d);

  Assert.assertEquals(ans, eval.eval(variables,
      "${time:extractStringFromDate(time:millisecondsToDateTime(record:value('/epochMS')), record:value('/format'))}", String.class));

  Assert.assertEquals(ans, eval.eval(variables,
      "${time:extractStringFromDate(record:value('/theDate'), record:value('/format'))}", String.class));
}
 
Example 9
Source Project: datacollector   Source File: Matchers.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean matches(Object item) {
  if (item instanceof Field) {
    Field field = (Field) item;
    if (field.getType().isOneOf(Field.Type.MAP, Field.Type.LIST_MAP)) {
      final Map<String, Field> childFields = field.getValueAsMap();
      if (!childFields.containsKey(nestedFieldName)) {
        return false;
      }
      final VT value = valueAccessor.getValue(childFields.get(nestedFieldName));
      if (expectedValue == null) {
        return value == null;
      } else {
        return expectedValue.equals(value);
      }
    }
  }
  return false;
}
 
Example 10
Source Project: datacollector   Source File: JdbcUtil.java    License: Apache License 2.0 6 votes vote down vote up
public LinkedHashMap<String, Field> resultSetToFields(
    ResultSet rs,
    CommonSourceConfigBean commonSourceBean,
    ErrorRecordHandler errorRecordHandler,
    UnknownTypeAction unknownTypeAction,
    Set<String> recordHeader,
    DatabaseVendor vendor
) throws SQLException, StageException {
  return resultSetToFields(
      rs,
      commonSourceBean.maxClobSize,
      commonSourceBean.maxBlobSize,
      Collections.emptyMap(),
      errorRecordHandler,
      unknownTypeAction,
      recordHeader,
      commonSourceBean.convertTimestampToString,
      vendor
  );
}
 
Example 11
Source Project: datacollector   Source File: TestJdbcMetadata.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInitialRecordStructure() throws Exception {
  JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString);

  ProcessorRunner processorRunner = getProcessorRunner(processor);

  processorRunner.runInit();

  List<Record> singleRecord = ImmutableList.of(makeRecord(fieldMap1));
  StageRunner.Output output = processorRunner.runProcess(singleRecord);
  Assert.assertEquals(1, output.getRecords().get("lane").size());

  Record record = output.getRecords().get("lane").get(0);
  for (Map.Entry<String, Field>entry : record.get().getValueAsListMap().entrySet()) {
    Assert.assertEquals(fieldMap1.get(entry.getKey()).getRight(), entry.getValue().getValue());
  }

  try (ResultSet metaDataColumns = jdbcUtil.getColumnMetadata(connection, null, tableName)) {
    while (metaDataColumns.next()) {
      Pair<Field.Type, Object> typeAndValue = fieldMap1.get(metaDataColumns.getString(JdbcSchemaReader.COLUMN_NAME).toLowerCase());
      Assert.assertEquals(typeAndValue.getLeft(), typeMap.get(metaDataColumns.getInt(JdbcSchemaReader.DATA_TYPE)));
    }
  }
}
 
Example 12
Source Project: datacollector   Source File: TestDelimitedDataGenerator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGeneratorReplaceNewLines() throws Exception {
  StringWriter writer = new StringWriter();
  DataGenerator gen = new DelimitedCharDataGenerator(writer, CsvMode.CSV.getFormat(), CsvHeader.NO_HEADER, "h", "d", " ");
  Record record = RecordCreator.create();
  List<Field> list = new ArrayList<>();
  Map<String,Field> map = new HashMap<>();
  map.put("h", Field.create("A"));
  map.put("d", Field.create("a\n\r"));
  list.add(Field.create(map));
  map.put("h", Field.create("B"));
  map.put("d", Field.create("b"));
  list.add(Field.create(map));
  map.put("h", Field.create("C"));
  map.put("d", Field.create((String)null));
  list.add(Field.create(map));
  record.set(Field.create(list));
  gen.write(record);
  gen.close();
  Assert.assertEquals("\"a  \",b,\r\n", writer.toString());
}
 
Example 13
Source Project: datacollector   Source File: TestAvroTypeUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateDoubleField() throws Exception {
  String schema = "{\"name\": \"name\", \"type\": \"double\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, 345823746923.863423, false);
  Assert.assertEquals(Field.Type.DOUBLE, field.getType());
  Assert.assertTrue(345823746923.863423 == field.getValueAsDouble());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof Double);
  Assert.assertTrue(345823746923.863423 == (Double) avroObject);

  //Check invalid type - String to Double
  makeBadType(Field.create("notDouble"), record, avroSchema);
}
 
Example 14
Source Project: datacollector   Source File: TestAvroTypeUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateListPrimitiveField() throws Exception {
  String schema = "{\"type\": \"array\", \"items\": \"string\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, Arrays.asList("Hari", "Kiran"), false);

  Assert.assertEquals(Field.Type.LIST, field.getType());
  List<Field> valueAsList = field.getValueAsList();
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(0).getType());
  Assert.assertEquals("Hari", valueAsList.get(0).getValueAsString());
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(1).getType());
  Assert.assertEquals("Kiran", valueAsList.get(1).getValueAsString());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof List<?>);

  List<String> listString = (List<String>) avroObject;
  Assert.assertEquals("Hari", listString.get(0));
  Assert.assertEquals("Kiran", listString.get(1));

  //Check invalid type - String to List
  makeBadType(Field.create("notList"), record, avroSchema);
}
 
Example 15
Source Project: datacollector   Source File: BasicIT.java    License: Apache License 2.0 6 votes vote down vote up
private static List<Record> createTransactionRecords(int noOfRecords) {
  List<Record> records = new ArrayList<>();
  long currentTime = (System.currentTimeMillis() / 1000) * 1000;
  for (int i = 0; i < noOfRecords; i++) {
    Record record = RecordCreator.create();
    LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
    fields.put("unique_int", Field.create(i + 1));
    fields.put("t_date", Field.create(Field.Type.LONG, currentTime));
    fields.put("random_string", Field.create(UUID.randomUUID().toString()));
    record.set(Field.createListMap(fields));
    records.add(record);
    //making sure time is at least off by a second.
    currentTime = currentTime + 1000;
  }
  return records;
}
 
Example 16
Source Project: datacollector   Source File: RecordConverter.java    License: Apache License 2.0 6 votes vote down vote up
private List<Record> toRecords(Table table,
                               EventHeader eventHeader,
                               DeleteRowsEventData eventData,
                               SourceOffset offset) {
  List<Record> res = new ArrayList<>(eventData.getRows().size());
  for (Serializable[] row : eventData.getRows()) {
    Record record = recordFactory.create(offset.format());
    Map<String, Field> fields = createHeader(table, eventHeader, offset);
    fields.put(TYPE_FIELD, create("DELETE"));
    record.getHeader().setAttribute(
        OperationType.SDC_OPERATION_TYPE,
        String.valueOf(OperationType.DELETE_CODE)
    );
    List<ColumnValue> columnValues = zipColumnsValues(eventData.getIncludedColumns(), table, row);
    Map<String, Field> data = toMap(columnValues);
    fields.put(OLD_DATA_FIELD, create(data));

    record.set(create(fields));
    res.add(record);
  }
  return res;
}
 
Example 17
Source Project: datacollector   Source File: TestSdcRecordDataGenerator.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IOException.class)
public void testWriteAfterClose() throws Exception {
  ByteArrayOutputStream writer = new ByteArrayOutputStream();
  DataGenerator gen = new SdcRecordDataGenerator(
      getContextExtensions().createRecordWriter(writer),
      getContextExtensions()
  );
  Record record = RecordCreator.create();
  record.set(Field.create("Hello"));
  gen.close();
  gen.write(record);
}
 
Example 18
Source Project: datacollector   Source File: StreamingXmlParser.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void addContent(Map<String, Object> contents, String name, Field field) throws
    XMLStreamException,
    ObjectLengthException {
  throwIfOverMaxObjectLength();
  List<Field> list = (List<Field>) contents.get(name);
  if (list == null) {
    list = new ArrayList<>();
    contents.put(name, list);
  }
  list.add(field);
}
 
Example 19
@Test
public void testFlattenEntireRecordDifferentSeparator() throws Exception {
  FieldFlattenerConfig config = new FieldFlattenerConfig();
  config.nameSeparator = "/";
  config.flattenType = FlattenType.ENTIRE_RECORD;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFlattenerDProcessor.class, new FieldFlattenerProcessor(config))
    .addOutputLane("a").build();
  runner.runInit();

  Record record = RecordCreator.create();
  record.set(Field.create(Field.Type.MAP, ImmutableMap.builder()
    .put("top-level", Field.create(Field.Type.STRING, "top-level"))
    .put("map", Field.create(Field.Type.MAP, ImmutableMap.builder()
      .put("map1", Field.create(Field.Type.STRING, "map1"))
      .put("map2", Field.create(Field.Type.STRING, "map2"))
      .build()))
    .put("list", Field.create(Field.Type.LIST, ImmutableList.of(
      Field.create(Field.Type.STRING, "list1"),
      Field.create(Field.Type.STRING, "list2"))))
    .build()
  ));

  StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
  assertEquals(1, output.getRecords().get("a").size());

  Field field = output.getRecords().get("a").get(0).get();
  assertEquals(Field.Type.MAP, field.getType());

  Map<String, Field> result = field.getValueAsMap();
  assertEquals(5, result.size());
  assertTrue(result.containsKey("top-level"));
  assertTrue(result.containsKey("map/map1"));
  assertTrue(result.containsKey("map/map2"));
  assertTrue(result.containsKey("list/0"));
  assertTrue(result.containsKey("list/1"));

  runner.runDestroy();
}
 
Example 20
Source Project: datacollector   Source File: FieldMapperProcessor.java    License: Apache License 2.0 5 votes vote down vote up
private void transformFieldValues(Record record) throws StageException {
  record.forEachField(fv -> {
    final String fieldPath = fv.getFieldPath();
    final String fieldName = fv.getFieldName();
    final Field field = fv.getField();
    final String parentFieldPath = fv.getParentFieldPath();
    final Field parentField = fv.getParentField();
    final int indexWithinParent = fv.getIndexInParent();
    if (checkSkipFieldAndSetContextVar(
        fieldPath,
        fieldName,
        field,
        parentFieldPath,
        parentField,
        indexWithinParent,
        true
    )) {
      return;
    }

    final Object newValue = mapperExpressionEval.eval(
        expressionVars,
        fieldMapperConfig.mappingExpression,
        Object.class
    );

    final Field.Type newType = FieldUtils.getTypeFromObject(newValue);

    record.set(fieldPath, Field.create(newType, newValue));
  });
}
 
Example 21
Source Project: datacollector   Source File: ForceLookupSoapLoader.java    License: Apache License 2.0 5 votes vote down vote up
private void addResult(List<Map<String, Field>> lookupItems, QueryResult queryResult) throws StageException {
  SObject[] records = queryResult.getRecords();

  LOG.info("Retrieved {} records", records.length);

  if (processor.getRecordCreator().isCountQuery()) {
    // Special case for old-style COUNT() query
    DataType dataType = (processor.columnsToTypes != null)
        ? processor.columnsToTypes.get(COUNT.toLowerCase())
        : DataType.USE_SALESFORCE_TYPE;
    if (dataType == null) {
      dataType = DataType.USE_SALESFORCE_TYPE;
    }
    Field.Type fieldType = (dataType == DataType.USE_SALESFORCE_TYPE)
        ? Field.Type.INTEGER
        : Field.Type.valueOf(dataType.getLabel());
    LinkedHashMap<String, Field> map = new LinkedHashMap<>();
    map.put(COUNT, Field.create(fieldType, queryResult.getSize()));
    lookupItems.add(map);
  } else {
    for (int i = 0; i < records.length; i++) {
      lookupItems.add(((SoapRecordCreator) processor.getRecordCreator()).addFields(
          records[i],
          processor.columnsToTypes));
    }
  }
}
 
Example 22
Source Project: datacollector   Source File: KinesisTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static List<Record> getProducerTestRecords(int i) {
  List<Record> records = new ArrayList<>(i);

  for (int j = 0; j < i; j++) {
    Record record = RecordCreator.create();
    Map<String, Field> root = new HashMap<>();
    root.put("a", Field.create(1));
    root.put("b", Field.create(2));
    root.put("seq", Field.create(j));
    record.set(Field.create(root));
    records.add(record);
  }

  return records;
}
 
Example 23
Source Project: datacollector-api   Source File: TestFieldDeepCopy.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testClone() {
  ListTypeSupport ts = new ListTypeSupport();
  List<Field> list = new ArrayList<>();
  list.add(Field.create("a"));
  List<Field> cList = (List) ts.clone(list);
  Assert.assertEquals(list, cList);
  Assert.assertNotSame(list, cList);
  Assert.assertNotSame(list.get(0), cList.get(0));
  List<Field> listX = new ArrayList<>();
  listX.add(Field.create(list));

  Map<String, Field> map = new HashMap<>();
  map.put("list", Field.create(listX));
  MapTypeSupport ms = new MapTypeSupport();
  Map<String, Field> cMap = (Map) ms.clone(map);
  Assert.assertEquals(map, cMap);
  Assert.assertNotSame(map, cMap);
  Assert.assertEquals(map.get("list"), cMap.get("list"));
  Assert.assertNotSame(map.get("list"), cMap.get("list"));
  Assert.assertEquals(map.get("list").getValueAsList().get(0), cMap.get("list").getValueAsList().get(0));
  Assert.assertNotSame(map.get("list").getValueAsList().get(0), cMap.get("list").getValueAsList().get(0));

  LinkedHashMap<String, Field> listMap = new LinkedHashMap<>();
  listMap.put("list", Field.create(listX));
  ListMapTypeSupport listMapTypeSupport = new ListMapTypeSupport();
  LinkedHashMap<String, Field> listMapClone = (LinkedHashMap) listMapTypeSupport.clone(listMap);
  Assert.assertEquals(listMap, listMapClone);
  Assert.assertNotSame(listMap, listMapClone);
  Assert.assertEquals(map.get("list"), listMapClone.get("list"));
  Assert.assertNotSame(map.get("list"), listMapClone.get("list"));
  Assert.assertEquals(map.get("list").getValueAsList().get(0), listMapClone.get("list").getValueAsList().get(0));
  Assert.assertNotSame(map.get("list").getValueAsList().get(0), listMapClone.get("list").getValueAsList().get(0));
}
 
Example 24
Source Project: datacollector   Source File: TestFieldMaskProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFieldMaskProcessorFiledDoesNotExist() throws StageException {

  FieldMaskConfig nameMaskConfig = new FieldMaskConfig();
  nameMaskConfig.fields = ImmutableList.of("/name");
  nameMaskConfig.maskType = MaskType.VARIABLE_LENGTH;
  nameMaskConfig.mask = null;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldMaskDProcessor.class)
      .addConfiguration("fieldMaskConfigs", ImmutableList.of(nameMaskConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("age", Field.create("12"));
    map.put("ssn", Field.create("123-45-6789"));
    map.put("phone", Field.create("9876543210"));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 3);
    Assert.assertTrue(result.containsKey("age"));
    Assert.assertEquals("12", result.get("age").getValue());
    Assert.assertTrue(result.containsKey("ssn"));
    Assert.assertEquals("123-45-6789", result.get("ssn").getValue());
    Assert.assertTrue(result.containsKey("phone"));
    Assert.assertEquals("9876543210", result.get("phone").getValue());
  } finally {
    runner.runDestroy();
  }
}
 
Example 25
Source Project: datacollector   Source File: HiveMetadataProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
Record generateSchemaChangeRecord(
    String database,
    String tableName,
    LinkedHashMap<String, HiveTypeInfo> columnList,
    LinkedHashMap<String, HiveTypeInfo> partitionTypeList,
    String location,
    String avroSchema,
    Map<String, String> metadataHeaderAttributes
) throws HiveStageCheckedException
{
  //creating a record with uuid as postfix so multiple SDCs won't generate the record with same id.
  Record metadataRecord = getContext().createRecord("Table Metadata Record" + UUID.randomUUID().toString());

  Field metadataField = HiveMetastoreUtil.newSchemaMetadataFieldBuilder(
      database,
      tableName,
      columnList,
      partitionTypeList,
      !externalTable,  // need to flip boolean to send if this is internal table
      location,
      avroSchema,
      dataFormat
  );
  metadataRecord.set(metadataField);

  for (Map.Entry<String, String> entry : metadataHeaderAttributes.entrySet()){
    metadataRecord.getHeader().setAttribute(entry.getKey(), entry.getValue());
  }

    return metadataRecord;
}
 
Example 26
Source Project: datacollector   Source File: DriftRuleEL.java    License: Apache License 2.0 5 votes vote down vote up
String composeTypeAlert(String fieldPath, Field.Type givenType, Set<Field.Type> supportedTypes) {
  return Utils.format("Field {} have unsupported type of {}. Supported types are {}",
    fieldPath,
    givenType.toString(),
    StringUtils.join(supportedTypes, ",")
  );
}
 
Example 27
Source Project: datacollector   Source File: TestFieldValueReplacer.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReplaceNullFloatFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/floatField");
  stringFieldReplacement.newValue = "10.345f";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("floatField", Field.create(Field.Type.FLOAT, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("floatField"));
    Assert.assertEquals(10.345f, result.get("floatField").getValue());
  } finally {
    runner.runDestroy();
  }
}
 
Example 28
Source Project: datacollector   Source File: ExtraOffsetConditionIT.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setupTables() throws Exception {
  try (Statement statement = connection.createStatement()) {

    Map<String, String> fieldsForCreate = new LinkedHashMap<>();

    for (Map.Entry<String, Field.Type> offsetFieldEntry : transactionOffsetFields.entrySet()) {
      fieldsForCreate.put(offsetFieldEntry.getKey(), FIELD_TYPE_TO_SQL_TYPE_AND_STRING.get(offsetFieldEntry.getValue()));
    }
    //extra field (other than offset column.)
    fieldsForCreate.put(RANDOM_STRING_COLUMN, FIELD_TYPE_TO_SQL_TYPE_AND_STRING.get(Field.Type.STRING));

    statement.addBatch(
        getCreateStatement(
            "TEST",
            tableName,
            Collections.<String, String>emptyMap(),
            fieldsForCreate
        )
    );

    createRecords(transactionOffsetFields);

    List<Record> recordsToInsert = new ArrayList<>();
    for (List<Record> records : batchRecords) {
      recordsToInsert.addAll(records);
    }

    for (Record recordToInsert : recordsToInsert) {
      Map<String, Field> fields = recordToInsert.get().getValueAsListMap();
      statement.addBatch(getInsertStatement(database, tableName, fields.values()));
    }
    statement.executeBatch();
  }
}
 
Example 29
@Test
public void testFlattenEntireRecordMapInLists() throws Exception {
  FieldFlattenerConfig config = new FieldFlattenerConfig();
  config.nameSeparator = ".";
  config.flattenType = FlattenType.ENTIRE_RECORD;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFlattenerDProcessor.class, new FieldFlattenerProcessor(config))
    .addOutputLane("a").build();
  runner.runInit();

  Record record = RecordCreator.create();
  record.set(Field.create(Field.Type.MAP, ImmutableMap.builder()
    .put("root-list", Field.create(Field.Type.LIST, ImmutableList.of(
      Field.create(Field.Type.MAP, ImmutableMap.builder()
        .put("a", Field.create(Field.Type.STRING, "a"))
        .put("b", Field.create(Field.Type.STRING, "b"))
        .build()),
      Field.create(Field.Type.MAP, ImmutableMap.builder()
        .put("c", Field.create(Field.Type.STRING, "c"))
        .put("d", Field.create(Field.Type.STRING, "d"))
        .build()))))
    .build()
  ));

  StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
  assertEquals(1, output.getRecords().get("a").size());

  Field field = output.getRecords().get("a").get(0).get();
  assertEquals(Field.Type.MAP, field.getType());

  Map<String, Field> result = field.getValueAsMap();
  assertEquals(4, result.size());
  assertTrue(result.containsKey("root-list.0.a"));
  assertTrue(result.containsKey("root-list.0.b"));
  assertTrue(result.containsKey("root-list.1.c"));
  assertTrue(result.containsKey("root-list.1.d"));

  runner.runDestroy();
}
 
Example 30
Source Project: datacollector   Source File: TestFieldMaskProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFieldMaskProcessorFixedLength() throws StageException {

  FieldMaskConfig ageMaskConfig = new FieldMaskConfig();
  ageMaskConfig.fields = ImmutableList.of("/name", "/age", "/ssn", "/phone");
  ageMaskConfig.maskType = MaskType.FIXED_LENGTH;
  ageMaskConfig.mask = null;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldMaskDProcessor.class)
      .addConfiguration("fieldMaskConfigs", ImmutableList.of(ageMaskConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("name", Field.create("streamsetsinc"));
    map.put("age", Field.create("12"));
    map.put("ssn", Field.create("123-45-6789"));
    map.put("phone", Field.create(Field.Type.STRING, null));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 4);
    Assert.assertTrue(result.containsKey("name"));
    Assert.assertEquals("xxxxxxxxxx", result.get("name").getValue());
    Assert.assertTrue(result.containsKey("age"));
    Assert.assertEquals("xxxxxxxxxx", result.get("age").getValue());
    Assert.assertTrue(result.containsKey("ssn"));
    Assert.assertEquals("xxxxxxxxxx", result.get("ssn").getValue());
    Assert.assertTrue(result.containsKey("phone"));
    Assert.assertEquals(null, result.get("phone").getValue());
  } finally {
    runner.runDestroy();
  }
}