Java Code Examples for com.streamsets.pipeline.api.Record#getEscapedFieldPaths()

The following examples show how to use com.streamsets.pipeline.api.Record#getEscapedFieldPaths() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestDelimitedCharDataParser.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
@Test
public void testParseWithExtraColumnsAllowed() throws Exception {
  OverrunReader reader = new OverrunReader(new StringReader("a,b,c\n1,2,3,4"), 1000, true, false);
  DelimitedDataParserSettings settings = DelimitedDataParserSettings.builder()
      .withSkipStartLines(0)
      .withFormat(CSVFormat.DEFAULT)
      .withHeader(CsvHeader.WITH_HEADER)
      .withMaxObjectLen(-1)
      .withRecordType(CsvRecordType.LIST_MAP)
      .withParseNull(false)
      .withNullConstant(null)
      .withAllowExtraColumns(true)
      .withExtraColumnPrefix("_abc_")
      .build();
  DataParser parser = new DelimitedCharDataParser(getContext(), "id", reader, 0, settings);

  Record record = parser.parse();
  Assert.assertNotNull(record);
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  Set<String> expectedFieldPaths = ImmutableSet.of("", "/a", "/b", "/c", "/_abc_01");
  assertTrue(fieldPaths.containsAll(expectedFieldPaths));
}
Example 2
Source File: CollectdRecordConverter.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
List<String> getValueFields(Record record) throws OnRecordErrorException {
  List<String> fields = new ArrayList<>();
  Set<String> fieldPaths = record.getEscapedFieldPaths();

  for (String fieldPath : fieldPaths) {
    if (!isValueField(fieldPath)) {
      continue;
    }
    fields.add(stripPathPrefix(fieldPath));
  }

  if (fields.isEmpty()) {
    throw new OnRecordErrorException(Errors.INFLUX_06);
  }

  return fields;
}
Example 3
Source File: TensorFlowProcessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
private void setInputConfigFields(Record record) {
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  for (TensorInputConfig inputConfig : conf.inputConfigs) {
    Set<String> inputConfigFields = new HashSet<>();
    for(String f: inputConfig.fields) {
      List<String> matchingFieldPaths = FieldPathExpressionUtil.evaluateMatchingFieldPaths(
          f,
          fieldPathEval,
          fieldPathVars,
          record,
          fieldPaths
      );
      inputConfigFields.addAll(matchingFieldPaths);
    }
    inputConfig.setResolvedFields(new ArrayList<>(inputConfigFields));
  }
}
Example 4
Source File: TestFieldFilterProcessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNestedKeep8() throws StageException {
  /*
   */
  Record record = createNestedRecord();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[2]/SantaClara/main/streets[*][*]/name12*"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);
    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertTrue(resultRecord.has("/USA[0]/SantaClara/main/streets[1][1]/name12AB"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SantaClara/main/streets[0][0]/name1"));


  } finally {
    runner.runDestroy();
  }
}
Example 5
Source File: HashingUtil.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
protected List<String> getFieldsToHash(Record record) {
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  List<String> fields = new ArrayList<>();
  if (fieldsToHash != null) {
    for(String field : fieldsToHash) {
      List<String> matchingFieldPaths = FieldRegexUtil.getMatchingFieldPaths(field, fieldPaths);
      Collections.sort(matchingFieldPaths);
      fields.addAll(matchingFieldPaths);
    }
  } else {
    fields = new ArrayList<>(record.getEscapedFieldPaths());
    Collections.sort(fields);
  }
  return fields;
}
Example 6
Source File: FileRefUtil.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
public static void validateWholeFileRecord(Record record) {
  Set<String> fieldPathsInRecord = record.getEscapedFieldPaths();
  Utils.checkArgument(
      fieldPathsInRecord.containsAll(MANDATORY_FIELD_PATHS),
      Utils.format(
          "Record does not contain the mandatory fields {} for Whole File Format.",
          COMMA_JOINER.join(Sets.difference(MANDATORY_FIELD_PATHS, fieldPathsInRecord))
      )
  );
}
Example 7
Source File: BaseTableJdbcSourceIT.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
protected static void insertRows(String insertTemplate, List<Record> records) throws SQLException {
  try (Statement st = connection.createStatement()) {
    for (Record record : records) {
      List<String> values = new ArrayList<>();
      for (String fieldPath : record.getEscapedFieldPaths()) {
        //Skip root field
        if (!fieldPath.equals("")) {
          values.add(getStringRepOfFieldValueForInsert(record.get(fieldPath)));
        }
      }
      st.addBatch(String.format(insertTemplate, values.toArray()));
    }
    st.executeBatch();
  }
}
Example 8
Source File: FieldTypeConverterProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
private void processByField(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  final Set<String> fieldPaths = record.getEscapedFieldPaths();
  for(FieldTypeConverterConfig fieldTypeConverterConfig : fieldTypeConverterConfigs) {
    for(String fieldToConvert : fieldTypeConverterConfig.fields) {
      final List<String> matchingFieldPaths = new LinkedList<>(FieldPathExpressionUtil.evaluateMatchingFieldPaths(
          fieldToConvert,
          fieldPathEval,
          fieldPathVars,
          record,
          fieldPaths
      ));
      if (matchingFieldPaths.isEmpty()) {
        // FieldPathExpressionUtil.evaluateMatchingFieldPaths does NOT return the supplied param in its result
        // regardless, like FieldRegexUtil#getMatchingFieldPaths did, so we add manually here
        matchingFieldPaths.add(fieldToConvert);
      }
      for (String matchingField : matchingFieldPaths) {
        Field field = record.get(matchingField);
        if(field == null) {
          LOG.trace("Record does not have field {}. Ignoring conversion.", matchingField);
        } else {
          record.set(matchingField, convertField(matchingField, field, fieldTypeConverterConfig));
        }
      }
    }
  }
  batchMaker.addRecord(record);
}
Example 9
Source File: FieldMaskProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  List<String> nonStringFields = new ArrayList<>();
  // For each individual configuration entry
  for(FieldMaskConfig fieldMaskConfig : activeFieldMaskConfigs) {
    // For each configured field expression
    for (String toMask : fieldMaskConfig.fields) {
      // Find all actual fields that matches given configured expression
      for (String matchingFieldPath : FieldPathExpressionUtil.evaluateMatchingFieldPaths(
          toMask,
          fieldPathEval,
          fieldPathVars,
          record,
          fieldPaths
      )) {
        if (record.has(matchingFieldPath)) {
          Field field = record.get(matchingFieldPath);
          if (field.getType() != Field.Type.STRING) {
            nonStringFields.add(matchingFieldPath);
          } else {
            if (field.getValue() != null) {
              Field newField = Field.create(maskField(field, fieldMaskConfig));
              record.set(matchingFieldPath, newField);
            }
          }
        }
      }
    }
  }
  if (nonStringFields.isEmpty()) {
    batchMaker.addRecord(record);
  } else {
    throw new OnRecordErrorException(Errors.MASK_00, StringUtils.join(nonStringFields, ", "), record.getHeader().getSourceId());
  }
}
Example 10
Source File: TestFieldFilterProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNestedKeep9() throws StageException {
  /*check for embedded *
   */
  Record record = createNestedRecord();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[2]/SantaClara/ma*/streets[*][*]/name1"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);
    Set <String> r = resultRecord.getEscapedFieldPaths();


    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertFalse(resultRecord.has("/USA[0]/SantaClara/main/streets[1][1]/name12AB"));
    Assert.assertTrue(resultRecord.has("/USA[0]/SantaClara/main/streets[0][0]/name1"));


  } finally {
    runner.runDestroy();
  }
}
Example 11
Source File: TestFieldFilterProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNestedKeep10() throws StageException {
  /* check for embedded ? and * at end of field and after /
   */
  Record record = createNestedRecordForRegex();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[?]/San*/*/streets[*][*]/name?2"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);

    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name12"));
    Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noel/streets[0][0]/name12"));
    Assert.assertTrue(resultRecord.has("/USA[1]/SantaClara/main/streets[0][1]/name22"));
    Assert.assertFalse(resultRecord.has("/USA[1]/SantaClara/main/streets[0][0]/name21"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name1234"));


  } finally {
    runner.runDestroy();
  }
}
Example 12
Source File: TestFieldFilterProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNestedKeep11() throws StageException {
  /*
   check for regex * at the end of the line
   */
  Record record = createNestedRecordForRegex();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[?]/San*/*/streets[*][*]/name2*"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);

    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name12"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noel/streets[0][0]/name12"));
    Assert.assertTrue(resultRecord.has("/USA[1]/SantaClara/main/streets[0][1]/name22"));
    Assert.assertTrue(resultRecord.has("/USA[1]/SantaClara/main/streets[0][0]/name21"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name1234"));


  } finally {
    runner.runDestroy();
  }
}
Example 13
Source File: AbstractHBaseProducer.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
private static void validateRootLevelType(Record record) throws OnRecordErrorException {
  for (String fieldPath : record.getEscapedFieldPaths()) {
    if (fieldPath.isEmpty()) {
      Field.Type type = record.get(fieldPath).getType();
      if (type != Field.Type.MAP && type != Field.Type.LIST_MAP) {
        throw new OnRecordErrorException(record, Errors.HBASE_29, type);
      }
      break;
    }
  }
}
Example 14
Source File: AbstractHBaseProducer.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
private void doImplicitFieldMapping(
    Put p,
    Record record,
    StringBuilder errorMsgBuilder,
    Set<String> explicitFields,
    String timeDriver,
    boolean ignoreMissingField,
    boolean ignoreInvalidColumn,
    String hbaseRowKey
) throws OnRecordErrorException {
  validateRootLevelType(record);
  Date recordTime = getRecordTime(record, timeDriver);
  for (String fieldPath : record.getEscapedFieldPaths()) {
    if (!fieldPath.isEmpty() && !fieldPath.equals(hbaseRowKey) && !explicitFields.contains(fieldPath)) {
      String fieldPathColumn = fieldPath;
      if (fieldPath.charAt(0) == '/') {
        fieldPathColumn = fieldPath.substring(1);
      }
      HBaseColumn hbaseColumn = hbaseConnectionHelper.getColumn(fieldPathColumn.replace("'", ""));
      if (hbaseColumn.getCf().isPresent() && hbaseColumn.getQualifier().isPresent()) {
        byte[] value = getBytesForValue(record, fieldPath, null, ignoreMissingField);
        addCell(p, hbaseColumn.getCf().get(), hbaseColumn.getQualifier().get(), recordTime, value);
      } else if (ignoreInvalidColumn) {
        String errorMessage = Utils.format(Errors.HBASE_28.getMessage(),
            fieldPathColumn,
            KeyValue.COLUMN_FAMILY_DELIMITER
        );
        LOG.warn(errorMessage);
        errorMsgBuilder.append(errorMessage);
      } else {
        throw new OnRecordErrorException(record, Errors.HBASE_28, fieldPathColumn, KeyValue.COLUMN_FAMILY_DELIMITER);
      }
    }
  }
}