Java Code Examples for com.streamsets.pipeline.api.Field#getValueAsList()

The following examples show how to use com.streamsets.pipeline.api.Field#getValueAsList() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RedisTarget.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private void doUpsertList(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException{
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.lpush(key, val);
        tempRecords.add(new ErrorRecord(record, "List", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 
Example 2
Source File: RedisTarget.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private void doUpsertSet(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.sadd(key, val);
        tempRecords.add(new ErrorRecord(record, "Set", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 
Example 3
Source File: FuzzyFieldProcessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private Map<String, Field> flattenRootField(Field rootField) throws StageException {
  Map<String, Field> fields;
  switch (rootField.getType()) {
    case LIST:
      // Flatten CSV input types.
      List<Field> fieldList = rootField.getValueAsList();
      fields = new HashMap<>();
      for (Field field : fieldList) {
        Map<String, Field> fieldWithHeader = field.getValueAsMap();
        if (!fieldWithHeader.containsKey(HEADER) || !fieldWithHeader.containsKey(VALUE)) {
          throw new StageException(Errors.FUZZY_00, rootField.getType());
        }
        fields.put(fieldWithHeader.get(HEADER).getValueAsString(), fieldWithHeader.get(VALUE));
      }
      break;
    case MAP:
      fields = rootField.getValueAsMap();
      break;
    default:
      throw new StageException(Errors.FUZZY_00, rootField.getType());
  }
  return fields;
}
 
Example 4
Source File: TestAvroTypeUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateListPrimitiveField() throws Exception {
  String schema = "{\"type\": \"array\", \"items\": \"string\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, Arrays.asList("Hari", "Kiran"), false);

  Assert.assertEquals(Field.Type.LIST, field.getType());
  List<Field> valueAsList = field.getValueAsList();
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(0).getType());
  Assert.assertEquals("Hari", valueAsList.get(0).getValueAsString());
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(1).getType());
  Assert.assertEquals("Kiran", valueAsList.get(1).getValueAsString());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof List<?>);

  List<String> listString = (List<String>) avroObject;
  Assert.assertEquals("Hari", listString.get(0));
  Assert.assertEquals("Kiran", listString.get(1));

  //Check invalid type - String to List
  makeBadType(Field.create("notList"), record, avroSchema);
}
 
Example 5
Source File: RecordEL.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private static int getDelimitedIndex(String header, int startIndex) {
  int index = -1;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (int i= startIndex; index == -1 && i < list.size(); i++) {
        Field element = list.get(i);
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                index = i;
              }
            }
          }
        }
      }
    }
  }
  return index;
}
 
Example 6
Source File: TestXmlParserProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testParsingMultipleValuesAsList() throws Exception {
  XmlParserConfig configs = new XmlParserConfig();
  configs.charset = "UTF-8";
  configs.fieldPathToParse = "/xml";
  configs.parsedFieldPath = "/data";
  configs.removeCtrlChars = false;
  configs.xmlRecordElement = "a";
  configs.multipleValuesBehavior = MultipleValuesBehavior.ALL_AS_LIST;

  XmlParserDProcessor processor = new XmlParserDProcessor();
  processor.configs = configs;

  ProcessorRunner runner = new ProcessorRunner.Builder(XmlParserDProcessor.class, processor)
      .addOutputLane("out").setOnRecordError(OnRecordError.TO_ERROR).build();
  Map<String, Field> map = new HashMap<>();
  map.put("xml", Field.create("<root><a>1</a><a>2</a><a>3</a></root>"));
  Record record = RecordCreator.create();
  record.set(Field.create(map));
  List<Record> input = new ArrayList<>();
  input.add(record);
  try {
    runner.runInit();
    StageRunner.Output output = runner.runProcess(input);
    Assert.assertTrue(output.getRecords().containsKey("out"));
    final List<Record> records = output.getRecords().get("out");
    Assert.assertEquals(1, records.size());
    final Field outputField = records.get(0).get("/data");
    Assert.assertNotNull(outputField);
    final List<Field> valueList = outputField.getValueAsList();
    Assert.assertNotNull(valueList);
    Assert.assertEquals(3, valueList.size());
    Assert.assertEquals("1", valueList.get(0).getValueAsMap().get("value").getValueAsString());
    Assert.assertEquals("2", valueList.get(1).getValueAsMap().get("value").getValueAsString());
    Assert.assertEquals("3", valueList.get(2).getValueAsMap().get("value").getValueAsString());

  } finally {
    runner.runDestroy();
  }
}
 
Example 7
Source File: MongoDBOplogSourceIT.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private void walkAndCheckField(String fieldName, Field expected, Field actual) {
  Assert.assertEquals(expected.getType(), actual.getType());
  Field.Type type = expected.getType();
  switch (type) {
    case MAP:
    case LIST_MAP:
      Map<String, Field> expectedFieldMap = expected.getValueAsMap();
      Map<String, Field> actualFieldMap = actual.getValueAsMap();
      Assert.assertTrue(getMismatchMessage(fieldName, type, "Expected Field has different keys from Actual Field"), Sets.difference(expectedFieldMap.keySet(), actualFieldMap.keySet()).isEmpty());
      Assert.assertTrue(getMismatchMessage(fieldName, type, "Actual Field has different keys from Expected Field"), Sets.difference(actualFieldMap.keySet(), expectedFieldMap.keySet()).isEmpty());
      for (String key : expectedFieldMap.keySet()) {
        walkAndCheckField(fieldName + "/" + key, expectedFieldMap.get(key), actualFieldMap.get(key));
      }
      break;
    case LIST:
      List<Field> expectedFieldList = expected.getValueAsList();
      List<Field> actualFieldList = actual.getValueAsList();
      Assert.assertEquals(getMismatchMessage(fieldName, type, "Size mistmatch"), expectedFieldList.size(), actualFieldList.size());
      for (int i=0; i < expectedFieldList.size(); i++) {
        walkAndCheckField(fieldName + "[" + i + "]", expectedFieldList.get(i), actualFieldList.get(i));
      }
      break;
    case BYTE_ARRAY:
      Assert.assertArrayEquals(getMismatchMessage(fieldName, type, "Byte Array value mismatch"), expected.getValueAsByteArray(), actual.getValueAsByteArray());
      break;
    default:
      Assert.assertEquals(getMismatchMessage(fieldName, type, "Field Value mismatch"),expected.getValue(), actual.getValue());
  }
}
 
Example 8
Source File: FieldTypeConverterProcessor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private Field processByType(String matchingPath, Field rootField) throws StageException {
  switch (rootField.getType()) {
    case MAP:
    case LIST_MAP:
      if(rootField.getValue() == null) {
        return rootField;
      }
      for (Map.Entry<String, Field> entry : rootField.getValueAsMap().entrySet()) {
        entry.setValue(processByType(matchingPath + "/" + entry.getKey(), entry.getValue()));
      }
      break;
    case LIST:
      if(rootField.getValue() == null) {
        return rootField;
      }
      List<Field> fields = rootField.getValueAsList();
      for(int i = 0; i < fields.size(); i++) {
        fields.set(i, processByType(matchingPath + "[" + i + "]", fields.get(i)));
      }
      break;
    default:
      for(WholeTypeConverterConfig converterConfig : wholeTypeConverterConfigs) {
        if(converterConfig.sourceType == rootField.getType()) {
          rootField = convertField(matchingPath, rootField, converterConfig);
        }
      }
  }

  // Return original field
  return rootField;
}
 
Example 9
Source File: RecordImpl.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private void visitFieldsInternal(
    RecordFieldImpl recordField,
    FieldVisitor visitor,
    String name,
    String path,
    Field currentField,
    Field parentField
) throws StageException {
  // For nested types, visit their children first
  switch (currentField.getType()) {
    case MAP:
    case LIST_MAP:
      for(Map.Entry<String, Field> entry : currentField.getValueAsMap().entrySet()) {
        visitFieldsInternal(
            recordField,
            visitor,
            entry.getKey(),
            path + "/" + escapeName(entry.getKey(), true),
            entry.getValue(),
            currentField
        );
      }
      break;
    case LIST:
      int index = 0;
      for(Field childField : currentField.getValueAsList()) {
        visitFieldsInternal(recordField, visitor, name, path + "[" + index + "]", childField, currentField);
        index++;
      }
      break;
    default:
  }

  // And always visit this field itself (whether it's terminal type of nested type doesn't matter)
  recordField.path = path;
  recordField.name = name;
  recordField.field = currentField;
  recordField.parentField = parentField;
  visitor.visit(recordField);
}
 
Example 10
Source File: RecordEL.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValueAt",
    description = "Returns the value of the specified header name")
public static String getDelimitedValueAt(
    @ElParam("index") int index) {
  String value = null;
  if (index >= 0) {
    Record record = getRecordInContext();
    if (record != null) {
      Field root = record.get();
      if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
        List<Field> list = root.getValueAsList();
        if (index < list.size()) {
          Field element = list.get(index);
          if (element.getType() == Field.Type.MAP && element.getValue() != null) {
            Map<String, Field> map = element.getValueAsMap();
            if (map.containsKey("value")) {
              Field valueField = map.get("value");
              if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                value = valueField.getValueAsString();
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 
Example 11
Source File: RecordEL.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValue",
    description = "Returns the value of the specified header name")
public static String getDelimitedValue(
    @ElParam("header") String header) {
  String value = null;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (Field element : list) {
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                if (map.containsKey("value")) {
                  Field valueField = map.get("value");
                  if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                    value = valueField.getValueAsString();
                    break;
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 
Example 12
Source File: BaseTableJdbcSourceIT.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private static void checkField(String fieldPath, Field expectedField, Field actualField) {
  String errorString = String.format("Error in Field Path: %s", fieldPath);
  Assert.assertEquals(errorString, expectedField.getType(), actualField.getType());
  errorString = errorString + " of type: " + expectedField.getType().name();
  switch (expectedField.getType()) {
    case MAP:
    case LIST_MAP:
      Map<String, Field> expectedFieldMap = expectedField.getValueAsMap();
      Map<String, Field> actualFieldMap = actualField.getValueAsMap();
      Assert.assertEquals(errorString, expectedFieldMap.keySet().size(), actualFieldMap.keySet().size());
      for (Map.Entry<String, Field> entry : actualFieldMap.entrySet()) {
        String actualFieldName = entry.getKey().toLowerCase();
        Assert.assertNotNull(errorString, expectedFieldMap.get(actualFieldName));
        checkField(fieldPath + "/" + actualFieldName, expectedFieldMap.get(actualFieldName), entry.getValue());
      }
      break;
    case LIST:
      List<Field> expectedFieldList = expectedField.getValueAsList();
      List<Field> actualFieldList = actualField.getValueAsList();
      Assert.assertEquals(errorString, expectedFieldList.size(), actualFieldList.size());
      for (int i = 0; i < expectedFieldList.size(); i++) {
        checkField(fieldPath + "[" + i + "]", expectedFieldList.get(i), actualFieldList.get(i));
      }
      break;
    case BYTE_ARRAY:
      Assert.assertArrayEquals(errorString, expectedField.getValueAsByteArray(), actualField.getValueAsByteArray());
      break;
    case ZONED_DATETIME:
      // H2 seems to throw away millis by default, so just make sure they are within 1 second
      final long millisDifference = expectedField.getValueAsZonedDateTime().toInstant().toEpochMilli()
          - actualField.getValueAsZonedDateTime().toInstant().toEpochMilli();
      assertThat(errorString, Math.abs(millisDifference), lessThanOrEqualTo(1000l));
      break;
    default:
      Assert.assertEquals(errorString, expectedField.getValue(), actualField.getValue());
  }
}
 
Example 13
Source File: MapRJsonDocumentLoader.java    From datacollector with Apache License 2.0 4 votes vote down vote up
/**
 * Map mode
 */
private void writeFieldToDocumentMap(Document doc, Field field, String name) throws IOException {
  if (field.getValue() == null) {
    // On the Map mode just set the null on the document using the name.
    doc.setNull(name);
  } else {
    switch (field.getType()) {
      case FILE_REF:
        throw new IOException("Cannot serialize FileRef fields.");
      case MAP:
      case LIST_MAP:
        Document newDoc = loader.createNewEmptyDocument();
        Map<String, Field> map = field.getValueAsMap();
        for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
          String fieldName = fieldEntry.getKey();
          Field newField = fieldEntry.getValue();
          // recursive call in map mode.
          writeFieldToDocumentMap(newDoc, newField, fieldName);
        }
        // Map the new doc
        doc.set(name, newDoc);
        break;
      case LIST:
        List<Field> listOfFields = field.getValueAsList();
        List<Object> objsList = new ArrayList<>();
        for (Field f : listOfFields) {
          // recursive call in a list mode.
          writeFieldToDocumentList(f, objsList);
        }
        doc.setArray(name, objsList.toArray());
        break;
      case BOOLEAN:
        doc.set(name, field.getValueAsBoolean());
        break;
      case CHAR:
        doc.set(name, String.valueOf(field.getValueAsChar()));
        break;
      case BYTE:
        doc.set(name, new byte[]{field.getValueAsByte()});
        break;
      case SHORT:
        doc.set(name, field.getValueAsShort());
        break;
      case INTEGER:
        doc.set(name, field.getValueAsInteger());
        break;
      case LONG:
        doc.set(name, field.getValueAsLong());
        break;
      case FLOAT:
        doc.set(name, field.getValueAsFloat());
        break;
      case DOUBLE:
        doc.set(name, field.getValueAsDouble());
        break;
      case DATE:
        doc.set(name, field.getValueAsDate().getTime());
        break;
      case DATETIME:
        doc.set(name, field.getValueAsDatetime().getTime());
        break;
      case TIME:
        doc.set(name, field.getValueAsTime().getTime());
        break;
      case DECIMAL:
        doc.set(name, field.getValueAsDecimal());
        break;
      case STRING:
      case ZONED_DATETIME:
        doc.set(name, field.getValueAsString());
        break;
      case BYTE_ARRAY:
        doc.set(name, field.getValueAsByteArray());
        break;
      default:
        throw new IllegalStateException(String.format("Unrecognized field type (%s) in field: %s",
            field.getType().name(),
            field.toString()
        ));
    }
  }
}
 
Example 14
Source File: JsonRecordWriterImpl.java    From datacollector with Apache License 2.0 4 votes vote down vote up
private void writeFieldToJsonObject(Field field) throws IOException {
  if (field == null || field.getValue() == null) {
    generator.writeNull();
    return;
  }
  switch (field.getType()) {
    case FILE_REF:
      throw new IOException("Cannot serialize FileRef fields.");
    case MAP:
    case LIST_MAP:
      generator.writeStartObject();
      Map<String, Field> map = field.getValueAsMap();
      for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
        generator.writeFieldName(fieldEntry.getKey());
        writeFieldToJsonObject(fieldEntry.getValue());
      }
      generator.writeEndObject();
      break;
    case LIST:
      generator.writeStartArray();
      List<Field> list = field.getValueAsList();
      for (Field f : list) {
        writeFieldToJsonObject(f);
      }
      generator.writeEndArray();
      break;
    case BOOLEAN:
      generator.writeBoolean(field.getValueAsBoolean());
      break;
    case CHAR:
      generator.writeString(String.valueOf(field.getValueAsChar()));
      break;
    case BYTE:
      generator.writeBinary(new byte[] {field.getValueAsByte()});
      break;
    case SHORT:
      generator.writeNumber(field.getValueAsShort());
      break;
    case INTEGER:
      generator.writeNumber(field.getValueAsInteger());
      break;
    case LONG:
      generator.writeNumber(field.getValueAsLong());
      break;
    case FLOAT:
      generator.writeNumber(field.getValueAsFloat());
      break;
    case DOUBLE:
      generator.writeNumber(field.getValueAsDouble());
      break;
    case DATE:
      generator.writeNumber(field.getValueAsDate().getTime());
      break;
    case DATETIME:
      generator.writeNumber(field.getValueAsDatetime().getTime());
      break;
    case TIME:
      generator.writeNumber(field.getValueAsTime().getTime());
      break;
    case DECIMAL:
      generator.writeNumber(field.getValueAsDecimal());
      break;
    case STRING:
      generator.writeString(field.getValueAsString());
      break;
    case BYTE_ARRAY:
      generator.writeBinary(field.getValueAsByteArray());
      break;
    case ZONED_DATETIME:
      generator.writeString(field.getValueAsString());
      break;
    default:
      throw new IllegalStateException(String.format(
          "Unrecognized field type (%s) in field: %s",
          field.getType().name(),
          field.toString())
      );
  }
}
 
Example 15
Source File: MapRJsonDocumentLoader.java    From datacollector with Apache License 2.0 4 votes vote down vote up
/**
 * List mode
 */
private void writeFieldToDocumentList(Field field, List list) throws IOException {
  if (field.getValue() == null) {
    list.add(null);
  } else {
    switch (field.getType()) {
      case FILE_REF:
        throw new IOException("Cannot serialize FileRef fields.");
      case MAP:
      case LIST_MAP:
        Document newDoc = loader.createNewEmptyDocument();
        Map<String, Field> map = field.getValueAsMap();
        for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
          String fieldName = fieldEntry.getKey();
          Field newField = fieldEntry.getValue();
          // recursive call in map mode.
          writeFieldToDocumentMap(newDoc, newField, fieldName);
        }
        // List mode
        list.add(newDoc);
        break;
      case LIST:
        List<Field> listOfFields = field.getValueAsList();
        List<Object> objsList = new ArrayList<>();
        for (Field f : listOfFields) {
          // recursive call in a list mode.
          writeFieldToDocumentList(f, objsList);
        }
        list.add(objsList);
        break;
      case BOOLEAN:
        list.add(field.getValueAsBoolean());
        break;
      case CHAR:
        list.add(String.valueOf(field.getValueAsChar()));
        break;
      case BYTE:
        list.add(new byte[]{field.getValueAsByte()});
        break;
      case SHORT:
        list.add(field.getValueAsShort());
        break;
      case INTEGER:
        list.add(field.getValueAsInteger());
        break;
      case LONG:
        list.add(field.getValueAsLong());
        break;
      case FLOAT:
        list.add(field.getValueAsFloat());
        break;
      case DOUBLE:
        list.add(field.getValueAsDouble());
        break;
      case DATE:
        list.add(field.getValueAsDate().getTime());
        break;
      case DATETIME:
        list.add(field.getValueAsDatetime().getTime());
        break;
      case TIME:
        list.add(field.getValueAsTime().getTime());
        break;
      case DECIMAL:
        list.add(field.getValueAsDecimal());
        break;
      case STRING:
      case ZONED_DATETIME:
        list.add(field.getValueAsString());
        break;
      case BYTE_ARRAY:
        list.add(field.getValueAsByteArray());
        break;
      default:
        throw new IllegalStateException(String.format(
            "Unrecognized field type (%s) in field: %s",
            field.getType().name(),
            field.toString())
        );
    }
  }
}
 
Example 16
Source File: PostgresWalRecord.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public List<Field> getChanges() {
  //Will return a 0...n length Field.LIST
  Field entry  = getEntryFromField("change");
  return entry.getValueAsList();
}
 
Example 17
Source File: ProtobufTypeUtil.java    From datacollector with Apache License 2.0 4 votes vote down vote up
private static void handleRepeatedField(
    Record record,
    Field field,
    String fieldPath,
    Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
    Map<String, Object> defaultValueMap,
    Descriptors.FieldDescriptor f,
    DynamicMessage.Builder builder
) throws DataGeneratorException {
  List<Object> toReturn = new ArrayList<>();
  List<Field> valueAsList = field.getValueAsList();
  if(valueAsList != null) {
    // According to proto 2 and 3 language guide repeated fields can have 0 elements.
    // Also null is treated as empty in case of json mappings so I guess we can ignore if it is null.
    for (int i = 0; i < valueAsList.size(); i++) {
      if (f.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) {
        // repeated field of type message
        toReturn.add(
          sdcFieldToProtobufMsg(
            record,
            valueAsList.get(i),
            fieldPath + FORWARD_SLASH + f.getName() + "[" + i + "]",
            f.getMessageType(),
            messageTypeToExtensionMap,
            defaultValueMap
          )
        );
      } else {
        // repeated field of primitive types
        toReturn.add(
          getValue(
            f,
            valueAsList.get(i),
            record,
            fieldPath + FORWARD_SLASH + f.getName(),
            messageTypeToExtensionMap,
            defaultValueMap
          )
        );
      }
    }
  }
  builder.setField(f, toReturn);
}
 
Example 18
Source File: TestExpressionProcessor.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Test
public void testEmptyMapAndEmptyList() throws StageException {
  ExpressionProcessorConfig expressionProcessorConfig1 = new ExpressionProcessorConfig();
  expressionProcessorConfig1.expression = "${emptyMap()}";
  expressionProcessorConfig1.fieldToSet = "/d";

  ExpressionProcessorConfig expressionProcessorConfig2 = new ExpressionProcessorConfig();
  expressionProcessorConfig2.expression = "${emptyList()}";
  expressionProcessorConfig2.fieldToSet = "/d/e";

  ExpressionProcessorConfig expressionProcessorConfig3 = new ExpressionProcessorConfig();
  expressionProcessorConfig3.expression = "${emptyMap()}";
  expressionProcessorConfig3.fieldToSet = "/d/e[0]";

  ExpressionProcessorConfig expressionProcessorConfig4 = new ExpressionProcessorConfig();
  expressionProcessorConfig4.expression = "${record:value('/a')}";
  expressionProcessorConfig4.fieldToSet = "/d/e[0]/f";

  ProcessorRunner runner = new ProcessorRunner.Builder(ExpressionDProcessor.class)
    .addConfiguration("expressionProcessorConfigs", ImmutableList.of(
      expressionProcessorConfig1, expressionProcessorConfig2, expressionProcessorConfig3, expressionProcessorConfig4))
    .addOutputLane("lane").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("a", Field.create("A"));
    map.put("b", Field.create("B"));
    map.put("c", Field.create("C"));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("lane").size());
    Field fieldD = output.getRecords().get("lane").get(0).get("/d");

    Assert.assertTrue(fieldD.getValue() instanceof Map);
    Map<String, Field> result = fieldD.getValueAsMap();
    Field fieldE = result.get("e");
    Assert.assertTrue(fieldE.getValue() instanceof List);
    List<Field> listField = fieldE.getValueAsList();
    Assert.assertTrue(listField.get(0).getValue() instanceof Map);
    Map<String, Field> fieldMap = listField.get(0).getValueAsMap();
    Assert.assertTrue(fieldMap.get("f").getValue() instanceof String);
    Assert.assertEquals(fieldMap.get("f").getValueAsString(), "A");
  } finally {
    runner.runDestroy();
  }
}
 
Example 19
Source File: TestFieldTypeConverterProcessorTypes.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Test
public void testListOfList() throws StageException {
  WholeTypeConverterConfig converterConfig = new WholeTypeConverterConfig();
  converterConfig.sourceType = Field.Type.STRING;
  converterConfig.targetType = Field.Type.BOOLEAN;
  converterConfig.dataLocale = "en";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
    .addConfiguration("convertBy", ConvertBy.BY_TYPE)
    .addConfiguration("wholeTypeConverterConfigs", ImmutableList.of(converterConfig))
    .addOutputLane("a").build();
  runner.runInit();

  try {
    List<Field> innerList = new LinkedList<>();
    innerList.add(Field.create("true"));
    innerList.add(Field.create("yes"));
    innerList.add(Field.create("122345566"));
    innerList.add(Field.create(Field.Type.STRING, null));
    innerList.add(Field.create(Field.Type.INTEGER, 10));
    innerList.add(Field.create(Field.Type.DECIMAL, BigDecimal.valueOf(1)));
    Field innerField = Field.create(innerList);

    List<Field> outerList = new LinkedList<>();
    outerList.add(innerField);

    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(outerList));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof List);
    List<Field> result = field.getValueAsList();
    Assert.assertTrue(result.size() == 1);

    field = result.get(0);

    Assert.assertTrue(field.getValue() instanceof List);
    result = field.getValueAsList();
    Assert.assertTrue(result.size() == 6);

    Assert.assertEquals(true, result.get(0).getValue());

    Assert.assertEquals(false, result.get(1).getValue());

    Assert.assertEquals(false, result.get(2).getValue());

    Assert.assertEquals(Field.Type.BOOLEAN, result.get(3).getType());
    Assert.assertEquals(null, result.get(3).getValue());

    Assert.assertEquals(10, result.get(4).getValue());

    Assert.assertEquals(BigDecimal.valueOf(1), result.get(5).getValue());
  } finally {
    runner.runDestroy();
  }
}
 
Example 20
Source File: HiveMetastoreUtil.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T> void extractInnerMapFromTheList(
    Record metadataRecord,
    String listFieldName,
    String innerPairFirstFieldName,
    String innerPairSecondFieldName,
    boolean isSecondFieldHiveType,
    LinkedHashMap<String, T> returnValMap,
    HiveStageCheckedException exception
) throws HiveStageCheckedException{
  boolean throwException = false;
  try {
    if (metadataRecord.has(SEP + listFieldName)) {
      Field columnField = metadataRecord.get(SEP + listFieldName);
      List<Field> columnList = columnField.getValueAsList();
      if (columnList != null) {
        for (Field listElementField : columnList) {
          if (listElementField.getType() != Field.Type.MAP && listElementField.getType() != Field.Type.LIST_MAP) {
            throwException = true;
            break;
          }
          LinkedHashMap<String, Field> innerPair = listElementField.getValueAsListMap();
          String innerPairFirstField = innerPair.get(innerPairFirstFieldName).getValueAsString();
          T retVal;
          if (isSecondFieldHiveType) {
            Field hiveTypeInfoField = innerPair.get(innerPairSecondFieldName);
            HiveType hiveType = HiveType.getHiveTypeFromString(
                hiveTypeInfoField.getValueAsMap().get(HiveMetastoreUtil.TYPE).getValueAsString()
            );
            retVal = (T) (hiveType.getSupport().generateHiveTypeInfoFromMetadataField(hiveTypeInfoField));
          } else {
            retVal = (T) innerPair.get(innerPairSecondFieldName).getValueAsString();
          }
          returnValMap.put(innerPairFirstField, retVal);
        }
      }
    } else {
      // we allow partition to be empty for non-partitioned table
      if (!listFieldName.equals(PARTITION_FIELD))
        throwException = true;
    }
  } catch(Exception e) {
    LOG.error("Can't parse metadata record", e);
    throwException = true;
    exception.initCause(e);
  }
  if (throwException) {
    throw exception;
  }
}