com.streamsets.pipeline.api.Field Java Examples
The following examples show how to use
com.streamsets.pipeline.api.Field.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDelimitedDataGenerator.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testGeneratorReplaceNewLines() throws Exception { StringWriter writer = new StringWriter(); DataGenerator gen = new DelimitedCharDataGenerator(writer, CsvMode.CSV.getFormat(), CsvHeader.NO_HEADER, "h", "d", " "); Record record = RecordCreator.create(); List<Field> list = new ArrayList<>(); Map<String,Field> map = new HashMap<>(); map.put("h", Field.create("A")); map.put("d", Field.create("a\n\r")); list.add(Field.create(map)); map.put("h", Field.create("B")); map.put("d", Field.create("b")); list.add(Field.create(map)); map.put("h", Field.create("C")); map.put("d", Field.create((String)null)); list.add(Field.create(map)); record.set(Field.create(list)); gen.write(record); gen.close(); Assert.assertEquals("\"a \",b,\r\n", writer.toString()); }
Example #2
Source File: TestDelimitedDataGenerator.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testGeneratorNoHeader() throws Exception { StringWriter writer = new StringWriter(); DataGenerator gen = new DelimitedCharDataGenerator(writer, CsvMode.CSV.getFormat(), CsvHeader.NO_HEADER, "h", "d", null); Record record = RecordCreator.create(); List<Field> list = new ArrayList<>(); Map<String,Field> map = new HashMap<>(); map.put("h", Field.create("A")); map.put("d", Field.create("a\n\r")); list.add(Field.create(map)); map.put("h", Field.create("B")); map.put("d", Field.create("b")); list.add(Field.create(map)); record.set(Field.create(list)); gen.write(record); gen.close(); Assert.assertEquals("\"a\n\r\",b\r\n", writer.toString()); }
Example #3
Source File: Matchers.java From datacollector with Apache License 2.0 | 6 votes |
@Override public boolean matches(Object item) { if (item instanceof Field) { Field field = (Field) item; if (field.getType() == type) { final Object valueFromField = getValueFromField(field); if (value == null) { return valueFromField == null; } else if (value.getClass().isArray()) { // arrays are a pain... if (byte.class.equals(value.getClass().getComponentType())) { return Arrays.equals((byte[]) value, (byte[]) valueFromField); } else { throw new IllegalStateException( "Incomparable array type: " + value.getClass().getComponentType().toString() ); } } else { return value.equals(valueFromField); } } } return false; }
Example #4
Source File: RecordConverter.java From datacollector with Apache License 2.0 | 6 votes |
private List<Record> toRecords(Table table, EventHeader eventHeader, DeleteRowsEventData eventData, SourceOffset offset) { List<Record> res = new ArrayList<>(eventData.getRows().size()); for (Serializable[] row : eventData.getRows()) { Record record = recordFactory.create(offset.format()); Map<String, Field> fields = createHeader(table, eventHeader, offset); fields.put(TYPE_FIELD, create("DELETE")); record.getHeader().setAttribute( OperationType.SDC_OPERATION_TYPE, String.valueOf(OperationType.DELETE_CODE) ); List<ColumnValue> columnValues = zipColumnsValues(eventData.getIncludedColumns(), table, row); Map<String, Field> data = toMap(columnValues); fields.put(OLD_DATA_FIELD, create(data)); record.set(create(fields)); res.add(record); } return res; }
Example #5
Source File: TestSyslogTarget.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testWriteSingleRecord() throws Exception { SyslogDTarget config = getDefaultConfig(); Target target = config.createTarget(); TargetRunner runner = new TargetRunner.Builder(SyslogDTarget.class, target) .setOnRecordError(OnRecordError.TO_ERROR) .addService(DataFormatGeneratorService.class, new SdkJsonDataFormatGeneratorService()) .build(); runner.runInit(); Record record = RecordCreator.create(); Map<String, Field> fields = new HashMap<>(); fields.put("text", Field.create("myTestMessage")); fields.put("hostname", Field.create("localhost")); fields.put("application", Field.create("myTestApp")); fields.put("facility", Field.create("1")); fields.put("severity", Field.create("1")); record.set(Field.create(fields)); runner.runWrite(Arrays.asList(record)); runner.runDestroy(); }
Example #6
Source File: StartJobSupplier.java From datacollector with Apache License 2.0 | 6 votes |
private void generateField(Map<String, Object> jobStatus) { String status = jobStatus.containsKey("status") ? (String) jobStatus.get("status") : null; String statusColor = jobStatus.containsKey("color") ? (String) jobStatus.get("color") : null; String errorMessage = jobStatus.containsKey("errorMessage") ? (String) jobStatus.get("errorMessage") : null; boolean success = ControlHubApiUtil.determineJobSuccess(status, statusColor); LinkedHashMap<String, Field> startOutput = new LinkedHashMap<>(); startOutput.put(Constants.JOB_ID_FIELD, Field.create(jobIdConfig.jobId)); startOutput.put(Constants.STARTED_SUCCESSFULLY_FIELD, Field.create(true)); if (!conf.runInBackground) { startOutput.put(Constants.FINISHED_SUCCESSFULLY_FIELD, Field.create(success)); MetricRegistryJson jobMetrics = ControlHubApiUtil.getJobMetrics( clientBuilder, conf.baseUrl, jobIdConfig.jobId, userAuthToken ); startOutput.put(Constants.JOB_METRICS_FIELD, CommonUtil.getMetricsField(jobMetrics)); } startOutput.put(Constants.JOB_STATUS_FIELD, Field.create(status)); startOutput.put(Constants.JOB_STATUS_COLOR_FIELD, Field.create(statusColor)); startOutput.put(Constants.ERROR_MESSAGE_FIELD, Field.create(errorMessage)); responseField = Field.createListMap(startOutput); }
Example #7
Source File: BasicIT.java From datacollector with Apache License 2.0 | 6 votes |
private static List<Record> createTransactionRecords(int noOfRecords) { List<Record> records = new ArrayList<>(); long currentTime = (System.currentTimeMillis() / 1000) * 1000; for (int i = 0; i < noOfRecords; i++) { Record record = RecordCreator.create(); LinkedHashMap<String, Field> fields = new LinkedHashMap<>(); fields.put("unique_int", Field.create(i + 1)); fields.put("t_date", Field.create(Field.Type.LONG, currentTime)); fields.put("random_string", Field.create(UUID.randomUUID().toString())); record.set(Field.createListMap(fields)); records.add(record); //making sure time is at least off by a second. currentTime = currentTime + 1000; } return records; }
Example #8
Source File: FieldHasherProcessor.java From datacollector with Apache License 2.0 | 6 votes |
private Set<String> validateAndExtractFieldsToHash( Record record, Set<String> fieldsDontExist, Set<String> fieldsWithListOrMapType, Set<String> fieldsWithNull, Collection<String> matchingFieldsPath ) { Set<String> validFieldsToHashForThisConfig = new HashSet<String>(); for (String matchingFieldPath : matchingFieldsPath) { if (record.has(matchingFieldPath)) { Field field = record.get(matchingFieldPath); if (UNSUPPORTED_FIELD_TYPES.contains(field.getType())) { fieldsWithListOrMapType.add(matchingFieldPath); } else if (field.getValue() == null) { fieldsWithNull.add(matchingFieldPath); } else { validFieldsToHashForThisConfig.add(matchingFieldPath); } } else { fieldsDontExist.add(matchingFieldPath); } } return validFieldsToHashForThisConfig; }
Example #9
Source File: SchedulerJob.java From datacollector with Apache License 2.0 | 6 votes |
@Override public void execute(JobExecutionContext jobExecutionContext) { try { SchedulerContext schedulerContext = jobExecutionContext.getScheduler().getContext(); PushSource.Context pushSourceContext = (PushSource.Context) schedulerContext.get( SchedulerPushSource.PUSH_SOURCE_CONTEXT ); if (pushSourceContext != null) { BatchContext batchContext = pushSourceContext.startBatch(); Record record = pushSourceContext.createRecord("cronRecord"); LinkedHashMap<String, Field> linkedHashMap = new LinkedHashMap<>(); linkedHashMap.put("timestamp", Field.createDatetime(new Date())); record.set(Field.createListMap(linkedHashMap)); batchContext.getBatchMaker().addRecord(record); pushSourceContext.processBatch(batchContext); } } catch (SchedulerException ex) { LOG.error(ex.getMessage(), ex); } }
Example #10
Source File: TestAvroTypeUtil.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testCreateListPrimitiveField() throws Exception { String schema = "{\"type\": \"array\", \"items\": \"string\"}"; Schema avroSchema = new Schema.Parser().parse(schema); Record record = RecordCreator.create(); Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, Arrays.asList("Hari", "Kiran"), false); Assert.assertEquals(Field.Type.LIST, field.getType()); List<Field> valueAsList = field.getValueAsList(); Assert.assertEquals(Field.Type.STRING, valueAsList.get(0).getType()); Assert.assertEquals("Hari", valueAsList.get(0).getValueAsString()); Assert.assertEquals(Field.Type.STRING, valueAsList.get(1).getType()); Assert.assertEquals("Kiran", valueAsList.get(1).getValueAsString()); record.set(field); Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>()); Assert.assertTrue(avroObject instanceof List<?>); List<String> listString = (List<String>) avroObject; Assert.assertEquals("Hari", listString.get(0)); Assert.assertEquals("Kiran", listString.get(1)); //Check invalid type - String to List makeBadType(Field.create("notList"), record, avroSchema); }
Example #11
Source File: TestAvroTypeUtil.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testCreateDoubleField() throws Exception { String schema = "{\"name\": \"name\", \"type\": \"double\"}"; Schema avroSchema = new Schema.Parser().parse(schema); Record record = RecordCreator.create(); Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, 345823746923.863423, false); Assert.assertEquals(Field.Type.DOUBLE, field.getType()); Assert.assertTrue(345823746923.863423 == field.getValueAsDouble()); record.set(field); Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>()); Assert.assertTrue(avroObject instanceof Double); Assert.assertTrue(345823746923.863423 == (Double) avroObject); //Check invalid type - String to Double makeBadType(Field.create("notDouble"), record, avroSchema); }
Example #12
Source File: TestWholeFileTransformerProcessor.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testNotWholeFileFormatRecord() throws StageException { Processor wholeFileTransofrmer = new TestWholeFileTransformerProcessorBuilder() .tempDir("${file:parentPath(record:attribute('file'))}/.parquet") .build(); ProcessorRunner runner = new ProcessorRunner.Builder(WholeFileTransformerDProcessor.class, wholeFileTransofrmer) .addOutputLane("a") .setOnRecordError(OnRecordError.TO_ERROR) .build(); Record record = RecordCreator.create(); record.set(Field.create("test")); try { runner.runInit(); StageRunner.Output output = runner.runProcess(Arrays.asList(record)); Assert.assertEquals(0, output.getRecords().get("a").size()); List<Record> errorRecords = runner.getErrorRecords(); Assert.assertEquals(1, errorRecords.size()); Assert.assertEquals(Errors.CONVERT_01.getCode(), errorRecords.get(0).getHeader().getErrorCode()); } finally { runner.runDestroy(); } }
Example #13
Source File: TestTimeNewEL.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testExtractStringFromDate() throws Exception { Record.Header header = Mockito.mock(Record.Header.class); Record record = Mockito.mock(Record.class); Mockito.when(record.getHeader()).thenReturn(header); long val = System.currentTimeMillis(); Date d = new Date(val); Mockito.when(record.get(Mockito.eq("/theDate"))).thenReturn(Field.createDate(d)); Mockito.when(record.get(Mockito.eq("/epochMS"))).thenReturn(Field.create(val)); String format = "G yyyy-MM-dd HH:mm:ss.SSS a zzzzzzz"; Mockito.when(record.get(Mockito.eq("/format"))).thenReturn(Field.create(format)); RecordEL.setRecordInContext(variables, record); SimpleDateFormat sdf = new SimpleDateFormat(format); String ans = sdf.format(d); Assert.assertEquals(ans, eval.eval(variables, "${time:extractStringFromDate(time:millisecondsToDateTime(record:value('/epochMS')), record:value('/format'))}", String.class)); Assert.assertEquals(ans, eval.eval(variables, "${time:extractStringFromDate(record:value('/theDate'), record:value('/format'))}", String.class)); }
Example #14
Source File: Matchers.java From datacollector with Apache License 2.0 | 6 votes |
@Override public boolean matches(Object item) { if (item instanceof Field) { Field field = (Field) item; if (field.getType().isOneOf(Field.Type.MAP, Field.Type.LIST_MAP)) { final Map<String, Field> childFields = field.getValueAsMap(); if (!childFields.containsKey(nestedFieldName)) { return false; } final VT value = valueAccessor.getValue(childFields.get(nestedFieldName)); if (expectedValue == null) { return value == null; } else { return expectedValue.equals(value); } } } return false; }
Example #15
Source File: JdbcUtil.java From datacollector with Apache License 2.0 | 6 votes |
public LinkedHashMap<String, Field> resultSetToFields( ResultSet rs, CommonSourceConfigBean commonSourceBean, ErrorRecordHandler errorRecordHandler, UnknownTypeAction unknownTypeAction, Set<String> recordHeader, DatabaseVendor vendor ) throws SQLException, StageException { return resultSetToFields( rs, commonSourceBean.maxClobSize, commonSourceBean.maxBlobSize, Collections.emptyMap(), errorRecordHandler, unknownTypeAction, recordHeader, commonSourceBean.convertTimestampToString, vendor ); }
Example #16
Source File: TestJdbcMetadata.java From datacollector with Apache License 2.0 | 6 votes |
@Test public void testInitialRecordStructure() throws Exception { JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString); ProcessorRunner processorRunner = getProcessorRunner(processor); processorRunner.runInit(); List<Record> singleRecord = ImmutableList.of(makeRecord(fieldMap1)); StageRunner.Output output = processorRunner.runProcess(singleRecord); Assert.assertEquals(1, output.getRecords().get("lane").size()); Record record = output.getRecords().get("lane").get(0); for (Map.Entry<String, Field>entry : record.get().getValueAsListMap().entrySet()) { Assert.assertEquals(fieldMap1.get(entry.getKey()).getRight(), entry.getValue().getValue()); } try (ResultSet metaDataColumns = jdbcUtil.getColumnMetadata(connection, null, tableName)) { while (metaDataColumns.next()) { Pair<Field.Type, Object> typeAndValue = fieldMap1.get(metaDataColumns.getString(JdbcSchemaReader.COLUMN_NAME).toLowerCase()); Assert.assertEquals(typeAndValue.getLeft(), typeMap.get(metaDataColumns.getInt(JdbcSchemaReader.DATA_TYPE))); } } }
Example #17
Source File: TestSdcRecordDataGenerator.java From datacollector with Apache License 2.0 | 5 votes |
@Test(expected = IOException.class) public void testWriteAfterClose() throws Exception { ByteArrayOutputStream writer = new ByteArrayOutputStream(); DataGenerator gen = new SdcRecordDataGenerator( getContextExtensions().createRecordWriter(writer), getContextExtensions() ); Record record = RecordCreator.create(); record.set(Field.create("Hello")); gen.close(); gen.write(record); }
Example #18
Source File: TestFieldTypeConverterProcessorFields.java From datacollector with Apache License 2.0 | 5 votes |
@Test public void testStringToByte() throws StageException { FieldTypeConverterConfig fieldTypeConverterConfig = new FieldTypeConverterConfig(); fieldTypeConverterConfig.fields = ImmutableList.of("/beginner", "/intermediate", "/null"); fieldTypeConverterConfig.targetType = Field.Type.BYTE; fieldTypeConverterConfig.dataLocale = "en"; ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class) .addConfiguration("convertBy", ConvertBy.BY_FIELD) .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(fieldTypeConverterConfig)) .addOutputLane("a").build(); runner.runInit(); try { Map<String, Field> map = new LinkedHashMap<>(); map.put("beginner", Field.create("1")); map.put("intermediate", Field.create("126")); map.put("null", Field.create(Field.Type.STRING, null)); Record record = RecordCreator.create("s", "s:1"); record.set(Field.create(map)); StageRunner.Output output = runner.runProcess(ImmutableList.of(record)); Assert.assertEquals(1, output.getRecords().get("a").size()); Field field = output.getRecords().get("a").get(0).get(); Assert.assertTrue(field.getValue() instanceof Map); Map<String, Field> result = field.getValueAsMap(); Assert.assertTrue(result.size() == 3); Assert.assertTrue(result.containsKey("beginner")); Assert.assertEquals((byte)1, result.get("beginner").getValue()); Assert.assertTrue(result.containsKey("intermediate")); Assert.assertEquals((byte)126, result.get("intermediate").getValue()); Assert.assertTrue(result.containsKey("null")); Assert.assertEquals(null, result.get("null").getValue()); } finally { runner.runDestroy(); } }
Example #19
Source File: DriftRuleEL.java From datacollector with Apache License 2.0 | 5 votes |
@Override protected Set<String> getValue(Field field) { Set<String> names; switch (field.getType()) { case MAP: case LIST_MAP: names = new HashSet<>(field.getValueAsMap().keySet()); break; default: names = null; break; } return names; }
Example #20
Source File: Matchers.java From datacollector with Apache License 2.0 | 5 votes |
public static Matcher<Field> fieldWithValue(final double value) { return new FieldMatcher(Field.Type.DOUBLE, value) { @Override protected Object getValueFromField(Field field) { return field.getValueAsDouble(); } }; }
Example #21
Source File: MapRDBCDCSource.java From datacollector with Apache License 2.0 | 5 votes |
private void setId(Record record, Value id) throws StageException { if(id.getType() == Value.Type.STRING) { record.set("/_id", Field.create(id.getString())); } else if(id.getType() == Value.Type.BINARY) { record.set("/_id", Field.create(id.getBinary().array())); } else { throw new OnRecordErrorException(record, MaprDBCDCErrors.MAPRDB_04, id.getType().name()); } }
Example #22
Source File: TestXMLFlatteningProcessor.java From datacollector with Apache License 2.0 | 5 votes |
@Test public void testNonMap() throws Exception { Record expected1 = RecordCreator.create(); expected1.set(Field.create(new ArrayList<Field>())); processor = new XMLFlatteningProcessor(ORIGINAL, true, false, "", "", "", "", false, false); ProcessorRunner runner = new ProcessorRunner.Builder(dProcessorClass, processor) .addOutputLane("xml").setOnRecordError(OnRecordError.STOP_PIPELINE).build(); try { runner.runInit(); runner.runProcess(ImmutableList.of(expected1)); Assert.fail(); } catch (StageException ex) { Assert.assertEquals(CMN_0100, ex.getErrorCode()); } }
Example #23
Source File: StreamingXmlParser.java From datacollector with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private void addContent(Map<String, Object> contents, String name, Field field) throws XMLStreamException, ObjectLengthException { throwIfOverMaxObjectLength(); List<Field> list = (List<Field>) contents.get(name); if (list == null) { list = new ArrayList<>(); contents.put(name, list); } list.add(field); }
Example #24
Source File: TestExpressionProcessor.java From datacollector with Apache License 2.0 | 5 votes |
@Test public void testIsEmptyMapNullable() throws Exception { ExpressionProcessorConfig expressionProcessorConfig; ProcessorRunner runner; Map<String, Field> map = new LinkedHashMap<>(); map.put("a", Field.create(123)); Record record = RecordCreator.create("s", "s:1"); record.set(Field.create(map)); expressionProcessorConfig = new ExpressionProcessorConfig(); expressionProcessorConfig.expression = "${isEmptyMap(NULL)}"; expressionProcessorConfig.fieldToSet = "/setme"; runner = new ProcessorRunner.Builder(ExpressionDProcessor.class) .addConfiguration("expressionProcessorConfigs", ImmutableList.of(expressionProcessorConfig)) .addOutputLane("a").setOnRecordError(OnRecordError.TO_ERROR).build(); runner.runInit(); try { runner.runProcess(ImmutableList.of(record)); Assert.assertEquals(0, runner.getErrorRecords().size()); } finally { runner.runDestroy(); } }
Example #25
Source File: CollectdParser.java From datacollector with Apache License 2.0 | 5 votes |
private void startNewRecord() { Record record = context.createRecord(fields.get(PART_TYPES.get(HOST)).getValueAsString() + "::" + recordId++); record.set(Field.create(fields)); records.add(record); fields = new HashMap<>(fields); }
Example #26
Source File: TestAggregationEL.java From datacollector with Apache License 2.0 | 5 votes |
@Test public void testMin() throws Exception { final ELEvaluator eval = new ELEvaluator("testMin", elDefinitionExtractor, AggregationEL.class); final ELVariables variables = makeTestVariables(); final Field result1 = eval.eval(variables, "${min(intFields)}", Field.class); assertThat(result1, Matchers.fieldWithValue(1l)); final Field result2 = eval.eval(variables, "${min(doubleFields)}", Field.class); assertThat(result2, Matchers.fieldWithValue(0.5d)); final Field result3 = eval.eval(variables, "${min(decimalFields)}", Field.class); assertThat(result3, Matchers.fieldWithValue(new BigDecimal("-8.05"))); }
Example #27
Source File: TestFieldFlattenerEntireRecordProcessor.java From datacollector with Apache License 2.0 | 5 votes |
@Test public void testFlattenEntireRecordDifferentSeparator() throws Exception { FieldFlattenerConfig config = new FieldFlattenerConfig(); config.nameSeparator = "/"; config.flattenType = FlattenType.ENTIRE_RECORD; ProcessorRunner runner = new ProcessorRunner.Builder(FieldFlattenerDProcessor.class, new FieldFlattenerProcessor(config)) .addOutputLane("a").build(); runner.runInit(); Record record = RecordCreator.create(); record.set(Field.create(Field.Type.MAP, ImmutableMap.builder() .put("top-level", Field.create(Field.Type.STRING, "top-level")) .put("map", Field.create(Field.Type.MAP, ImmutableMap.builder() .put("map1", Field.create(Field.Type.STRING, "map1")) .put("map2", Field.create(Field.Type.STRING, "map2")) .build())) .put("list", Field.create(Field.Type.LIST, ImmutableList.of( Field.create(Field.Type.STRING, "list1"), Field.create(Field.Type.STRING, "list2")))) .build() )); StageRunner.Output output = runner.runProcess(ImmutableList.of(record)); assertEquals(1, output.getRecords().get("a").size()); Field field = output.getRecords().get("a").get(0).get(); assertEquals(Field.Type.MAP, field.getType()); Map<String, Field> result = field.getValueAsMap(); assertEquals(5, result.size()); assertTrue(result.containsKey("top-level")); assertTrue(result.containsKey("map/map1")); assertTrue(result.containsKey("map/map2")); assertTrue(result.containsKey("list/0")); assertTrue(result.containsKey("list/1")); runner.runDestroy(); }
Example #28
Source File: StartPipelineSource.java From datacollector with Apache License 2.0 | 5 votes |
@Override public String produce(String s, int i, BatchMaker batchMaker) throws StageException { List<CompletableFuture<Field>> startPipelineFutures = new ArrayList<>(); Executor executor = Executors.newCachedThreadPool(); for (PipelineIdConfig pipelineIdConfig: conf.pipelineIdConfigList) { StartPipelineSupplier startPipelineSupplier = startPipelineCommon.getStartPipelineSupplier( pipelineIdConfig, null ); CompletableFuture<Field> future = CompletableFuture.supplyAsync(startPipelineSupplier, executor); startPipelineFutures.add(future); } try { LinkedHashMap<String, Field> outputField = startPipelineCommon.startPipelineInParallel(startPipelineFutures); Record outputRecord = CommonUtil.createOrchestratorTaskRecord( null, getContext(), conf.taskName, outputField ); batchMaker.addRecord(outputRecord); } catch (Exception ex) { LOG.error(ex.toString(), ex); errorRecordHandler.onError(StartPipelineErrors.START_PIPELINE_04, ex.toString(), ex); } return null; }
Example #29
Source File: TestFieldDeepCopy.java From datacollector-api with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("unchecked") public void testClone() { ListTypeSupport ts = new ListTypeSupport(); List<Field> list = new ArrayList<>(); list.add(Field.create("a")); List<Field> cList = (List) ts.clone(list); Assert.assertEquals(list, cList); Assert.assertNotSame(list, cList); Assert.assertNotSame(list.get(0), cList.get(0)); List<Field> listX = new ArrayList<>(); listX.add(Field.create(list)); Map<String, Field> map = new HashMap<>(); map.put("list", Field.create(listX)); MapTypeSupport ms = new MapTypeSupport(); Map<String, Field> cMap = (Map) ms.clone(map); Assert.assertEquals(map, cMap); Assert.assertNotSame(map, cMap); Assert.assertEquals(map.get("list"), cMap.get("list")); Assert.assertNotSame(map.get("list"), cMap.get("list")); Assert.assertEquals(map.get("list").getValueAsList().get(0), cMap.get("list").getValueAsList().get(0)); Assert.assertNotSame(map.get("list").getValueAsList().get(0), cMap.get("list").getValueAsList().get(0)); LinkedHashMap<String, Field> listMap = new LinkedHashMap<>(); listMap.put("list", Field.create(listX)); ListMapTypeSupport listMapTypeSupport = new ListMapTypeSupport(); LinkedHashMap<String, Field> listMapClone = (LinkedHashMap) listMapTypeSupport.clone(listMap); Assert.assertEquals(listMap, listMapClone); Assert.assertNotSame(listMap, listMapClone); Assert.assertEquals(map.get("list"), listMapClone.get("list")); Assert.assertNotSame(map.get("list"), listMapClone.get("list")); Assert.assertEquals(map.get("list").getValueAsList().get(0), listMapClone.get("list").getValueAsList().get(0)); Assert.assertNotSame(map.get("list").getValueAsList().get(0), listMapClone.get("list").getValueAsList().get(0)); }
Example #30
Source File: TestFieldValueReplacer.java From datacollector with Apache License 2.0 | 5 votes |
@Test public void testReplaceNullLongFields() throws StageException { FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig(); stringFieldReplacement.fields = ImmutableList.of("/longField"); stringFieldReplacement.newValue = "523456789345"; ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class) .addConfiguration("nullReplacerConditionalConfigs", null) .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement)) .addConfiguration("fieldsToConditionallyReplace", null) .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE) .addOutputLane("a").build(); runner.runInit(); try { Map<String, Field> map = new LinkedHashMap<>(); map.put("longField", Field.create(Field.Type.LONG, null)); Record record = RecordCreator.create("s", "s:1"); record.set(Field.create(map)); StageRunner.Output output = runner.runProcess(ImmutableList.of(record)); Assert.assertEquals(1, output.getRecords().get("a").size()); Field field = output.getRecords().get("a").get(0).get(); Assert.assertTrue(field.getValue() instanceof Map); Map<String, Field> result = field.getValueAsMap(); Assert.assertTrue(result.size() == 1); Assert.assertTrue(result.containsKey("longField")); Assert.assertEquals(523456789345L, result.get("longField").getValue()); } finally { runner.runDestroy(); } }