Java Code Examples for com.streamsets.pipeline.api.Record

The following examples show how to use com.streamsets.pipeline.api.Record. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: datacollector   Source File: RabbitTarget.java    License: Apache License 2.0 6 votes vote down vote up
private void writeRecord(DataGenerator generator, Record record) throws StageException {
  try {
    generator.write(record);
  } catch (IOException e) {
    //Record Error
    LOG.error("Record Write error", e);
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.RABBITMQ_07,
            e.toString(),
            e
        )
    );
  }
}
 
Example 2
Source Project: datacollector   Source File: FieldMapperProcessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  RecordEL.setRecordInContext(expressionVars, record);
  TimeNowEL.setTimeNowInContext(expressionVars, new Date());

  switch (fieldMapperConfig.operateOn) {
    case FIELD_PATHS:
      transformFieldPaths(record);
      break;
    case FIELD_VALUES:
      transformFieldValues(record);
      break;
    case FIELD_NAMES:
      transformFieldNames(record);
      break;
    default:
      throw new IllegalStateException(String.format(
          "Unrecognized operateOn value of %s",
          fieldMapperConfig.operateOn
      ));
  }
  batchMaker.addRecord(record);
}
 
Example 3
Source Project: datacollector   Source File: BatchMakerImpl.java    License: Apache License 2.0 6 votes vote down vote up
public BatchMakerImpl(StagePipe stagePipe, boolean keepSnapshot, int recordAllowance) {
  this.stagePipe = stagePipe;
  this.instanceName= stagePipe.getStage().getInfo().getInstanceName();
  outputLanes = ImmutableList.copyOf(stagePipe.getStage().getConfiguration().getOutputLanes());
  singleOutputLane = (outputLanes.size() == 1) ? outputLanes.iterator().next() : null;
  stageOutput = new HashMap<>();
  stageOutputSnapshot = (keepSnapshot) ? new HashMap<String, List<Record>>() : null;
  for (String outputLane : outputLanes) {
    stageOutput.put(outputLane, new ArrayList<Record>());
    if (stageOutputSnapshot != null) {
      stageOutputSnapshot.put(outputLane, new ArrayList<Record>());
    }
  }
  this.recordAllowance = recordAllowance;
  // if the stage is annotated as recordsByRef it means it does not reuse the records it creates, thus
  // we can skip one copy here (just here though), except if we are in preview
  recordByRef = !stagePipe.getStage().getContext().isPreview() &&
                stagePipe.getStage().getDefinition().getRecordsByRef();
}
 
Example 4
Source Project: datacollector   Source File: HBaseLookupProcessor.java    License: Apache License 2.0 6 votes vote down vote up
private Pair<String, HBaseColumn> getKey(Record record, HBaseLookupParameterConfig config) throws ELEvalException {
  if (config.rowExpr.isEmpty()) {
    throw new IllegalArgumentException(Utils.format("Empty lookup Key Expression"));
  }

  ELVars elVars = getContext().createELVars();
  RecordEL.setRecordInContext(elVars, record);

  String rowKey = keyExprEval.eval(elVars, config.rowExpr, String.class);
  String column = columnExprEval.eval(elVars, config.columnExpr, String.class);
  Date timestamp = timestampExprEval.eval(elVars, config.timestampExpr, Date.class);
  HBaseColumn hBaseColumn = hbaseConnectionHelper.getColumn(column);
  if(timestamp != null) {
    hBaseColumn.setTimestamp(timestamp.getTime());
  }
  return Pair.of(rowKey, hBaseColumn);
}
 
Example 5
Source Project: datacollector   Source File: BasicIT.java    License: Apache License 2.0 6 votes vote down vote up
private static List<Record> createTransactionRecords(int noOfRecords) {
  List<Record> records = new ArrayList<>();
  long currentTime = (System.currentTimeMillis() / 1000) * 1000;
  for (int i = 0; i < noOfRecords; i++) {
    Record record = RecordCreator.create();
    LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
    fields.put("unique_int", Field.create(i + 1));
    fields.put("t_date", Field.create(Field.Type.LONG, currentTime));
    fields.put("random_string", Field.create(UUID.randomUUID().toString()));
    record.set(Field.createListMap(fields));
    records.add(record);
    //making sure time is at least off by a second.
    currentTime = currentTime + 1000;
  }
  return records;
}
 
Example 6
Source Project: datacollector   Source File: TestDelimitedCharDataParser.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testParserNull() throws Exception {
  OverrunReader reader = new OverrunReader(new StringReader("A,B\r\nnull,b"), 1000, true, false);
  DelimitedDataParserSettings settings = DelimitedDataParserSettings.builder()
    .withSkipStartLines(0)
    .withFormat(CSVFormat.DEFAULT)
    .withHeader(CsvHeader.WITH_HEADER)
    .withMaxObjectLen(-1)
    .withRecordType(CsvRecordType.LIST_MAP)
    .withParseNull(true)
    .withNullConstant("null")
    .build();
  DataParser parser = new DelimitedCharDataParser(getContext(), "id", reader, 0, settings);

  Record record = parser.parse();
  Assert.assertNotNull(record);
  Assert.assertTrue(record.has("/A"));
  Assert.assertTrue(record.has("/B"));
  Assert.assertEquals(null, record.get("/A").getValueAsString());
  Assert.assertEquals("b", record.get("/B").getValueAsString());

  record = parser.parse();
  Assert.assertNull(record);
  Assert.assertEquals("-1", parser.getOffset());
  parser.close();
}
 
Example 7
Source Project: datacollector   Source File: RecordSampler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean sample(Record record) {
  boolean isSampled = false;
  if (isOrigin) {
    String sampled = record.getHeader().getAttribute(SdcRecordConstants.SDC_SAMPLED_RECORD);
    isSampled = (null != sampled && SdcRecordConstants.TRUE.equals(sampled));
    if (isSampled) {
      updateTimer(record);
    }
  } else if (sampleSize > 0){
    if (sampleSet.remove(recordCounter)) {
      updateRecordHeader(record);
      isSampled = true;
    }
    recordCounter++;
    if (sampleSet.isEmpty()) {
      recordCounter = 0;
      chooseSampleFromPopulation();
    }
  }
  return isSampled;
}
 
Example 8
Source Project: datacollector   Source File: AbstractMysqlSource.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void shouldIncludeAndIgnoreTables() throws Exception {
  MysqlSourceConfig config = createConfig("root");
  MysqlSource source = createMysqlSource(config);
  config.includeTables = "test.foo,t%.foo2";
  config.ignoreTables = "test.foo";
  runner = new SourceRunner.Builder(MysqlDSource.class, source)
      .addOutputLane(LANE)
      .build();
  runner.runInit();

  final String lastSourceOffset = null;
  StageRunner.Output output = runner.runProduce(lastSourceOffset, MAX_BATCH_SIZE);
  assertThat(output.getRecords().get(LANE), is(IsEmptyCollection.<Record>empty()));

  execute(ds, "INSERT INTO foo (bar) VALUES (1)");
  execute(ds, "INSERT INTO foo2 VALUES (1, 2, 3)");

  output = runner.runProduce(output.getNewOffset(), MAX_BATCH_SIZE);
  List<Record> records = output.getRecords().get(LANE);
  assertThat(records, hasSize(1));
  assertThat(records.get(0).get("/Table").getValueAsString(), is("foo2"));
  execute(ds, "TRUNCATE foo");
  execute(ds, "TRUNCATE foo2");
}
 
Example 9
Source Project: datacollector   Source File: TestJsonRecordWriterImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleObjects() throws Exception {
  StringWriter writer = new StringWriter();

  JsonRecordWriterImpl jsonRecordWriter = new JsonRecordWriterImpl(writer, Mode.MULTIPLE_OBJECTS);

  Record record = new RecordImpl("stage", "id", null, null);
  record.set(Field.create(10));

  jsonRecordWriter.write(record);
  jsonRecordWriter.write(record);
  jsonRecordWriter.write(record);
  jsonRecordWriter.close();

  Assert.assertEquals("10\n10\n10", writer.toString());
}
 
Example 10
Source Project: datacollector   Source File: TestRestServicePushSource.java    License: Apache License 2.0 6 votes vote down vote up
private void testEmptyPayloadRequest(String method, String httpServerUrl, List<Record> requestRecords) {
  Response response = ClientBuilder.newClient()
      .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true)
      .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true)
      .target(httpServerUrl)
      .request()
      .header(Constants.X_SDC_APPLICATION_ID_HEADER, "id")
      .method(method);

  Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getStatus());
  String responseBody = response.readEntity(String.class);

  Assert.assertEquals(1, requestRecords.size());
  Record.Header emptyPayloadRecordHeader = requestRecords.get(0).getHeader();
  Assert.assertEquals(
      "true",
      emptyPayloadRecordHeader.getAttribute(RestServiceReceiver.EMPTY_PAYLOAD_RECORD_HEADER_ATTR_NAME)
  );
  Assert.assertEquals(method, emptyPayloadRecordHeader.getAttribute(RestServiceReceiver.METHOD_HEADER));


  // check custom HTTP Response header
  Assert.assertNotNull(response.getHeaders().getFirst("test"));
  Assert.assertEquals("value", response.getHeaders().getFirst("test"));
}
 
Example 11
Source Project: datacollector   Source File: MapReduceExecutorIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleJobExecutionThatFails() throws Exception{
  MapReduceExecutor executor = generateExecutor(ImmutableMap.<String, String>builder()
    .put("mapreduce.job.inputformat.class", SimpleTestInputFormat.class.getCanonicalName())
    .put("mapreduce.output.fileoutputformat.outputdir", getOutputDir())
    .put(SimpleTestInputFormat.THROW_EXCEPTION, "true")
  .build());

  ExecutorRunner runner = new ExecutorRunner.Builder(MapReduceDExecutor.class, executor)
    .setOnRecordError(OnRecordError.TO_ERROR)
    .build();
  runner.runInit();

  Record record = RecordCreator.create();
  record.set(Field.create(ImmutableMap.of(
    "key", Field.create("value")
  )));

  runner.runWrite(ImmutableList.of(record));

  Assert.assertEquals(1, runner.getErrorRecords().size());

  runner.runDestroy();
}
 
Example 12
Source Project: datacollector   Source File: PushCoapReceiver.java    License: Apache License 2.0 6 votes vote down vote up
private boolean process(DataParser parser) throws IOException, DataParserException {
  BatchContext batchContext = getContext().startBatch();
  List<Record> records = new ArrayList<>();
  Record parsedRecord = parser.parse();
  while (parsedRecord != null) {
    records.add(parsedRecord);
    parsedRecord = parser.parse();
  }

  // dispatch records to batch
  for (Record record : records) {
    batchContext.getBatchMaker().addRecord(record);
  }

  return getContext().processBatch(batchContext);
}
 
Example 13
Source Project: datacollector   Source File: ScriptingProcessorTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static <C extends Processor> void verifyTypedFieldWithNullValue(
    Class<C> clazz,
    Processor processor,
    Record record
) throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(clazz, processor)
      .addOutputLane("lane")
      .build();

  runner.runInit();
  StageRunner.Output output;
  try{
    output = runner.runProcess(Collections.singletonList(record));
  } finally {
    runner.runDestroy();
  }
  Record outRec = output.getRecords().get("lane").get(0);
  assertEquals(record.get().getValueAsMap().size(), outRec.get().getValueAsMap().size());
  Map<String, Field> outMap = outRec.get().getValueAsMap();
  for(Map.Entry<String, Field> entry : outMap.entrySet()) {
    assertFieldUtil(entry.getKey(), entry.getValue(), null);
  }
}
 
Example 14
Source Project: datacollector   Source File: TestJdbcMetadata.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMissingScale() throws Exception {
  JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString);

  ProcessorRunner processorRunner = getProcessorRunner(processor);

  processorRunner.runInit();

  Record record = makeRecord(fieldMap2);
  record.get("/column5").deleteAttribute("scale");
  List<Record> recordList = ImmutableList.of(record);

  StageRunner.Output output = processorRunner.runProcess(recordList);
  Assert.assertEquals(0, output.getRecords().get("lane").size());

  Assert.assertEquals(1, processorRunner.getErrorRecords().size());
  Record errorRecord = processorRunner.getErrorRecords().get(0);
  Assert.assertEquals(JdbcErrors.JDBC_304.name(), errorRecord.getHeader().getErrorCode());
}
 
Example 15
Source Project: datacollector   Source File: HiveQueryExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void write(Batch batch) throws StageException {
  Iterator<Record> it = batch.getRecords();
  ELVars variables = getContext().createELVars();
  while(it.hasNext()) {
    Record record = it.next();
    RecordEL.setRecordInContext(variables, record);
    try {
      List<String> queriesToExecute = getEvaluatedQueriesForTheRecord(record, variables);
      executeQueries(record, queriesToExecute);
    } catch (OnRecordErrorException e) {
      LOG.error("Error when processing record: {}", e.toString(), e);
      errorRecordHandler.onError(e);
    }
  }
}
 
Example 16
Source Project: datacollector   Source File: ProtobufTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void checkRecordForUnknownFields(Record record, int i) throws IOException {
  // unknown fields are expected in paths for person and employee
  String attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/");
  UnknownFieldSet.Builder builder = UnknownFieldSet.newBuilder();
  builder.mergeDelimitedFrom(new ByteArrayInputStream(org.apache.commons.codec.binary.Base64.decodeBase64(attribute.getBytes())));
  UnknownFieldSet unknownFieldSet = builder.build();

  UnknownFieldsUtil.checkEmployeeUnknownFields(unknownFieldSet);

  if(i%2 == 0) {
    attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/engineer/person");
  } else {
    attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/exec/person");
  }
  builder = UnknownFieldSet.newBuilder();
  builder.mergeDelimitedFrom(new ByteArrayInputStream(org.apache.commons.codec.binary.Base64.decodeBase64(attribute.getBytes())));
  unknownFieldSet = builder.build();

  UnknownFieldsUtil.checkPersonUnknownFields(unknownFieldSet);
}
 
Example 17
Source Project: datacollector   Source File: GenericRecordConverter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<Point> getPoints(Record record) throws OnRecordErrorException {
  List<Point> points = new ArrayList<>();

  verifyRequireFieldsPresent(record);

  final String measurementName = record.get(conf.measurementField).getValueAsString();

  for (String fieldPath : conf.valueFields) {
    if (!record.has(fieldPath)) {
      continue;
    }
    Point.Builder point = Point
        .measurement(measurementName)
        .tag(RecordConverterUtil.getTags(conf.tagFields, record))
        .field(CollectdRecordConverter.stripPathPrefix(fieldPath), record.get(fieldPath).getValue());

    if (!conf.timeField.isEmpty()) {
      point.time(getTime(record), conf.timeUnit);
    }

    points.add(point.build());
  }
  return points;
}
 
Example 18
Source Project: datacollector   Source File: TestAvroTypeUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateEnumField() throws Exception {
  String schema = "{ \"type\": \"enum\",\n" +
    "  \"name\": \"Suit\",\n" +
    "  \"symbols\" : [\"SPADES\", \"HEARTS\", \"DIAMONDS\", \"CLUBS\"]\n" +
    "}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  GenericData.EnumSymbol enumSymbol = new GenericData.EnumSymbol(avroSchema, "CLUBS");
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, enumSymbol, false);

  Assert.assertEquals(Field.Type.STRING, field.getType());
  Assert.assertEquals("CLUBS", field.getValueAsString());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof GenericData.EnumSymbol);
  Assert.assertEquals("CLUBS", avroObject.toString());
}
 
Example 19
Source Project: datacollector   Source File: ProductionPipelineRunner.java    License: Apache License 2.0 6 votes vote down vote up
private void retainErrorRecordsInMemory(Map<String, List<Record>> errorRecords) {
  // Shortcut to avoid synchronization
  if(errorRecords.isEmpty()) {
    return;
  }

  synchronized (stageToErrorRecordsMap) {
    for (Map.Entry<String, List<Record>> e : errorRecords.entrySet()) {
      EvictingQueue<Record> errorRecordList = stageToErrorRecordsMap.computeIfAbsent(e.getKey(),
          k -> EvictingQueue.create(configuration.get(Constants.MAX_ERROR_RECORDS_PER_STAGE_KEY,
              Constants.MAX_ERROR_RECORDS_PER_STAGE_DEFAULT
          ))
      );
      // replace with a data structure with an upper cap
      errorRecordList.addAll(errorRecords.get(e.getKey()));
    }
  }
}
 
Example 20
Source Project: datacollector   Source File: ScriptingProcessorTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static <C extends Processor> void verifyStateObjectJavaScript(Class<C> clazz, Processor processor)
    throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(clazz, processor)
      .addOutputLane("lane")
      .build();
  runner.runInit();
  try {
    Record record = RecordCreator.create();
    Map<String, Field> map = new HashMap<>();
    map.put("count", Field.create(0));
    record.set(Field.create(map));
    List<Record> input = Collections.singletonList(record);
    runner.runProcess(input);
    StageRunner.Output output = runner.runProcess(input);
    assertEquals(1, output.getRecords().get("lane").size());
    // JavaScript only has a single number type, which is a double.
    assertEquals(2.0d, output.getRecords().get("lane").get(0).get("/count").getValue());
  } finally {
    runner.runDestroy();
  }
}
 
Example 21
Source Project: datacollector   Source File: KinesisTarget.java    License: Apache License 2.0 6 votes vote down vote up
private Record generateDestResponseRecord(UserRecordResult result) {
  Record record = getContext().createRecord("responseRecord");
  List<Field> attemptsVal = new ArrayList<>();
  for(Attempt attempt: result.getAttempts()) {
    LinkedHashMap<String, Field> attemptVal = new LinkedHashMap<>();
    attemptVal.put("delay", Field.create(attempt.getDelay()));
    attemptVal.put("duration", Field.create(attempt.getDuration()));
    attemptVal.put("errorMessage", Field.create(attempt.getErrorMessage()));
    attemptVal.put("errorCode", Field.create(attempt.getErrorCode()));
    attemptVal.put("success", Field.create(attempt.isSuccessful()));
    attemptsVal.add(Field.createListMap(attemptVal));
  }
  LinkedHashMap<String, Field> resultVal = new LinkedHashMap<>();
  resultVal.put("sequenceNumber", Field.create(result.getSequenceNumber()));
  resultVal.put("shardId", Field.create(result.getShardId()));
  resultVal.put("successful", Field.create(result.isSuccessful()));
  resultVal.put("attempts", Field.create(attemptsVal));
  record.set(Field.createListMap(resultVal));
  return record;
}
 
Example 22
Source Project: datacollector   Source File: HttpClientCommon.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Evaluates any EL expressions in the headers section of the stage configuration.
 *
 * @param record current record in context for EL evaluation
 * @return Map of headers that can be added to the Jersey Client request
 * @throws StageException if an expression could not be evaluated
 */
public MultivaluedMap<String, Object> resolveHeaders(
    Map<String, String> headers,
    Record record
) throws StageException {
  RecordEL.setRecordInContext(headerVars, record);

  MultivaluedMap<String, Object> requestHeaders = new MultivaluedHashMap<>();
  for (Map.Entry<String, String> entry : headers.entrySet()) {
    List<Object> header = new ArrayList<>(1);
    Object resolvedValue = headerEval.eval(headerVars, entry.getValue(), String.class);
    header.add(resolvedValue);
    requestHeaders.put(entry.getKey(), header);
  }

  return requestHeaders;
}
 
Example 23
Source Project: datacollector   Source File: TestAvroSchemaGenerator.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGenerateDateTime() throws OnRecordErrorException {
  Record record = RecordCreator.create();
  record.set(Field.create(Field.Type.LIST_MAP, ImmutableMap.of(
    "datetime", Field.create(Field.Type.DATETIME, new Date())
  )));

  generateAndValidateSchema(
    record,
    "{\"name\":\"datetime\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}}"
  );
}
 
Example 24
@Test
public void testGetOperationFromRecordWithOperationCodeInHeader() throws Exception {
  Record record = RecordCreator.create();
  Map<String, Field> insert = new HashMap<>();
  insert.put("P_ID", Field.create(300));
  record.set(Field.create(insert));
  record.getHeader().setAttribute(OperationType.SDC_OPERATION_TYPE, String.valueOf(OperationType.INSERT_CODE));

  List< OnRecordErrorException > errorRecords = new ArrayList<>();

  JdbcRecordReader msReader = new JdbcOracleCDCRecordReader();
  assertEquals(
      OperationType.INSERT_CODE,
      msReader.getOperationFromRecord(record, JDBCOperationType.DELETE, UnsupportedOperationAction.DISCARD, errorRecords)
  );

  Record record2 = RecordCreator.create();
  record2.set(Field.create(insert));
  record2.getHeader().setAttribute(OperationType.SDC_OPERATION_TYPE, String.valueOf(OperationType.UPDATE_CODE));
  assertEquals(
      OperationType.UPDATE_CODE,
      msReader.getOperationFromRecord(record2, JDBCOperationType.DELETE, UnsupportedOperationAction.DISCARD, errorRecords)
  );

  Record record3 = RecordCreator.create();
  record3.set(Field.create(insert));
  record3.getHeader().setAttribute(OperationType.SDC_OPERATION_TYPE, String.valueOf(OperationType.DELETE_CODE));
  assertEquals(
      OperationType.DELETE_CODE,
      msReader.getOperationFromRecord(record3, JDBCOperationType.INSERT, UnsupportedOperationAction.DISCARD, errorRecords)
  );
  assertEquals(0, errorRecords.size());
}
 
Example 25
Source Project: datacollector   Source File: SampleExecutor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Executes an action for given record.
 *
 * @param record the record that will parametrize the action
 * @throws OnRecordErrorException when action can't be executed
 */
private void execute(Record record) throws OnRecordErrorException {
  // This is a contrived example, normally you may be performing an operation that could throw
  // an exception or produce an error condition. In that case you can throw an OnRecordErrorException
  // to send this record to the error pipeline with some details.
  if (!record.has("/someField")) {
    throw new OnRecordErrorException(Errors.SAMPLE_01, record, "exception detail message.");
  }

  // TODO: execute action
}
 
Example 26
Source Project: datacollector   Source File: TestXMLFlatteningProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleRecords() throws Exception {
  String xml = "<contacts>" + getXML("0") + getXML("1") + "</contacts>";
  Record expected1 = RecordCreator.create();
  expected1.set(Field.create(createExpectedRecord("", "0", "", ".", "#", "", false, false)));
  Record expected2 = RecordCreator.create();
  expected2.set(Field.create(createExpectedRecord("", "1", "", ".", "#", "", false, false)));
  List<Record> expected = ImmutableList.of(expected1, expected2);
  doTest(xml, "contact", expected, Collections.EMPTY_LIST, OnRecordError.DISCARD, true, true, false, false);
}
 
Example 27
Source Project: datacollector   Source File: AbstractHBaseProducer.java    License: Apache License 2.0 5 votes vote down vote up
private void doImplicitFieldMapping(
    Put p,
    Record record,
    StringBuilder errorMsgBuilder,
    Set<String> explicitFields,
    String timeDriver,
    boolean ignoreMissingField,
    boolean ignoreInvalidColumn,
    String hbaseRowKey
) throws OnRecordErrorException {
  validateRootLevelType(record);
  Date recordTime = getRecordTime(record, timeDriver);
  for (String fieldPath : record.getEscapedFieldPaths()) {
    if (!fieldPath.isEmpty() && !fieldPath.equals(hbaseRowKey) && !explicitFields.contains(fieldPath)) {
      String fieldPathColumn = fieldPath;
      if (fieldPath.charAt(0) == '/') {
        fieldPathColumn = fieldPath.substring(1);
      }
      HBaseColumn hbaseColumn = hbaseConnectionHelper.getColumn(fieldPathColumn.replace("'", ""));
      if (hbaseColumn.getCf().isPresent() && hbaseColumn.getQualifier().isPresent()) {
        byte[] value = getBytesForValue(record, fieldPath, null, ignoreMissingField);
        addCell(p, hbaseColumn.getCf().get(), hbaseColumn.getQualifier().get(), recordTime, value);
      } else if (ignoreInvalidColumn) {
        String errorMessage = Utils.format(Errors.HBASE_28.getMessage(),
            fieldPathColumn,
            KeyValue.COLUMN_FAMILY_DELIMITER
        );
        LOG.warn(errorMessage);
        errorMsgBuilder.append(errorMessage);
      } else {
        throw new OnRecordErrorException(record, Errors.HBASE_28, fieldPathColumn, KeyValue.COLUMN_FAMILY_DELIMITER);
      }
    }
  }
}
 
Example 28
Source Project: datacollector   Source File: MapRJsonDocumentLoader.java    License: Apache License 2.0 5 votes vote down vote up
public static Document createDocument(Record record) throws IOException{
  if(isTest) {
    Preconditions.checkNotNull(testDelegate);
    return testDelegate.createDocumentInternal(record);
  } else {
    Preconditions.checkNotNull(delegate);
    return delegate.createDocumentInternal(record);
  }
}
 
Example 29
Source Project: datacollector   Source File: MongoDBSourceIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStringOffsetCappedCollection() throws Exception {

  MongoDBSource origin = new MongoDBSourceBuilder()
      .connectionString("mongodb://" + mongoContainerIp + ":"  + mongoContainerMappedPort)
      .database(DATABASE_NAME)
      .collection(CAPPED_STRING_ID_COLLECTION)
      .offsetField("_id")
      .isCapped(false)
      .setOffsetType(OffsetFieldType.STRING)
      .maxBatchWaitTime(100)
      .readPreference(ReadPreferenceLabel.NEAREST)
      .build();

  SourceRunner runner = new SourceRunner.Builder(MongoDBSource.class, origin)
      .addOutputLane("lane")
      .build();

  List<Stage.ConfigIssue> issues = runner.runValidateConfigs();
  assertEquals(0, issues.size());

  runner.runInit();
  insertDocsWithStringID(CAPPED_STRING_ID_COLLECTION);

  final int maxBatchSize = 2;
  StageRunner.Output output = runner.runProduce(null, maxBatchSize);
  List<Record> parsedRecords = output.getRecords().get("lane");
  assertEquals("First batch should contain 2 records",2, parsedRecords.size());

  output = runner.runProduce(output.getNewOffset(), maxBatchSize);
  parsedRecords = output.getRecords().get("lane");
  assertEquals("Second batch should contain 2 records",1, parsedRecords.size());

  output = runner.runProduce(output.getNewOffset(), maxBatchSize);
  parsedRecords = output.getRecords().get("lane");
  assertEquals("Last batch should have 0 records",0, parsedRecords.size());
}
 
Example 30
Source Project: datacollector   Source File: StandaloneRunner.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<Record> getErrorRecords(String stage, int max) {
  if (prodPipeline != null) {
    return prodPipeline.getErrorRecords(stage, max);
  }
  return Collections.emptyList();
}