com.streamsets.pipeline.api.Record Java Examples

@Test
public void testParserNull() throws Exception {
  OverrunReader reader = new OverrunReader(new StringReader("A,B\r\nnull,b"), 1000, true, false);
  DelimitedDataParserSettings settings = DelimitedDataParserSettings.builder()
    .withSkipStartLines(0)
    .withFormat(CSVFormat.DEFAULT)
    .withHeader(CsvHeader.WITH_HEADER)
    .withMaxObjectLen(-1)
    .withRecordType(CsvRecordType.LIST_MAP)
    .withParseNull(true)
    .withNullConstant("null")
    .build();
  DataParser parser = new DelimitedCharDataParser(getContext(), "id", reader, 0, settings);

  Record record = parser.parse();
  Assert.assertNotNull(record);
  Assert.assertTrue(record.has("/A"));
  Assert.assertTrue(record.has("/B"));
  Assert.assertEquals(null, record.get("/A").getValueAsString());
  Assert.assertEquals("b", record.get("/B").getValueAsString());

  record = parser.parse();
  Assert.assertNull(record);
  Assert.assertEquals("-1", parser.getOffset());
  parser.close();
}
 

public static <C extends Processor> void verifyTypedFieldWithNullValue(
    Class<C> clazz,
    Processor processor,
    Record record
) throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(clazz, processor)
      .addOutputLane("lane")
      .build();

  runner.runInit();
  StageRunner.Output output;
  try{
    output = runner.runProcess(Collections.singletonList(record));
  } finally {
    runner.runDestroy();
  }
  Record outRec = output.getRecords().get("lane").get(0);
  assertEquals(record.get().getValueAsMap().size(), outRec.get().getValueAsMap().size());
  Map<String, Field> outMap = outRec.get().getValueAsMap();
  for(Map.Entry<String, Field> entry : outMap.entrySet()) {
    assertFieldUtil(entry.getKey(), entry.getValue(), null);
  }
}
 

private boolean process(DataParser parser) throws IOException, DataParserException {
  BatchContext batchContext = getContext().startBatch();
  List<Record> records = new ArrayList<>();
  Record parsedRecord = parser.parse();
  while (parsedRecord != null) {
    records.add(parsedRecord);
    parsedRecord = parser.parse();
  }

  // dispatch records to batch
  for (Record record : records) {
    batchContext.getBatchMaker().addRecord(record);
  }

  return getContext().processBatch(batchContext);
}
 

@Test
public void testMissingScale() throws Exception {
  JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString);

  ProcessorRunner processorRunner = getProcessorRunner(processor);

  processorRunner.runInit();

  Record record = makeRecord(fieldMap2);
  record.get("/column5").deleteAttribute("scale");
  List<Record> recordList = ImmutableList.of(record);

  StageRunner.Output output = processorRunner.runProcess(recordList);
  Assert.assertEquals(0, output.getRecords().get("lane").size());

  Assert.assertEquals(1, processorRunner.getErrorRecords().size());
  Record errorRecord = processorRunner.getErrorRecords().get(0);
  Assert.assertEquals(JdbcErrors.JDBC_304.name(), errorRecord.getHeader().getErrorCode());
}
 

/**
 * Evaluates any EL expressions in the headers section of the stage configuration.
 *
 * @param record current record in context for EL evaluation
 * @return Map of headers that can be added to the Jersey Client request
 * @throws StageException if an expression could not be evaluated
 */
public MultivaluedMap<String, Object> resolveHeaders(
    Map<String, String> headers,
    Record record
) throws StageException {
  RecordEL.setRecordInContext(headerVars, record);

  MultivaluedMap<String, Object> requestHeaders = new MultivaluedHashMap<>();
  for (Map.Entry<String, String> entry : headers.entrySet()) {
    List<Object> header = new ArrayList<>(1);
    Object resolvedValue = headerEval.eval(headerVars, entry.getValue(), String.class);
    header.add(resolvedValue);
    requestHeaders.put(entry.getKey(), header);
  }

  return requestHeaders;
}
 

private Record generateDestResponseRecord(UserRecordResult result) {
  Record record = getContext().createRecord("responseRecord");
  List<Field> attemptsVal = new ArrayList<>();
  for(Attempt attempt: result.getAttempts()) {
    LinkedHashMap<String, Field> attemptVal = new LinkedHashMap<>();
    attemptVal.put("delay", Field.create(attempt.getDelay()));
    attemptVal.put("duration", Field.create(attempt.getDuration()));
    attemptVal.put("errorMessage", Field.create(attempt.getErrorMessage()));
    attemptVal.put("errorCode", Field.create(attempt.getErrorCode()));
    attemptVal.put("success", Field.create(attempt.isSuccessful()));
    attemptsVal.add(Field.createListMap(attemptVal));
  }
  LinkedHashMap<String, Field> resultVal = new LinkedHashMap<>();
  resultVal.put("sequenceNumber", Field.create(result.getSequenceNumber()));
  resultVal.put("shardId", Field.create(result.getShardId()));
  resultVal.put("successful", Field.create(result.isSuccessful()));
  resultVal.put("attempts", Field.create(attemptsVal));
  record.set(Field.createListMap(resultVal));
  return record;
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  RecordEL.setRecordInContext(expressionVars, record);
  TimeNowEL.setTimeNowInContext(expressionVars, new Date());

  switch (fieldMapperConfig.operateOn) {
    case FIELD_PATHS:
      transformFieldPaths(record);
      break;
    case FIELD_VALUES:
      transformFieldValues(record);
      break;
    case FIELD_NAMES:
      transformFieldNames(record);
      break;
    default:
      throw new IllegalStateException(String.format(
          "Unrecognized operateOn value of %s",
          fieldMapperConfig.operateOn
      ));
  }
  batchMaker.addRecord(record);
}
 

private void writeRecord(DataGenerator generator, Record record) throws StageException {
  try {
    generator.write(record);
  } catch (IOException e) {
    //Record Error
    LOG.error("Record Write error", e);
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.RABBITMQ_07,
            e.toString(),
            e
        )
    );
  }
}
 

@Test
public void testSimpleJobExecutionThatFails() throws Exception{
  MapReduceExecutor executor = generateExecutor(ImmutableMap.<String, String>builder()
    .put("mapreduce.job.inputformat.class", SimpleTestInputFormat.class.getCanonicalName())
    .put("mapreduce.output.fileoutputformat.outputdir", getOutputDir())
    .put(SimpleTestInputFormat.THROW_EXCEPTION, "true")
  .build());

  ExecutorRunner runner = new ExecutorRunner.Builder(MapReduceDExecutor.class, executor)
    .setOnRecordError(OnRecordError.TO_ERROR)
    .build();
  runner.runInit();

  Record record = RecordCreator.create();
  record.set(Field.create(ImmutableMap.of(
    "key", Field.create("value")
  )));

  runner.runWrite(ImmutableList.of(record));

  Assert.assertEquals(1, runner.getErrorRecords().size());

  runner.runDestroy();
}
 

public static void checkRecordForUnknownFields(Record record, int i) throws IOException {
  // unknown fields are expected in paths for person and employee
  String attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/");
  UnknownFieldSet.Builder builder = UnknownFieldSet.newBuilder();
  builder.mergeDelimitedFrom(new ByteArrayInputStream(org.apache.commons.codec.binary.Base64.decodeBase64(attribute.getBytes())));
  UnknownFieldSet unknownFieldSet = builder.build();

  UnknownFieldsUtil.checkEmployeeUnknownFields(unknownFieldSet);

  if(i%2 == 0) {
    attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/engineer/person");
  } else {
    attribute = record.getHeader().getAttribute(ProtobufTypeUtil.PROTOBUF_UNKNOWN_FIELDS_PREFIX + "/exec/person");
  }
  builder = UnknownFieldSet.newBuilder();
  builder.mergeDelimitedFrom(new ByteArrayInputStream(org.apache.commons.codec.binary.Base64.decodeBase64(attribute.getBytes())));
  unknownFieldSet = builder.build();

  UnknownFieldsUtil.checkPersonUnknownFields(unknownFieldSet);
}
 

public BatchMakerImpl(StagePipe stagePipe, boolean keepSnapshot, int recordAllowance) {
  this.stagePipe = stagePipe;
  this.instanceName= stagePipe.getStage().getInfo().getInstanceName();
  outputLanes = ImmutableList.copyOf(stagePipe.getStage().getConfiguration().getOutputLanes());
  singleOutputLane = (outputLanes.size() == 1) ? outputLanes.iterator().next() : null;
  stageOutput = new HashMap<>();
  stageOutputSnapshot = (keepSnapshot) ? new HashMap<String, List<Record>>() : null;
  for (String outputLane : outputLanes) {
    stageOutput.put(outputLane, new ArrayList<Record>());
    if (stageOutputSnapshot != null) {
      stageOutputSnapshot.put(outputLane, new ArrayList<Record>());
    }
  }
  this.recordAllowance = recordAllowance;
  // if the stage is annotated as recordsByRef it means it does not reuse the records it creates, thus
  // we can skip one copy here (just here though), except if we are in preview
  recordByRef = !stagePipe.getStage().getContext().isPreview() &&
                stagePipe.getStage().getDefinition().getRecordsByRef();
}
 

private void testEmptyPayloadRequest(String method, String httpServerUrl, List<Record> requestRecords) {
  Response response = ClientBuilder.newClient()
      .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true)
      .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true)
      .target(httpServerUrl)
      .request()
      .header(Constants.X_SDC_APPLICATION_ID_HEADER, "id")
      .method(method);

  Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getStatus());
  String responseBody = response.readEntity(String.class);

  Assert.assertEquals(1, requestRecords.size());
  Record.Header emptyPayloadRecordHeader = requestRecords.get(0).getHeader();
  Assert.assertEquals(
      "true",
      emptyPayloadRecordHeader.getAttribute(RestServiceReceiver.EMPTY_PAYLOAD_RECORD_HEADER_ATTR_NAME)
  );
  Assert.assertEquals(method, emptyPayloadRecordHeader.getAttribute(RestServiceReceiver.METHOD_HEADER));


  // check custom HTTP Response header
  Assert.assertNotNull(response.getHeaders().getFirst("test"));
  Assert.assertEquals("value", response.getHeaders().getFirst("test"));
}
 

@Override
public void write(Batch batch) throws StageException {
  Iterator<Record> it = batch.getRecords();
  ELVars variables = getContext().createELVars();
  while(it.hasNext()) {
    Record record = it.next();
    RecordEL.setRecordInContext(variables, record);
    try {
      List<String> queriesToExecute = getEvaluatedQueriesForTheRecord(record, variables);
      executeQueries(record, queriesToExecute);
    } catch (OnRecordErrorException e) {
      LOG.error("Error when processing record: {}", e.toString(), e);
      errorRecordHandler.onError(e);
    }
  }
}
 

private void retainErrorRecordsInMemory(Map<String, List<Record>> errorRecords) {
  // Shortcut to avoid synchronization
  if(errorRecords.isEmpty()) {
    return;
  }

  synchronized (stageToErrorRecordsMap) {
    for (Map.Entry<String, List<Record>> e : errorRecords.entrySet()) {
      EvictingQueue<Record> errorRecordList = stageToErrorRecordsMap.computeIfAbsent(e.getKey(),
          k -> EvictingQueue.create(configuration.get(Constants.MAX_ERROR_RECORDS_PER_STAGE_KEY,
              Constants.MAX_ERROR_RECORDS_PER_STAGE_DEFAULT
          ))
      );
      // replace with a data structure with an upper cap
      errorRecordList.addAll(errorRecords.get(e.getKey()));
    }
  }
}
 

@Override
public boolean sample(Record record) {
  boolean isSampled = false;
  if (isOrigin) {
    String sampled = record.getHeader().getAttribute(SdcRecordConstants.SDC_SAMPLED_RECORD);
    isSampled = (null != sampled && SdcRecordConstants.TRUE.equals(sampled));
    if (isSampled) {
      updateTimer(record);
    }
  } else if (sampleSize > 0){
    if (sampleSet.remove(recordCounter)) {
      updateRecordHeader(record);
      isSampled = true;
    }
    recordCounter++;
    if (sampleSet.isEmpty()) {
      recordCounter = 0;
      chooseSampleFromPopulation();
    }
  }
  return isSampled;
}
 

@Test
public void testCreateEnumField() throws Exception {
  String schema = "{ \"type\": \"enum\",\n" +
    "  \"name\": \"Suit\",\n" +
    "  \"symbols\" : [\"SPADES\", \"HEARTS\", \"DIAMONDS\", \"CLUBS\"]\n" +
    "}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  GenericData.EnumSymbol enumSymbol = new GenericData.EnumSymbol(avroSchema, "CLUBS");
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, enumSymbol, false);

  Assert.assertEquals(Field.Type.STRING, field.getType());
  Assert.assertEquals("CLUBS", field.getValueAsString());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof GenericData.EnumSymbol);
  Assert.assertEquals("CLUBS", avroObject.toString());
}
 

@Test
public void shouldIncludeAndIgnoreTables() throws Exception {
  MysqlSourceConfig config = createConfig("root");
  MysqlSource source = createMysqlSource(config);
  config.includeTables = "test.foo,t%.foo2";
  config.ignoreTables = "test.foo";
  runner = new SourceRunner.Builder(MysqlDSource.class, source)
      .addOutputLane(LANE)
      .build();
  runner.runInit();

  final String lastSourceOffset = null;
  StageRunner.Output output = runner.runProduce(lastSourceOffset, MAX_BATCH_SIZE);
  assertThat(output.getRecords().get(LANE), is(IsEmptyCollection.<Record>empty()));

  execute(ds, "INSERT INTO foo (bar) VALUES (1)");
  execute(ds, "INSERT INTO foo2 VALUES (1, 2, 3)");

  output = runner.runProduce(output.getNewOffset(), MAX_BATCH_SIZE);
  List<Record> records = output.getRecords().get(LANE);
  assertThat(records, hasSize(1));
  assertThat(records.get(0).get("/Table").getValueAsString(), is("foo2"));
  execute(ds, "TRUNCATE foo");
  execute(ds, "TRUNCATE foo2");
}
 

@Test
public void testMultipleObjects() throws Exception {
  StringWriter writer = new StringWriter();

  JsonRecordWriterImpl jsonRecordWriter = new JsonRecordWriterImpl(writer, Mode.MULTIPLE_OBJECTS);

  Record record = new RecordImpl("stage", "id", null, null);
  record.set(Field.create(10));

  jsonRecordWriter.write(record);
  jsonRecordWriter.write(record);
  jsonRecordWriter.write(record);
  jsonRecordWriter.close();

  Assert.assertEquals("10\n10\n10", writer.toString());
}
 

@Override
public List<Point> getPoints(Record record) throws OnRecordErrorException {
  List<Point> points = new ArrayList<>();

  verifyRequireFieldsPresent(record);

  final String measurementName = record.get(conf.measurementField).getValueAsString();

  for (String fieldPath : conf.valueFields) {
    if (!record.has(fieldPath)) {
      continue;
    }
    Point.Builder point = Point
        .measurement(measurementName)
        .tag(RecordConverterUtil.getTags(conf.tagFields, record))
        .field(CollectdRecordConverter.stripPathPrefix(fieldPath), record.get(fieldPath).getValue());

    if (!conf.timeField.isEmpty()) {
      point.time(getTime(record), conf.timeUnit);
    }

    points.add(point.build());
  }
  return points;
}
 

public static <C extends Processor> void verifyStateObjectJavaScript(Class<C> clazz, Processor processor)
    throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(clazz, processor)
      .addOutputLane("lane")
      .build();
  runner.runInit();
  try {
    Record record = RecordCreator.create();
    Map<String, Field> map = new HashMap<>();
    map.put("count", Field.create(0));
    record.set(Field.create(map));
    List<Record> input = Collections.singletonList(record);
    runner.runProcess(input);
    StageRunner.Output output = runner.runProcess(input);
    assertEquals(1, output.getRecords().get("lane").size());
    // JavaScript only has a single number type, which is a double.
    assertEquals(2.0d, output.getRecords().get("lane").get(0).get("/count").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

private static List<Record> createTransactionRecords(int noOfRecords) {
  List<Record> records = new ArrayList<>();
  long currentTime = (System.currentTimeMillis() / 1000) * 1000;
  for (int i = 0; i < noOfRecords; i++) {
    Record record = RecordCreator.create();
    LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
    fields.put("unique_int", Field.create(i + 1));
    fields.put("t_date", Field.create(Field.Type.LONG, currentTime));
    fields.put("random_string", Field.create(UUID.randomUUID().toString()));
    record.set(Field.createListMap(fields));
    records.add(record);
    //making sure time is at least off by a second.
    currentTime = currentTime + 1000;
  }
  return records;
}
 

private Pair<String, HBaseColumn> getKey(Record record, HBaseLookupParameterConfig config) throws ELEvalException {
  if (config.rowExpr.isEmpty()) {
    throw new IllegalArgumentException(Utils.format("Empty lookup Key Expression"));
  }

  ELVars elVars = getContext().createELVars();
  RecordEL.setRecordInContext(elVars, record);

  String rowKey = keyExprEval.eval(elVars, config.rowExpr, String.class);
  String column = columnExprEval.eval(elVars, config.columnExpr, String.class);
  Date timestamp = timestampExprEval.eval(elVars, config.timestampExpr, Date.class);
  HBaseColumn hBaseColumn = hbaseConnectionHelper.getColumn(column);
  if(timestamp != null) {
    hBaseColumn.setTimestamp(timestamp.getTime());
  }
  return Pair.of(rowKey, hBaseColumn);
}
 

@Test
public void testReplaceNullDecimalFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/decimalField");
  stringFieldReplacement.newValue = "12345678";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("decimalField", Field.create(Field.Type.DECIMAL, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("decimalField"));
    Assert.assertEquals(new BigDecimal(12345678), result.get("decimalField").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testQuoting() throws Exception  {
  TableConfigBeanImpl tableConfigBean =  new TableJdbcSourceTestBuilder.TableConfigBeanTestBuilder()
      .tablePattern(QUOTED_TABLE_NAME_WITHOUT_QUOTES)
      .schema(database)
      .build();

  TableJdbcSource tableJdbcSource = new TableJdbcSourceTestBuilder(JDBC_URL, true, USER_NAME, PASSWORD)
      .tableConfigBeans(ImmutableList.of(tableConfigBean))
      .quoteChar(QuoteChar.BACKTICK)
      .build();
  List<Record> records = runProduceSingleBatchAndGetRecords(tableJdbcSource, new HashMap<>(), 20);
  checkRecords(EXPECTED_QUOTE_TESTING_RECORDS, records);
}
 

@Test
public void testBestEffortResolve2() throws StageException, IOException {

  String schemaString = "{\n" +
    "  \"type\" : \"record\",\n" +
    "  \"name\" : \"TopLevel\",\n" +
    "  \"namespace\" : \"com.streamsets.demo\",\n" +
    "  \"fields\" : [ {\n" +
    "    \"name\" : \"foo\",\n" +
    "    \"type\" : [ {\n" +
    "      \"type\" : \"record\",\n" +
    "      \"name\" : \"Foo\",\n" +
    "      \"fields\" : [ {\n" +
    "        \"name\" : \"bar\",\n" +
    "        \"type\" : \"int\"\n" +
    "      } ]\n" +
    "    }, {\"type\" : \"map\", \"values\" : \"int\"} ]\n" +
    "  } ]\n" +
    "}";

  // create sdc record matching the above schema
  Map<String, Field> barField = new HashMap<>();
  barField.put("bar", Field.create(45237));
  Map<String, Field> fooField = new HashMap<>();
  fooField.put("foo", Field.create(barField));
  Record record = RecordCreator.create();
  record.set(Field.create(fooField));

  Schema schema = new Schema.Parser().parse(schemaString);

  // convert record to avro record
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, schema, new HashMap<String, Object>());

  GenericRecord avroRecord = (GenericRecord) avroObject;

  // Avro will match this to map schema type from the union before going to the best effort resolve by sdc
  Map<String, Integer> foo = (Map<String, Integer>) avroRecord.get("foo");
  Object bar = foo.get("bar");
  Assert.assertEquals(45237, bar);
}
 

@Test
public void testParse() throws Exception {
  DataParser parser = getDataParser(LOG_LINE, 1000, 0);
  Assert.assertEquals(0, Long.parseLong(parser.getOffset()));
  Record record = parser.parse();
  Assert.assertNotNull(record);

  Assert.assertEquals("id::0", record.getHeader().getSourceId());

  Assert.assertEquals(LOG_LINE, record.get().getValueAsMap().get("originalLine").getValueAsString());

  Assert.assertFalse(record.has("/truncated"));

  Assert.assertEquals(88, Long.parseLong(parser.getOffset()));

  Assert.assertTrue(record.has("/remoteHost"));
  Assert.assertEquals("127.0.0.1", record.get("/remoteHost").getValueAsString());

  Assert.assertTrue(record.has("/logName"));
  Assert.assertEquals("ss", record.get("/logName").getValueAsString());

  Assert.assertTrue(record.has("/remoteUser"));
  Assert.assertEquals("h", record.get("/remoteUser").getValueAsString());

  Assert.assertTrue(record.has("/requestTime"));
  Assert.assertEquals("10/Oct/2000:13:55:36 -0700", record.get("/requestTime").getValueAsString());

  Assert.assertTrue(record.has("/request"));
  Assert.assertEquals("GET /apache_pb.gif HTTP/1.0", record.get("/request").getValueAsString());

  Assert.assertTrue(record.has("/status"));
  Assert.assertEquals("200", record.get("/status").getValueAsString());

  Assert.assertTrue(record.has("/bytesSent"));
  Assert.assertEquals("2326", record.get("/bytesSent").getValueAsString());

  parser.close();
}
 

@Test
public void testGetParserReaderWithOffset() throws Exception {
  DataParserFactoryBuilder dataParserFactoryBuilder = new DataParserFactoryBuilder(getContext(), DataParserFormat.TEXT);
  DataParserFactory factory = dataParserFactoryBuilder
    .setMaxDataLen(1000)
    .build();
  InputStream is = new ByteArrayInputStream("Hello\nBye".getBytes());
  DataParser parser = factory.getParser("id", is, "6");
  Assert.assertEquals(6, Long.parseLong(parser.getOffset()));
  Record record = parser.parse();
  Assert.assertTrue(record.has("/text"));
  Assert.assertEquals(9, Long.parseLong(parser.getOffset()));
  parser.close();
}
 

@Test
public void testMultipleOperations() throws Exception {
  MysqlBinLogSourceConfig config = createConfig("root");
  MysqlBinLogSource source = createMysqlSource(config);
  runner = new SourceRunner.Builder(MySQLBinLogDSource.class, source)
      .addOutputLane(LANE)
      .build();
  runner.runInit();

  StageRunner.Output output = runner.runProduce(null, MAX_BATCH_SIZE);
  List<Record> records = new ArrayList<>(output.getRecords().get(LANE));
  assertThat(records, is(Matchers.<Record>empty()));

  // add one more
  execute(ds, Arrays.asList(
      "INSERT INTO foo (bar) VALUES (2)",
      "UPDATE foo set bar = 3 where bar = 2",
      "DELETE from foo where bar = 3")
  );
  output = runner.runProduce(null, MAX_BATCH_SIZE);
  records = new ArrayList<>(output.getRecords().get(LANE));
  assertThat(records, hasSize(3));

  assertThat(records.get(0).get("/Type").getValueAsString(), is("INSERT"));
  assertThat(
      records.get(0).getHeader().getAttribute(OperationType.SDC_OPERATION_TYPE),
      is(String.valueOf(OperationType.INSERT_CODE))
  );
  assertThat(records.get(1).get("/Type").getValueAsString(), is("UPDATE"));
  assertThat(
      records.get(1).getHeader().getAttribute(OperationType.SDC_OPERATION_TYPE),
      is(String.valueOf(OperationType.UPDATE_CODE))
  );
  assertThat(records.get(2).get("/Type").getValueAsString(), is("DELETE"));
  assertThat(
      records.get(2).getHeader().getAttribute(OperationType.SDC_OPERATION_TYPE),
      is(String.valueOf(OperationType.DELETE_CODE))
  );
}
 

private void orderToListMap(Record record) {
  LinkedHashMap<String, Field> list = new LinkedHashMap<>(fields.size());
  for(String fieldPath : config.fields) {
    list.put(
      toFieldName(fieldPath),
      record.has(fieldPath) ? record.get(fieldPath) : defaultField
    );
  }

  record.set(Field.create(Field.Type.LIST_MAP, list));
}
 

@Test(timeout = 10000)
public void testInsertOrReplaceDuplicateBinaryKey() throws Exception {

  MapRJsonConfigBean conf = new MapRJsonConfigBean();
  basicConfiguration(conf);
  conf.createTable = true;
  conf.insertOrReplace = InsertOrReplace.REPLACE;
  conf.isBinaryRowKey = true;

  MapRJsonTarget target = new MapRJsonTarget(conf);

  TargetRunner runner = new TargetRunner.Builder(MapRJsonDTarget.class,
      target
  ).setOnRecordError(OnRecordError.DISCARD).build();

  List<Stage.ConfigIssue> configIssues = runner.runValidateConfigs();
  assertEquals(0, configIssues.size());

  List<Record> twoRecords = ImmutableList.of(newRecord(27,0), newRecord(27, 1));

  runner.runInit();
  runner.runWrite(twoRecords);

  assertEquals(1, recordCount());
  assertNotNull(fetchDocument());

}