com.google.privacy.dlp.v2.Table Java Examples

The following examples show how to use com.google.privacy.dlp.v2.Table. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: DeIdentifyTableConditionMasking.java From java-docs-samples with Apache License 2.0

6 votes

public static void deIdentifyTableConditionMasking() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  deIdentifyTableConditionMasking(projectId, tableToDeIdentify);
}

Example #2

Source File: DLPTextToBigQueryStreaming.java From DataflowTemplates with Apache License 2.0

6 votes

@ProcessElement
public void processElement(ProcessContext c) {

  Table tokenizedData = c.element().getValue();
  List<String> headers =
      tokenizedData.getHeadersList().stream()
          .map(fid -> fid.getName())
          .collect(Collectors.toList());
  List<Table.Row> outputRows = tokenizedData.getRowsList();
  if (outputRows.size() > 0) {
    for (Table.Row outputRow : outputRows) {
      if (outputRow.getValuesCount() != headers.size()) {
        throw new IllegalArgumentException(
            "CSV file's header count must exactly match with data element count");
      }
      c.output(
          KV.of(
              c.element().getKey(),
              createBqRow(outputRow, headers.toArray(new String[headers.size()]))));
    }
  }
}

Example #3

Source File: DLPTextToBigQueryStreaming.java From DataflowTemplates with Apache License 2.0

6 votes

private static TableRow createBqRow(Table.Row tokenizedValue, String[] headers) {
  TableRow bqRow = new TableRow();
  AtomicInteger headerIndex = new AtomicInteger(0);
  List<TableCell> cells = new ArrayList<>();
  tokenizedValue
      .getValuesList()
      .forEach(
          value -> {
            String checkedHeaderName =
                checkHeaderName(headers[headerIndex.getAndIncrement()].toString());
            bqRow.set(checkedHeaderName, value.getStringValue());
            cells.add(new TableCell().set(checkedHeaderName, value.getStringValue()));
          });
  bqRow.setF(cells);
  return bqRow;
}

Example #4

Source File: ReIdentifyTableWithFpe.java From java-docs-samples with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  String kmsKeyName =
      "projects/YOUR_PROJECT/"
          + "locations/YOUR_KEYRING_REGION/"
          + "keyRings/YOUR_KEYRING_NAME/"
          + "cryptoKeys/YOUR_KEY_NAME";
  String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY";
  Table tableToReIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addRows(
          Row.newBuilder().addValues(
              Value.newBuilder().setStringValue("28777").build())
              .build())
      .build();
  reIdentifyTableWithFpe(projectId, tableToReIdentify, kmsKeyName, wrappedAesKey);
}

Example #5

Source File: CSVStreamingPipelineTest.java From dlp-dataflow-deidentification with Apache License 2.0

6 votes

@Test
public void testCSVContentProcessorDoFn() {
  List<String> sampleStringList = new ArrayList<String>();
  sampleStringList.add("A,a");
  sampleStringList.add("B,b");
  sampleStringList.add("C,c");
  sampleStringList.add("D,c");

  PCollection<KV<String, List<String>>> input =
      pipeline.apply(Create.of(KV.of("test", sampleStringList)));

  PCollection<KV<String, Table>> outputTables =
      input.apply(
          "ContentHandler",
          ParDo.of(new CSVContentProcessorDoFn(ValueProvider.StaticValueProvider.of(1))));

  PCollection<String> outputKeys = outputTables.apply(Keys.create());
  PAssert.that(outputKeys).containsInAnyOrder("test_1", "test_2", "test_3");
  pipeline.run();
}

Example #6

Source File: DeIdentifyTableBucketing.java From java-docs-samples with Apache License 2.0

6 votes

public static void deIdentifyTableBucketing() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  deIdentifyTableBucketing(projectId, tableToDeIdentify);
}

Example #7

Source File: DLPTokenizationDoFnTest.java From dlp-dataflow-deidentification with Apache License 2.0

6 votes

@Test
public void testConvertTableRowToRow() {
  DLPTokenizationDoFn dlp =
      new DLPTokenizationDoFn(
          "Project Name",
          ValueProvider.StaticValueProvider.of("DeidentifyTemplateName"),
          ValueProvider.StaticValueProvider.of("IdentifyTemplateName"));

  String[] header = {"header0", "header1"};
  String key = "Key name";
  Table.Row.Builder tableRowBuilder = Table.Row.newBuilder();
  tableRowBuilder.addValues(0, Value.newBuilder().setStringValue("value0"));
  tableRowBuilder.addValues(1, Value.newBuilder().setStringValue("value1"));
  Table.Row row = tableRowBuilder.build();

  Row result = dlp.convertTableRowToRow(header, key, row);
  assertEquals(result.getTableId(), key);
  assertEquals(result.getHeader()[0], "header0");
  assertEquals(result.getValue()[1], "value1");
}

Example #8

Source File: DLPDeidentifyText.java From beam with Apache License 2.0

6 votes

@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  String fileName = c.element().getKey();
  List<FieldId> dlpTableHeaders;
  if (headerColumns != null) {
    dlpTableHeaders =
        c.sideInput(headerColumns).stream()
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
  } else {
    // handle unstructured input
    dlpTableHeaders = new ArrayList<>();
    dlpTableHeaders.add(FieldId.newBuilder().setName("value").build());
  }
  Table table =
      Table.newBuilder()
          .addAllHeaders(dlpTableHeaders)
          .addAllRows(c.element().getValue())
          .build();
  ContentItem contentItem = ContentItem.newBuilder().setTable(table).build();
  this.requestBuilder.setItem(contentItem);
  DeidentifyContentResponse response =
      dlpServiceClient.deidentifyContent(this.requestBuilder.build());
  c.output(KV.of(fileName, response));
}

Example #9

Source File: MapStringToDlpRowTest.java From beam with Apache License 2.0

6 votes

@Test
public void mapsDelimitedStringToRow() {
  PCollection<KV<String, Table.Row>> rowCollection =
      testPipeline
          .apply(Create.of(KV.of("key", "value,secondValue")))
          .apply(ParDo.of(new MapStringToDlpRow(",")));
  PAssert.that(rowCollection)
      .containsInAnyOrder(
          KV.of(
              "key",
              Table.Row.newBuilder()
                  .addValues(Value.newBuilder().setStringValue("value").build())
                  .addValues(Value.newBuilder().setStringValue("secondValue").build())
                  .build()));
  testPipeline.run().waitUntilFinish();
}

Example #10

Source File: DeIdentifyTableRowSuppress.java From java-docs-samples with Apache License 2.0

6 votes

public static void deIdentifyTableRowSuppress() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  deIdentifyTableRowSuppress(projectId, tableToDeIdentify);
}

Example #11

Source File: DLPInspectText.java From beam with Apache License 2.0

6 votes

@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  List<FieldId> tableHeaders;
  if (headerColumns != null) {
    tableHeaders =
        c.sideInput(headerColumns).stream()
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
  } else {
    tableHeaders = new ArrayList<>();
    tableHeaders.add(FieldId.newBuilder().setName("value").build());
  }
  Table table =
      Table.newBuilder().addAllHeaders(tableHeaders).addAllRows(c.element().getValue()).build();
  ContentItem contentItem = ContentItem.newBuilder().setTable(table).build();
  this.requestBuilder.setItem(contentItem);
  InspectContentResponse response =
      dlpServiceClient.inspectContent(this.requestBuilder.build());
  c.output(KV.of(c.element().getKey(), response));
}

Example #12

Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0

6 votes

private static TableRow createBqRow(Table.Row tokenizedValue, String[] headers) {
  TableRow bqRow = new TableRow();
  AtomicInteger headerIndex = new AtomicInteger(0);
  List<TableCell> cells = new ArrayList<>();
  tokenizedValue
      .getValuesList()
      .forEach(
          value -> {
            String checkedHeaderName =
                checkHeaderName(headers[headerIndex.getAndIncrement()].toString());
            bqRow.set(checkedHeaderName, value.getStringValue());
            cells.add(new TableCell().set(checkedHeaderName, value.getStringValue()));
          });
  bqRow.setF(cells);
  return bqRow;
}

Example #13

Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0

6 votes

@ProcessElement
public void processElement(ProcessContext c) {

  Table tokenizedData = c.element().getValue();
  List<String> headers =
      tokenizedData.getHeadersList().stream()
          .map(fid -> fid.getName())
          .collect(Collectors.toList());
  List<Table.Row> outputRows = tokenizedData.getRowsList();
  if (outputRows.size() > 0) {
    for (Table.Row outputRow : outputRows) {
      if (outputRow.getValuesCount() != headers.size()) {
        throw new IllegalArgumentException(
            "CSV file's header count must exactly match with data element count");
      }
      c.output(
          KV.of(
              c.element().getKey(),
              createBqRow(outputRow, headers.toArray(new String[headers.size()]))));
    }
  }
}

Example #14

Source File: DLPReidentifyText.java From beam with Apache License 2.0

6 votes

@ProcessElement
public void processElement(ProcessContext context) throws IOException {
  List<FieldId> tableHeaders;
  if (headerColumns != null) {
    tableHeaders =
        context.sideInput(headerColumns).stream()
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
  } else {
    // handle unstructured input.
    tableHeaders = new ArrayList<>();
    tableHeaders.add(FieldId.newBuilder().setName("value").build());
  }
  Table table =
      Table.newBuilder()
          .addAllHeaders(tableHeaders)
          .addAllRows(context.element().getValue())
          .build();
  ContentItem contentItem = ContentItem.newBuilder().setTable(table).build();
  this.requestBuilder.setItem(contentItem);
  ReidentifyContentResponse response =
      dlpServiceClient.reidentifyContent(requestBuilder.build());
  context.output(KV.of(context.element().getKey(), response));
}

Example #15

Source File: MapStringToDlpRowTest.java From beam with Apache License 2.0

5 votes

@Test
public void mapsStringToRow() {
  PCollection<KV<String, Table.Row>> rowCollection =
      testPipeline
          .apply(Create.of(KV.of("key", "value")))
          .apply(ParDo.of(new MapStringToDlpRow(null)));
  PAssert.that(rowCollection)
      .containsInAnyOrder(
          KV.of(
              "key",
              Table.Row.newBuilder()
                  .addValues(Value.newBuilder().setStringValue("value").build())
                  .build()));
  testPipeline.run().waitUntilFinish();
}

Example #16

Source File: BatchRequestForDlpTest.java From beam with Apache License 2.0

5 votes

@Override
public Void apply(Iterable<KV<String, Iterable<Table.Row>>> input) {
  List<KV<String, Iterable<Table.Row>>> itemList = new ArrayList<>();
  input.forEach(itemList::add);
  assertEquals(1, itemList.size());
  return null;
}

Example #17

Source File: BatchRequestForDlpTest.java From beam with Apache License 2.0

5 votes

@Test
public void batchesRequests() {
  PCollection<KV<String, Iterable<Table.Row>>> batchedRows =
      testPipeline
          .apply(Create.of(KV.of("key", "value1"), KV.of("key", "value2")))
          .apply(ParDo.of(new MapStringToDlpRow(null)))
          .apply(ParDo.of(new BatchRequestForDLP(524000)));
  PAssert.that(batchedRows).satisfies(new VerifyPCollectionSize());
  testPipeline.run().waitUntilFinish();
}

Example #18

Source File: DeIdentifyTableWithFpe.java From java-docs-samples with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  String kmsKeyName =
      "projects/YOUR_PROJECT/"
          + "locations/YOUR_KEYRING_REGION/"
          + "keyRings/YOUR_KEYRING_NAME/"
          + "cryptoKeys/YOUR_KEY_NAME";
  String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addHeaders(FieldId.newBuilder().setName("Date").build())
      .addHeaders(FieldId.newBuilder().setName("Compensation").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2015").build())
          .addValues(Value.newBuilder().setStringValue("$10").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$20").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22222").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$15").build())
          .build())
      .build();
  deIdentifyTableWithFpe(projectId, tableToDeIdentify, kmsKeyName, wrappedAesKey);
}

Example #19

Source File: DeIdentifyWithDateShift.java From java-docs-samples with Apache License 2.0

5 votes

public static Table.Row parseLineAsRow(String line) {
  List<String> values = Splitter.on(",").splitToList(line);
  Value name = Value.newBuilder().setStringValue(values.get(0)).build();
  Value birthDate = Value.newBuilder().setDateValue(parseAsDate(values.get(1))).build();
  Value creditCardNumber = Value.newBuilder().setStringValue(values.get(2)).build();
  Value registerDate = Value.newBuilder().setDateValue(parseAsDate(values.get(3))).build();
  return Table.Row.newBuilder()
      .addValues(name)
      .addValues(birthDate)
      .addValues(creditCardNumber)
      .addValues(registerDate)
      .build();
}

Example #20

Source File: DeIdentifyTableInfoTypes.java From java-docs-samples with Apache License 2.0

5 votes

public static void deIdentifyTableInfoTypes() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "Charles Dickens name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();

  deIdentifyTableInfoTypes(projectId, tableToDeIdentify);
}

Example #21

Source File: DeIdentifyTableConditionInfoTypes.java From java-docs-samples with Apache License 2.0

5 votes

public static void deIdentifyTableConditionInfoTypes() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "Charles Dickens name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();

  deIdentifyTableConditionInfoTypes(projectId, tableToDeIdentify);
}

Example #22

Source File: InspectTable.java From java-docs-samples with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToInspect = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("name").build())
      .addHeaders(FieldId.newBuilder().setName("phone").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("John Doe").build())
          .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build()))
      .build();

  inspectTable(projectId, tableToInspect);
}

Example #23

Source File: DeIdentificationTests.java From java-docs-samples with Apache License 2.0

5 votes

@Test
public void testDeIdentifyTableWithFpe() throws IOException {
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addHeaders(FieldId.newBuilder().setName("Date").build())
      .addHeaders(FieldId.newBuilder().setName("Compensation").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2015").build())
          .addValues(Value.newBuilder().setStringValue("$10").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$20").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22222").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$15").build())
          .build())
      .build();

  DeIdentifyTableWithFpe.deIdentifyTableWithFpe(
      PROJECT_ID, tableToDeIdentify, kmsKeyName, wrappedKey);

  String output = bout.toString();
  assertThat(output).contains("Table after format-preserving encryption:");
}

Example #24

Source File: DeIdentificationTests.java From java-docs-samples with Apache License 2.0

5 votes

@Test
public void testReIdentifyTableWithFpe() throws IOException {
  Table tableToReIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("28777").build()).build())
      .build();

  ReIdentifyTableWithFpe.reIdentifyTableWithFpe(
      PROJECT_ID, tableToReIdentify, kmsKeyName, wrappedKey);

  String output = bout.toString();
  assertThat(output).contains("Table after re-identification:");
}

Example #25

Source File: InspectTests.java From java-docs-samples with Apache License 2.0

5 votes

@Test
public void testInspectTable() {
  Table tableToInspect = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("name").build())
      .addHeaders(FieldId.newBuilder().setName("phone").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("John Doe").build())
          .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build()))
      .build();
  InspectTable.inspectTable(PROJECT_ID, tableToInspect);

  String output = bout.toString();
  assertThat(output).contains("Info type: PHONE_NUMBER");
}

Example #26

Source File: Util.java From dlp-dataflow-deidentification with Apache License 2.0

5 votes

public static Table createDLPTable(List<FieldId> headers, List<String> lines) {

    List<Table.Row> rows = new ArrayList<>();
    lines.forEach(
        line -> {
          rows.add(convertCsvRowToTableRow(line));
        });
    Table table = Table.newBuilder().addAllHeaders(headers).addAllRows(rows).build();

    return table;
  }

Example #27

Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0

5 votes

private Table.Row convertCsvRowToTableRow(CSVRecord csvRow) {
  /** convert from CSV row to DLP Table Row */
  Iterator<String> valueIterator = csvRow.iterator();
  Table.Row.Builder tableRowBuilder = Table.Row.newBuilder();
  while (valueIterator.hasNext()) {
    String value = valueIterator.next();
    if (value != null) {
      tableRowBuilder.addValues(Value.newBuilder().setStringValue(value.toString()).build());
    } else {
      tableRowBuilder.addValues(Value.newBuilder().setStringValue("").build());
    }
  }

  return tableRowBuilder.build();
}

Example #28

Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0

5 votes

@ProcessElement
public void processElement(ProcessContext c) {
  String key = c.element().getKey();
  Table nonEncryptedData = c.element().getValue();
  ContentItem tableItem = ContentItem.newBuilder().setTable(nonEncryptedData).build();
  this.requestBuilder.setItem(tableItem);
  DeidentifyContentResponse response =
      dlpServiceClient.deidentifyContent(this.requestBuilder.build());

  Table tokenizedData = response.getItem().getTable();
  numberOfRowsTokenized.update(tokenizedData.getRowsList().size());
  numberOfBytesTokenized.update(tokenizedData.toByteArray().length);
  c.output(KV.of(key, tokenizedData));
}

Example #29

Source File: DLPTokenizationDoFn.java From dlp-dataflow-deidentification with Apache License 2.0

5 votes

public Row convertTableRowToRow(String[] header, String key, Table.Row outputRow) {
  String dlpRow =
      outputRow.getValuesList().stream()
          .map(value -> value.getStringValue())
          .collect(Collectors.joining(","));
  String[] values = dlpRow.split(",");
  Row row = new Row(key, header, values);
  return row;
}

Example #30

Source File: DLPTokenizationDoFn.java From dlp-dataflow-deidentification with Apache License 2.0

5 votes

@ProcessElement
public void processElement(ProcessContext c) {

  String key = c.element().getKey();
  Table nonEncryptedData = c.element().getValue();
  setInspectTemplateExist();
  ContentItem tableItem = ContentItem.newBuilder().setTable(nonEncryptedData).build();

  DeidentifyContentResponse response;
  DeidentifyContentRequest request = buildDeidentifyContentRequest(tableItem);
  response = dlpServiceClient.deidentifyContent(request);
  Table encryptedData = response.getItem().getTable();
  LOG.info(
      "Request Size Successfully Tokenized:{} rows {} bytes ",
      encryptedData.getRowsList().size(),
      request.toByteString().size());

  List<String> outputHeaders =
      encryptedData.getHeadersList().stream().map(FieldId::getName).collect(Collectors.toList());
  String[] header = new String[outputHeaders.size()];

  for (int i = 0; i < header.length; i++) {
    header[i] = Util.checkHeaderName(outputHeaders.get(i));
  }
  List<Table.Row> outputRows = encryptedData.getRowsList();

  for (Table.Row outputRow : outputRows) {
    Row row = convertTableRowToRow(header, key, outputRow);
    c.output(row);
  }
}