com.google.privacy.dlp.v2.FieldId Java Examples

The following examples show how to use com.google.privacy.dlp.v2.FieldId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DeIdentifyTableRowSuppress.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
public static void deIdentifyTableRowSuppress() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  deIdentifyTableRowSuppress(projectId, tableToDeIdentify);
}
 
Example #2
Source File: DLPDeidentifyText.java    From beam with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  String fileName = c.element().getKey();
  List<FieldId> dlpTableHeaders;
  if (headerColumns != null) {
    dlpTableHeaders =
        c.sideInput(headerColumns).stream()
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
  } else {
    // handle unstructured input
    dlpTableHeaders = new ArrayList<>();
    dlpTableHeaders.add(FieldId.newBuilder().setName("value").build());
  }
  Table table =
      Table.newBuilder()
          .addAllHeaders(dlpTableHeaders)
          .addAllRows(c.element().getValue())
          .build();
  ContentItem contentItem = ContentItem.newBuilder().setTable(table).build();
  this.requestBuilder.setItem(contentItem);
  DeidentifyContentResponse response =
      dlpServiceClient.deidentifyContent(this.requestBuilder.build());
  c.output(KV.of(fileName, response));
}
 
Example #3
Source File: DLPInspectText.java    From beam with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  List<FieldId> tableHeaders;
  if (headerColumns != null) {
    tableHeaders =
        c.sideInput(headerColumns).stream()
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
  } else {
    tableHeaders = new ArrayList<>();
    tableHeaders.add(FieldId.newBuilder().setName("value").build());
  }
  Table table =
      Table.newBuilder().addAllHeaders(tableHeaders).addAllRows(c.element().getValue()).build();
  ContentItem contentItem = ContentItem.newBuilder().setTable(table).build();
  this.requestBuilder.setItem(contentItem);
  InspectContentResponse response =
      dlpServiceClient.inspectContent(this.requestBuilder.build());
  c.output(KV.of(c.element().getKey(), response));
}
 
Example #4
Source File: DLPReidentifyText.java    From beam with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext context) throws IOException {
  List<FieldId> tableHeaders;
  if (headerColumns != null) {
    tableHeaders =
        context.sideInput(headerColumns).stream()
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
  } else {
    // handle unstructured input.
    tableHeaders = new ArrayList<>();
    tableHeaders.add(FieldId.newBuilder().setName("value").build());
  }
  Table table =
      Table.newBuilder()
          .addAllHeaders(tableHeaders)
          .addAllRows(context.element().getValue())
          .build();
  ContentItem contentItem = ContentItem.newBuilder().setTable(table).build();
  this.requestBuilder.setItem(contentItem);
  ReidentifyContentResponse response =
      dlpServiceClient.reidentifyContent(requestBuilder.build());
  context.output(KV.of(context.element().getKey(), response));
}
 
Example #5
Source File: DLPTextOperationsIT.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public Void apply(Iterable<KV<String, DeidentifyContentResponse>> input) {
  List<Boolean> matches = new ArrayList<>();
  input.forEach(
      item -> {
        item.getValue()
            .getItem()
            .getTable()
            .getRowsList()
            .forEach(
                row ->
                    matches.add(
                        row.getValuesList().stream()
                            .anyMatch(value -> value.getStringValue().equals(expectedValue))));
        assertTrue(
            item.getValue()
                .getItem()
                .getTable()
                .getHeadersList()
                .contains(FieldId.newBuilder().setName("value").build()));
      });
  assertTrue(matches.contains(Boolean.TRUE));
  return null;
}
 
Example #6
Source File: DeIdentifyTableConditionMasking.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
public static void deIdentifyTableConditionMasking() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  deIdentifyTableConditionMasking(projectId, tableToDeIdentify);
}
 
Example #7
Source File: ReIdentifyTableWithFpe.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  String kmsKeyName =
      "projects/YOUR_PROJECT/"
          + "locations/YOUR_KEYRING_REGION/"
          + "keyRings/YOUR_KEYRING_NAME/"
          + "cryptoKeys/YOUR_KEY_NAME";
  String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY";
  Table tableToReIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addRows(
          Row.newBuilder().addValues(
              Value.newBuilder().setStringValue("28777").build())
              .build())
      .build();
  reIdentifyTableWithFpe(projectId, tableToReIdentify, kmsKeyName, wrappedAesKey);
}
 
Example #8
Source File: DeIdentifyTableBucketing.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
public static void deIdentifyTableBucketing() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  deIdentifyTableBucketing(projectId, tableToDeIdentify);
}
 
Example #9
Source File: DLPTokenizationDoFn.java    From dlp-dataflow-deidentification with Apache License 2.0 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) {

  String key = c.element().getKey();
  Table nonEncryptedData = c.element().getValue();
  setInspectTemplateExist();
  ContentItem tableItem = ContentItem.newBuilder().setTable(nonEncryptedData).build();

  DeidentifyContentResponse response;
  DeidentifyContentRequest request = buildDeidentifyContentRequest(tableItem);
  response = dlpServiceClient.deidentifyContent(request);
  Table encryptedData = response.getItem().getTable();
  LOG.info(
      "Request Size Successfully Tokenized:{} rows {} bytes ",
      encryptedData.getRowsList().size(),
      request.toByteString().size());

  List<String> outputHeaders =
      encryptedData.getHeadersList().stream().map(FieldId::getName).collect(Collectors.toList());
  String[] header = new String[outputHeaders.size()];

  for (int i = 0; i < header.length; i++) {
    header[i] = Util.checkHeaderName(outputHeaders.get(i));
  }
  List<Table.Row> outputRows = encryptedData.getRowsList();

  for (Table.Row outputRow : outputRows) {
    Row row = convertTableRowToRow(header, key, outputRow);
    c.output(row);
  }
}
 
Example #10
Source File: InspectTests.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
@Test
public void testInspectTable() {
  Table tableToInspect = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("name").build())
      .addHeaders(FieldId.newBuilder().setName("phone").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("John Doe").build())
          .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build()))
      .build();
  InspectTable.inspectTable(PROJECT_ID, tableToInspect);

  String output = bout.toString();
  assertThat(output).contains("Info type: PHONE_NUMBER");
}
 
Example #11
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
@Test
public void testReIdentifyTableWithFpe() throws IOException {
  Table tableToReIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("28777").build()).build())
      .build();

  ReIdentifyTableWithFpe.reIdentifyTableWithFpe(
      PROJECT_ID, tableToReIdentify, kmsKeyName, wrappedKey);

  String output = bout.toString();
  assertThat(output).contains("Table after re-identification:");
}
 
Example #12
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeIdentifyTableWithFpe() throws IOException {
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addHeaders(FieldId.newBuilder().setName("Date").build())
      .addHeaders(FieldId.newBuilder().setName("Compensation").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2015").build())
          .addValues(Value.newBuilder().setStringValue("$10").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$20").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22222").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$15").build())
          .build())
      .build();

  DeIdentifyTableWithFpe.deIdentifyTableWithFpe(
      PROJECT_ID, tableToDeIdentify, kmsKeyName, wrappedKey);

  String output = bout.toString();
  assertThat(output).contains("Table after format-preserving encryption:");
}
 
Example #13
Source File: InspectTable.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToInspect = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("name").build())
      .addHeaders(FieldId.newBuilder().setName("phone").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("John Doe").build())
          .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build()))
      .build();

  inspectTable(projectId, tableToInspect);
}
 
Example #14
Source File: DeIdentifyTableConditionInfoTypes.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
public static void deIdentifyTableConditionInfoTypes() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "Charles Dickens name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();

  deIdentifyTableConditionInfoTypes(projectId, tableToDeIdentify);
}
 
Example #15
Source File: DeIdentifyTableInfoTypes.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
public static void deIdentifyTableInfoTypes() throws IOException {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "Charles Dickens name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();

  deIdentifyTableInfoTypes(projectId, tableToDeIdentify);
}
 
Example #16
Source File: DeIdentifyTableWithFpe.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
  // TODO(developer): Replace these variables before running the sample.
  String projectId = "your-project-id";
  String kmsKeyName =
      "projects/YOUR_PROJECT/"
          + "locations/YOUR_KEYRING_REGION/"
          + "keyRings/YOUR_KEYRING_NAME/"
          + "cryptoKeys/YOUR_KEY_NAME";
  String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY";
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("Employee ID").build())
      .addHeaders(FieldId.newBuilder().setName("Date").build())
      .addHeaders(FieldId.newBuilder().setName("Compensation").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2015").build())
          .addValues(Value.newBuilder().setStringValue("$10").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("11111").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$20").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22222").build())
          .addValues(Value.newBuilder().setStringValue("2016").build())
          .addValues(Value.newBuilder().setStringValue("$15").build())
          .build())
      .build();
  deIdentifyTableWithFpe(projectId, tableToDeIdentify, kmsKeyName, wrappedAesKey);
}
 
Example #17
Source File: Util.java    From dlp-dataflow-deidentification with Apache License 2.0 5 votes vote down vote up
public static Table createDLPTable(List<FieldId> headers, List<String> lines) {

    List<Table.Row> rows = new ArrayList<>();
    lines.forEach(
        line -> {
          rows.add(convertCsvRowToTableRow(line));
        });
    Table table = Table.newBuilder().addAllHeaders(headers).addAllRows(rows).build();

    return table;
  }
 
Example #18
Source File: CSVContentProcessorDoFn.java    From dlp-dataflow-deidentification with Apache License 2.0 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c, OffsetRangeTracker tracker) {
  for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
    String fileName = c.element().getKey();
    String key = String.format("%s_%d", fileName, i);
    List<String> rows = c.element().getValue().stream().skip(1).collect(Collectors.toList());
    List<FieldId> headers =
        Arrays.stream(c.element().getValue().get(0).split(","))
            .map(header -> FieldId.newBuilder().setName(header).build())
            .collect(Collectors.toList());
    KV<Integer, Integer> lineRange = createStartEnd(rows.size(), i);
    int startOfLine = lineRange.getKey();
    int endOfLine = lineRange.getValue();

    List<String> lines = new ArrayList<>();

    for (int index = startOfLine - 1; index < endOfLine; index++) {

      lines.add(rows.get(index));
    }
    Table batchData = Util.createDLPTable(headers, lines);

    if (batchData.getRowsCount() > 0) {
      LOG.info(
          "Current Restriction From: {}, Current Restriction To: {}, StartofLine: {}, End Of Line {}, BatchData {}",
          tracker.currentRestriction().getFrom(),
          tracker.currentRestriction().getTo(),
          startOfLine,
          endOfLine,
          batchData.getRowsCount());
      c.output(KV.of(key, batchData));
      lines.clear();
    }
  }
}
 
Example #19
Source File: DLPTextToBigQueryStreaming.java    From DataflowTemplates with Apache License 2.0 4 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c, RestrictionTracker<OffsetRange, Long> tracker)
    throws IOException {
  for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
    String fileKey = c.element().getKey();
    try (BufferedReader br = getReader(c.element().getValue())) {

      csvHeaders = getHeaders(c.sideInput(headerMap), fileKey);
      if (csvHeaders != null) {
        List<FieldId> dlpTableHeaders =
            csvHeaders.stream()
                .map(header -> FieldId.newBuilder().setName(header).build())
                .collect(Collectors.toList());
        List<Table.Row> rows = new ArrayList<>();
        Table dlpTable = null;
        /** finding out EOL for this restriction so that we know the SOL */
        int endOfLine = (int) (i * batchSize.get().intValue());
        int startOfLine = (endOfLine - batchSize.get().intValue());
        /** skipping all the rows that's not part of this restriction */
        br.readLine();
        Iterator<CSVRecord> csvRows =
            CSVFormat.DEFAULT.withSkipHeaderRecord().parse(br).iterator();
        for (int line = 0; line < startOfLine; line++) {
          if (csvRows.hasNext()) {
            csvRows.next();
          }
        }
        /** looping through buffered reader and creating DLP Table Rows equals to batch */
        while (csvRows.hasNext() && lineCount <= batchSize.get()) {

          CSVRecord csvRow = csvRows.next();
          rows.add(convertCsvRowToTableRow(csvRow));
          lineCount += 1;
        }
        /** creating DLP table and output for next transformation */
        dlpTable = Table.newBuilder().addAllHeaders(dlpTableHeaders).addAllRows(rows).build();
        c.output(KV.of(fileKey, dlpTable));

        LOG.debug(
            "Current Restriction From: {}, Current Restriction To: {},"
                + " StartofLine: {}, End Of Line {}, BatchData {}",
            tracker.currentRestriction().getFrom(),
            tracker.currentRestriction().getTo(),
            startOfLine,
            endOfLine,
            dlpTable.getRowsCount());

      } else {

        throw new RuntimeException("Header Values Can't be found For file Key " + fileKey);
      }
    }
  }
}
 
Example #20
Source File: DLPTextToBigQueryStreaming.java    From dlp-dataflow-deidentification with Apache License 2.0 4 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c, RestrictionTracker<OffsetRange, Long> tracker)
    throws IOException {
  for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
    String fileKey = c.element().getKey();
    try (BufferedReader br = getReader(c.element().getValue())) {

      csvHeaders = getHeaders(c.sideInput(headerMap), fileKey);
      if (csvHeaders != null) {
        List<FieldId> dlpTableHeaders =
            csvHeaders.stream()
                .map(header -> FieldId.newBuilder().setName(header).build())
                .collect(Collectors.toList());
        List<Table.Row> rows = new ArrayList<>();
        Table dlpTable = null;
        /** finding out EOL for this restriction so that we know the SOL */
        int endOfLine = (int) (i * batchSize.get().intValue());
        int startOfLine = (endOfLine - batchSize.get().intValue());
        /** skipping all the rows that's not part of this restriction */
        br.readLine();
        Iterator<CSVRecord> csvRows =
            CSVFormat.DEFAULT.withSkipHeaderRecord().parse(br).iterator();
        for (int line = 0; line < startOfLine; line++) {
          if (csvRows.hasNext()) {
            csvRows.next();
          }
        }
        /** looping through buffered reader and creating DLP Table Rows equals to batch */
        while (csvRows.hasNext() && lineCount <= batchSize.get()) {

          CSVRecord csvRow = csvRows.next();
          rows.add(convertCsvRowToTableRow(csvRow));
          lineCount += 1;
        }
        /** creating DLP table and output for next transformation */
        dlpTable = Table.newBuilder().addAllHeaders(dlpTableHeaders).addAllRows(rows).build();
        c.output(KV.of(fileKey, dlpTable));

        LOG.debug(
            "Current Restriction From: {}, Current Restriction To: {},"
                + " StartofLine: {}, End Of Line {}, BatchData {}",
            tracker.currentRestriction().getFrom(),
            tracker.currentRestriction().getTo(),
            startOfLine,
            endOfLine,
            dlpTable.getRowsCount());

      } else {

        throw new RuntimeException("Header Values Can't be found For file Key " + fileKey);
      }
    }
  }
}
 
Example #21
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeIdentifyTableConditionsInfoTypes() throws IOException {
  // Transform findings only when specific conditions are met on another field
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "Charles Dickens name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();
  Table expectedTable = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "[PERSON_NAME] name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();

  Table table = DeIdentifyTableConditionInfoTypes.deIdentifyTableConditionInfoTypes(
      PROJECT_ID, tableToDeIdentify);

  String output = bout.toString();
  assertThat(output).contains("Table after de-identification:");
  assertThat(table).isEqualTo(expectedTable);
}
 
Example #22
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeIdentifyTableRowSuppress() throws IOException {
  // Suppress a row based on the content of a column
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();
  Table expectedTable = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  Table table = DeIdentifyTableRowSuppress.deIdentifyTableRowSuppress(
      PROJECT_ID, tableToDeIdentify);

  String output = bout.toString();
  assertThat(output).contains("Table after de-identification:");
  assertThat(table).isEqualTo(expectedTable);
}
 
Example #23
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeIdentifyTableInfoTypes() throws IOException {
  // Transform findings found in column
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "Charles Dickens name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in Jane Austen's novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build())
          .build())
      .build();
  Table expectedTable = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addHeaders(FieldId.newBuilder().setName("FACTOID").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .addValues(Value.newBuilder().setStringValue(
              "[PERSON_NAME] name was a curse, possibly invented by Shakespeare.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .addValues(Value.newBuilder().setStringValue(
              "There are 14 kisses in [PERSON_NAME] novels.").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .addValues(Value.newBuilder().setStringValue("[PERSON_NAME] loved cats.").build())
          .build())
      .build();

  Table table = DeIdentifyTableInfoTypes.deIdentifyTableInfoTypes(PROJECT_ID, tableToDeIdentify);

  String output = bout.toString();
  assertThat(output).contains("Table after de-identification:");
  assertThat(table).isEqualTo(expectedTable);
}
 
Example #24
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeIdentifyTableConditionMasking() throws IOException {
  // Transform a column based on the value of another column
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();
  Table expectedTable = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("**").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();

  Table table = DeIdentifyTableConditionMasking.deIdentifyTableConditionMasking(
      PROJECT_ID, tableToDeIdentify);

  String output = bout.toString();
  assertThat(output).contains("Table after de-identification:");
  assertThat(table).isEqualTo(expectedTable);
}
 
Example #25
Source File: DeIdentificationTests.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeIdentifyTableBucketing() throws IOException {
  // Transform a column based on the value of another column
  Table tableToDeIdentify = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("95").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("21").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("75").build())
          .build())
      .build();
  Table expectedTable = Table.newBuilder()
      .addHeaders(FieldId.newBuilder().setName("AGE").build())
      .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
      .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("101").build())
          .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
          .addValues(Value.newBuilder().setStringValue("90:100").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("22").build())
          .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
          .addValues(Value.newBuilder().setStringValue("20:30").build())
          .build())
      .addRows(Row.newBuilder()
          .addValues(Value.newBuilder().setStringValue("55").build())
          .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
          .addValues(Value.newBuilder().setStringValue("70:80").build())
          .build())
      .build();

  Table table = DeIdentifyTableBucketing.deIdentifyTableBucketing(PROJECT_ID, tableToDeIdentify);

  String output = bout.toString();
  assertThat(output).contains("Table after de-identification:");
  assertThat(table).isEqualTo(expectedTable);
}
 
Example #26
Source File: DeIdentifyTableRowSuppress.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
public static Table deIdentifyTableRowSuppress(String projectId, Table tableToDeIdentify)
    throws IOException {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (DlpServiceClient dlp = DlpServiceClient.create()) {
    // Specify what content you want the service to de-identify.
    ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build();

    // Specify when the content should be de-identified.
    Condition condition = Condition.newBuilder()
        .setField(FieldId.newBuilder().setName("AGE").build())
        .setOperator(RelationalOperator.GREATER_THAN)
        .setValue(Value.newBuilder().setIntegerValue(89).build()).build();
    // Apply the condition to record suppression.
    RecordSuppression recordSuppressions =
        RecordSuppression.newBuilder()
            .setCondition(RecordCondition.newBuilder()
                .setExpressions(Expressions.newBuilder()
                    .setConditions(Conditions.newBuilder().addConditions(condition).build())
                    .build())
                .build())
            .build();
    // Use record suppression as the only transformation
    RecordTransformations transformations =
        RecordTransformations.newBuilder()
            .addRecordSuppressions(recordSuppressions)
            .build();

    DeidentifyConfig deidentifyConfig =
        DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build();

    // Combine configurations into a request for the service.
    DeidentifyContentRequest request =
        DeidentifyContentRequest.newBuilder()
            .setParent(LocationName.of(projectId, "global").toString())
            .setItem(contentItem)
            .setDeidentifyConfig(deidentifyConfig)
            .build();

    // Send the request and receive response from the service.
    DeidentifyContentResponse response = dlp.deidentifyContent(request);

    // Print the results.
    System.out.println(
        "Table after de-identification: " + response.getItem().getTable());

    return response.getItem().getTable();
  }
}
 
Example #27
Source File: ReIdentifyTableWithFpe.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
public static void reIdentifyTableWithFpe(
    String projectId, Table tableToReIdentify, String kmsKeyName, String wrappedAesKey)
    throws IOException {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (DlpServiceClient dlp = DlpServiceClient.create()) {
    // Specify what content you want the service to re-identify.
    ContentItem contentItem = ContentItem.newBuilder().setTable(tableToReIdentify).build();

    // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it.
    KmsWrappedCryptoKey kmsWrappedCryptoKey =
        KmsWrappedCryptoKey.newBuilder()
            .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey)))
            .setCryptoKeyName(kmsKeyName)
            .build();
    CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build();

    // Specify how to un-encrypt the previously de-identified information.
    CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig =
        CryptoReplaceFfxFpeConfig.newBuilder()
            .setCryptoKey(cryptoKey)
            // Set of characters in the input text. For more info, see
            // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet
            .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC)
            .build();
    PrimitiveTransformation primitiveTransformation =
        PrimitiveTransformation.newBuilder()
            .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig)
            .build();

    // Specify field to be decrypted.
    FieldId fieldId = FieldId.newBuilder().setName("Employee ID").build();

    // Associate the decryption with the specified field.
    FieldTransformation fieldTransformation =
        FieldTransformation.newBuilder()
            .setPrimitiveTransformation(primitiveTransformation)
            .addFields(fieldId)
            .build();
    RecordTransformations transformations =
        RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build();

    DeidentifyConfig reidentifyConfig =
        DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build();

    // Combine configurations into a request for the service.
    ReidentifyContentRequest request =
        ReidentifyContentRequest.newBuilder()
            .setParent(LocationName.of(projectId, "global").toString())
            .setItem(contentItem)
            .setReidentifyConfig(reidentifyConfig)
            .build();

    // Send the request and receive response from the service
    ReidentifyContentResponse response = dlp.reidentifyContent(request);

    // Print the results
    System.out.println("Table after re-identification: " + response.getItem().getValue());
  }
}
 
Example #28
Source File: InspectBigQueryTableWithSampling.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
public static void inspectBigQueryTableWithSampling(
    String projectId, String topicId, String subscriptionId)
    throws ExecutionException, InterruptedException, IOException {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (DlpServiceClient dlp = DlpServiceClient.create()) {
    // Specify the BigQuery table to be inspected.
    BigQueryTable tableReference =
        BigQueryTable.newBuilder()
            .setProjectId("bigquery-public-data")
            .setDatasetId("usa_names")
            .setTableId("usa_1910_current")
            .build();

    BigQueryOptions bigQueryOptions =
        BigQueryOptions.newBuilder()
            .setTableReference(tableReference)
            .setRowsLimit(1000)
            .setSampleMethod(SampleMethod.RANDOM_START)
            .addIdentifyingFields(FieldId.newBuilder().setName("name"))
            .build();

    StorageConfig storageConfig =
        StorageConfig.newBuilder().setBigQueryOptions(bigQueryOptions).build();

    // Specify the type of info the inspection will look for.
    // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types
    InfoType infoType = InfoType.newBuilder().setName("PERSON_NAME").build();

    // Specify how the content should be inspected.
    InspectConfig inspectConfig =
        InspectConfig.newBuilder()
            .addInfoTypes(infoType)
            .setIncludeQuote(true)
            .build();

    // Specify the action that is triggered when the job completes.
    String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId);
    Action.PublishToPubSub publishToPubSub =
        Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build();
    Action action = Action.newBuilder().setPubSub(publishToPubSub).build();

    // Configure the long running job we want the service to perform.
    InspectJobConfig inspectJobConfig =
        InspectJobConfig.newBuilder()
            .setStorageConfig(storageConfig)
            .setInspectConfig(inspectConfig)
            .addActions(action)
            .build();

    // Create the request for the job configured above.
    CreateDlpJobRequest createDlpJobRequest =
        CreateDlpJobRequest.newBuilder()
            .setParent(LocationName.of(projectId, "global").toString())
            .setInspectJob(inspectJobConfig)
            .build();

    // Use the client to send the request.
    final DlpJob dlpJob = dlp.createDlpJob(createDlpJobRequest);
    System.out.println("Job created: " + dlpJob.getName());

    // Set up a Pub/Sub subscriber to listen on the job completion status
    final SettableApiFuture<Boolean> done = SettableApiFuture.create();

    ProjectSubscriptionName subscriptionName =
        ProjectSubscriptionName.of(projectId, subscriptionId);

    MessageReceiver messageHandler =
        (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> {
          handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer);
        };
    Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build();
    subscriber.startAsync();

    // Wait for job completion semi-synchronously
    // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
    try {
      done.get(15, TimeUnit.MINUTES);
    } catch (TimeoutException e) {
      System.out.println("Job was not completed after 15 minutes.");
      return;
    } finally {
      subscriber.stopAsync();
      subscriber.awaitTerminated();
    }

    // Get the latest state of the job from the service
    GetDlpJobRequest request = GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build();
    DlpJob completedJob = dlp.getDlpJob(request);

    // Parse the response and process results.
    System.out.println("Job status: " + completedJob.getState());
    InspectDataSourceDetails.Result result = completedJob.getInspectDetails().getResult();
    System.out.println("Findings: ");
    for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) {
      System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName());
      System.out.println("\tCount: " + infoTypeStat.getCount());
    }
  }
}
 
Example #29
Source File: DeIdentifyTableConditionMasking.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
public static Table deIdentifyTableConditionMasking(String projectId, Table tableToDeIdentify)
    throws IOException {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (DlpServiceClient dlp = DlpServiceClient.create()) {
    // Specify what content you want the service to de-identify.
    ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build();

    // Specify how the content should be de-identified.
    CharacterMaskConfig characterMaskConfig =
        CharacterMaskConfig.newBuilder()
            .setMaskingCharacter("*")
            .build();
    PrimitiveTransformation primitiveTransformation =
        PrimitiveTransformation.newBuilder()
            .setCharacterMaskConfig(characterMaskConfig)
            .build();

    // Specify field to be de-identified.
    FieldId fieldId = FieldId.newBuilder().setName("HAPPINESS SCORE").build();

    // Specify when the above field should be de-identified.
    Condition condition = Condition.newBuilder()
        .setField(FieldId.newBuilder().setName("AGE").build())
        .setOperator(RelationalOperator.GREATER_THAN)
        .setValue(Value.newBuilder().setIntegerValue(89).build())
        .build();
    // Apply the condition to records
    RecordCondition recordCondition = RecordCondition.newBuilder()
        .setExpressions(Expressions.newBuilder()
            .setConditions(Conditions.newBuilder()
                .addConditions(condition)
                .build())
            .build())
        .build();

    // Associate the de-identification and conditions with the specified field.
    FieldTransformation fieldTransformation =
        FieldTransformation.newBuilder()
            .setPrimitiveTransformation(primitiveTransformation)
            .addFields(fieldId)
            .setCondition(recordCondition)
            .build();
    RecordTransformations transformations =
        RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build();

    DeidentifyConfig deidentifyConfig =
        DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build();

    // Combine configurations into a request for the service.
    DeidentifyContentRequest request =
        DeidentifyContentRequest.newBuilder()
            .setParent(LocationName.of(projectId, "global").toString())
            .setItem(contentItem)
            .setDeidentifyConfig(deidentifyConfig)
            .build();

    // Send the request and receive response from the service.
    DeidentifyContentResponse response = dlp.deidentifyContent(request);

    // Print the results.
    System.out.println(
        "Table after de-identification: " + response.getItem().getTable());

    return response.getItem().getTable();
  }
}
 
Example #30
Source File: DeIdentifyTableConditionInfoTypes.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
public static Table deIdentifyTableConditionInfoTypes(String projectId, Table tableToDeIdentify)
    throws IOException {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (DlpServiceClient dlp = DlpServiceClient.create()) {
    // Specify what content you want the service to de-identify.
    ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build();

    // Specify how the content should be de-identified.
    // Select type of info to be replaced.
    InfoType infoType = InfoType.newBuilder().setName("PERSON_NAME").build();
    // Specify that findings should be replaced with corresponding info type name.
    ReplaceWithInfoTypeConfig replaceWithInfoTypeConfig =
        ReplaceWithInfoTypeConfig.getDefaultInstance();
    PrimitiveTransformation primitiveTransformation = PrimitiveTransformation.newBuilder()
        .setReplaceWithInfoTypeConfig(replaceWithInfoTypeConfig).build();
    // Associate info type with the replacement strategy
    InfoTypeTransformation infoTypeTransformation =
        InfoTypeTransformation.newBuilder()
            .addInfoTypes(infoType)
            .setPrimitiveTransformation(primitiveTransformation)
            .build();
    InfoTypeTransformations infoTypeTransformations =
        InfoTypeTransformations.newBuilder()
            .addTransformations(infoTypeTransformation)
            .build();

    // Specify fields to be de-identified.
    List<FieldId> fieldIds = Stream.of("PATIENT", "FACTOID")
        .map(id -> FieldId.newBuilder().setName(id).build())
        .collect(Collectors.toList());

    // Specify when the above fields should be de-identified.
    Condition condition = Condition.newBuilder()
        .setField(FieldId.newBuilder().setName("AGE").build())
        .setOperator(RelationalOperator.GREATER_THAN)
        .setValue(Value.newBuilder().setIntegerValue(89).build())
        .build();
    // Apply the condition to records
    RecordCondition recordCondition = RecordCondition.newBuilder()
        .setExpressions(Expressions.newBuilder()
            .setConditions(Conditions.newBuilder()
                .addConditions(condition)
                .build())
            .build())
        .build();

    // Associate the de-identification and conditions with the specified fields.
    FieldTransformation fieldTransformation =
        FieldTransformation.newBuilder()
            .setInfoTypeTransformations(infoTypeTransformations)
            .addAllFields(fieldIds)
            .setCondition(recordCondition)
            .build();
    RecordTransformations transformations =
        RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build();

    DeidentifyConfig deidentifyConfig =
        DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build();

    // Combine configurations into a request for the service.
    DeidentifyContentRequest request =
        DeidentifyContentRequest.newBuilder()
            .setParent(LocationName.of(projectId, "global").toString())
            .setItem(contentItem)
            .setDeidentifyConfig(deidentifyConfig)
            .build();

    // Send the request and receive response from the service.
    DeidentifyContentResponse response = dlp.deidentifyContent(request);

    // Print the results.
    System.out.println(
        "Table after de-identification: " + response.getItem().getTable());

    return response.getItem().getTable();
  }
}