com.microsoft.azure.storage.table.TableOperation Java Examples

@Override
public PortabilityJob findJob(UUID jobId) {
  Preconditions.checkNotNull(jobId, "Job id is null");
  try {

    CloudTable table = tableClient.getTableReference(JOB_TABLE);

    TableOperation retrieve =
        TableOperation.retrieve(
            configuration.getPartitionKey(), jobId.toString(), DataWrapper.class);
    TableResult result = table.execute(retrieve);
    DataWrapper wrapper = result.getResultAsType();
    return configuration.getMapper().readValue(wrapper.getSerialized(), PortabilityJob.class);
  } catch (StorageException | URISyntaxException | IOException e) {
    throw new MicrosoftStorageException("Error finding job: " + jobId, e);
  }
}
 

private void create(String rowKey, String tableName, String state, Object type)
    throws IOException {
  try {

    CloudTable table = tableClient.getTableReference(tableName);

    String serializedJob = configuration.getMapper().writeValueAsString(type);
    DataWrapper wrapper =
        new DataWrapper(
            configuration.getPartitionKey(), rowKey, state, serializedJob); // job id used as key
    TableOperation insert = TableOperation.insert(wrapper);
    table.execute(insert);
  } catch (JsonProcessingException | StorageException | URISyntaxException e) {
    throw new IOException("Error creating data for rowKey: " + rowKey, e);
  }
}
 

private void remove(UUID jobId, String tableName) throws IOException {
  try {

    CloudTable table = tableClient.getTableReference(tableName);
    TableOperation retrieve =
        TableOperation.retrieve(
            configuration.getPartitionKey(), jobId.toString(), DataWrapper.class);
    TableResult result = table.execute(retrieve);
    DataWrapper wrapper = result.getResultAsType();

    TableOperation delete = TableOperation.delete(wrapper);
    table.execute(delete);

  } catch (StorageException | URISyntaxException e) {
    throw new IOException("Error removing data for job: " + jobId, e);
  }
}
 

/**
 * Deletes a specified row in the processor table.
 *
 * Note: Table service uses optimistic locking by default. In order to disable it, set the ETag on the ProcessorEntity
 * to "*" before invoking this method.
 *
 * @param entity ProcessorEntity that has to be deleted
 * @throws AzureException If an Azure storage service error occurred.
 */
public void deleteProcessorEntity(ProcessorEntity entity) {
  try {
    TableOperation remove = TableOperation.delete(entity);
    table.execute(remove);
  } catch (StorageException e) {
    LOG.error("Azure storage exception while deleting processor entity with job model version: " +
        entity.getJobModelVersion() + "and pid: " + entity.getProcessorId(), e);
    throw new AzureException(e);
  }
}
 

@Test
@Category({ DevFabricTests.class, DevStoreTests.class, SecondaryTests.class })
public void testTableMaximumExecutionTime() throws URISyntaxException, StorageException {
    OperationContext opContext = new OperationContext();
    setDelay(opContext, 2500);
    
    opContext.getResponseReceivedEventHandler().addListener(new StorageEvent<ResponseReceivedEvent>() {

        @Override
        public void eventOccurred(ResponseReceivedEvent eventArg) {
            // Set status code to 500 to force a retry
            eventArg.getRequestResult().setStatusCode(500);
        }
    });

    // set the maximum execution time
    TableRequestOptions options = new TableRequestOptions();
    options.setMaximumExecutionTimeInMs(2000);
    options.setTimeoutIntervalInMs(1000);

    CloudTableClient tableClient = TestHelper.createCloudTableClient();
    CloudTable table = tableClient.getTableReference(generateRandomName("share"));

    try {
        // 1. insert entity will fail as the table does not exist
        // 2. the executor will attempt to retry as we set the status code to 500
        // 3. maximum execution time should prevent the retry from being made
        DynamicTableEntity ent = new DynamicTableEntity("partition", "row");
        TableOperation insert = TableOperation.insert(ent);
        table.execute(insert, options, opContext);
        fail("Maximum execution time was reached but request did not fail.");
    }
    catch (StorageException e) {
        assertEquals(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION, e.getMessage());
    }
}
 

@Test(expected = ComponentException.class)
public void testWriteToUnavailableSinkDieOnError() {

    properties.dieOnError.setValue(true);

    assertEquals(ValidationResult.Result.OK, sink.initialize(container, properties).getStatus());
    assertEquals(ValidationResult.Result.OK, sink.validate(container).getStatus());
    WriteOperation<?> writeOperation = sink.createWriteOperation();
    writeOperation.initialize(container);
    writer = (AzureStorageTableWriter) writeOperation.createWriter(container);

    // mock
    writer.tableservice = tableService;
    try {
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                return null;
            }
        }).when(tableService).handleActionOnTable(anyString(), any(ActionOnTable.class));
        when(tableService.executeOperation(anyString(), any(TableOperation.class)))
                .thenThrow(new StorageException("500", "insertion problem", new RuntimeException()));

        // assert
        writer.open(RandomStringUtils.random(12));
        for (int i = 0; i < 2; i++) {
            writer.write(TableHelper.getRecord(i));
        }
        writer.close();

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }

}
 

@Test
public void testWriteToUnavailableSinkHandleError() {

    assertEquals(ValidationResult.Result.OK, sink.initialize(container, properties).getStatus());
    assertEquals(ValidationResult.Result.OK, sink.validate(container).getStatus());
    WriteOperation<?> writeOperation = sink.createWriteOperation();
    writeOperation.initialize(container);
    writer = (AzureStorageTableWriter) writeOperation.createWriter(container);

    // mock
    writer.tableservice = tableService;
    try {
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                return null;
            }
        }).when(tableService).handleActionOnTable(anyString(), any(ActionOnTable.class));
        when(tableService.executeOperation(anyString(), any(TableOperation.class)))
                .thenThrow(new StorageException("500", "insertion problem", new RuntimeException()));

        // assert
        writer.open(RandomStringUtils.random(12));
        for (int i = 0; i < 2; i++) {
            writer.write(TableHelper.getRecord(i));
        }
        writer.close();

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }

}
 

@Test
public void testWriteRecordWithDynamicSchemaToAvailableSink() {
    // setup
    properties.schema.schema.setValue(TableHelper.getDynamicWriteSchema());

    assertEquals(ValidationResult.Result.OK, sink.initialize(container, properties).getStatus());
    assertEquals(ValidationResult.Result.OK, sink.validate(container).getStatus());
    WriteOperation<?> writeOperation = sink.createWriteOperation();
    writeOperation.initialize(container);
    writer = (AzureStorageTableWriter) writeOperation.createWriter(container);

    // mock
    writer.tableservice = tableService;
    try {
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                return null;
            }
        }).when(tableService).handleActionOnTable(anyString(), any(ActionOnTable.class));
        when(tableService.executeOperation(anyString(), any(TableOperation.class))).thenReturn(new TableResult(200));

        // assert
        writer.open(RandomStringUtils.random(12));
        writer.write(TableHelper.getRecord(0));
        writer.close();

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testWriteValidRecordsToAvailableSink() {
    // setup
    assertEquals(ValidationResult.Result.OK, sink.initialize(container, properties).getStatus());
    assertEquals(ValidationResult.Result.OK, sink.validate(container).getStatus());
    WriteOperation<?> writeOperation = sink.createWriteOperation();
    writeOperation.initialize(container);
    writer = (AzureStorageTableWriter) writeOperation.createWriter(container);

    // mock
    writer.tableservice = tableService;
    try {
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                return null;
            }
        }).when(tableService).handleActionOnTable(Mockito.<String>any(), any(ActionOnTable.class));
        when(tableService.executeOperation(Mockito.<String>any(), any(TableOperation.class))).thenReturn(new TableResult(200));

        // assert
        writer.open(RandomStringUtils.random(12));
        for (int i = 0; i < 500; i++) {
            writer.write(TableHelper.getRecord(i));
        }
        writer.close();

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

private void addOperationToBatch(DynamicTableEntity entity, IndexedRecord record) throws IOException {
    if (latestPartitionKey == null || latestPartitionKey.isEmpty()) {
        latestPartitionKey = entity.getPartitionKey();
    }
    // we reached the threshold for batch OR changed PartitionKey
    if (batchOperationsCount == 100 || !entity.getPartitionKey().equals(latestPartitionKey)) {
        processBatch();
        latestPartitionKey = entity.getPartitionKey();
    }
    TableOperation to = getTableOperation(entity);
    batchOperations.add(to);
    batchRecords.add(record);
    batchOperationsCount++;
    latestPartitionKey = entity.getPartitionKey();
}
 

private TableOperation getTableOperation(DynamicTableEntity entity) {
    TableOperation tableOpe = null;
    switch (actionData) {
    case Insert:
        tableOpe = TableOperation.insert(entity);
        break;
    case Insert_Or_Merge:
        tableOpe = TableOperation.insertOrMerge(entity);
        break;
    case Insert_Or_Replace:
        tableOpe = TableOperation.insertOrReplace(entity);
        break;
    case Merge:
        tableOpe = TableOperation.merge(entity);
        break;
    case Replace:
        tableOpe = TableOperation.replace(entity);
        break;
    case Delete:
        tableOpe = TableOperation.delete(entity);
        break;
    default:
        LOGGER.error("No specified operation for table");
    }

    return tableOpe;
}
 

@Override
protected void updateJob(UUID jobId, PortabilityJob job, JobUpdateValidator validator)
    throws IOException {

  Preconditions.checkNotNull(jobId, "Job is null");

  Preconditions.checkNotNull(job, "Job is null");

  try {

    CloudTable table = tableClient.getTableReference(JOB_TABLE);

    String serializedJob = configuration.getMapper().writeValueAsString(job);
    DataWrapper wrapper =
        new DataWrapper(
            configuration.getPartitionKey(),
            jobId.toString(),
            job.jobAuthorization().state().name(),
            serializedJob);

    if (validator != null) {
      PortabilityJob previousJob = findJob(jobId);
      if (previousJob == null) {
        throw new IOException("Could not find record for jobId: " + jobId);
      }

      validator.validate(previousJob, job);
    }

    TableOperation insert = TableOperation.insertOrReplace(wrapper);
    table.execute(insert);

  } catch (JsonProcessingException | StorageException | URISyntaxException e) {
    throw new IOException("Error updating job: " + jobId, e);
  }
}
 

/**
 * Deletes a specified row in the processor table.
 *
 * Note: Table service uses optimistic locking by default. Hence, if there is an update after retrieving the entity,
 * then the delete operation will fail.
 *
 * @param jmVersion Job model version of the processor row to be deleted.
 * @param pid Unique processor ID of the processor row to be deleted.
 * @param force True, to disable optimistic locking on the table. False, otherwise. Setting to false may result in
 *              AzureException when there is concurrent access to the table.
 *
 * @throws AzureException If an Azure storage service error occurred.
 */
public void deleteProcessorEntity(String jmVersion, String pid, boolean force) {
  try {
    TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
    ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType();
    if (force) {
      entity.setEtag("*");
    }
    TableOperation remove = TableOperation.delete(entity);
    table.execute(remove);
  } catch (StorageException e) {
    LOG.error("Azure storage exception while deleting processor entity with job model version: " + jmVersion + "and pid: " + pid, e);
    throw new AzureException(e);
  }
}
 

/**
 * Updates the isLeader value when the processor starts or stops being a leader.
 * @param jmVersion Job model version of the processor row to be updated.
 * @param pid Unique processor ID of the processor row to be updated.
 * @param isLeader Denotes whether the processor is a leader or not.
 * @throws AzureException If an Azure storage service error occurred.
 */
public void updateIsLeader(String jmVersion, String pid, boolean isLeader) {
  try {
    TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
    ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType();
    entity.setIsLeader(isLeader);
    TableOperation update = TableOperation.replace(entity);
    table.execute(update);
  } catch (StorageException e) {
    LOG.error("Azure storage exception while updating isLeader value for job model version: " + jmVersion + "and pid: " + pid, e);
    throw new AzureException(e);
  }
}
 

/**
 * Updates the liveness value of a particular processor with a randomly generated integer, which in turn updates the last modified since timestamp of the row.
 * @param jmVersion Job model version of the processor row to be updated.
 * @param pid Unique processor ID of the processor row to be updated.
 */
public void updateHeartbeat(String jmVersion, String pid) {
  try {
    TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
    ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType();
    entity.updateLiveness();
    TableOperation update = TableOperation.replace(entity);
    table.execute(update);
  } catch (StorageException e) {
    LOG.error("Azure storage exception while updating heartbeat for job model version: " + jmVersion + "and pid: " + pid, e);
  }
}
 

/**
 * Retrieve a particular row in the processor table, given the partition key and the row key.
 * @param jmVersion Job model version of the processor row to be retrieved.
 * @param pid Unique processor ID of the processor row to be retrieved.
 * @return An instance of required processor entity. Null if does not exist.
 * @throws AzureException If an Azure storage service error occurred.
 */
public ProcessorEntity getEntity(String jmVersion, String pid) {
  try {
    TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
    return table.execute(retrieveEntity).getResultAsType();
  } catch (StorageException e) {
    LOG.error("Azure storage exception while retrieving processor entity with job model version: " + jmVersion + "and pid: " + pid, e);
    throw new AzureException(e);
  }
}
 

/**
 * Add a row which denotes an active processor to the processor table.
 * @param jmVersion Job model version that the processor is operating on.
 * @param pid Unique processor ID.
 * @param isLeader Denotes whether the processor is a leader or not.
 * @throws AzureException If an Azure storage service error occurred.
 */
public void addProcessorEntity(String jmVersion, String pid, boolean isLeader) {
  ProcessorEntity entity = new ProcessorEntity(jmVersion, pid);
  entity.setIsLeader(isLeader);
  entity.updateLiveness();
  TableOperation add = TableOperation.insert(entity);
  try {
    table.execute(add);
  } catch (StorageException e) {
    LOG.error("Azure storage exception while adding processor entity with job model version: " + jmVersion + "and pid: " + pid, e);
    throw new AzureException(e);
  }
}
 

@Override
public List<IngestionStatus> getIngestionStatusCollection() throws StorageException, URISyntaxException {
    List<IngestionStatus> results = new LinkedList<>();
    for (IngestionStatusInTableDescription descriptor : descriptors) {
        CloudTable table = new CloudTable(new URI(descriptor.TableConnectionString));
        TableOperation operation = TableOperation.retrieve(descriptor.PartitionKey, descriptor.RowKey,
                IngestionStatus.class);
        results.add(table.execute(operation).getResultAsType());
    }

    return results;
}
 

void azureTableInsertEntity(String tableUri, TableServiceEntity entity) throws StorageException,
        URISyntaxException {
    // Ensure
    Ensure.stringIsNotBlank(tableUri, "tableUri");
    Ensure.argIsNotNull(entity, "entity");

    CloudTable table = new CloudTable(new URI(tableUri));
    // Create an operation to add the new customer to the table basics table.
    TableOperation insert = TableOperation.insert(entity);
    // Submit the operation to the table service.
    table.execute(insert);
}
 

private <T> T find(Class<T> type, String rowKey, String tableName) {
  try {

    CloudTable table = tableClient.getTableReference(tableName);
    TableOperation retrieve =
        TableOperation.retrieve(configuration.getPartitionKey(), rowKey, DataWrapper.class);
    TableResult result = table.execute(retrieve);
    DataWrapper wrapper = result.getResultAsType();
    return configuration.getMapper().readValue(wrapper.getSerialized(), type);
  } catch (StorageException | IOException | URISyntaxException e) {
    throw new MicrosoftStorageException("Error finding data for rowKey: " + rowKey, e);
  }
}
 

public TableResult executeOperation(String tableName, TableOperation ope)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudTable cloudTable = connection.getCloudStorageAccount().createCloudTableClient().getTableReference(tableName);
    return cloudTable.execute(ope, null, AzureStorageUtils.getTalendOperationContext());
}
 

protected Optional<DynamicTableEntity> retrieve(String partitionKey, String rowKey) throws StorageException {
  DynamicTableEntity data = table
    .execute(TableOperation.retrieve(partitionKey, rowKey, DynamicTableEntity.class))
    .getResultAsType();
  return Optional.ofNullable(data);
}
 

protected void create(String partitionKey, String rowKey, Map<String, EntityProperty> properties)
  throws StorageException {
  table.execute(TableOperation.insert(new DynamicTableEntity(partitionKey, rowKey, new HashMap<>(properties))));
}
 

protected void delete(String partitionKey, String rowKey) throws StorageException {
  table.execute(TableOperation.delete(new DynamicTableEntity(partitionKey, rowKey)));
}