com.google.cloud.bigquery.Job Java Examples

The following examples show how to use com.google.cloud.bigquery.Job. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ITBigQuerySnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
@Test
public void testJob() throws ExecutionException, InterruptedException {
  Job job1 = bigquerySnippets.createJob(QUERY);
  Job job2 = bigquerySnippets.createJob(QUERY);
  assertNotNull(job1);
  assertNotNull(job2);
  assertEquals(job1.getJobId(), bigquerySnippets.getJob(job1.getJobId().getJob()).getJobId());
  assertEquals(
      job2.getJobId(), bigquerySnippets.getJobFromId(job2.getJobId().getJob()).getJobId());
  Set<JobId> jobs =
      Sets.newHashSet(
          Iterators.transform(
              bigquerySnippets.listJobs().iterateAll().iterator(), TO_JOB_ID_FUNCTION));
  while (!jobs.contains(job1.getJobId()) || !jobs.contains(job2.getJobId())) {
    Thread.sleep(500);
    jobs =
        Sets.newHashSet(
            Iterators.transform(
                bigquerySnippets.listJobs().iterateAll().iterator(), TO_JOB_ID_FUNCTION));
  }
  assertTrue(bigquerySnippets.cancelJob(job1.getJobId().getJob()));
  assertTrue(bigquerySnippets.cancelJobFromId(job2.getJobId().getJob()));
}
 
Example #2
Source File: BigQueryTemplate.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
private SettableListenableFuture<Job> createJobFuture(Job pendingJob) {
	// Prepare the polling task for the ListenableFuture result returned to end-user
	SettableListenableFuture<Job> result = new SettableListenableFuture<>();

	ScheduledFuture<?> scheduledFuture = taskScheduler.scheduleAtFixedRate(() -> {
		Job job = pendingJob.reload();
		if (State.DONE.equals(job.getStatus().getState())) {
			if (job.getStatus().getError() != null) {
				result.setException(
						new BigQueryException(job.getStatus().getError().getMessage()));
			}
			else {
				result.set(job);
			}
		}
	}, this.jobPollInterval);

	result.addCallback(
			response -> scheduledFuture.cancel(true),
			response -> {
				pendingJob.cancel();
				scheduledFuture.cancel(true);
			});

	return result;
}
 
Example #3
Source File: BigQueryFileMessageHandlerIntegrationTests.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadFile_sync() throws InterruptedException {
	this.messageHandler.setSync(true);

	HashMap<String, Object> messageHeaders = new HashMap<>();
	messageHeaders.put(BigQuerySpringMessageHeaders.TABLE_NAME, TABLE_NAME);
	messageHeaders.put(BigQuerySpringMessageHeaders.FORMAT_OPTIONS, FormatOptions.csv());

	Message<File> message = MessageBuilder.createMessage(
			new File("src/test/resources/data.csv"),
			new MessageHeaders(messageHeaders));

	Job job = (Job) this.messageHandler.handleRequestMessage(message);
	assertThat(job).isNotNull();

	QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration
			.newBuilder("SELECT * FROM test_dataset.test_table").build();
	TableResult result = this.bigquery.query(queryJobConfiguration);
	assertThat(result.getTotalRows()).isEqualTo(1);
}
 
Example #4
Source File: BigQueryFileMessageHandlerIntegrationTests.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadFile_cancel() {
	HashMap<String, Object> messageHeaders = new HashMap<>();
	messageHeaders.put(BigQuerySpringMessageHeaders.TABLE_NAME, TABLE_NAME);
	messageHeaders.put(BigQuerySpringMessageHeaders.FORMAT_OPTIONS, FormatOptions.csv());

	Message<File> message = MessageBuilder.createMessage(
			new File("src/test/resources/data.csv"),
			new MessageHeaders(messageHeaders));

	ListenableFuture<Job> jobFuture =
			(ListenableFuture<Job>) this.messageHandler.handleRequestMessage(message);
	jobFuture.cancel(true);

	await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
		// This asserts that the BigQuery job polling task is no longer in the scheduler after cancel.
		assertThat(this.taskScheduler.getScheduledThreadPoolExecutor().getQueue()).hasSize(0);
	});
}
 
Example #5
Source File: BigQueryTemplateIntegrationTests.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadFile() throws IOException, ExecutionException, InterruptedException {
	ListenableFuture<Job> bigQueryJobFuture =
			bigQueryTemplate.writeDataToTable(TABLE_NAME, dataFile.getInputStream(), FormatOptions.csv());

	Job job = bigQueryJobFuture.get();
	assertThat(job.getStatus().getState()).isEqualTo(JobStatus.State.DONE);

	QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration
			.newBuilder("SELECT * FROM test_dataset.template_test_table").build();
	TableResult result = this.bigQuery.query(queryJobConfiguration);

	assertThat(result.getTotalRows()).isEqualTo(1);
	assertThat(
			result.getValues().iterator().next().get("State").getStringValue()).isEqualTo("Alabama");
}
 
Example #6
Source File: BigQueryTemplateIntegrationTests.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadBytes() throws ExecutionException, InterruptedException {
	byte[] byteArray =
			"CountyId,State,County\n1001,Alabama,Autauga County\n".getBytes();
	ByteArrayInputStream byteStream = new ByteArrayInputStream(byteArray);

	ListenableFuture<Job> bigQueryJobFuture =
			bigQueryTemplate.writeDataToTable(TABLE_NAME, byteStream, FormatOptions.csv());

	Job job = bigQueryJobFuture.get();
	assertThat(job.getStatus().getState()).isEqualTo(JobStatus.State.DONE);

	QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration
			.newBuilder("SELECT * FROM test_dataset.template_test_table").build();
	TableResult result = this.bigQuery.query(queryJobConfiguration);

	assertThat(result.getTotalRows()).isEqualTo(1);
	assertThat(
			result.getValues().iterator().next().get("State").getStringValue()).isEqualTo("Alabama");
}
 
Example #7
Source File: TestBigQueryDelegate.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void runQuery() throws Exception {
  QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder("SELECT * FROM [sample:table] LIMIT 1000")
      .setUseQueryCache(true)
      .setUseLegacySql(useLegacySql)
      .build();

  TableResult mockQueryResponse = mock(TableResult.class);
  Job mockJob = mock(Job.class);
  JobStatus mockJobStatus = mock(JobStatus.class);

  // First pretend we haven't finished running the query, second time around its completed.
  when(mockJob.isDone()).thenReturn(false).thenReturn(true);
  when(mockJob.getJobId()).thenReturn(jobId);
  when(mockJobStatus.getError()).thenReturn(null);
  when(mockJob.getStatus()).thenReturn(mockJobStatus);

  when(mockBigquery.create((JobInfo)any())).thenReturn(mockJob);
  when(mockBigquery.cancel(jobId)).thenReturn(true);
  when(mockJob.getQueryResults()).thenReturn(mockQueryResponse);

  BigQueryDelegate delegate = new BigQueryDelegate(mockBigquery, useLegacySql);
  delegate.runQuery(queryConfig, 1000, 1000);
}
 
Example #8
Source File: CreateTableAndLoadData.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
public static void main(String... args) throws InterruptedException, TimeoutException {
  BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
  TableId tableId = TableId.of("dataset", "table");
  Table table = bigquery.getTable(tableId);
  if (table == null) {
    System.out.println("Creating table " + tableId);
    Field integerField = Field.of("fieldName", LegacySQLTypeName.INTEGER);
    Schema schema = Schema.of(integerField);
    table = bigquery.create(TableInfo.of(tableId, StandardTableDefinition.of(schema)));
  }
  System.out.println("Loading data into table " + tableId);
  Job loadJob = table.load(FormatOptions.csv(), "gs://bucket/path");
  loadJob = loadJob.waitFor();
  if (loadJob.getStatus().getError() != null) {
    System.out.println("Job completed with errors");
  } else {
    System.out.println("Job succeeded");
  }
}
 
Example #9
Source File: JobSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example usage of {@code waitFor()}. */
// [TARGET waitFor(RetryOption...)]
public boolean waitFor() throws InterruptedException {
  try {
    // [START ]
    Job completedJob = job.waitFor();
    if (completedJob == null) {
      // job no longer exists
    } else if (completedJob.getStatus().getError() != null) {
      // job failed, handle error
    } else {
      // job completed successfully
    }
    // [END ]
  } catch (BigQueryException e) {
    // Timeouts shouldn't happen without a timeout option.
    if (e.getCause() instanceof PollException) {
      return false;
    }
    throw e;
  }
  return true;
}
 
Example #10
Source File: JobSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example usage of {@code waitFor()} with checking period and timeout. */
// [TARGET waitFor(RetryOption...)]
public boolean waitForWithOptions() throws InterruptedException {
  try {
    // [START ]
    Job completedJob =
        job.waitFor(
            RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
            RetryOption.totalTimeout(Duration.ofMinutes(1)));
    if (completedJob == null) {
      // job no longer exists
    } else if (completedJob.getStatus().getError() != null) {
      // job failed, handle error
    } else {
      // job completed successfully
    }
    // [END ]
  } catch (BigQueryException e) {
    if (e.getCause() instanceof PollException) {
      return false;
    }
    throw e;
  }
  return true;
}
 
Example #11
Source File: TableSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example copying the table to a destination table. */
// [TARGET copy(TableId, JobOption...)]
// [VARIABLE "my_dataset"]
// [VARIABLE "my_destination_table"]
public Job copyTableId(String dataset, String tableName) throws BigQueryException {
  // [START bigquery_copy_table]
  TableId destinationId = TableId.of(dataset, tableName);
  JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL);
  Job job = table.copy(destinationId, options);
  // Wait for the job to complete.
  try {
    Job completedJob =
        job.waitFor(
            RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
            RetryOption.totalTimeout(Duration.ofMinutes(3)));
    if (completedJob != null && completedJob.getStatus().getError() == null) {
      // Job completed successfully.
    } else {
      // Handle error case.
    }
  } catch (InterruptedException e) {
    // Handle interrupted wait
  }
  // [END bigquery_copy_table]
  return job;
}
 
Example #12
Source File: ReadSessionCreator.java    From presto with Apache License 2.0 6 votes vote down vote up
TableInfo createTableFromQuery()
{
    TableId destinationTable = bigQueryClient.createDestinationTable(table);
    log.debug("destinationTable is %s", destinationTable);
    JobInfo jobInfo = JobInfo.of(
            QueryJobConfiguration
                    .newBuilder(query)
                    .setDestinationTable(destinationTable)
                    .build());
    log.debug("running query %s", jobInfo);
    Job job = waitForJob(bigQueryClient.create(jobInfo));
    log.debug("job has finished. %s", job);
    if (job.getStatus().getError() != null) {
        throw convertToBigQueryException(job.getStatus().getError());
    }
    // add expiration time to the table
    TableInfo createdTable = bigQueryClient.getTable(destinationTable);
    long expirationTime = createdTable.getCreationTime() +
            TimeUnit.HOURS.toMillis(config.viewExpirationTimeInHours);
    Table updatedTable = bigQueryClient.update(createdTable.toBuilder()
            .setExpirationTime(expirationTime)
            .build());
    return updatedTable;
}
 
Example #13
Source File: BigQueryExample.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
@Override
void run(BigQuery bigquery, JobInfo job) throws Exception {
  System.out.println("Creating job");
  Job startedJob = bigquery.create(job);
  while (!startedJob.isDone()) {
    System.out.println("Waiting for job " + startedJob.getJobId().getJob() + " to complete");
    Thread.sleep(1000L);
  }
  startedJob = startedJob.reload();
  if (startedJob.getStatus().getError() == null) {
    System.out.println("Job " + startedJob.getJobId().getJob() + " succeeded");
  } else {
    System.out.println("Job " + startedJob.getJobId().getJob() + " failed");
    System.out.println("Error: " + startedJob.getStatus().getError());
  }
}
 
Example #14
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of creating a query job. */
// [TARGET create(JobInfo, JobOption...)]
// [VARIABLE "SELECT field FROM my_dataset_name.my_table_name"]
public Job createJob(String query) {
  // [START ]
  Job job = null;
  JobConfiguration jobConfiguration = QueryJobConfiguration.of(query);
  JobInfo jobInfo = JobInfo.of(jobConfiguration);
  try {
    job = bigquery.create(jobInfo);
  } catch (BigQueryException e) {
    // the job was not created
  }
  // [END ]
  return job;
}
 
Example #15
Source File: CloudSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of copying multiple tables to a destination. */
public void copyTables(String datasetId, String destinationTableId) throws InterruptedException {
  generateTableWithDdl(datasetId, "table1");
  generateTableWithDdl(datasetId, "table2");

  // [START bigquery_copy_table_multiple_source]
  TableId destinationTable = TableId.of(datasetId, destinationTableId);
  CopyJobConfiguration configuration =
      CopyJobConfiguration.newBuilder(
              destinationTable,
              Arrays.asList(TableId.of(datasetId, "table1"), TableId.of(datasetId, "table2")))
          .build();

  // Copy the tables.
  Job job = bigquery.create(JobInfo.of(configuration));
  job = job.waitFor();

  // Check the table
  StandardTableDefinition table = bigquery.getTable(destinationTable).getDefinition();
  System.out.println("State: " + job.getStatus().getState());
  System.out.printf("Copied %d rows.\n", table.getNumRows());
  // [END bigquery_copy_table_multiple_source]
}
 
Example #16
Source File: CloudSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of loading a parquet file from GCS to a table. */
public void loadTableGcsParquet(String datasetName) throws InterruptedException {
  // [START bigquery_load_table_gcs_parquet]
  String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet";
  TableId tableId = TableId.of(datasetName, "us_states");
  LoadJobConfiguration configuration =
      LoadJobConfiguration.builder(tableId, sourceUri)
          .setFormatOptions(FormatOptions.parquet())
          .build();
  // Load the table
  Job loadJob = bigquery.create(JobInfo.of(configuration));
  loadJob = loadJob.waitFor();
  // Check the table
  StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition();
  System.out.println("State: " + loadJob.getStatus().getState());
  System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows());
  // [END bigquery_load_table_gcs_parquet]
}
 
Example #17
Source File: TableSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example loading data from a single Google Cloud Storage file. */
// [TARGET load(FormatOptions, String, JobOption...)]
// [VARIABLE "gs://my_bucket/filename.csv"]
public Job loadSingle(String sourceUri) {
  // [START bigquery_load_table_gcs_csv]
  Job job = table.load(FormatOptions.csv(), sourceUri);
  // Wait for the job to complete
  try {
    Job completedJob =
        job.waitFor(
            RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
            RetryOption.totalTimeout(Duration.ofMinutes(3)));
    if (completedJob != null && completedJob.getStatus().getError() == null) {
      // Job completed successfully
    } else {
      // Handle error case
    }
  } catch (InterruptedException e) {
    // Handle interrupted wait
  }
  // [END bigquery_load_table_gcs_csv]
  return job;
}
 
Example #18
Source File: TableSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example extracting data to single Google Cloud Storage file. */
// [TARGET extract(String, String, JobOption...)]
// [VARIABLE "CSV"]
// [VARIABLE "gs://my_bucket/filename.csv"]
public Job extractSingle(String format, String gcsUrl) {
  // [START bigquery_extract_table]
  Job job = table.extract(format, gcsUrl);
  // Wait for the job to complete
  try {
    Job completedJob =
        job.waitFor(
            RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
            RetryOption.totalTimeout(Duration.ofMinutes(3)));
    if (completedJob != null && completedJob.getStatus().getError() == null) {
      // Job completed successfully
    } else {
      // Handle error case
    }
  } catch (InterruptedException e) {
    // Handle interrupted wait
  }
  // [END bigquery_extract_table]
  return job;
}
 
Example #19
Source File: TableSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of copying the table to a destination table. */
// [TARGET copy(String, String, JobOption...)]
// [VARIABLE "my_dataset"]
// [VARIABLE "my_destination_table"]
public Job copy(String datasetName, String tableName) {
  // [START ]
  Job job = table.copy(datasetName, tableName);
  // Wait for the job to complete.
  try {
    Job completedJob =
        job.waitFor(
            RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
            RetryOption.totalTimeout(Duration.ofMinutes(3)));
    if (completedJob != null && completedJob.getStatus().getError() == null) {
      // Job completed successfully
    } else {
      // Handle error case
    }
  } catch (InterruptedException e) {
    // Handle interrupted wait
  }
  // [END ]
  return job;
}
 
Example #20
Source File: ITTableSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
@Test
public void testExtractAndLoadList() {
  String gcsFile1 = "gs://" + BUCKET_NAME + "/extractTestA_*.csv";
  String gcsFile2 = "gs://" + BUCKET_NAME + "/extractTestB_*.csv";
  Job extractJob = tableSnippets.extractList("CSV", gcsFile1, gcsFile2);
  gcsFile1 = gcsFile1.replace("*", "000000000000");
  gcsFile2 = gcsFile2.replace("*", "000000000000");
  assertSuccessful(extractJob);
  Job loadJob = tableSnippets.loadList(gcsFile1, gcsFile2);
  assertSuccessful(loadJob);
}
 
Example #21
Source File: BigQueryStatementIssuingFn.java    From DataflowTemplates with Apache License 2.0 5 votes vote down vote up
private Job issueQueryToBQ(String statement) throws InterruptedException {
  QueryJobConfiguration jobConfiguration = QueryJobConfiguration.newBuilder(statement)
      .build();

  String jobId = makeJobId(jobIdPrefix, statement);

  LOG.info("Triggering job {} for statement |{}|", jobId, statement);

  TableResult result = bigQueryClient.query(jobConfiguration, JobId.of(jobId));
  return bigQueryClient.getJob(JobId.of(jobId));
}
 
Example #22
Source File: CloudSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
private void generateTableWithDdl(String datasetId, String tableId) throws InterruptedException {
  String sql =
      String.format(
          "CREATE TABLE %s.%s "
              + "AS "
              + "SELECT "
              + "2000 + CAST(18 * RAND() as INT64) AS year, "
              + "IF(RAND() > 0.5,\"foo\",\"bar\") AS token "
              + "FROM "
              + "UNNEST(GENERATE_ARRAY(0,5,1)) AS r",
          datasetId, tableId);
  Job job = bigquery.create(JobInfo.of(QueryJobConfiguration.newBuilder(sql).build()));
  job.waitFor();
}
 
Example #23
Source File: BigQueryFileMessageHandlerIntegrationTests.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Test
public void testLoadFile() throws InterruptedException, ExecutionException {
	HashMap<String, Object> messageHeaders = new HashMap<>();
	this.messageHandler.setTableName(TABLE_NAME);
	this.messageHandler.setFormatOptions(FormatOptions.csv());

	Message<File> message = MessageBuilder.createMessage(
			new File("src/test/resources/data.csv"),
			new MessageHeaders(messageHeaders));

	ListenableFuture<Job> jobFuture =
			(ListenableFuture<Job>) this.messageHandler.handleRequestMessage(message);

	// Assert that a BigQuery polling task is scheduled successfully.
	await().atMost(Duration.FIVE_SECONDS)
			.untilAsserted(
					() -> assertThat(
							this.taskScheduler.getScheduledThreadPoolExecutor().getQueue()).hasSize(1));
	jobFuture.get();

	QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration
			.newBuilder("SELECT * FROM test_dataset.test_table").build();
	TableResult result = this.bigquery.query(queryJobConfiguration);

	assertThat(result.getTotalRows()).isEqualTo(1);
	assertThat(
			result.getValues().iterator().next().get("State").getStringValue()).isEqualTo("Alabama");

	// This asserts that the BigQuery job polling task is no longer in the scheduler.
	assertThat(this.taskScheduler.getScheduledThreadPoolExecutor().getQueue()).hasSize(0);
}
 
Example #24
Source File: CloudSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of undeleting a table. */
public void undeleteTable(String datasetId) throws InterruptedException {
  generateTableWithDdl(datasetId, "oops_undelete_me");

  // [START bigquery_undelete_table]
  // String datasetId = "my_dataset";
  String tableId = "oops_undelete_me";

  // Record the current time.  We'll use this as the snapshot time
  // for recovering the table.
  long snapTime = Instant.now().toEpochMilli();

  // "Accidentally" delete the table.
  bigquery.delete(TableId.of(datasetId, tableId));

  // Construct the restore-from tableID using a snapshot decorator.
  String snapshotTableId = String.format("%s@%d", tableId, snapTime);
  // Choose a new table ID for the recovered table data.
  String recoverTableId = String.format("%s_recovered", tableId);

  // Construct and run a copy job.
  CopyJobConfiguration configuration =
      CopyJobConfiguration.newBuilder(
              TableId.of(datasetId, recoverTableId), TableId.of(datasetId, snapshotTableId))
          .build();
  Job job = bigquery.create(JobInfo.of(configuration));
  job = job.waitFor();

  // Check the table
  StandardTableDefinition table =
      bigquery.getTable(TableId.of(datasetId, recoverTableId)).getDefinition();
  System.out.println("State: " + job.getStatus().getState());
  System.out.printf("Recovered %d rows.\n", table.getNumRows());
  // [END bigquery_undelete_table]
}
 
Example #25
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of writing a local file to a table. */
// [TARGET writer(WriteChannelConfiguration)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
// [VARIABLE FileSystems.getDefault().getPath(".", "my-data.csv")]
// [VARIABLE "us"]
public long writeFileToTable(String datasetName, String tableName, Path csvPath, String location)
    throws IOException, InterruptedException, TimeoutException {
  // [START bigquery_load_from_file]
  TableId tableId = TableId.of(datasetName, tableName);
  WriteChannelConfiguration writeChannelConfiguration =
      WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build();
  // Generally, location can be inferred based on the location of the referenced dataset.
  // However,
  // it can also be set explicitly to force job execution to be routed to a specific processing
  // location.  See https://cloud.google.com/bigquery/docs/locations for more info.
  JobId jobId = JobId.newBuilder().setLocation(location).build();
  TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration);
  // Write data to writer
  try (OutputStream stream = Channels.newOutputStream(writer)) {
    Files.copy(csvPath, stream);
  } finally {
    writer.close();
  }
  // Get load job
  Job job = writer.getJob();
  job = job.waitFor();
  LoadStatistics stats = job.getStatistics();
  return stats.getOutputRows();
  // [END bigquery_load_from_file]
}
 
Example #26
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of loading a newline-delimited-json file with textual fields from GCS to a table. */
// [TARGET create(JobInfo, JobOption...)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
public Long writeRemoteFileToTable(String datasetName, String tableName)
    throws InterruptedException {
  // [START bigquery_load_table_gcs_json]
  String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.json";
  TableId tableId = TableId.of(datasetName, tableName);
  // Table field definition
  Field[] fields =
      new Field[] {
        Field.of("name", LegacySQLTypeName.STRING),
        Field.of("post_abbr", LegacySQLTypeName.STRING)
      };
  // Table schema definition
  Schema schema = Schema.of(fields);
  LoadJobConfiguration configuration =
      LoadJobConfiguration.builder(tableId, sourceUri)
          .setFormatOptions(FormatOptions.json())
          .setCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
          .setSchema(schema)
          .build();
  // Load the table
  Job loadJob = bigquery.create(JobInfo.of(configuration));
  loadJob = loadJob.waitFor();
  // Check the table
  System.out.println("State: " + loadJob.getStatus().getState());
  return ((StandardTableDefinition) bigquery.getTable(tableId).getDefinition()).getNumRows();
  // [END bigquery_load_table_gcs_json]
}
 
Example #27
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of listing jobs, specifying the page size. */
// [TARGET listJobs(JobListOption...)]
public Page<Job> listJobs() {
  // [START bigquery_list_jobs]
  Page<Job> jobs = bigquery.listJobs(JobListOption.pageSize(100));
  for (Job job : jobs.iterateAll()) {
    // do something with the job
  }
  // [END bigquery_list_jobs]
  return jobs;
}
 
Example #28
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of getting a job. */
// [TARGET getJob(String, JobOption...)]
// [VARIABLE "my_job_name"]
public Job getJob(String jobName) {
  // [START ]
  Job job = bigquery.getJob(jobName);
  if (job == null) {
    // job was not found
  }
  // [END ]
  return job;
}
 
Example #29
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of getting a job. */
// [TARGET getJob(JobId, JobOption...)]
// [VARIABLE "my_job_name"]
public Job getJobFromId(String jobName) {
  // [START ]
  JobId jobIdObject = JobId.of(jobName);
  Job job = bigquery.getJob(jobIdObject);
  if (job == null) {
    // job was not found
  }
  // [END ]
  return job;
}
 
Example #30
Source File: ITJobSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
@Test
public void testReloadStatus() throws Exception {
  JobConfiguration jobConfig =
      QueryJobConfiguration.newBuilder(QUERY).setUseLegacySql(false).build();
  JobInfo jobInfo = JobInfo.newBuilder(jobConfig).build();
  Job job = bigquery.create(jobInfo);
  JobSnippets jobSnippets = new JobSnippets(job);
  JobStatus.State result = jobSnippets.reloadStatus();
  assertEquals(JobStatus.State.DONE, result);
}