Java Code Examples for com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem

The following examples show how to use com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: GoogleGcsConfigurationInitializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeConfiguration(Configuration config)
{
    config.set("fs.gs.impl", GoogleHadoopFileSystem.class.getName());

    if (useGcsAccessToken) {
        // use oauth token to authenticate with Google Cloud Storage
        config.set(AUTH_SERVICE_ACCOUNT_ENABLE.getKey(), "false");
        config.set(AUTHENTICATION_PREFIX + ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX, GcsAccessTokenProvider.class.getName());
    }
    else if (jsonKeyFilePath != null) {
        // use service account key file
        config.set(AUTH_SERVICE_ACCOUNT_ENABLE.getKey(), "true");
        config.set(AUTHENTICATION_PREFIX + JSON_KEYFILE_SUFFIX, jsonKeyFilePath);
    }
}
 
Example 2
public AccessToken refresh(Configuration conf) {
    BrokerDelegationTokenBinding binding = new BrokerDelegationTokenBinding();
    Text service = new Text(MOCK_BUCKET);
    binding.bindToFileSystem(new GoogleHadoopFileSystem(), service);
    BrokerAccessTokenProvider provider;
    try {
        provider = (BrokerAccessTokenProvider) binding.deployUnbonded();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    provider.setConf(conf);
    provider.refresh();
    return provider.getAccessToken();
}
 
Example 3
public String getSessionToken(Configuration conf) {
    BrokerDelegationTokenBinding binding = new BrokerDelegationTokenBinding();
    Text service = new Text(MOCK_BUCKET);
    Text owner = new Text(ALICE);
    Text renewer = new Text(YARN);
    binding.bindToFileSystem(new GoogleHadoopFileSystem(), service);
    BrokerTokenIdentifier identifier = new BrokerTokenIdentifier(conf, owner, renewer, null, service);
    return identifier.getSessionToken();
}
 
Example 4
Source Project: circus-train   Source File: BindGoogleHadoopFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
public void bindFileSystem(Configuration configuration) {
  LOG.debug("Binding GoogleHadoopFileSystem");
  configuration.set(GS_FS_IMPLEMENTATION, GoogleHadoopFileSystem.class.getName());
  configuration.set(GS_ABSTRACT_FS, GoogleHadoopFS.class.getName());
  configuration.set(GCP_SERVICE_ACCOUNT_ENABLE, "true");
  configuration.set(GCP_PROJECT_ID, "_THIS_VALUE_DOESNT_MATTER");
  loadGSFileSystem(configuration);
}
 
Example 5
Source Project: circus-train   Source File: BindGoogleHadoopFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
private void loadGSFileSystem(Configuration configuration) {
  try {
    new LibJarDeployer()
        .libjars(configuration, com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem.class,
            com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS.class);
  } catch (IOException e) {
    throw new CircusTrainException(e);
  }
}
 
Example 6
@Test
public void bindFileSystemTest() throws Exception {
  Configuration conf = new Configuration();
  BindGoogleHadoopFileSystem binder = new BindGoogleHadoopFileSystem();
  binder.bindFileSystem(conf);
  assertNotNull(conf.get(GCP_PROJECT_ID));
  assertEquals("true", conf.get(GCP_SERVICE_ACCOUNT_ENABLE));
  assertEquals(GoogleHadoopFileSystem.class.getName(), conf.get(GS_FS_IMPLEMENTATION));
  assertEquals(GoogleHadoopFS.class.getName(), conf.get(GS_ABSTRACT_FS));
}
 
Example 7
Source Project: hadoop-connectors   Source File: TestGoogleContractOpen.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 8
Source Project: hadoop-connectors   Source File: TestGoogleContractMkdir.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 9
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 10
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 11
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 12
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 13
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 14
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 15
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}
 
Example 16
Source Project: hadoop-connectors   Source File: TestGoogleContractSeek.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void before() {
  fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem());
}