Python google.cloud() Examples

The following are 30 code examples for showing how to use google.cloud(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module google , or try the search function .

Example 1
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 6 votes vote down vote up
def copy_file_to(self, local_path_or_handle, remote_path, metadata=None):
    """Copy file from a local path to a remote path."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      if metadata:
        blob.metadata = metadata

      if isinstance(local_path_or_handle, basestring):
        blob.upload_from_filename(local_path_or_handle)
      else:
        blob.upload_from_file(local_path_or_handle, rewind=True)

    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy local file %s to cloud storage file %s.' %
                    (local_path_or_handle, remote_path))
      raise

    return True 
Example 2
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 6 votes vote down vote up
def copy_blob(self, remote_source, remote_target):
    """Copy a remote file to another remote location."""
    source_bucket_name, source_path = get_bucket_name_and_path(remote_source)
    target_bucket_name, target_path = get_bucket_name_and_path(remote_target)

    client = _storage_client()
    try:
      source_bucket = client.bucket(source_bucket_name)
      source_blob = source_bucket.blob(source_path)
      target_bucket = client.bucket(target_bucket_name)
      source_bucket.copy_blob(source_blob, target_bucket, target_path)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy cloud storage file %s to cloud storage '
                    'file %s.' % (remote_source, remote_target))
      raise

    return True 
Example 3
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 6 votes vote down vote up
def write_data(self, data, remote_path, metadata=None):
    """Write the data of a remote file."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      if metadata:
        blob.metadata = metadata
      blob.upload_from_string(data)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to write cloud storage file %s.' % remote_path)
      raise

    return True 
Example 4
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 6 votes vote down vote up
def generate_life_cycle_config(action, age=None, num_newer_versions=None):
  """Generate GCS lifecycle management config.

  For the reference, see https://cloud.google.com/storage/docs/lifecycle and
  https://cloud.google.com/storage/docs/managing-lifecycles.
  """
  rule = {}
  rule['action'] = {'type': action}
  rule['condition'] = {}
  if age is not None:
    rule['condition']['age'] = age
  if num_newer_versions is not None:
    rule['condition']['numNewerVersions'] = num_newer_versions

  config = {'rule': [rule]}
  return config 
Example 5
Project: sregistry-cli   Author: singularityhub   File: __init__.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def _get_bucket(self):
        """get a bucket based on a bucket name. If it doesn't exist, create it.
        """

        # Case 1: The bucket already exists
        try:
            self._bucket = self._bucket_service.get_bucket(self._bucket_name)

        # Case 2: The bucket needs to be created
        except google.cloud.exceptions.NotFound:
            self._bucket = self._bucket_service.create_bucket(self._bucket_name)

        # Case 3: The bucket name is already taken
        except:
            bot.exit("Cannot get or create %s" % self._bucket_name)

        return self._bucket 
Example 6
Project: sregistry-cli   Author: singularityhub   File: __init__.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def _get_bucket(self, bucket_name):
        """get a bucket based on a bucket name. If it doesn't exist, create it.

           Parameters
           ==========
           bucket_name: the name of the bucket to get (or create). It should
                        not contain google, and should be all lowercase with -
                        or underscores.
        """

        # Case 1: The bucket already exists
        try:
            bucket = self._bucket_service.get_bucket(bucket_name)

        # Case 2: The bucket needs to be created
        except google.cloud.exceptions.NotFound:
            bucket = self._bucket_service.create_bucket(bucket_name)

        # Case 2: The bucket name is already taken
        except:
            bot.exit("Cannot get or create %s, is the name taken?" % bucket_name)

        return bucket 
Example 7
Project: python-storage   Author: googleapis   File: snippets.py    License: Apache License 2.0 6 votes vote down vote up
def delete_blob(to_delete):
    # [START delete_blob]
    from google.cloud.exceptions import NotFound

    client = storage.Client()
    bucket = client.get_bucket("my-bucket")
    blobs = list(bucket.list_blobs())
    assert len(blobs) > 0
    # [<Blob: my-bucket, my-file.txt>]
    bucket.delete_blob("my-file.txt")
    try:
        bucket.delete_blob("doesnt-exist")
    except NotFound:
        pass
    # [END delete_blob]

    blob = None
    # [START delete_blobs]
    bucket.delete_blobs([blob], on_error=lambda blob: None)
    # [END delete_blobs]

    to_delete.append(bucket) 
Example 8
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def copy_file_from(self, remote_path, local_path):
    """Copy file from a remote path to a local path."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      blob.download_to_filename(local_path)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy cloud storage file %s to local file %s.' %
                    (remote_path, local_path))
      raise

    return True 
Example 9
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def read_data(self, remote_path):
    """Read the data of a remote file."""
    bucket_name, path = get_bucket_name_and_path(remote_path)

    client = _storage_client()
    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      return blob.download_as_string()
    except google.cloud.exceptions.GoogleCloudError as e:
      if e.code == 404:
        return None

      logs.log_warn('Failed to read cloud storage file %s.' % remote_path)
      raise 
Example 10
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def get_bucket_name_and_path(cloud_storage_file_path):
  """Return bucket name and path given a full cloud storage path."""
  filtered_path = utils.strip_from_left(cloud_storage_file_path, GS_PREFIX)
  _, bucket_name_and_path = filtered_path.split('/', 1)

  if '/' in bucket_name_and_path:
    bucket_name, path = bucket_name_and_path.split('/', 1)
  else:
    bucket_name = bucket_name_and_path
    path = ''

  return bucket_name, path 
Example 11
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def copy_file_from(cloud_storage_file_path, local_file_path, use_cache=False):
  """Saves a cloud storage file locally."""
  if use_cache and get_file_from_cache_if_exists(local_file_path):
    logs.log('Copied file %s from local cache.' % cloud_storage_file_path)
    return True

  if not _provider().copy_file_from(cloud_storage_file_path, local_file_path):
    return False

  if use_cache:
    store_file_in_cache(local_file_path)

  return True 
Example 12
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def copy_file_to(local_file_path_or_handle,
                 cloud_storage_file_path,
                 metadata=None):
  """Copy local file to a cloud storage path."""
  if (isinstance(local_file_path_or_handle, basestring) and
      not os.path.exists(local_file_path_or_handle)):
    logs.log_error('Local file %s not found.' % local_file_path_or_handle)
    return False

  return _provider().copy_file_to(
      local_file_path_or_handle, cloud_storage_file_path, metadata=metadata) 
Example 13
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def copy_blob(cloud_storage_source_path, cloud_storage_target_path):
  """Copy two blobs on GCS 'in the cloud' without touching local disk."""
  return _provider().copy_blob(cloud_storage_source_path,
                               cloud_storage_target_path) 
Example 14
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def exists(cloud_storage_file_path, ignore_errors=False):
  """Return whether if a cloud storage file exists."""
  try:
    return bool(_provider().get(cloud_storage_file_path))
  except HttpError:
    if not ignore_errors:
      logs.log_error('Failed when trying to find cloud storage file %s.' %
                     cloud_storage_file_path)

    return False 
Example 15
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def last_updated(cloud_storage_file_path):
  """Return last updated value by parsing stats for all blobs under a cloud
  storage path."""
  last_update = None
  for blob in _provider().list_blobs(cloud_storage_file_path):
    if not last_update or blob['updated'] > last_update:
      last_update = blob['updated']
  if last_update:
    # Remove UTC tzinfo to make these comparable.
    last_update = last_update.replace(tzinfo=None)
  return last_update 
Example 16
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def read_data(cloud_storage_file_path):
  """Return content of a cloud storage file."""
  return _provider().read_data(cloud_storage_file_path) 
Example 17
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def write_data(data, cloud_storage_file_path, metadata=None):
  """Return content of a cloud storage file."""
  return _provider().write_data(
      data, cloud_storage_file_path, metadata=metadata) 
Example 18
Project: clusterfuzz   Author: google   File: storage.py    License: Apache License 2.0 5 votes vote down vote up
def get_blobs(cloud_storage_path, recursive=True):
  """Return blobs under the given cloud storage path."""
  for blob in _provider().list_blobs(cloud_storage_path, recursive=recursive):
    yield blob 
Example 19
Project: ztag   Author: zmap   File: stream.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, logger=None, destination=None, *args, **kwargs):
        import google
        from google.cloud import pubsub, pubsub_v1
        self.logger = logger
        if logger is None:
            self.logger = logging.getLogger('null-logger')
            self.logger.setLevel(9999)
        if destination == "full_ipv4":
            self.topic_url = os.environ.get('PUBSUB_IPV4_TOPIC_URL')
        elif destination == "alexa_top1mil":
            self.topic_url = os.environ.get('PUBSUB_ALEXA_TOPIC_URL')
        self.cert_topic_url = os.environ.get('PUBSUB_CERT_TOPIC_URL')
        if not self.topic_url:
            raise Exception('missing $PUBSUB_[IPV4|ALEXA]_TOPIC_URL')
        if not self.cert_topic_url:
            raise Exception('missing $PUBSUB_CERT_TOPIC_URL')
        batch_settings = pubsub_v1.types.BatchSettings(
            # "The entire request including one or more messages must
            #  be smaller than 10MB, after decoding."
            max_bytes=8192000,  # 8 MB
            max_latency=15,     # 15 seconds
        )
        self.publisher = pubsub.PublisherClient(batch_settings)
        self.publish_count = {}
        try:
            self.publisher.get_topic(self.topic_url)
            self.publisher.get_topic(self.cert_topic_url)
        except google.api_core.exceptions.GoogleAPICallError as e:
            logger.error(e.message)
            raise
        self._state = PubsubState() 
Example 20
Project: python-storage   Author: googleapis   File: snippets.py    License: Apache License 2.0 5 votes vote down vote up
def download_to_file(to_delete):
    # [START download_to_file]
    from google.cloud.storage import Blob

    client = storage.Client(project="my-project")
    bucket = client.get_bucket("my-bucket")
    encryption_key = "c7f32af42e45e85b9848a6a14dd2a8f6"
    blob = Blob("secure-data", bucket, encryption_key=encryption_key)
    blob.upload_from_string("my secret message.")
    with open("/tmp/my-secure-file", "wb") as file_obj:
        blob.download_to_file(file_obj)
    # [END download_to_file]

    to_delete.append(blob) 
Example 21
Project: python-storage   Author: googleapis   File: snippets.py    License: Apache License 2.0 5 votes vote down vote up
def upload_from_file(to_delete):
    # [START upload_from_file]
    from google.cloud.storage import Blob

    client = storage.Client(project="my-project")
    bucket = client.get_bucket("my-bucket")
    encryption_key = "aa426195405adee2c8081bb9e7e74b19"
    blob = Blob("secure-data", bucket, encryption_key=encryption_key)
    with open("my-file", "rb") as my_file:
        blob.upload_from_file(my_file)
    # [END upload_from_file]

    to_delete.append(blob) 
Example 22
Project: python-storage   Author: googleapis   File: snippets.py    License: Apache License 2.0 5 votes vote down vote up
def get_blob(to_delete):
    from google.cloud.storage.blob import Blob

    # [START get_blob]
    client = storage.Client()
    bucket = client.get_bucket("my-bucket")
    assert isinstance(bucket.get_blob("/path/to/blob.txt"), Blob)
    # <Blob: my-bucket, /path/to/blob.txt>
    assert not bucket.get_blob("/does-not-exist.txt")
    # None
    # [END get_blob]

    to_delete.append(bucket) 
Example 23
Project: python-storage   Author: googleapis   File: snippets.py    License: Apache License 2.0 5 votes vote down vote up
def get_bucket(client, to_delete):
    import google

    # [START get_bucket]
    try:
        bucket = client.get_bucket("my-bucket")
    except google.cloud.exceptions.NotFound:
        print("Sorry, that bucket does not exist!")
    # [END get_bucket]
    to_delete.append(bucket) 
Example 24
Project: python-storage   Author: googleapis   File: snippets.py    License: Apache License 2.0 5 votes vote down vote up
def create_bucket(client, to_delete):
    from google.cloud.storage import Bucket

    # [START create_bucket]
    bucket = client.create_bucket("my-bucket")
    assert isinstance(bucket, Bucket)
    # <Bucket: my-bucket>
    # [END create_bucket]

    to_delete.append(bucket) 
Example 25
Project: pipelines   Author: kubeflow   File: component.py    License: Apache License 2.0 5 votes vote down vote up
def automl_export_data_to_gcs(
    dataset_path: str,
    gcs_output_uri_prefix: str = None,
    #retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout: float = None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = {},
) -> NamedTuple('Outputs', [('gcs_output_uri_prefix', str)]):
    """Exports dataset data to GCS."""
    import sys
    import subprocess
    subprocess.run([sys.executable, "-m", "pip", "install", "google-cloud-automl==0.4.0", "--quiet", "--no-warn-script-location"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True)

    import google
    from google.cloud import automl
    client = automl.AutoMlClient()

    output_config = {"gcs_destination": {"output_uri_prefix": gcs_output_uri_prefix}}

    response = client.export_data(
        name=dataset_path,
        output_config=output_config,
        #retry=retry or google.api_core.gapic_v1.method.DEFAULT
        timeout=timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata=metadata,
    )
    print('Operation started:')
    print(response.operation)
    result = response.result()
    metadata = response.metadata
    print('Operation finished:')
    print(metadata)
    return (gcs_output_uri_prefix, ) 
Example 26
Project: pipelines   Author: kubeflow   File: component.py    License: Apache License 2.0 5 votes vote down vote up
def automl_import_data_from_bigquery(
    dataset_path,
    input_uri: str,
    retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout=None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = None,
) -> NamedTuple('Outputs', [('dataset_path', str)]):
    import sys
    import subprocess
    subprocess.run([sys.executable, '-m', 'pip', 'install', 'google-cloud-automl==0.4.0', '--quiet', '--no-warn-script-location'], env={'PIP_DISABLE_PIP_VERSION_CHECK': '1'}, check=True)

    import google
    from google.cloud import automl
    client = automl.AutoMlClient()
    input_config = {
        'bigquery_source': {
            'input_uri': input_uri,
        },
    }
    response = client.import_data(
        dataset_path,
        input_config,
        retry or google.api_core.gapic_v1.method.DEFAULT,
        timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata,
    )
    result = response.result()
    print(result)
    metadata = response.metadata
    print(metadata)
    return (dataset_path) 
Example 27
Project: pipelines   Author: kubeflow   File: component.py    License: Apache License 2.0 5 votes vote down vote up
def automl_import_data_from_gcs(
    dataset_path: str,
    input_uris: list,
    retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout=None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = None,
) -> NamedTuple('Outputs', [('dataset_path', str)]):
    import sys
    import subprocess
    subprocess.run([sys.executable, '-m', 'pip', 'install', 'google-cloud-automl==0.4.0', '--quiet', '--no-warn-script-location'], env={'PIP_DISABLE_PIP_VERSION_CHECK': '1'}, check=True)

    import google
    from google.cloud import automl
    client = automl.AutoMlClient()
    input_config = {
        'gcs_source': {
            'input_uris': input_uris,
        },
    }
    response = client.import_data(
        dataset_path,
        input_config,
        retry or google.api_core.gapic_v1.method.DEFAULT,
        timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata,
    )
    result = response.result()
    print(result)
    metadata = response.metadata
    print(metadata)
    return (dataset_path) 
Example 28
Project: pipelines   Author: kubeflow   File: component.py    License: Apache License 2.0 5 votes vote down vote up
def automl_create_dataset_for_tables(
    gcp_project_id: str,
    gcp_region: str,
    display_name: str,
    description: str = None,
    tables_dataset_metadata: dict = {},
    retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout: float = None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = None,
) -> NamedTuple('Outputs', [('dataset_path', str), ('create_time', str), ('dataset_id', str), ('dataset_url', 'URI')]):
    '''automl_create_dataset_for_tables creates an empty Dataset for AutoML tables
    '''
    import google
    from google.cloud import automl
    client = automl.AutoMlClient()

    location_path = client.location_path(gcp_project_id, gcp_region)
    dataset_dict = {
        'display_name': display_name,
        'description': description,
        'tables_dataset_metadata': tables_dataset_metadata,
    }
    dataset = client.create_dataset(
        location_path,
        dataset_dict,
        retry or google.api_core.gapic_v1.method.DEFAULT,
        timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata,
    )
    print(dataset)
    dataset_id = dataset.name.rsplit('/', 1)[-1]
    dataset_url = 'https://console.cloud.google.com/automl-tables/locations/{region}/datasets/{dataset_id}/schemav2?project={project_id}'.format(
        project_id=gcp_project_id,
        region=gcp_region,
        dataset_id=dataset_id,
    )
    return (dataset.name, dataset.create_time, dataset_id, dataset_url) 
Example 29
Project: cgat-core   Author: cgat-developers   File: google_cloud.py    License: MIT License 5 votes vote down vote up
def bucket_exists(self, bucket_name):
        try:
            self.GC.get_bucket(bucket_name)
            return True
        except KeyError as e:
            print("%s bucket does not exist in google cloud" % (e)) 
Example 30
Project: cgat-core   Author: cgat-developers   File: google_cloud.py    License: MIT License 5 votes vote down vote up
def remote_upload(self,
                      bucket_name,
                      file_dir,
                      key=None):
        '''Upload data/file (blob) to a google cloud  bucket.'''

        file_path = os.path.realpath(os.path.expanduser(file_dir))

        if not bucket_name:
            raise ValueError("Bucket name must be specified to upload file")
        if not os.path.exists(file_dir):
            raise ValueError(
                "File path specified does not exitis: {}".format(file_path))
        if not os.path.isfile(file_dir):
            raise ValueError(
                "File path specified is not a file: {}".format(file_path))

        if not self.bucket_exists(bucket_name):
            self.GC.create_bucket(bucket_name)

        b = self.GC.get_bucket(bucket_name)
        blob = b.blob(key)

        try:
            blob.upload_from_filename(file_path)
        except:
            raise Exception(
                "filename is not correctly specified: {}".format(file_dir))