Python google.cloud() Examples

def copy_file_to(self, local_path_or_handle, remote_path, metadata=None):
    """Copy file from a local path to a remote path."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      if metadata:
        blob.metadata = metadata

      if isinstance(local_path_or_handle, basestring):
        blob.upload_from_filename(local_path_or_handle)
      else:
        blob.upload_from_file(local_path_or_handle, rewind=True)

    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy local file %s to cloud storage file %s.' %
                    (local_path_or_handle, remote_path))
      raise

    return True 

def delete_blob(to_delete):
    # [START delete_blob]
    from google.cloud.exceptions import NotFound

    client = storage.Client()
    bucket = client.get_bucket("my-bucket")
    blobs = list(bucket.list_blobs())
    assert len(blobs) > 0
    # [<Blob: my-bucket, my-file.txt>]
    bucket.delete_blob("my-file.txt")
    try:
        bucket.delete_blob("doesnt-exist")
    except NotFound:
        pass
    # [END delete_blob]

    blob = None
    # [START delete_blobs]
    bucket.delete_blobs([blob], on_error=lambda blob: None)
    # [END delete_blobs]

    to_delete.append(bucket) 

def _get_bucket(self, bucket_name):
        """get a bucket based on a bucket name. If it doesn't exist, create it.

           Parameters
           ==========
           bucket_name: the name of the bucket to get (or create). It should
                        not contain google, and should be all lowercase with -
                        or underscores.
        """

        # Case 1: The bucket already exists
        try:
            bucket = self._bucket_service.get_bucket(bucket_name)

        # Case 2: The bucket needs to be created
        except google.cloud.exceptions.NotFound:
            bucket = self._bucket_service.create_bucket(bucket_name)

        # Case 2: The bucket name is already taken
        except:
            bot.exit("Cannot get or create %s, is the name taken?" % bucket_name)

        return bucket 

def _get_bucket(self):
        """get a bucket based on a bucket name. If it doesn't exist, create it.
        """

        # Case 1: The bucket already exists
        try:
            self._bucket = self._bucket_service.get_bucket(self._bucket_name)

        # Case 2: The bucket needs to be created
        except google.cloud.exceptions.NotFound:
            self._bucket = self._bucket_service.create_bucket(self._bucket_name)

        # Case 3: The bucket name is already taken
        except:
            bot.exit("Cannot get or create %s" % self._bucket_name)

        return self._bucket 

def generate_life_cycle_config(action, age=None, num_newer_versions=None):
  """Generate GCS lifecycle management config.

  For the reference, see https://cloud.google.com/storage/docs/lifecycle and
  https://cloud.google.com/storage/docs/managing-lifecycles.
  """
  rule = {}
  rule['action'] = {'type': action}
  rule['condition'] = {}
  if age is not None:
    rule['condition']['age'] = age
  if num_newer_versions is not None:
    rule['condition']['numNewerVersions'] = num_newer_versions

  config = {'rule': [rule]}
  return config 

def write_data(self, data, remote_path, metadata=None):
    """Write the data of a remote file."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      if metadata:
        blob.metadata = metadata
      blob.upload_from_string(data)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to write cloud storage file %s.' % remote_path)
      raise

    return True 

def copy_blob(self, remote_source, remote_target):
    """Copy a remote file to another remote location."""
    source_bucket_name, source_path = get_bucket_name_and_path(remote_source)
    target_bucket_name, target_path = get_bucket_name_and_path(remote_target)

    client = _storage_client()
    try:
      source_bucket = client.bucket(source_bucket_name)
      source_blob = source_bucket.blob(source_path)
      target_bucket = client.bucket(target_bucket_name)
      source_bucket.copy_blob(source_blob, target_bucket, target_path)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy cloud storage file %s to cloud storage '
                    'file %s.' % (remote_source, remote_target))
      raise

    return True 

def upload_file(module, client, src, dest):
    try:
        bucket = client.get_bucket(module.params['bucket'])
        blob = Blob(dest, bucket)
        with open(src, "r") as file_obj:
            blob.upload_from_file(file_obj)
        return blob_to_dict(blob)
    except google.cloud.exceptions.GoogleCloudError as e:
        module.fail_json(msg=str(e)) 

def download_file(module, client, name, dest):
    try:
        bucket = client.get_bucket(module.params['bucket'])
        blob = Blob(name, bucket)
        with open(dest, "wb") as file_obj:
            blob.download_to_file(file_obj)
        return blob_to_dict(blob)
    except google.cloud.exceptions.NotFound as e:
        module.fail_json(msg=str(e)) 

def main():
    # tests "extras" dependencies, as well as native dependencies
    # google.cloud is also an "implicit" namespace package: the tool needs to make it a "real"
    # package
    print 'datatore:', google.cloud.datastore 

def remote_upload(self,
                      bucket_name,
                      file_dir,
                      key=None):
        '''Upload data/file (blob) to a google cloud  bucket.'''

        file_path = os.path.realpath(os.path.expanduser(file_dir))

        if not bucket_name:
            raise ValueError("Bucket name must be specified to upload file")
        if not os.path.exists(file_dir):
            raise ValueError(
                "File path specified does not exitis: {}".format(file_path))
        if not os.path.isfile(file_dir):
            raise ValueError(
                "File path specified is not a file: {}".format(file_path))

        if not self.bucket_exists(bucket_name):
            self.GC.create_bucket(bucket_name)

        b = self.GC.get_bucket(bucket_name)
        blob = b.blob(key)

        try:
            blob.upload_from_filename(file_path)
        except:
            raise Exception(
                "filename is not correctly specified: {}".format(file_dir)) 

def bucket_exists(self, bucket_name):
        try:
            self.GC.get_bucket(bucket_name)
            return True
        except KeyError as e:
            print("%s bucket does not exist in google cloud" % (e)) 

def automl_create_dataset_for_tables(
    gcp_project_id: str,
    gcp_region: str,
    display_name: str,
    description: str = None,
    tables_dataset_metadata: dict = {},
    retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout: float = None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = None,
) -> NamedTuple('Outputs', [('dataset_path', str), ('create_time', str), ('dataset_id', str), ('dataset_url', 'URI')]):
    '''automl_create_dataset_for_tables creates an empty Dataset for AutoML tables
    '''
    import google
    from google.cloud import automl
    client = automl.AutoMlClient()

    location_path = client.location_path(gcp_project_id, gcp_region)
    dataset_dict = {
        'display_name': display_name,
        'description': description,
        'tables_dataset_metadata': tables_dataset_metadata,
    }
    dataset = client.create_dataset(
        location_path,
        dataset_dict,
        retry or google.api_core.gapic_v1.method.DEFAULT,
        timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata,
    )
    print(dataset)
    dataset_id = dataset.name.rsplit('/', 1)[-1]
    dataset_url = 'https://console.cloud.google.com/automl-tables/locations/{region}/datasets/{dataset_id}/schemav2?project={project_id}'.format(
        project_id=gcp_project_id,
        region=gcp_region,
        dataset_id=dataset_id,
    )
    return (dataset.name, dataset.create_time, dataset_id, dataset_url) 

def automl_import_data_from_gcs(
    dataset_path: str,
    input_uris: list,
    retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout=None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = None,
) -> NamedTuple('Outputs', [('dataset_path', str)]):
    import sys
    import subprocess
    subprocess.run([sys.executable, '-m', 'pip', 'install', 'google-cloud-automl==0.4.0', '--quiet', '--no-warn-script-location'], env={'PIP_DISABLE_PIP_VERSION_CHECK': '1'}, check=True)

    import google
    from google.cloud import automl
    client = automl.AutoMlClient()
    input_config = {
        'gcs_source': {
            'input_uris': input_uris,
        },
    }
    response = client.import_data(
        dataset_path,
        input_config,
        retry or google.api_core.gapic_v1.method.DEFAULT,
        timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata,
    )
    result = response.result()
    print(result)
    metadata = response.metadata
    print(metadata)
    return (dataset_path) 

def automl_import_data_from_bigquery(
    dataset_path,
    input_uri: str,
    retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout=None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = None,
) -> NamedTuple('Outputs', [('dataset_path', str)]):
    import sys
    import subprocess
    subprocess.run([sys.executable, '-m', 'pip', 'install', 'google-cloud-automl==0.4.0', '--quiet', '--no-warn-script-location'], env={'PIP_DISABLE_PIP_VERSION_CHECK': '1'}, check=True)

    import google
    from google.cloud import automl
    client = automl.AutoMlClient()
    input_config = {
        'bigquery_source': {
            'input_uri': input_uri,
        },
    }
    response = client.import_data(
        dataset_path,
        input_config,
        retry or google.api_core.gapic_v1.method.DEFAULT,
        timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata,
    )
    result = response.result()
    print(result)
    metadata = response.metadata
    print(metadata)
    return (dataset_path) 

def automl_export_data_to_gcs(
    dataset_path: str,
    gcs_output_uri_prefix: str = None,
    #retry=None, #=google.api_core.gapic_v1.method.DEFAULT,
    timeout: float = None, #=google.api_core.gapic_v1.method.DEFAULT,
    metadata: dict = {},
) -> NamedTuple('Outputs', [('gcs_output_uri_prefix', str)]):
    """Exports dataset data to GCS."""
    import sys
    import subprocess
    subprocess.run([sys.executable, "-m", "pip", "install", "google-cloud-automl==0.4.0", "--quiet", "--no-warn-script-location"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True)

    import google
    from google.cloud import automl
    client = automl.AutoMlClient()

    output_config = {"gcs_destination": {"output_uri_prefix": gcs_output_uri_prefix}}

    response = client.export_data(
        name=dataset_path,
        output_config=output_config,
        #retry=retry or google.api_core.gapic_v1.method.DEFAULT
        timeout=timeout or google.api_core.gapic_v1.method.DEFAULT,
        metadata=metadata,
    )
    print('Operation started:')
    print(response.operation)
    result = response.result()
    metadata = response.metadata
    print('Operation finished:')
    print(metadata)
    return (gcs_output_uri_prefix, ) 

def copy_file_from(self, remote_path, local_path):
    """Copy file from a remote path to a local path."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      blob.download_to_filename(local_path)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy cloud storage file %s to local file %s.' %
                    (remote_path, local_path))
      raise

    return True 

def create_bucket(client, to_delete):
    from google.cloud.storage import Bucket

    # [START create_bucket]
    bucket = client.create_bucket("my-bucket")
    assert isinstance(bucket, Bucket)
    # <Bucket: my-bucket>
    # [END create_bucket]

    to_delete.append(bucket) 

def get_bucket(client, to_delete):
    import google

    # [START get_bucket]
    try:
        bucket = client.get_bucket("my-bucket")
    except google.cloud.exceptions.NotFound:
        print("Sorry, that bucket does not exist!")
    # [END get_bucket]
    to_delete.append(bucket) 

def delete_file(module, client, name):
    try:
        bucket = client.get_bucket(module.params['bucket'])
        blob = Blob(name, bucket)
        blob.delete()
        return {}
    except google.cloud.exceptions.NotFound as e:
        module.fail_json(msg=str(e)) 

def exists(cloud_storage_file_path, ignore_errors=False):
  """Return whether if a cloud storage file exists."""
  try:
    return bool(_provider().get(cloud_storage_file_path))
  except HttpError:
    if not ignore_errors:
      logs.log_error('Failed when trying to find cloud storage file %s.' %
                     cloud_storage_file_path)

    return False 

def read_data(self, remote_path):
    """Read the data of a remote file."""
    bucket_name, path = get_bucket_name_and_path(remote_path)

    client = _storage_client()
    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      return blob.download_as_string()
    except google.cloud.exceptions.GoogleCloudError as e:
      if e.code == 404:
        return None

      logs.log_warn('Failed to read cloud storage file %s.' % remote_path)
      raise 

def get_bucket_name_and_path(cloud_storage_file_path):
  """Return bucket name and path given a full cloud storage path."""
  filtered_path = utils.strip_from_left(cloud_storage_file_path, GS_PREFIX)
  _, bucket_name_and_path = filtered_path.split('/', 1)

  if '/' in bucket_name_and_path:
    bucket_name, path = bucket_name_and_path.split('/', 1)
  else:
    bucket_name = bucket_name_and_path
    path = ''

  return bucket_name, path 

def copy_file_from(cloud_storage_file_path, local_file_path, use_cache=False):
  """Saves a cloud storage file locally."""
  if use_cache and get_file_from_cache_if_exists(local_file_path):
    logs.log('Copied file %s from local cache.' % cloud_storage_file_path)
    return True

  if not _provider().copy_file_from(cloud_storage_file_path, local_file_path):
    return False

  if use_cache:
    store_file_in_cache(local_file_path)

  return True 

def copy_file_to(local_file_path_or_handle,
                 cloud_storage_file_path,
                 metadata=None):
  """Copy local file to a cloud storage path."""
  if (isinstance(local_file_path_or_handle, basestring) and
      not os.path.exists(local_file_path_or_handle)):
    logs.log_error('Local file %s not found.' % local_file_path_or_handle)
    return False

  return _provider().copy_file_to(
      local_file_path_or_handle, cloud_storage_file_path, metadata=metadata) 

def copy_blob(cloud_storage_source_path, cloud_storage_target_path):
  """Copy two blobs on GCS 'in the cloud' without touching local disk."""
  return _provider().copy_blob(cloud_storage_source_path,
                               cloud_storage_target_path) 

def get_blob(to_delete):
    from google.cloud.storage.blob import Blob

    # [START get_blob]
    client = storage.Client()
    bucket = client.get_bucket("my-bucket")
    assert isinstance(bucket.get_blob("/path/to/blob.txt"), Blob)
    # <Blob: my-bucket, /path/to/blob.txt>
    assert not bucket.get_blob("/does-not-exist.txt")
    # None
    # [END get_blob]

    to_delete.append(bucket) 

def last_updated(cloud_storage_file_path):
  """Return last updated value by parsing stats for all blobs under a cloud
  storage path."""
  last_update = None
  for blob in _provider().list_blobs(cloud_storage_file_path):
    if not last_update or blob['updated'] > last_update:
      last_update = blob['updated']
  if last_update:
    # Remove UTC tzinfo to make these comparable.
    last_update = last_update.replace(tzinfo=None)
  return last_update 

def read_data(cloud_storage_file_path):
  """Return content of a cloud storage file."""
  return _provider().read_data(cloud_storage_file_path) 

def write_data(data, cloud_storage_file_path, metadata=None):
  """Return content of a cloud storage file."""
  return _provider().write_data(
      data, cloud_storage_file_path, metadata=metadata)