Python googleapiclient.errors() Examples

The following are code examples for showing how to use googleapiclient.errors(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: dsub   Author: DataBiosphere   File: google_base.py    Apache License 2.0 6 votes vote down vote up
def retry_auth_check(exception, verbose):
  """Specific check for auth error codes.

  Return True if we should retry.

  False otherwise.
  Args:
    exception: An exception to test for transience.
    verbose: If true, output retry messages

  Returns:
    True if we should retry. False otherwise.
  """
  if isinstance(exception, googleapiclient.errors.HttpError):
    if exception.resp.status in HTTP_AUTH_ERROR_CODES:
      _print_retry_error(exception, verbose)
      return True

  return False 
Example 2
Project: listenbrainz-server   Author: metabrainz   File: __init__.py    GNU General Public License v2.0 6 votes vote down vote up
def wait_for_completion(projectId, jobId):
    """ Make requests periodically until the passed job has been completed """

    while True:

        try:
            job = bigquery.jobs().get(projectId=projectId, jobId=jobId).execute(num_retries=5)
        except googleapiclient.errors.HttpError as err:
            current_app.logger.error("HttpError while waiting for completion of job: {}".format(err), exc_info=True)
            time.sleep(JOB_COMPLETION_CHECK_DELAY)
            continue

        if job["status"]["state"] == "DONE":
            return
        else:
            time.sleep(JOB_COMPLETION_CHECK_DELAY) 
Example 3
Project: podpublish   Author: rikai   File: main.py    GNU Lesser General Public License v2.1 5 votes vote down vote up
def parse_options_error(parser, options):
    """Check errors in options."""
    required_options = ["title"]
    missing = [opt for opt in required_options if not getattr(options, opt)]
    if missing:
        parser.print_usage()
        msg = "Some required option are missing: {0}".format(", ".join(missing))
        raise OptionsError(msg) 
Example 4
Project: pulse-data   Author: Recidiviz   File: cloudsql_export_test.py    GNU General Public License v3.0 5 votes vote down vote up
def test_wait_until_operation_finished_surfaces_errors(self, mock_time):
        """Test that errors are logged from wait_until_operation_finished."""
        mock_time.side_effect = None
        mock_get_operation_calls = [
            {'status': 'PENDING'},
            {'status': 'RUNNING'},
            {'status': 'RUNNING'},
            {
                'status': 'DONE',
                'error': {
                    'errors': [{
                        'kind': 'error_kind',
                        'code': 'error_code',
                        'message': 'error_message'
                    }]
                }
            }
        ]
        mock_get_op = self.mock_client.operations.return_value.get.return_value

        mock_get_op.execute.side_effect = mock_get_operation_calls

        with self.assertLogs(level='ERROR'):
            cloudsql_export.wait_until_operation_finished('fake-op')

    # pylint: disable=line-too-long 
Example 5
Project: pulse-data   Author: Recidiviz   File: cloudsql_export_test.py    GNU General Public License v3.0 5 votes vote down vote up
def test_export_table_api_fail(self, mock_wait):
        """Test that export_table fails if the export API request fails."""
        mock_export = self.mock_client.instances.return_value.export
        mock_export_request = mock_export.return_value
        mock_export_request.execute.side_effect = \
            googleapiclient.errors.HttpError('', content=b'')

        with self.assertLogs(level='ERROR'):
            success = cloudsql_export.export_table(self.schema_type,
                                                   self.mock_table_id,
                                                   self.mock_table_query)

        self.assertFalse(success)
        mock_wait.assert_not_called() 
Example 6
Project: podpublish   Author: ubuntupodcast   File: main.py    GNU Lesser General Public License v2.1 5 votes vote down vote up
def parse_options_error(parser, options):
    """Check errors in options."""
    required_options = ["title"]
    missing = [opt for opt in required_options if not getattr(options, opt)]
    if missing:
        parser.print_usage()
        msg = "Some required option are missing: {0}".format(", ".join(missing))
        raise OptionsError(msg) 
Example 7
Project: dsub   Author: DataBiosphere   File: google_base.py    Apache License 2.0 4 votes vote down vote up
def retry_api_check(exception, verbose):
  """Return True if we should retry. False otherwise.

  Args:
    exception: An exception to test for transience.
    verbose: If true, output retry messages

  Returns:
    True if we should retry. False otherwise.
  """
  if isinstance(exception, googleapiclient.errors.HttpError):
    if exception.resp.status in TRANSIENT_HTTP_ERROR_CODES:
      _print_retry_error(exception, verbose)
      return True

  if isinstance(exception, socket.error):
    if exception.errno in TRANSIENT_SOCKET_ERROR_CODES:
      _print_retry_error(exception, verbose)
      return True

  if isinstance(exception, google.auth.exceptions.RefreshError):
    _print_retry_error(exception, verbose)
    return True

  # For a given installation, this could be a permanent error, but has only
  # been observed as transient.
  if isinstance(exception, SSLError):
    _print_retry_error(exception, verbose)
    return True

  # This has been observed as a transient error:
  #   ServerNotFoundError: Unable to find the server at genomics.googleapis.com
  if isinstance(exception, ServerNotFoundError):
    _print_retry_error(exception, verbose)
    return True

  # Observed to be thrown transiently from auth libraries which use httplib2
  # Use the one from six because httlib no longer exists in Python3
  # https://docs.python.org/2/library/httplib.html
  if isinstance(exception, six.moves.http_client.ResponseNotReady):
    _print_retry_error(exception, verbose)
    return True

  return False 
Example 8
Project: podpublish   Author: rikai   File: main.py    GNU Lesser General Public License v2.1 4 votes vote down vote up
def main(arguments):
    """Upload videos to Youtube."""
    usage = """Usage: %prog [OPTIONS] VIDEO [VIDEO2 ...]

    Upload videos to Youtube."""
    parser = optparse.OptionParser(usage)

    # Video metadata
    parser.add_option('-t', '--title', dest='title', type="string",
        help='Video title')
    parser.add_option('-c', '--category', dest='category', type="string",
        help='Video category')
    parser.add_option('-d', '--description', dest='description', type="string",
        help='Video description')
    parser.add_option('', '--tags', dest='tags', type="string",
        help='Video tags (separated by commas: "tag1, tag2,...")')
    parser.add_option('', '--privacy', dest='privacy', metavar="STRING",
        default="public", help='Privacy status (public | unlisted | private)')
    parser.add_option('', '--publish-at', dest='publish_at', metavar="datetime",
       default=None, help='Publish date (ISO 8601): YYYY-MM-DDThh:mm:ss.sZ')
    parser.add_option('', '--location', dest='location', type="string",
        default=None, metavar="latitude=VAL,longitude=VAL[,altitude=VAL]",
        help='Video location"')
    parser.add_option('', '--recording-date', dest='recording_date', metavar="datetime",
        default=None, help="Recording date (ISO 8601): YYYY-MM-DDThh:mm:ss.sZ")
    parser.add_option('', '--default-language', dest='default_language', type="string",
        default=None, metavar="string", 
        help="Default language (ISO 639-1: en | fr | de | ...)")
    parser.add_option('', '--default-audio-language', dest='default_audio_language', type="string",
        default=None, metavar="string", 
        help="Default audio language (ISO 639-1: en | fr | de | ...)")
    parser.add_option('', '--thumbnail', dest='thumb', type="string",
        help='Video thumbnail')
    parser.add_option('', '--playlist', dest='playlist', type="string",
        help='Playlist title (if it does not exist, it will be created)')
    parser.add_option('', '--title-template', dest='title_template',
        type="string", default="{title} [{n}/{total}]", metavar="string",
        help='Template for multiple videos (default: {title} [{n}/{total}])')

    # Authentication
    parser.add_option('', '--client-secrets', dest='client_secrets',
        type="string", help='Client secrets JSON file')
    parser.add_option('', '--credentials-file', dest='credentials_file',
        type="string", help='Credentials JSON file')
    parser.add_option('', '--auth-browser', dest='auth_browser', action='store_true',
        help='Open a GUI browser to authenticate if required')

    #Additional options
    parser.add_option('', '--open-link', dest='open_link', action='store_true',
        help='Opens a url in a web browser to display the uploaded video')

    options, args = parser.parse_args(arguments)
    try:
        run_main(parser, options, args)
    except googleapiclient.errors.HttpError as error:
        raise RequestError("Server response: {0}".format(bytes.decode(error.content).strip())) 
Example 9
Project: pulse-data   Author: Recidiviz   File: cloudsql_export.py    GNU General Public License v3.0 4 votes vote down vote up
def wait_until_operation_finished(operation_id: str) -> bool:
    """Monitor a Cloud SQL operation's progress and wait until it completes.

    We must wait until completion becuase only one Cloud SQL operation can run
    at a time.

    Args:
        operation_id: Cloud SQL Operation ID.
    Returns:
        True if operation succeeded without errors, False if not.

    See here for details:
    https://cloud.google.com/sql/docs/postgres/admin-api/v1beta4/operations/get
    """
    operation_in_progress = True
    operation_success = False

    while operation_in_progress:
        get_operation = sqladmin_client().operations().get(
            project=metadata.project_id(), operation=operation_id)
        operation = get_operation.execute()
        operation_status = operation['status']

        if operation_status in {'PENDING', 'RUNNING', 'UNKNOWN'}:
            time.sleep(SECONDS_BETWEEN_OPERATION_STATUS_CHECKS)
        elif operation_status == 'DONE':
            operation_in_progress = False

        logging.debug("Operation [%s] status: [%s]",
                      operation_id, operation_status)

    if 'error' in operation:
        errors = operation['error'].get('errors', [])
        for error in errors:
            logging.error(
                "Operation %s finished with error: %s, %s\n%s",
                operation_id,
                error.get('kind'),
                error.get('code'),
                error.get('message'))
    else:
        logging.info("Operation [%s] succeeded.", operation_id)
        operation_success = True

    return operation_success 
Example 10
Project: pulse-data   Author: Recidiviz   File: cloudsql_export.py    GNU General Public License v3.0 4 votes vote down vote up
def export_table(schema_type: SchemaType, table_name: str, export_query: str) \
        -> bool:
    """Export a Cloud SQL table to a CSV file on GCS.

    Given a table name and export_query, retrieve the export URI from
    export_config, then execute the export operation and wait until it
    completes.

    Args:
        schema_type: The schema, either SchemaType.JAILS or
            SchemaType.STATE, where this table lives.
        table_name: Table to export.
        export_query: Corresponding query for the table.
    Returns:
        True if operation succeeded without errors, False if not.
    """

    export_uri = export_config.gcs_export_uri(table_name)
    export_context = create_export_context(
        schema_type, export_uri, export_query)

    project_id = metadata.project_id()
    instance_id = \
        SQLAlchemyEngineManager.get_stripped_cloudql_instance_id(schema_type)
    export_request = sqladmin_client().instances().export(
        project=project_id,
        instance=instance_id,
        body=export_context)

    logging.info("Starting export: [%s]", str(export_request.to_json()))
    try:
        response = export_request.execute()
    except googleapiclient.errors.HttpError:
        logging.exception("Failed to export table [%s]", table_name)
        return False

    # We need to block until the operation is done because
    # the Cloud SQL API only supports one operation at a time.
    operation_id = response['name']
    logging.info("Waiting for export operation [%s] to complete for table [%s] "
                 "in database [%s] in project [%s]",
                 operation_id, table_name, instance_id, project_id)
    operation_success = wait_until_operation_finished(operation_id)

    return operation_success 
Example 11
Project: podpublish   Author: ubuntupodcast   File: main.py    GNU Lesser General Public License v2.1 4 votes vote down vote up
def main(arguments):
    """Upload videos to Youtube."""
    usage = """Usage: %prog [OPTIONS] VIDEO [VIDEO2 ...]

    Upload videos to Youtube."""
    parser = optparse.OptionParser(usage)

    # Video metadata
    parser.add_option('-t', '--title', dest='title', type="string",
        help='Video title')
    parser.add_option('-c', '--category', dest='category', type="string",
        help='Video category')
    parser.add_option('-d', '--description', dest='description', type="string",
        help='Video description')
    parser.add_option('', '--tags', dest='tags', type="string",
        help='Video tags (separated by commas: "tag1, tag2,...")')
    parser.add_option('', '--privacy', dest='privacy', metavar="STRING",
        default="public", help='Privacy status (public | unlisted | private)')
    parser.add_option('', '--publish-at', dest='publish_at', metavar="datetime",
       default=None, help='Publish date (ISO 8601): YYYY-MM-DDThh:mm:ss.sZ')
    parser.add_option('', '--location', dest='location', type="string",
        default=None, metavar="latitude=VAL,longitude=VAL[,altitude=VAL]",
        help='Video location"')
    parser.add_option('', '--recording-date', dest='recording_date', metavar="datetime",
        default=None, help="Recording date (ISO 8601): YYYY-MM-DDThh:mm:ss.sZ")
    parser.add_option('', '--default-language', dest='default_language', type="string",
        default=None, metavar="string", 
        help="Default language (ISO 639-1: en | fr | de | ...)")
    parser.add_option('', '--default-audio-language', dest='default_audio_language', type="string",
        default=None, metavar="string", 
        help="Default audio language (ISO 639-1: en | fr | de | ...)")
    parser.add_option('', '--thumbnail', dest='thumb', type="string",
        help='Video thumbnail')
    parser.add_option('', '--playlist', dest='playlist', type="string",
        help='Playlist title (if it does not exist, it will be created)')
    parser.add_option('', '--title-template', dest='title_template',
        type="string", default="{title} [{n}/{total}]", metavar="string",
        help='Template for multiple videos (default: {title} [{n}/{total}])')

    # Authentication
    parser.add_option('', '--client-secrets', dest='client_secrets',
        type="string", help='Client secrets JSON file')
    parser.add_option('', '--credentials-file', dest='credentials_file',
        type="string", help='Credentials JSON file')
    parser.add_option('', '--auth-browser', dest='auth_browser', action='store_true',
        help='Open a GUI browser to authenticate if required')

    #Additional options
    parser.add_option('', '--open-link', dest='open_link', action='store_true',
        help='Opens a url in a web browser to display the uploaded video')

    options, args = parser.parse_args(arguments)
    try:
        run_main(parser, options, args)
    except googleapiclient.errors.HttpError as error:
        raise RequestError("Server response: {0}".format(bytes.decode(error.content).strip()))