Python django.db.OperationalError() Examples

def check_migrations():
    """
    Check the status of database migrations.
    The koku API server is responsible for running all database migrations.  This method
    will return the state of the database and whether or not all migrations have been completed.
    Hat tip to the Stack Overflow contributor: https://stackoverflow.com/a/31847406
    Returns:
        Boolean - True if database is available and migrations have completed.  False otherwise.
    """
    try:
        connection = connections[DEFAULT_DB_ALIAS]
        connection.prepare_database()
        executor = MigrationExecutor(connection)
        targets = executor.loader.graph.leaf_nodes()
        return not executor.migration_plan(targets)
    except OperationalError:
        return False 

def destroy_source_event(source_id):
    """
    Destroy a Sources database object.

    Args:
        source_id (Integer) - Platform-Sources identifier

    Returns:
        None

    """
    koku_uuid = None
    try:
        source = Sources.objects.get(source_id=source_id)
        koku_uuid = source.koku_uuid
        source.delete()
        LOG.info(f"source.storage.destroy_source_event destroyed Source ID: {source_id}")
    except Sources.DoesNotExist:
        LOG.debug("Source ID: %s already removed.", str(source_id))
    except (InterfaceError, OperationalError) as error:
        LOG.error(f"source.storage.destroy_provider_event {type(error).__name__}: {error}")
        raise error

    return koku_uuid 

def is_known_source(source_id):
    """
    Check if source exists in database.

    Args:
        source_id (Integer) - Platform-Sources identifier

    Returns:
        source_exists (Boolean) - True if source is known

    """
    try:
        Sources.objects.get(source_id=source_id)
        source_exists = True
    except Sources.DoesNotExist:
        source_exists = False
    except (InterfaceError, OperationalError) as error:
        LOG.error(f"Accessing Sources resulting in {type(error).__name__}: {error}")
        raise error
    return source_exists 

def test_process_synchronize_sources_msg_db_error(self, mock_process_message):
        """Test processing synchronize messages with database errors."""
        provider = Sources.objects.create(**self.aws_source)
        provider.save()
        future_mock = asyncio.Future()
        future_mock.set_result("test result")

        test_queue = queue.PriorityQueue()

        test_matrix = [
            {"test_value": {"operation": "update", "provider": provider}, "side_effect": InterfaceError},
            {"test_value": {"operation": "update", "provider": provider}, "side_effect": OperationalError},
        ]

        for i, test in enumerate(test_matrix):
            mock_process_message.side_effect = test.get("side_effect")
            with patch("sources.kafka_listener.connection.close") as close_mock:
                with patch.object(Config, "RETRY_SECONDS", 0):
                    process_synchronize_sources_msg((i, test["test_value"]), test_queue)
                    close_mock.assert_called()
        for i in range(2):
            priority, _ = test_queue.get_nowait()
            self.assertEqual(priority, i) 

def process(self):
        """
        Process the current cost usage report.

        Args:
            None

        Returns:
            (List) List of filenames downloaded.

        """
        try:
            return self._processor.process()
        except (InterfaceError, DjangoInterfaceError, OperationalError) as err:
            raise ReportProcessorDBError(str(err))
        except Exception as err:
            raise ReportProcessorError(str(err)) 

def close_on_exception(func):
    """
    A wrapper to close the database connection if a DB error occurs,
    so that it will get re-opened on the next use.

    Squashes the exception and logs it.
    """

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            func(*args, **kwargs)
        except OperationalError:
            logger.error('Database error, closing connection', exc_info=True)
            db.connection.close()
            assert db.connection.closed_in_transaction is False, \
                'Could not close connection, probably because this wrapper ' \
                'was used inside an transaction.atomic() block.'

    return wrapper 

def test_process_task_handles_failed_db(
            self, close_old_connections, handle_task, logger):
        """
        Ensure that process_task() gracefully handles closed DB connections
        """
        for err in (InterfaceError, OperationalError):
            handle_task.side_effect = err('Broken DB')

            body = MagicMock()
            msg = MagicMock()

            # First time will fail, will log error, and not ack message
            self.mock_worker.process_task(body, msg)
            assert not msg.ack.called
            assert logger.exception.calls[0][0]
            assert close_old_connections.called
            close_old_connections.reset_mock() 

def process_task(self, body, message):
        logger.info('Processing message: %r', message)
        try:
            self._handle_task(body, message)
        except (OperationalError, InterfaceError):
            # Lost DB connection, close DB and don't ack() msg.
            # A new DB connection will be re-opened next time we
            # try to access the DB. Msg will be re-processed
            # after SQS visibility timeout passes.
            logger.exception("DB connection lost. Cleaning up connections")
            return close_old_connections()
        except:  # NOQA
            logger.exception("Failed to process message: %r", message)

        logger.info("ACKing message %r", message)
        if self.connection.as_uri().lower().startswith('sqs://'):
            # HACK: Can't seem to get message.ack() to work for SQS
            # backend. Without this hack, messages will keep
            # re-appearing after the visibility_timeout expires.
            # See https://github.com/celery/kombu/issues/758
            return self._sqs_ack(message)
        return message.ack() 

def check_users(app_configs=None, **kwargs):
    from django.contrib.auth import get_user_model

    errors = []

    User = get_user_model()
    try:
        admin_user = User.objects.get(username="admin")
    except (User.DoesNotExist, OperationalError, ProgrammingError):
        pass
    else:
        if admin_user.check_password("admin"):
            errors.append(
                checks.Warning(
                    _("The default 'admin' user still has a password set to 'admin'."),
                    hint=_("Remove the 'admin' user or change its password."),
                    id="pootle.W016",
                )
            )

    return errors 

def check_revision(app_configs=None, **kwargs):
    from pootle.core.models import Revision
    from pootle_store.models import Unit

    errors = []
    revision = Revision.get()
    try:
        max_revision = Unit.max_revision()
    except (OperationalError, ProgrammingError):
        return errors
    if revision is None or revision < max_revision:
        errors.append(
            checks.Critical(
                _("Revision is missing or has an incorrect value."),
                hint=_("Run `revision --restore` to reset the revision counter."),
                id="pootle.C016",
            )
        )

    return errors 

def get_context_data(self, **kwargs):
        context = super(PointDetailView, self).get_context_data(**kwargs)
        context['grafana_url'] = settings.GRAFANA_BASE_URL + "/d/"
        # add the parent Site (optional)
        try:
            context['site'] = SiteView.objects.get(object_id=context['object'].site_id)
        except SiteView.DoesNotExist:
            pass

        # add the parent Equipment (optional)
        try:
            context['equipment'] = EquipmentView.objects.get(object_id=context['object'].equipment_id)
        except EquipmentView.DoesNotExist:
            pass

        if context['object']:
            charts = []
            try:
                charts = utils.charts_for_points([context['object']])
            except OperationalError:
                logging.warning('Crate database unavailable')

            context['charts'] = charts

        return context 

def delete_tags_from_crate_entity(row):
    # we only sync for entity linked to a topic
    if not row.topic:
        return

    try:
        with connections['crate'].cursor() as c:
            # make sure the topic is in CrateDB
            sql = """DELETE {0} WHERE topic = %s;""".format("topic")
            try:
                c.execute(sql, [row.topic])
            except Exception:
                # ignore if the entity did not exist
                pass
    except OperationalError:
        logging.warning('Crate database unavailable') 

def get_queryset(self):
        """Select all the answer distribution response having to do with this usage of the problem."""
        problem_id = self.kwargs.get('problem_id')

        try:
            queryset = list(ProblemResponseAnswerDistribution.objects.filter(module_id=problem_id).order_by('part_id'))
        except OperationalError:
            self.serializer_class = ConsolidatedFirstLastAnswerDistributionSerializer
            queryset = list(ProblemFirstLastResponseAnswerDistribution.objects.filter(
                module_id=problem_id).order_by('part_id'))

        consolidated_rows = []

        for _, part in groupby(queryset, lambda x: x.part_id):
            consolidated_rows += self.consolidate_answers(list(part))

        return consolidated_rows 

def process_content(self):
        for content in self.pre_processed_content:  # type: Reference
            try:
                # First save (some processing steps require ids)
                content.full_clean()  # Validate model
                content.save()

                self.call_processing_steps(content)

                # Save again
                content.save()

                logger.debug('Completed: %s' % content)

                self.doc_counter += 1
                self.processed_content.append(content)

            except (ValidationError, DataError, OperationalError, IntegrityError, ProcessingError) as e:
                logger.error('Cannot process: %s; %s' % (content, e))
                self.processing_errors.append(e)
                self.doc_failed_counter += 1 

def process_content_item(self, content: Case) -> Case:
        try:
            # First save (some processing steps require ids)
            # content.full_clean()  # Validate model
            content.save()

            self.call_processing_steps(content)

            # Save again
            content.save()

            logger.debug('Completed: %s' % content)

            self.doc_counter += 1
            self.processed_content.append(content)

        except (ValidationError, DataError, OperationalError, IntegrityError, ProcessingError) as e:
            logger.error('Cannot process case: %s; %s' % (content, e))
            self.processing_errors.append(e)
            self.doc_failed_counter += 1

        return content 

def process_content(self):
        for content in self.pre_processed_content:  # type: Court
            try:
                # First save (some processing steps require ids)
                content.full_clean()  # Validate model
                content.save()

                self.call_processing_steps(content)

                # Save again
                content.save()

                logger.debug('Completed: %s' % content)

                self.doc_counter += 1
                self.processed_content.append(content)

            except (ValidationError, DataError, OperationalError, IntegrityError, ProcessingError) as e:
                logger.error('Cannot process court: %s; %s' % (content, e))
                self.processing_errors.append(e)
                self.doc_failed_counter += 1 

def LatestEvent():
    from tracker.models import Event

    if Event.objects.exists():
        try:
            return Event.objects.latest()
        except (Event.DoesNotExist, OperationalError):
            return None
    return None 

def setUp(self) -> None:
        _log.debug("[FullErrorTests] patching for setup")
        self.s_connect = BaseDatabaseWrapper.connect
        BaseDatabaseWrapper.connect = Mock(side_effect=OperationalError('fail testing'))
        BaseDatabaseWrapper.connection = property(lambda x: None, lambda x, y: None)  # type: ignore 

def test_prehook(self) -> None:
        cb = Mock(name='pre_reconnect_hook')
        ddr.pre_reconnect.connect(cb)
        self.assertRaises(OperationalError, connection.ensure_connection)
        self.assertTrue(cb.called)
        del connection._connection_retries 

def test_posthook(self) -> None:
        cb = Mock(name='post_reconnect_hook')
        ddr.post_reconnect.connect(cb)
        self.assertRaises(OperationalError, connection.ensure_connection)
        self.assertTrue(cb.called)
        del connection._connection_retries 

def test_prehook(self) -> None:
        cb = Mock(name='pre_reconnect_hook')
        ddr.pre_reconnect.connect(fix_connection)
        ddr.pre_reconnect.connect(cb)
        from django.db import connection
        connection.close()
        connection.s_connect = connection.connect
        connection.connect = Mock(side_effect=OperationalError('reconnect testing'))
        connection.ensure_connection()
        ReconnectTests.cls_atomics['default'] = transaction.atomic(using='default')
        ReconnectTests.cls_atomics['default'].__enter__()
        self.assertTrue(cb.called)
        self.assertTrue(connection.is_usable()) 

def test_posthook(self) -> None:
        cb = Mock(name='post_reconnect_hook')
        ddr.pre_reconnect.connect(fix_connection)
        ddr.post_reconnect.connect(cb)
        from django.db import connection
        connection.close()
        connection.s_connect = connection.connect
        connection.connect = Mock(side_effect=OperationalError('reconnect testing'))
        connection.ensure_connection()
        ReconnectTests.cls_atomics['default'] = transaction.atomic(using='default')
        ReconnectTests.cls_atomics['default'].__enter__()
        self.assertTrue(cb.called)
        self.assertTrue(connection.is_usable()) 

def connection_check(self):
        """Check DB connection."""
        try:
            connection.cursor()
            LOG.debug("DatabaseStatus.connection_check: DB connected!")
        except OperationalError as error:
            LOG.error("DatabaseStatus.connection_check: No connection to DB: %s", str(error))
            DB_CONNECTION_ERRORS_COUNTER.inc() 

def query(self, query, query_tag):
        """Execute a SQL query, format the results.

        Returns:
            [
                {col_1: <value>, col_2: value, ...},
                {col_1: <value>, col_2: value, ...},
                {col_1: <value>, col_2: value, ...},
            ]

        """
        rows = None
        for _ in range(3):
            try:
                with connection.cursor() as cursor:
                    cursor.execute(query)
                    rows = cursor.fetchall()
            except (OperationalError, InterfaceError) as exc:
                LOG.warning("DatabaseStatus.query exception: %s", exc)
                time.sleep(2)
            else:
                break
        else:
            LOG.error("DatabaseStatus.query (query: %s): Query failed to return results.", query_tag)
            return []

        if not rows:
            LOG.info("DatabaseStatus.query (query: %s): Query returned no results.", query_tag)
            return []

        # get column names
        names = [desc[0] for desc in cursor.description]

        # transform list-of-lists into list-of-dicts including column names.
        result = [dict(zip(names, row)) for row in rows if len(row) == 2]

        LOG.info("DatabaseStatus.query (query: %s): query returned.", query_tag)

        return result 

def test_db_is_not_connected(self, mock_connection):
        """Test that db is not connected, log at ERROR level and counter increments."""
        mock_connection.cursor.side_effect = OperationalError("test exception")
        logging.disable(logging.NOTSET)
        before = REGISTRY.get_sample_value("db_connection_errors_total")
        self.assertIsNotNone(before)
        with self.assertLogs(logger="koku.metrics", level=logging.ERROR):
            DatabaseStatus().connection_check()
        after = REGISTRY.get_sample_value("db_connection_errors_total")
        self.assertIsNotNone(after)
        self.assertEqual(1, after - before) 

def test_celery_task(self, mock_collect, mock_push):
        """Test celery task to increment prometheus counter."""
        before = REGISTRY.get_sample_value("db_connection_errors_total")
        self.assertIsNotNone(before)
        with mock.patch("django.db.backends.utils.CursorWrapper") as mock_cursor:
            mock_cursor.side_effect = OperationalError("test exception")
            mock_collect.return_value = []
            task = collect_metrics.s().apply()
            self.assertTrue(task.successful())
        after = REGISTRY.get_sample_value("db_connection_errors_total")
        self.assertIsNotNone(after)
        self.assertEqual(1, after - before) 

def listen_for_messages(msg, consumer, application_source_id):  # noqa: C901
    """
    Listen for Platform-Sources kafka messages.

    Args:
        consumer (Consumer): Kafka consumer object
        application_source_id (Integer): Cost Management's current Application Source ID. Used for
            kafka message filtering.

    Returns:
        None

    """
    try:
        try:
            msg = get_sources_msg_data(msg, application_source_id)
            offset = msg.get("offset")
            partition = msg.get("partition")
        except SourcesMessageError:
            return
        if msg:
            LOG.info(f"Processing message offset: {offset} partition: {partition}")
            topic_partition = TopicPartition(topic=Config.SOURCES_TOPIC, partition=partition, offset=offset)
            LOG.info(f"Cost Management Message to process: {str(msg)}")
            try:
                with transaction.atomic():
                    process_message(application_source_id, msg)
                    consumer.commit()
            except (IntegrityError, InterfaceError, OperationalError) as err:
                connection.close()
                LOG.error(f"{type(err).__name__}: {err}")
                rewind_consumer_to_retry(consumer, topic_partition)
            except SourcesHTTPClientError as err:
                LOG.error(err)
                rewind_consumer_to_retry(consumer, topic_partition)
            except SourceNotFoundError:
                LOG.warning(f"Source not found in platform sources. Skipping msg: {msg}")
                consumer.commit()

    except KafkaError as error:
        LOG.error(f"[listen_for_messages] Kafka error encountered: {type(error).__name__}: {error}", exc_info=True)
    except Exception as error:
        LOG.error(f"[listen_for_messages] UNKNOWN error encountered: {type(error).__name__}: {error}", exc_info=True) 

def database_status(self):
        """Collect database connection information.

        :returns: A dict of db connection info.
        """
        try:
            with connection.cursor() as cursor:
                cursor.execute(
                    """
                    SELECT datname AS database,
                        numbackends as database_connections
                    FROM pg_stat_database
                    """
                )
                raw = cursor.fetchall()

                # get pg_stat_database column names
                names = [desc[0] for desc in cursor.description]
        except (InterfaceError, NotSupportedError, OperationalError, ProgrammingError) as exc:
            LOG.warning("Unable to connect to DB: %s", str(exc))
            return {"ERROR": str(exc)}

        # transform list-of-lists into list-of-dicts including column names.
        result = [dict(zip(names, row)) for row in raw]

        return result 

def get_source(source_id, err_msg, logger):
    """Access Sources, log err on DoesNotExist, close connection on InterfaceError."""
    try:
        return Sources.objects.get(source_id=source_id)
    except Sources.DoesNotExist:
        logger(err_msg)
    except (InterfaceError, OperationalError) as error:
        LOG.error(f"Accessing sources resulted in {type(error).__name__}: {error}")
        raise error 

def enqueue_source_delete(source_id, offset, allow_out_of_order=False):
    """
    Queues a source destroy event to be processed by the synchronize_sources method.

    Args:
        queue (Asyncio Queue) - process_queue containing all pending Souces-koku events.
        source_id (Integer) - Platform-Sources identifier.
        allow_out_of_order (Bool) - Allow for out of order delete events (Application or Source).

    Returns:
        None

    """
    try:
        source = Sources.objects.get(source_id=source_id)
        if not source.pending_delete and not source.out_of_order_delete:
            source.pending_delete = True
            source.save()
    except Sources.DoesNotExist:
        if allow_out_of_order:
            LOG.info(f"Source ID: {source_id} not known.  Marking as out of order delete.")
            new_event = Sources(source_id=source_id, offset=offset, out_of_order_delete=True)
            new_event.save()
            LOG.info(f"source.storage.create_source_event created Source ID as pending delete: {source_id}")
    except (InterfaceError, OperationalError) as error:
        LOG.error(f"Accessing sources resulted in {type(error).__name__}: {error}")
        raise error