Python gc.get_objects() Examples

def test_teardown_issue1649(testdir):
    """
    Are TestCase objects cleaned up? Often unittest TestCase objects set
    attributes that are large and expensive during setUp.

    The TestCase will not be cleaned up if the test fails, because it
    would then exist in the stackframe.
    """
    testpath = testdir.makepyfile(
        """
        import unittest
        class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase):
            def setUp(self):
                self.an_expensive_object = 1
            def test_demo(self):
                pass

    """
    )
    testdir.inline_run("-s", testpath)
    gc.collect()
    for obj in gc.get_objects():
        assert type(obj).__name__ != "TestCaseObjectsShouldBeCleanedUp" 

def print_muppy_sumary():
    # http://pythonhosted.org/Pympler/index.html
    try:
        from pympler import muppy, summary
    except ImportError:
        print("WARNING: pympler not installed")
        return
    # from pympler.classtracker import ClassTracker
    # from pympler.classtracker_stats import HtmlStats
    global all_objects, obj_summary, class_tracker
    if all_objects is None:
        all_objects = muppy.get_objects()
        obj_summary = summary.summarize(all_objects)
        summary.print_(obj_summary)

        # class_tracker = ClassTracker()
        # class_tracker.track_class(FICSPlayer, trace=1)
        # class_tracker.track_class(ICGameModel, resolution_level=2, trace=1)
    else:
        obj_summary2 = summary.summarize(muppy.get_objects())
        diff = summary.get_diff(obj_summary, obj_summary2)
        summary.print_(diff, limit=200)

        # class_tracker.create_snapshot('usage')
        # HtmlStats(tracker=class_tracker).create_html('profile.html') 

def count(typename, objects=None):
    """Count objects tracked by the garbage collector with a given class name.

    Example:

        >>> count('dict')
        42
        >>> count('MyClass', get_leaking_objects())
        3

    Note that the GC does not track simple objects like int or str.

    .. versionchanged:: 1.7
       New parameter: ``objects``.

    """
    if objects is None:
        objects = gc.get_objects()
    return sum(1 for o in objects if type(o).__name__ == typename) 

def get_leaking_objects(objects=None):
    """Return objects that do not have any referents.

    These could indicate reference-counting bugs in C code.  Or they could
    be legitimate.

    Note that the GC does not track simple objects like int or str.

    .. versionadded:: 1.7
    """
    if objects is None:
        gc.collect()
        objects = gc.get_objects()
    try:
        ids = set(id(i) for i in objects)
        for i in objects:
            ids.difference_update(id(j) for j in gc.get_referents(i))
        # this then is our set of objects without referrers
        return [i for i in objects if id(i) in ids]
    finally:
        objects = i = j = None # clear cyclic references to frame 

def by_type(typename, objects=None):
    """Return objects tracked by the garbage collector with a given class name.

    Example:

        >>> by_type('MyClass')
        [<mymodule.MyClass object at 0x...>]

    Note that the GC does not track simple objects like int or str.

    .. versionchanged:: 1.7
       New parameter: ``objects``.

    """
    if objects is None:
        objects = gc.get_objects()
    return [o for o in objects if type(o).__name__ == typename] 

def obj_referrers(klass):
    find_obj = False
    for obj in gc.get_objects():
        # closures are evil !
        if isinstance(obj, types.FunctionType) and obj.__closure__ is not None:
            for c in obj.__closure__:
                try:
                    if isinstance(c.cell_contents, klass):
                        print('!!!', obj, c.cell_contents)
                except ValueError:
                    print("Cell is empty...")
        if isinstance(obj, klass):
            find_obj = True
            rs = gc.get_referrers(obj)
            print("---------------------------referrers of %s" % klass.__name__)
            for ob in rs:
                print(type(ob), ob.__name__ if type(ob) is type else repr(ob)[:140])
                rs1 = gc.get_referrers(ob)
                for ob1 in rs1:
                    print('    ', type(ob1), ob1.__name__ if type(ob1) is type else repr(ob1)[:140])
            print("---------------------------")
    if not find_obj:
        print("Nothing refrences %s" % klass.__name__) 

def memory_used():
    result = {}
    for obj in gc.get_objects():
        try:
            if torch.is_tensor(obj):
                if type(obj) not in result:
                    result[type(obj)] = 0
                count = 1
                for elem in list(obj.data.size()):
                    count *= elem
                result[type(obj.data)] += count * obj.data.element_size()
            elif hasattr(obj, 'data') and torch.is_tensor(obj.data):
                if type(obj.data) not in result:
                    result[type(obj.data)] = 0
                count = 1
                for elem in list(obj.data.size()):
                    count *= elem
                result[type(obj.data)] += count * obj.data.element_size()
        except:
            print("could not track ...")

    return result 

def observation_containers(self):
        # Return the current set of 'observation containers'
        # as discussed in Notes Oct 27 2005.
        # returns a nodeset, not an idset, to avoid recursive referenes

        objs = self.gc.get_objects()
        cli = self.hv.cli_type()
        objs = (cli.select(objs, self.NodeSet, '<=') +
                cli.select(objs, ObservationList, '<=') +
                cli.select(
                    objs, self._parent.UniSet.IdentitySetSingleton, '<=')
                )
        r = self.immnodeset([x for x in objs if getattr(
            x, '_hiding_tag_', None) is self._hiding_tag_])
        del cli, objs
        return r 

def cleanup_gc(self):
        gc.collect()
        leaked = [obj for obj in gc.get_objects() + gc.garbage
                  if isinstance(obj,
                                (qubes.Qubes, qubes.vm.BaseVM,
                                 libvirt.virConnect, libvirt.virDomain))]

        if leaked:
            try:
                import objgraph
                objgraph.show_backrefs(leaked,
                                       max_depth=15, extra_info=extra_info,
                                       filename='/tmp/objgraph-{}.png'.format(
                                           self.id()))
            except ImportError:
                pass

        # do not keep leaked object references in locals()
        leaked = bool(leaked)
        assert not leaked 

def host_memory_usage_in_gb():
    gc.collect()
    gc.disable() # Avoids accessing gc'd objects during traversal.
    objects = gc.get_objects()
    tensors = [obj for obj in objects if torch.is_tensor(obj)] # Debug
    host_tensors = [t for t in tensors if not t.is_cuda]
    total_mem_mb = 0
    visited_data = []

    for tensor in host_tensors:
        if tensor.is_sparse:
            continue
        # a data_ptr indicates a memory block allocated
        data_ptr = tensor.storage().data_ptr()
        if data_ptr in visited_data:
            continue
        visited_data.append(data_ptr)

        numel = tensor.storage().size()
        element_size = tensor.storage().element_size()
        mem_mb = numel*element_size /1024/1024 # 32bit=4Byte, MByte
        total_mem_mb += mem_mb

    gc.enable()
    return total_mem_mb / 1024 # in 

def disconnectAllHdfDBs():
    """
    Forcibly disconnect all instances of HdfDB objects

    Notes
    -----
    This is a hack to help ARMI exit gracefully when the garbage collector and h5py have
    issues destroying objects. After lots of investigation, the root cause for why this
    was having issues was never identified. It appears that when several HDF5 files are
    open in the same run (e.g.  when calling armi.init() multiple times from a
    post-processing script), when these h5py File objects were closed, the garbage
    collector would raise an exception related to the repr'ing the object. We
    get around this by using the garbage collector to manually disconnect all open HdfDB
    objects.
    """
    h5dbs = [db for db in gc.get_objects() if isinstance(db, Database3)]
    for db in h5dbs:
        db.close() 

def test_cycle_through_dict(self):
        # See bug #1469629
        class X(dict):
            def __init__(self):
                dict.__init__(self)
                self.__dict__ = self
        x = X()
        x.attr = 42
        wr = weakref.ref(x)
        del x
        support.gc_collect()
        self.assertIsNone(wr())
        for o in gc.get_objects():
            self.assertIsNot(type(o), X) 

def getobjects(name, cls=True):
    """
    :param name:
    :param cls:
    :return:
    """
    objects = []
    for obj in get_objects():
        if (isinstance(obj, QObject) and
                ((cls and obj.inherits(name)) or
                 (not cls and obj.objectName() == name))):
            objects.append(obj)
    return objects
#endregion
#region QT Manipulation 

def gather_sample(self):
        try:
            # collect everything that can be collected
            if not self._config.get("disable_garbage_collection_before_dump"):
                gc.collect()

            # dump garbage objects
            if self._monitor_garbage:
                garbage = gc.garbage
                self._dump_objects(
                    all_objects=garbage,
                    object_dump_types=self._object_dump_types,
                    max_type_dump=self._max_type_dump,
                    max_object_dump=self._max_object_dump,
                    dump_kind="garbage",
                )

            # dump live objects
            if self._monitor_live:
                objects = gc.get_objects()
                self._dump_objects(
                    all_objects=objects,
                    object_dump_types=self._object_dump_types,
                    max_type_dump=self._max_type_dump,
                    max_object_dump=self._max_object_dump,
                    dump_kind="live",
                )

        except Exception:
            global_log.info("error gathering sample %s", traceback.format_exc()) 

def test_get_objects(self):
        if is_cli:
            self.assertRaises(NotImplementedError, gc.get_objects)
        else:
            gc.get_objects() 

def test_cycle_through_dict(self):
        # See bug #1469629
        class X(dict):
            def __init__(self):
                dict.__init__(self)
                self.__dict__ = self
        x = X()
        x.attr = 42
        wr = weakref.ref(x)
        del x
        support.gc_collect()
        self.assertIsNone(wr())
        for o in gc.get_objects():
            self.assertIsNot(type(o), X) 

def instancesCounts():
        counts = collections.defaultdict(int)
        for item in gc.get_objects():
            if hasattr(item, "__class__"):
                counts[item.__class__.__name__] += 1
        for k, v in sorted(counts.items(), key=lambda x: x[1], reverse=True):
            print("{}\t\t{}".format(v, k)) 

def test_issue_7959(self):
        proto = self.functype.__func__(None)

        class X(object):
            def func(self): pass
            def __init__(self):
                self.v = proto(self.func)

        import gc
        for i in range(32):
            X()
        gc.collect()
        live = [x for x in gc.get_objects()
                if isinstance(x, X)]
        self.assertEqual(len(live), 0) 

def test_markup_leaks(self):
        counts = set()
        for count in range(20):
            for item in range(1000):
                escape("foo")
                escape("<foo>")
                escape(u"foo")
                escape(u"<foo>")
            counts.add(len(gc.get_objects()))
        assert len(counts) == 1, 'ouch, c extension seems to leak objects' 

def __enter__(self):
        gc.disable()
        _gc_lock.acquire()
        loc = flask._request_ctx_stack._local

        # Force Python to track this dictionary at all times.
        # This is necessary since Python only starts tracking
        # dicts if they contain mutable objects.  It's a horrible,
        # horrible hack but makes this kinda testable.
        loc.__storage__['FOOO'] = [1, 2, 3]

        gc.collect()
        self.old_objects = len(gc.get_objects()) 

def obj_at(self, addr):
        try:
            return self.immnodeset(self.hv.static_types).obj_at(addr)
        except ValueError:
            pass
        try:
            return self.immnodeset(self.gc.get_objects()).obj_at(addr)
        except ValueError:
            pass
        try:
            return self.immnodeset(self.hv.heap()).obj_at(addr)
        except ValueError:
            raise ValueError('No object found at address %s' % hex(addr)) 

def heapu(self, rma=1):
        self.gc.collect()
        self.gc.collect()
        r = self.gc.get_objects()

        exclude = (self.Type(self.gchook_type) |
                   self.Type(ClearCallback)
                   )

        if rma:
            exclude |= self.idset(self.heapyc.HeapView(
                self.heapyc.RootState,
                self.heapdefs
            ).reachable_x(
                self.immnodeset([self.heapyc.RootState]),
                self.immnodeset([r])
            ))

        r = self.retset(r) - exclude
        ref = r.referents - exclude
        while not ref <= r:
            r |= ref
            ref = ref.referents - exclude

        del ref, exclude

        r = r.bytype  # Avoid memoizing for complicated classification
        return r 

def heapg(self, rma=1):
        # Almost the same as gc.get_objects(),
        # except:
        # 1. calls gc.collect() first (twice)
        # 2. removes objects of type gchook
        # 3. removes objects of type ClearCallback
        # 4. removes all objects of type types.FrameType
        # 5. removes all objects of weakref type
        # 6. If rma = 1,
        #    removes all that is in the reachable heap
        #    except what is in the set itself.

        # . wraps the result in an IdSet

        self.gc.collect()
        self.gc.collect()
        objs = self.gc.get_objects()
        cli = self.hv.cli_type()
        objs = cli.select(objs, self.gchook_type, '!=')
        objs = cli.select(objs, ClearCallback, '!=')
        objs = cli.select(objs, self._root.types.FrameType, '!=')
        objs = cli.select(objs, self._root.weakref.ReferenceType, '!=')
        r = self.retset(objs)
        del cli, objs

        if rma:
            r = (r - self.idset(self.heapyc.HeapView(
                self.heapyc.RootState,
                self.heapdefs
            ).reachable_x(
                self.immnodeset([self.heapyc.RootState]),
                self.observation_containers()
            ))
            )

        return r 

def __exit__(self, exc_type, exc_value, tb):
        if not hasattr(sys, 'getrefcount'):
            gc.collect()
        new_objects = len(gc.get_objects())
        if new_objects > self.old_objects:
            self.testcase.fail('Example code leaked')
        _gc_lock.release()
        gc.enable() 

def mark(self):
        self.R.gc.collect()
        h = self.V.horizon()
        h.update(gc.get_objects())
        self.h = h 

def _walkReferrers(o, maxLevel=0, level=0, memo=None, whitelist=None):
    """
    Walk down the tree of objects that refer to the passed object, printing diagnostics along the
    way.
    """
    if maxLevel and level > maxLevel:
        return
    if level == 0:
        gc.collect()
        whitelist = {id(obj) for obj in gc.get_objects()}
        whitelist.remove(id(_getFunctionObject()))
        whitelist.remove(id(_getFunctionObject))

    if memo is None:
        memo = set()
    gc.collect()
    referrers = [
        (referrer, id(referrer), id(referrer) in memo)
        for referrer in gc.get_referrers(o)
        if referrer.__class__.__name__ != "frame" and id(referrer) in whitelist
    ]
    memo.update({oid for (_obj, oid, _seen) in referrers})
    for (obj, oid, seen) in referrers:
        runLog.important(
            "{}{}    {} at {:x} seen: {}".format(
                "-" * level, type(obj), "{}".format(obj)[:100], id(obj), seen
            )
        )
        if seen:
            return
        _walkReferrers(
            obj, maxLevel=maxLevel, level=level + 1, memo=memo, whitelist=whitelist
        ) 

def test_markup_leaks(self):
        counts = set()
        for count in range(20):
            for item in range(1000):
                escape("foo")
                escape("<foo>")
                escape(u"foo")
                escape(u"<foo>")
            counts.add(len(gc.get_objects()))
        assert len(counts) == 1, 'ouch, c extension seems to leak objects' 

def test_forward(self):
    knn = KNearestNeighbor(2)
    while(1):
        D, N, M = 128, 100, 1000
        ref = Variable(torch.rand(2, D, N))
        query = Variable(torch.rand(2, D, M))

        inds = knn(ref, query)
        for obj in gc.get_objects():
            if torch.is_tensor(obj):
                print(functools.reduce(op.mul, obj.size()) if len(obj.size()) > 0 else 0, type(obj), obj.size())
        #ref = ref.cpu()
        #query = query.cpu()
        print(inds) 

def get_instances(cls):
    return [x for x in gc.get_objects() if isinstance(x, cls)] 

def get_objects(remove_dups=True, include_frames=False):
    """Return a list of all known objects excluding frame objects.

    If (outer) frame objects shall be included, pass `include_frames=True`.  In
    order to prevent building reference cycles, the current frame object (of
    the caller of get_objects) is ignored. This will not prevent creating
    reference cycles if the object list is passed up the call-stack. Therefore,
    frame objects are not included by default.

    Keyword arguments:
    remove_dups -- if True, all duplicate objects will be removed.
    include_frames -- if True, includes frame objects.
    """
    gc.collect()

    # Do not initialize local variables before calling gc.get_objects or those
    # will be included in the list. Furthermore, ignore frame objects to
    # prevent reference cycles.
    tmp = gc.get_objects()
    tmp = [o for o in tmp if not isframe(o)]

    res = []
    for o in tmp:
        # gc.get_objects returns only container objects, but we also want
        # the objects referenced by them
        refs = get_referents(o)
        for ref in refs:
            if not _is_containerobject(ref):
                # we already got the container objects, now we only add
                # non-container objects
                res.append(ref)
    res.extend(tmp)
    if remove_dups:
        res = _remove_duplicates(res)

    if include_frames:
        for sf in stack()[2:]:
            res.append(sf[0])
    return res