Python twisted.internet.task.cooperate() Examples

def setUp(self):
        """
        Create a cooperator with a fake scheduler and a termination predicate
        that ensures only one unit of work will take place per tick.
        """
        self._doDeferNext = False
        self._doStopNext = False
        self._doDieNext = False
        self.work = []
        self.scheduler = FakeScheduler()
        self.cooperator = task.Cooperator(
            scheduler=self.scheduler,
            # Always stop after one iteration of work (return a function which
            # returns a function which always returns True)
            terminationPredicateFactory=lambda: lambda: True)
        self.task = self.cooperator.cooperate(self.worker())
        self.cooperator.start() 

def setUp(self):
        """
        Create a cooperator with a fake scheduler and a termination predicate
        that ensures only one unit of work will take place per tick.
        """
        self._doDeferNext = False
        self._doStopNext = False
        self._doDieNext = False
        self.work = []
        self.scheduler = FakeScheduler()
        self.cooperator = task.Cooperator(
            scheduler=self.scheduler,
            # Always stop after one iteration of work (return a function which
            # returns a function which always returns True)
            terminationPredicateFactory=lambda: lambda: True)
        self.task = self.cooperator.cooperate(self.worker())
        self.cooperator.start() 

def startStreaming(self):
        """
        This should be called by the consumer when the producer is registered.

        Start streaming data to the consumer.
        """
        self._coopTask = cooperate(self._pull()) 

def startService(self):
        """Open the queue and start processing database tasks.

        :return: `None`
        """
        super().startService()
        self.queue.size = None  # Open queue to puts.
        self.coop = cooperate(self._generateTasks()) 

def test_cooperate(self):
        """
        L{twisted.internet.task.cooperate} ought to run the generator that it is
        """
        d = defer.Deferred()
        def doit():
            yield 1
            yield 2
            yield 3
            d.callback("yay")
        it = doit()
        theTask = task.cooperate(it)
        self.assertIn(theTask, task._theCooperator._tasks)
        return d 

def _appendMessages(self, mbox, messages):
        """
        Deliver the given messages one at a time.  Delivery is serialized to
        guarantee a predictable order in the mailbox (overlapped message deliver
        makes no guarantees about which message which appear first).
        """
        results = []
        def append():
            for m in messages:
                d = mbox.appendMessage(m)
                d.addCallback(results.append)
                yield d
        d = task.cooperate(append()).whenDone()
        d.addCallback(lambda ignored: results)
        return d 

def create_cooperate(iterObj):
        return cooperate(TxnProcessor.iterator(iterObj)) 

def test_cooperate(self):
        """
        L{twisted.internet.task.cooperate} ought to run the generator that it is
        """
        d = defer.Deferred()
        def doit():
            yield 1
            yield 2
            yield 3
            d.callback("yay")
        it = doit()
        theTask = task.cooperate(it)
        self.assertIn(theTask, task._theCooperator._tasks)
        return d 

def test_removingLastTaskStopsScheduledCall(self):
        """
        If the last task in a Cooperator is removed, the scheduled call for
        the next tick is cancelled, since it is no longer necessary.

        This behavior is useful for tests that want to assert they have left
        no reactor state behind when they're done.
        """
        calls = [None]
        def sched(f):
            calls[0] = FakeDelayedCall(f)
            return calls[0]
        coop = task.Cooperator(scheduler=sched)

        # Add two task; this should schedule the tick:
        task1 = coop.cooperate(iter([1, 2]))
        task2 = coop.cooperate(iter([1, 2]))
        self.assertEqual(calls[0].func, coop._tick)

        # Remove first task; scheduled call should still be going:
        task1.stop()
        self.assertFalse(calls[0].cancelled)
        self.assertEqual(coop._delayedCall, calls[0])

        # Remove second task; scheduled call should be cancelled:
        task2.stop()
        self.assertTrue(calls[0].cancelled)
        self.assertIsNone(coop._delayedCall)

        # Add another task; scheduled call will be recreated:
        coop.cooperate(iter([1, 2]))
        self.assertFalse(calls[0].cancelled)
        self.assertEqual(coop._delayedCall, calls[0]) 

def _driveWorker(self, worker, result, testCases, cooperate):
        """
        Drive a L{LocalWorkerAMP} instance, iterating the tests and calling
        C{run} for every one of them.

        @param worker: The L{LocalWorkerAMP} to drive.

        @param result: The global L{DistReporter} instance.

        @param testCases: The global list of tests to iterate.

        @param cooperate: The cooperate function to use, to be customized in
            tests.
        @type cooperate: C{function}

        @return: A C{Deferred} firing when all the tests are finished.
        """

        def resultErrback(error, case):
            result.original.addFailure(case, error)
            return error

        def task(case):
            d = worker.run(case, result)
            d.addErrback(resultErrback, case)
            return d

        return cooperate(task(case) for case in testCases).whenDone() 

def test_defaultCooperator(self):
        """
        If no L{Cooperator} instance is passed to L{FileBodyProducer}, the
        global cooperator is used.
        """
        producer = FileBodyProducer(BytesIO(b""))
        self.assertEqual(task.cooperate, producer._cooperate) 

def test_defaultCooperator(self):
        """
        If no L{Cooperator} instance is passed to L{FileBodyProducer}, the
        global cooperator is used.
        """
        producer = FileBodyProducer(BytesIO(b""))
        self.assertEqual(task.cooperate, producer._cooperate) 

def test_removingLastTaskStopsScheduledCall(self):
        """
        If the last task in a Cooperator is removed, the scheduled call for
        the next tick is cancelled, since it is no longer necessary.

        This behavior is useful for tests that want to assert they have left
        no reactor state behind when they're done.
        """
        calls = [None]
        def sched(f):
            calls[0] = FakeDelayedCall(f)
            return calls[0]
        coop = task.Cooperator(scheduler=sched)

        # Add two task; this should schedule the tick:
        task1 = coop.cooperate(iter([1, 2]))
        task2 = coop.cooperate(iter([1, 2]))
        self.assertEqual(calls[0].func, coop._tick)

        # Remove first task; scheduled call should still be going:
        task1.stop()
        self.assertFalse(calls[0].cancelled)
        self.assertEqual(coop._delayedCall, calls[0])

        # Remove second task; scheduled call should be cancelled:
        task2.stop()
        self.assertTrue(calls[0].cancelled)
        self.assertIsNone(coop._delayedCall)

        # Add another task; scheduled call will be recreated:
        coop.cooperate(iter([1, 2]))
        self.assertFalse(calls[0].cancelled)
        self.assertEqual(coop._delayedCall, calls[0]) 

def _appendMessages(self, mbox, messages):
        """
        Deliver the given messages one at a time.  Delivery is serialized to
        guarantee a predictable order in the mailbox (overlapped message deliver
        makes no guarantees about which message which appear first).
        """
        results = []
        def append():
            for m in messages:
                d = mbox.appendMessage(m)
                d.addCallback(results.append)
                yield d
        d = task.cooperate(append()).whenDone()
        d.addCallback(lambda ignored: results)
        return d 

def _driveWorker(self, worker, result, testCases, cooperate):
        """
        Drive a L{LocalWorkerAMP} instance, iterating the tests and calling
        C{run} for every one of them.

        @param worker: The L{LocalWorkerAMP} to drive.

        @param result: The global L{DistReporter} instance.

        @param testCases: The global list of tests to iterate.

        @param cooperate: The cooperate function to use, to be customized in
            tests.
        @type cooperate: C{function}

        @return: A C{Deferred} firing when all the tests are finished.
        """

        def resultErrback(error, case):
            result.original.addFailure(case, error)
            return error

        def task(case):
            d = worker.run(case, result)
            d.addErrback(resultErrback, case)
            return d

        return cooperate(task(case) for case in testCases).whenDone() 

def startStreaming(self):
        """
        This should be called by the consumer when the producer is registered.

        Start streaming data to the consumer.
        """
        self._coopTask = cooperate(self._pull()) 

def test_snapshots(self):
        """
        The ``Deferred`` returned by ``Filesystem.snapshots`` fires with a
        ``list`` of ``Snapshot`` instances corresponding to the snapshots that
        exist for the ZFS filesystem to which the ``Filesystem`` instance
        corresponds.
        """
        expected_names = [b"foo", b"bar"]

        # Create a filesystem and a couple snapshots.
        pool = build_pool(self)
        service = service_for_pool(self, pool)
        volume = service.get(MY_VOLUME)
        creating = pool.create(volume)

        def created(filesystem):
            # Save it for later.
            self.filesystem = filesystem

            # Take a couple snapshots now that there is a filesystem.
            return cooperate(
                zfs_command(
                    reactor, [
                        b"snapshot",
                        u"{}@{}".format(filesystem.name, name).encode("ascii"),
                    ]
                )
                for name in expected_names
            ).whenDone()

        snapshotting = creating.addCallback(created)

        def snapshotted(ignored):
            # Now that some snapshots exist, interrogate the system.
            return self.filesystem.snapshots()

        loading = snapshotting.addCallback(snapshotted)

        def loaded(snapshots):
            self.assertEqual(
                list(Snapshot(name=name) for name in expected_names),
                snapshots)

        loading.addCallback(loaded)
        return loading 

def benchmark(scenario, operation, metric, num_samples):
    """
    Perform benchmarking of the operation within a scenario.

    :param IScenario scenario: A load scenario.
    :param IOperation operation: An operation to perform.
    :param IMetric metric: A quantity to measure.
    :param int num_samples: Number of samples to take.
    :return: Deferred firing with a tuple containing one list of
        benchmark samples and one scenario metrics result. See the
        ``sample`` function for the structure of the samples.  The
        scenario metrics are a dictionary containing information about
        the scenario.
    """
    scenario_established = scenario.start()

    samples = []

    def collect_samples(ignored):
        collecting = Deferred()
        task = cooperate(
            sample(operation, metric, i).addCallback(samples.append)
            for i in range(num_samples))

        # If the scenario collapses, stop sampling
        def stop_sampling_on_scenario_collapse(failure):
            task.stop()
            collecting.errback(failure)
        scenario.maintained().addErrback(stop_sampling_on_scenario_collapse)

        # Leaving the errback unhandled makes tests fail
        task.whenDone().addCallbacks(
            lambda ignored: collecting.callback(samples),
            lambda ignored: None)

        return collecting

    benchmarking = scenario_established.addCallback(collect_samples)

    def stop_scenario(samples):
        d = scenario.stop()

        def combine_results(scenario_metrics):
            return (samples, scenario_metrics)
        d.addCallback(combine_results)

        return d
    benchmarking.addCallbacks(
        stop_scenario,
        bypass, errbackArgs=[scenario.stop]
    )

    return benchmarking