Python gevent.GreenletExit() Examples

def _command_age_out(self):
        with self.state_lock:
            if self.state != ft_states.STARTED:
                return

            try:
                now = utc_millisec()

                for i, v in self.table.query(index='_age_out',
                                             to_key=now-1,
                                             include_value=True):
                    indicator, _ = i.split('\0', 1)

                    self.emit_withdraw(indicator=indicator, value=v)
                    self.table.delete(i)

                    self.statistics['aged_out'] += 1

                self.last_ageout_run = now

            except gevent.GreenletExit:
                raise

            except:
                LOG.exception('Exception in _age_out_loop') 

def _age_out_loop(self):
        while True:
            with self.state_lock:
                if self.state != ft_states.STARTED:
                    break

            self._actor_queue.put(
                (utc_millisec(), 'age_out')
            )

            if self.age_out['interval'] is None:
                break

            try:
                gevent.sleep(self.age_out['interval'])
            except gevent.GreenletExit:
                break 

def _process_minute_slices(self, flush=False):
        """ Goes through our internal aggregated share data structures and
        reports them to our external storage. If asked to flush it will report
        all one minute shares, otherwise it will only report minutes that have
        passed. """
        self.logger.info("Reporting one minute shares for address/workers")
        t = time.time()
        if not flush:
            upper = (int(t) // 60) * 60
        for stamp, data in self._minute_slices.items():
            if flush or stamp < upper:
                for (address, worker, algo, typ), amount in data.iteritems():
                    self.log_one_minute(address, worker, algo, stamp, typ, amount)
                    # XXX: GreenletExit getting raised here might cause some
                    # double reporting!
                del self._minute_slices[stamp]

        self.logger.info("One minute shares reported in {}"
                         .format(time_format(time.time() - t)))

        # Clean up old per address slices as well
        ten_ago = ((time.time() // 60) * 60) - 600
        for stamp in self._per_address_slices.keys():
            if stamp < ten_ago:
                del self._per_address_slices[stamp] 

def _actor_loop(self):
        while True:
            acommand = self._actor_queue.get()

            if acommand.command == 'checkpoint':
                method = super(ActorBaseFT, self).checkpoint
            elif acommand.command == 'update':
                method = super(ActorBaseFT, self).update
            elif acommand.command == 'withdraw':
                method = super(ActorBaseFT, self).withdraw
            elif acommand.command == 'rebuild':
                method = self._rebuild
            else:
                LOG.error('{} - unknown command {}'.format(self.name, acommand.command))

            try:
                method(**acommand.kwargs_)
            except gevent.GreenletExit:
                break
            except:
                LOG.exception('{} - error executing {!r}'.format(self.name, acommand)) 

def _publisher(self):
        while True:
            self._xmpp_client_ready.wait()

            try:
                while True:
                    cmd, indicator, value = self.q.peek()
                    if value is None:
                        value = {}
                    value['origins'] = [self.jid]
                    self._xmpp_publish(cmd, {
                        'indicator': indicator,
                        'value': value
                    })
                    _ = self.q.get()

            except gevent.GreenletExit:
                break

            except Exception as e:
                LOG.exception('%s - Exception in publishing message', self.name)
                gevent.sleep(30)
                self.statistics['xmpp.publish_error'] += 1 

def _device_pusher_died(self, g):
        try:
            g.get()

        except gevent.GreenletExit:
            pass

        except Exception:
            LOG.exception('%s - exception in greenlet for %s, '
                          'respawning in 60 seconds',
                          self.name, g.device['hostname'])

            for idx in range(len(self.device_pushers)):
                if self.device_pushers[idx].device == g.device:
                    break
            else:
                LOG.info('%s - device pusher for %s removed,' +
                         ' respawning aborted',
                         self.name, g.device['hostname'])
                g = None
                return

            dp = self._spawn_device_pusher(g.device)
            self.device_pushers[idx] = dp
            dp.start_later(60) 

def _age_out_loop(self, fieldidx):
        interval = self.fields[fieldidx].get('age_out', '30d')
        interval = _age_out_in_usecs(interval)
        t = self.tables[fieldidx]

        while True:
            try:
                now = utc_millisec()
                for i, v in t.query(index='last_seen', to_key=now-interval,
                                    include_value=True):
                    LOG.debug('%s - %s %s aged out', self.name, i, v)
                    self.emit_withdraw(indicator=i)
                    t.delete(i)

            except gevent.GreenletExit:
                break

            except:
                LOG.exception('Exception in _age_out_loop')

            gevent.sleep(self.age_out_interval) 

def _user_id(self, cmd=None):
        try:
            self.xapi.user_id(cmd=cmd,
                              vsys=self.device.get('vsys', None))

        except gevent.GreenletExit:
            raise

        except pan.xapi.PanXapiError as e:
            LOG.debug('%s', e)
            if 'already exists, ignore' in str(e):
                pass
            elif 'does not exist, ignore unreg' in str(e):
                pass
            elif 'Failed to register' in str(e):
                pass
            else:
                LOG.exception('XAPI exception in pusher for device %s: %s',
                              self.device.get('hostname', None), str(e))
                raise 

def _OpenImpl(self):
    while self._state != ChannelState.Closed:
      try:
        self._server_set_provider.Initialize(
            self.__OnServerSetJoin,
            self.__OnServerSetLeave)
        server_set = self._server_set_provider.GetServers()
      except gevent.GreenletExit:
        return
      except:
        self._log.exception("Unable to initialize serverset, retrying in 5 seconds.")
        gevent.sleep(5)
        continue

      random.shuffle(server_set)
      self._servers = {}
      [self.__AddServer(m) for m in server_set]
      self.__init_done.set()
      self._OpenInitialChannels()
      self._open_greenlet = None
      self._state = ChannelState.Open
      return True 

def _run(self):
        self._init_resync()

        while True:
            try:
                op, address, value = self.q.peek()
                self._push(op, address, value)
                self.q.get()  # discard processed message

            except gevent.GreenletExit:
                break

            except pan.xapi.PanXapiError as e:
                LOG.exception('XAPI exception in pusher for device %s: %s',
                              self.device.get('hostname', None), str(e))
                raise 

def _user_id(self, cmd=None):
        try:
            self.xapi.user_id(cmd=cmd,
                              vsys=self.device.get('vsys', None))

        except gevent.GreenletExit:
            raise

        except pan.xapi.PanXapiError as e:
            LOG.debug('%s', e)
            if 'already exists, ignore' in str(e):
                pass
            elif 'does not exist, ignore unreg' in str(e):
                pass
            elif 'Failed to register' in str(e):
                pass
            else:
                LOG.exception('XAPI exception in pusher for device %s: %s',
                              self.device.get('hostname', None), str(e))
                raise 

def _age_out(self):
        with self.state_lock:
            if self.state != ft_states.STARTED:
                return

            try:
                now = utc_millisec()

                for i, v in self.table.query(index='_age_out',
                                             to_key=now-1,
                                             include_value=True):
                    LOG.debug('%s - %s %s aged out', self.name, i, v)

                    if v.get('_withdrawn', None) is not None:
                        continue

                    self._controlled_emit_withdraw(
                        indicator=i,
                        value=v
                    )
                    v['_withdrawn'] = now
                    self.table.put(i, v)

                    self.statistics['aged_out'] += 1

                self.last_ageout_run = now

            except gevent.GreenletExit:
                raise

            except:
                LOG.exception('Exception in _age_out') 

def _amqp_callback(self, msg):
        try:
            LOG.info(u'{}'.format(msg.body))
            message = ujson.loads(msg.body)
            self._msg_queue.put(message)
            self._do_process.set()

        except gevent.GreenletExit:
            raise

        except:
            LOG.exception(
                "%s - exception handling syslog message",
                self.name
            ) 

def _flush(self):
        with self.state_lock:
            if self.state != ft_states.STARTED:
                return

            try:
                now = utc_millisec()

                for i, v in self.table.query(include_value=True):
                    if v.get('_withdrawn', None) is not None:
                        continue

                    self._controlled_emit_withdraw(
                        indicator=i,
                        value=v
                    )
                    v['_withdrawn'] = now
                    v['_last_run'] = 0
                    self.table.put(i, v)

                    self.statistics['flushed'] += 1

            except gevent.GreenletExit:
                raise

            except:
                LOG.exception('Exception in _flush') 

def _aggregate_iterator(self, iterator):
        self.agg_table = _bptable_factory(
            '{}.aggregate-temp'.format(self.name),
            truncate=True,
            type_in_key=True
        )

        for nitem, item in enumerate(iterator):
            if nitem != 0 and nitem % 1024 == 0:
                gevent.sleep(0.001)

            with self.state_lock:
                if self.state != ft_states.STARTED:
                    LOG.info(
                        '%s - state not STARTED, aggregation not performed',
                        self.name
                    )
                    self.agg_table.close()
                    return False

                try:
                    ipairs = self._process_item(item)

                except gevent.GreenletExit:
                    raise

                except:
                    self.statistics['error.parsing'] += 1
                    LOG.exception('%s - Exception parsing %s', self.name, item)
                    continue

                for indicator, attributes in ipairs:
                    self.agg_table.put(indicator, attributes)

        return True 

def start(self):
        super(Console, self).start()
        self.console_locals = {}
        self.console_locals.update(self.app.services)
        self.console_locals['app'] = self.app

        def stop_app():
            try:
                self.app.stop()
            except gevent.GreenletExit:
                pass
        self.console_locals['stop'] = stop_app 

def __init__(self, resource=None, collection=None):
        super(GreenletExit, self).__init__()
        self.r = resource
        self.c = collection 

def _TryResurrect(self):
    if not self._down_on:
      return

    last_attempt = self._down_on
    wait_interval = self._initial_wait_interval
    self._log.info('Attempting to reopen faulted channel')
    while True:
      gevent.sleep(wait_interval)
      now = time.time()
      down_time, last_attempt = now - last_attempt, now
      self._varz.time_failed(down_time)

      sink = self._next_factory.CreateSink(self._properties)
      try:
        self._varz.reconnect_attempts()
        sink.Open().get()
        sink.on_faulted.Subscribe(self._OnSinkFaulted)
        self.next_sink = sink
        self._down_on = None
        self._log.info('Reopened channel.')
        return
      except GreenletExit:
        return
      except:
        sink.Close()

      wait_interval **= self._backoff_exponent
      wait_interval = min(wait_interval, self._max_wait_interval) 

def handle_request(self, *args):
        try:
            super(GeventWorker, self).handle_request(*args)
        except gevent.GreenletExit:
            pass
        except SystemExit:
            pass 

def save_search_result(p, queue, retry=0):
    proxy = Proxy.get_random()['address']
    url = SEARCH_URL.format(SEARCH_TEXT, p)

    try:
        r = fetch(url, proxy=proxy)
    except (Timeout, ConnectionError):
        sleep(0.1)
        retry += 1
        if retry > 5:
            queue.put(url)
            raise GreenletExit()
        try:
            p = Proxy.objects.get(address=proxy)
            if p:
                p.delete()
        except DoesNotExist:
            pass

        return save_search_result(url, queue, retry)
    soup = BeautifulSoup(r.text, 'lxml')
    results = soup.find(class_='results')
    if results is None:
        # 此代理已经被封, 换其他的代理
        sleep(0.1)
        retry += 1
        if retry > 5:
            queue.put(url)
            raise GreenletExit()
        return save_search_result(url, queue, retry)
    articles = results.find_all(
        'div', lambda x: 'wx-rb' in x)
    for article in articles:
        save_article(article) 

def _run(self):
        """Run the wrapped function periodically"""
        try:
            while True:
                ts = time.time()
                if self.last_timestamp + self.interval_secs <= ts:
                    self.last_timestamp = ts
                    try:
                        self.f(*self.args, **self.kwargs)
                    except gevent.GreenletExit:
                        # We are notified to exit.
                        raise
                    except BaseException as e:
                        # We ignore other exceptions.
                        log.error("Exception %s caught in Periodical %s " % (
                            repr(e), self.name))

                # sleep until the time for the next run.
                sleep_secs = self.last_timestamp + self.interval_secs \
                    - time.time()

                if sleep_secs < 0:
                    sleep_secs = 0

                gevent.sleep(sleep_secs)

        except gevent.GreenletExit:
            log.info("Periodical %s stopped." % self.name) 

def result(self, timeout=None):
        try:
            res = self.get(block=True, timeout=timeout)
            if res is gevent.GreenletExit:
                raise res

            return res
        except gevent.Timeout:
            raise Timeout()
        except Exception:
            raise 

def perform_sync(self):
        sync = TimelineWebSync(self.runner.urls, self.oauth)
        for type, pin in sync.update_iter():
            try:
                self.handle_update(type, pin)
            except (gevent.GreenletExit, KeyboardInterrupt):
                raise
            except Exception:
                traceback.print_exc()
                self.logger.warning("Skipped invalid pin %s.", pin) 

def test_no_logging_on_greenlet_exit():
    logger = MockLogger()
    failing_function = FailingFunction(GreenletExit)
    with pytest.raises(GreenletExit):
        retry_with_logging(failing_function, logger=logger)
    assert logger.call_count == 0
    assert failing_function.call_count == 1 

def test_no_logging_on_greenlet_exit():
    logger = MockLogger()
    failing_function = FailingFunction(GreenletExit)
    with pytest.raises(GreenletExit):
        retry_with_logging(failing_function, logger=logger)
    assert logger.call_count == 0
    assert failing_function.call_count == 1 

def _run(self):
        try:
            return retry_with_logging(self._run_impl,
                                      account_id=self.account_id,
                                      provider=self.provider_name,
                                      logger=self.log)
        except GreenletExit:
            self._cleanup()
            raise 

def _age_out_run(self):
        while True:
            try:
                now = utc_millisec()

                LOG.debug('now: %s', now)

                for i, v in self.table.query(index='_age_out',
                                             to_key=now-1,
                                             include_value=True):
                    LOG.debug('%s - %s %s aged out', self.name, i, v)

                    for dp in self.device_pushers:
                        dp.put(
                            op='unregister',
                            address=i,
                            value=v
                        )

                    self.statistics['aged_out'] += 1
                    self.table.delete(i)

                self.last_ageout_run = now
                LOG.debug('%s - #indicators: %d', self.name, self.length())

            except gevent.GreenletExit:
                break

            except Exception:
                LOG.exception('Exception in _age_out_loop')

            try:
                gevent.sleep(self.age_out_interval)
            except gevent.GreenletExit:
                break 

def _retry_wrap(self):
        while True:
            try:
                self._listen()

            except gevent.GreenletExit:
                break

            except:
                LOG.exception('Exception in event listener') 

def _config_monitor(config_path):
    api_config_path = os.path.join(config_path, 'api')
    dirsnapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE)
    while True:
        try:
            with API_CONFIG_LOCK.acquire(timeout=600):
                new_snapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE)

                if new_snapshot != dirsnapshot:
                    try:
                        _load_config(config_path)
                        _load_auth_dbs(config_path)

                    except gevent.GreenletExit:
                        break

                    except:
                        LOG.exception('Error loading config')

                    dirsnapshot = new_snapshot

        except filelock.Timeout:
            LOG.error('Timeout locking config in config monitor')

        gevent.sleep(1)


# initialization 

def run(self):
        if self.socket is None:
            LOG.error('Run called with invalid socket in ZMQ Pub channel: {}'.format(self.name))

        while True:
            LOG.debug('ZMQPub {} receiving'.format(self.name))
            body = self.socket.recv_json()
            LOG.debug('ZMQPub {} recvd: {!r}'.format(self.name, body))

            method = body.get('method', None)
            id_ = body.get('id', None)
            params = body.get('params', {})

            if method is None:
                LOG.error('No method in msg body')
                return
            if id_ is None:
                LOG.error('No id in msg body')
                return

            method = self.method_prefix+method

            if method not in self.allowed_methods:
                LOG.error('Method not allowed in RPC server channel {}: {}'.format(self.name, method))
                continue

            m = getattr(self.obj, method, None)
            if m is None:
                LOG.error('Method {} not defined in RPC server channel {}'.format(method, self.name))
                continue

            try:
                m(**params)

            except gevent.GreenletExit:
                raise

            except Exception:
                LOG.exception('Exception in ZMQPub {}'.format(self.name))