Python oslo_concurrency.processutils.get_worker_count() Examples

def __init__(self, name, use_ssl=False):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param use_ssl: Wraps the socket in an SSL context if True.
        :returns: None
        """
        self.name = name
        self.app = app.load_app()
        self.workers = (CONF.api.workers or processutils.get_worker_count())
        if self.workers and self.workers < 1:
            raise exception.ConfigInvalid(
                _("api_workers value of %d is invalid, "
                  "must be greater than 0.") % self.workers)

        self.server = wsgi.Server(CONF, name, self.app,
                                  host=CONF.api.host_ip,
                                  port=CONF.api.port,
                                  use_ssl=use_ssl) 

def __init__(self, name):
        self._host = CONF.api_listen
        self._port = CONF.api_listen_port

        if platform.system() == "Windows":
            self._workers = 1
        else:
            self._workers = (
                CONF.api_workers or processutils.get_worker_count())

        self._loader = wsgi.Loader(CONF)
        self._app = self._loader.load_app(name)

        self._server = wsgi.Server(CONF,
                                   name,
                                   self._app,
                                   host=self._host,
                                   port=self._port) 

def test_api_https(self, mock_prep, mock_app, mock_run,
                       mock_exist, mock_base):
        self.config(enabled_ssl=True,
                    ssl_cert_file='tmp_crt',
                    ssl_key_file='tmp_key',
                    group='api')
        mock_exist.side_effect = [True, True]

        api.main()

        app = mock_app.load_app.return_value
        mock_prep.assert_called_once_with(mock.ANY)
        mock_app.load_app.assert_called_once_with()
        mock_exist.assert_has_calls([mock.call('tmp_crt'),
                                     mock.call('tmp_key')])
        workers = processutils.get_worker_count()
        mock_run.assert_called_once_with(base.CONF.api.host,
                                         base.CONF.api.port, app,
                                         processes=workers,
                                         ssl_context=('tmp_crt', 'tmp_key')) 

def _get_executor(self):
        """Get executor type based on lock object

        We check to see if the lock object is not set or if it is a threading
        lock. We cannot check if it is a ProcessLock due to the side effect
        of trying to include ProcessLock when running under Mistral breaks
        Mistral.
        """
        if not self.lock or isinstance(self.lock, threadinglock.ThreadingLock):
            # workers will scale from 2 to 8 based on the cpu count // 2
            workers = min(max(2, processutils.get_worker_count() // 2), 8)
            return futures.ThreadPoolExecutor(max_workers=workers)
        else:
            # there really isn't an improvement with > 4 workers due to the
            # container layer overlaps. The higher the workers, the more
            # RAM required which can lead to OOMs. It's best to limit to 4
            return futures.ProcessPoolExecutor(max_workers=4) 

def run_tasks(self):
        if not self.upload_tasks:
            return
        local_images = []

        # Pull a single image first, to avoid duplicate pulls of the
        # same base layers
        local_images.extend(upload_task(args=self.upload_tasks.pop()))

        # workers will be half the CPU count, to a minimum of 2
        workers = max(2, (processutils.get_worker_count() - 1))
        with futures.ThreadPoolExecutor(max_workers=workers) as p:
            for result in p.map(upload_task, self.upload_tasks):
                local_images.extend(result)
        LOG.info('result %s' % local_images)

        # Do cleanup after all the uploads so common layers don't get deleted
        # repeatedly
        self.cleanup(local_images) 

def __init__(self):
        self.app = app.setup_app()

        self.workers = CONF.api.api_workers
        if self.workers is not None and self.workers < 1:
            LOG.warning(
                "Value of config option api_workers must be integer "
                "greater than 1.  Input value ignored."
            )
            self.workers = None
        self.workers = self.workers or processutils.get_worker_count()

        self.server = wsgi.Server(
            cfg.CONF,
            "qinling_api",
            self.app,
            host=cfg.CONF.api.host,
            port=cfg.CONF.api.port,
            use_ssl=cfg.CONF.api.enable_ssl_api
        ) 

def __init__(self, name, use_ssl=False):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param use_ssl: Wraps the socket in an SSL context if True.
        :returns: None
        """
        self.name = name
        self.app = app.load_app()
        self.workers = (CONF.api.api_workers or
                        processutils.get_worker_count())
        if self.workers and self.workers < 1:
            raise exception.ConfigInvalid(
                _("api_workers value of %d is invalid, "
                  "must be greater than 0.") % self.workers)

        self.server = wsgi.Server(CONF, self.name, self.app,
                                  host=CONF.api.host_ip,
                                  port=CONF.api.port,
                                  use_ssl=use_ssl) 

def __init__(self, name):
        self._host = CONF.api_migration_listen
        self._port = CONF.api_migration_listen_port

        if platform.system() == "Windows":
            self._workers = 1
        else:
            self._workers = (
                CONF.api_migration_workers or processutils.get_worker_count())

        self._loader = wsgi.Loader(CONF)
        self._app = self._loader.load_app(name)

        self._server = wsgi.Server(CONF,
                                   name,
                                   self._app,
                                   host=self._host,
                                   port=self._port) 

def get_config_opts(cls):
        return [
            cfg.IntOpt(
                'max_workers',
                default=processutils.get_worker_count(),
                min=1,
                required=True,
                help='Number of workers for taskflow engine '
                     'to execute actions.'),
            cfg.DictOpt(
                'action_execution_rule',
                default={},
                help='The execution rule for linked actions,'
                     'the key is strategy name and '
                     'value ALWAYS means all actions will be executed,'
                     'value ANY means if previous action executes '
                     'success, the next action will be ignored.'
                     'None means ALWAYS.')
            ] 

def __init__(self, name, loader=None):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param loader: Loads the WSGI application using the given name.
        :returns: None

        """
        self.name = name
        self.manager = self._get_manager()
        self.loader = loader or wsgi.Loader(CONF)
        self.app = self.loader.load_app(name)
        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
        self.port = getattr(CONF, '%s_listen_port' % name, 0)
        self.workers = (getattr(CONF, '%s_workers' % name, None) or
                        processutils.get_worker_count())
        if self.workers and self.workers < 1:
            worker_name = '%s_workers' % name
            msg = (_("%(worker_name)s value of %(workers)d is invalid, "
                     "must be greater than 0.") %
                   {'worker_name': worker_name,
                    'workers': self.workers})
            raise exception.InvalidInput(msg)

        self.server = wsgi.Server(CONF,
                                  name,
                                  self.app,
                                  host=self.host,
                                  port=self.port)
        super(WSGIService, self).__init__() 

def test_workers_set_zero_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        self.override_config('osapi_karbor_workers', 0)
        test_service = service.WSGIService("osapi_karbor")
        # If a value less than 1 is used, defaults to number of procs
        # available
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called) 

def test_workers_set_default(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        test_service = service.WSGIService("osapi_karbor")
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called) 

def __init__(self, topic, endpoints, version, worker_count=None):
        target = messaging.Target(topic=topic,
                                  server=utils.get_hostname(),
                                  version=version)
        self._server = rpc.get_server(target, endpoints)

        self._workers = (worker_count or CONF.messaging_workers or
                         processutils.get_worker_count()) 

def test_get_worker_count_cpu_count_not_implemented(self,
                                                        mock_cpu_count):
        self.assertEqual(1, processutils.get_worker_count()) 

def test_get_worker_count(self, mock_cpu_count):
        self.assertEqual(8, processutils.get_worker_count()) 

def __init__(self, service_name, use_ssl=False):
        """Initialize, but do not start the WSGI server.

        :param service_name: The service name of the WSGI server.
        :param use_ssl: Wraps the socket in an SSL context if True.
        """
        self.service_name = service_name
        self.app = app.VersionSelectorApplication()
        self.workers = (CONF.api.workers or
                        processutils.get_worker_count())
        self.server = wsgi.Server(CONF, self.service_name, self.app,
                                  host=CONF.api.host,
                                  port=CONF.api.port,
                                  use_ssl=use_ssl,
                                  logger_name=self.service_name) 

def __init__(self, name, loader=None, use_ssl=False, max_url_len=None):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param loader: Loads the WSGI application using the given name.
        :returns: None
        """
        self.name = name
        self.binary = 'masakari-%s' % name
        self.topic = None
        self.loader = loader or wsgi.Loader()
        self.app = self.loader.load_app(name)
        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
        self.port = getattr(CONF, '%s_listen_port' % name, 0)

        self.workers = (getattr(CONF, '%s_workers' % name, None) or
                        processutils.get_worker_count())

        if self.workers and self.workers < 1:
            worker_name = '%s_workers' % name
            msg = (_("%(worker_name)s value of %(workers)s is invalid, "
                     "must be greater than 0") %
                   {'worker_name': worker_name,
                    'workers': str(self.workers)})
            raise exception.InvalidInput(msg)

        self.use_ssl = use_ssl
        self.server = wsgi.Server(name,
                                  self.app,
                                  host=self.host,
                                  port=self.port,
                                  use_ssl=self.use_ssl,
                                  max_url_len=max_url_len) 

def test_workers_set_zero_user_setting(self):
        CONF.set_override('masakari_api_workers', 0)
        test_service = service.WSGIService("masakari_api")
        # If a value less than 1 is used, defaults to number of procs available
        self.assertEqual(test_service.workers, processutils.get_worker_count()) 

def test_workers_set_default(self):
        test_service = service.WSGIService("masakari_api")
        self.assertEqual(test_service.workers, processutils.get_worker_count()) 

def main():
    service.prepare_service(sys.argv)

    gmr.TextGuruMeditation.setup_autorun(version)

    # Enable object backporting via the conductor
    base.MagnumObject.indirection_api = base.MagnumObjectIndirectionAPI()

    app = api_app.load_app()

    # Setup OSprofiler for WSGI service
    profiler.setup('magnum-api', CONF.host)

    # SSL configuration
    use_ssl = CONF.api.enabled_ssl

    # Create the WSGI server and start it
    host, port = CONF.api.host, CONF.api.port

    LOG.info('Starting server in PID %s', os.getpid())
    LOG.debug("Configuration:")
    CONF.log_opt_values(LOG, logging.DEBUG)

    LOG.info('Serving on %(proto)s://%(host)s:%(port)s',
             dict(proto="https" if use_ssl else "http", host=host, port=port))

    workers = CONF.api.workers
    if not workers:
        workers = processutils.get_worker_count()
    LOG.info('Server will handle each request in a new process up to'
             ' %s concurrent processes', workers)
    serving.run_simple(host, port, app, processes=workers,
                       ssl_context=_get_ssl_configs(use_ssl)) 

def main():
    magnum_service.prepare_service(sys.argv)

    gmr.TextGuruMeditation.setup_autorun(version)

    LOG.info('Starting server in PID %s', os.getpid())
    LOG.debug("Configuration:")
    CONF.log_opt_values(LOG, logging.DEBUG)

    conductor_id = short_id.generate_id()
    endpoints = [
        indirection_api.Handler(),
        cluster_conductor.Handler(),
        conductor_listener.Handler(),
        ca_conductor.Handler(),
        federation_conductor.Handler(),
        nodegroup_conductor.Handler(),
    ]

    server = rpc_service.Service.create(CONF.conductor.topic,
                                        conductor_id, endpoints,
                                        binary='magnum-conductor')
    workers = CONF.conductor.workers
    if not workers:
        workers = processutils.get_worker_count()
    launcher = service.launch(CONF, server, workers=workers)

    # NOTE(mnaser): We create the periodic tasks here so that they
    #               can be attached to the main process and not
    #               duplicated in all the children if multiple
    #               workers are being used.
    server.create_periodic_tasks()
    server.start()

    launcher.wait() 

def test_conductor(self, mock_prep, mock_rpc, mock_launch):
        conductor.main()

        server = mock_rpc.Service.create.return_value
        launcher = mock_launch.return_value
        mock_prep.assert_called_once_with(mock.ANY)
        mock_rpc.Service.create.assert_called_once_with(
            base.CONF.conductor.topic,
            mock.ANY, mock.ANY, binary='magnum-conductor')
        workers = processutils.get_worker_count()
        mock_launch.assert_called_once_with(base.CONF, server,
                                            workers=workers)
        launcher.wait.assert_called_once_with() 

def test_api_http(self, mock_prep, mock_app, mock_run, mock_base):
        api.main()

        app = mock_app.load_app.return_value
        mock_prep.assert_called_once_with(mock.ANY)
        mock_app.load_app.assert_called_once_with()
        workers = processutils.get_worker_count()
        mock_run.assert_called_once_with(base.CONF.api.host,
                                         base.CONF.api.port,
                                         app, processes=workers,
                                         ssl_context=None) 

def __init__(self, topic, endpoints, version):
        target = messaging.Target(topic=topic,
                                  server=utils.get_hostname(),
                                  version=version)
        self._server = rpc.get_server(target, endpoints)

        self._workers = (CONF.messaging_workers or
                         processutils.get_worker_count()) 

def __init__(self, name, loader=None, max_url_len=None):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param loader: Loads the WSGI application using the given name.
        :returns: None

        """
        self.name = name
        self.manager = self._get_manager()
        self.loader = loader or wsgi.Loader()
        self.app = self.loader.load_app(name)
        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
        self.port = getattr(CONF, '%s_listen_port' % name, 0)
        self.use_ssl = getattr(CONF, '%s_use_ssl' % name, False)
        self.workers = (getattr(CONF, '%s_workers' % name, None) or
                        processutils.get_worker_count())
        if self.workers and self.workers < 1:
            worker_name = '%s_workers' % name
            msg = (_("%(worker_name)s value of %(workers)s is invalid, "
                     "must be greater than 0") %
                   {'worker_name': worker_name,
                    'workers': str(self.workers)})
            raise exception.InvalidInput(msg)
        self.server = wsgi.Server(name,
                                  self.app,
                                  host=self.host,
                                  port=self.port,
                                  use_ssl=self.use_ssl,
                                  max_url_len=max_url_len)
        # Pull back actual port used
        self.port = self.server.port