Python consul.Consul() Examples

The following are code examples for showing how to use consul.Consul(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the ones you don't like. You can also save this page to your account.

Example 1
Project: znappy   Author: eBayClassifiedsGroup   File: keyvalue.py    (license) View Source Project 7 votes vote down vote up
def connect(self):
        logger.info('Connecting to consul and starting new session')
        self._session_id = self._consul.session.create(name="znappy-agent", ttl=30, lock_delay=0)

        logger.debug("Connected to Consul: {}".format(self._session_id))

        # not atomic, could in theory fail
        try:
            info = self._consul.session.info(self._session_id)
            logger.debug(info)
            self.node = info[1]['Node']
        except Exception:
            return False

        if not self.dc:
            try:
                self.dc = self._consul.agent.self()['Config']['Datacenter']
            except Exception:
                return False

        return True 
Example 2
Project: mongodb   Author: autopilotpattern   File: manage.py    (license) View Source Project 7 votes vote down vote up
def create_session(ttl=None):
    """
    We can't rely on storing Consul session IDs in memory because
    `health` and `onChange` handler calls happen in a subsequent
    process. Here we creates a session on Consul and cache the
    session ID to disk. Returns the session ID.
    """
    session_id = consul.session.create(name=SESSION_NAME,
                                       behavior='release',
                                       ttl=ttl)
    with open(SESSION_CACHE_FILE, 'w') as f:
        f.write(session_id)
    return session_id

# ---------------------------------------------------------
# utility functions 
Example 3
Project: mongodb   Author: autopilotpattern   File: manage.py    (license) View Source Project 6 votes vote down vote up
def get_session(no_cache=False):
    """
    Gets a Consul session ID from the on-disk cache or calls into
    `create_session` to generate and cache a new one.
    Also, renews the session TTL of on-disk key, to ensure it is valid
    in Consul
    """
    if no_cache:
        return create_session()

    try:
        with open(SESSION_CACHE_FILE, 'r') as f:
            session_id = f.read()

        # ensure the session_id is valid and refresh it
        consul.session.renew(session_id)
    except (IOError, pyconsul.base.NotFound) as e:
        # this means we have don't have a key locally, or
        # it is expired, so get a new one
        session_id = create_session()

    return session_id 
Example 4
Project: prom_finagle_exporter   Author: maginetv   File: discovery.py    (license) View Source Project 6 votes vote down vote up
def register_consul(host, port, service_id=str):
    service_address = get_ip_address()
    conn = consul_conn_check(host)
    if conn:
        c = consul.Consul(host=host)
        result = c.agent.service.register(
            'finagle_exporter',
            service_id=service_id,
            address=service_address,
            port=port,
            check={
                "DeregisterCriticalServiceAfter": "3m",
                "http": "http://{}:{}/health".format(service_address, port),
                "interval": "5s",
                "timeout": "1s"
                }
            )
        return result
    else:
        return False 
Example 5
Project: vagrant-ansible-docker-swarm   Author: jamesdmorgan   File: consul-notifier.py    (license) View Source Project 6 votes vote down vote up
def main():
    '''
        Register / De-register containers that have
        CONSUL_SERVICE_PORT env variable defined
    '''
    args = handler_args()
    setup_logging(args.verbose)

    # create a docker client object that talks to the local docker daemon
    cli = docker.Client(base_url='unix://var/run/docker.sock')
    con = consul.Consul()

    logger.info("Consul notifier processing {0}".format(args.action))

    if args.action == 'stream':
        stream(cli, con)
    elif args.action in ['register', 'deregister']:
        s = Service(cli, con, args.name)
        s.handle(args.action)
    else:
        logger.error("Unknown action {0}".format(args.action))
        sys.exit(1) 
Example 6
Project: authserver   Author: jdelic   File: _common.py    (license) View Source Project 6 votes vote down vote up
def _add_publishing_args(parser: argparse.ArgumentParser) -> None:
    out_gr = parser.add_argument_group("Config output options")
    out_gr.add_argument("--publish-to-stdout", dest="publish_to_stdout", action="store_true", default=False,
                        help="write the OAuth2 credentials to stdout as a JSON object.")
    out_gr.add_argument("--publish-to-consulkv", dest="publish_to_consulkv", default=None,
                        help="write the OAuth2 credentials to Hashicorp Consul. The credentials will be stored in "
                             "key/value pairs under the specified path.")
    out_gr.add_argument("--publish-to-vault", dest="publish_to_vault", default=None,
                        help="write the OAuth2 credentials to Hashicorp Vault. The credentials will be stored in "
                             "key/value paris under the specified path. The path must reside in a Vault 'secret' "
                             "backend. Set environment variables as specified by '12factor-vault' to configure "
                             "Vault authentication in the Authserver settings.")
    out_gr.add_argument("--consul-url", dest="consul_url",
                        default=os.getenv("CONSUL_HTTP_ADDR", "http://127.0.0.1:8500/"),
                        help="URL to use to contact the local Consul agent. Will use $CONSUL_HTTP_ADDR from the "
                             "environment if it exists.")
    out_gr.add_argument("--consul-token", dest="consul_token",
                        default=None,
                        help="An optional Consul ACL token. Will use the value of $CONSUL_TOKEN from the "
                             "environment if it exists.") 
Example 7
Project: testing   Author: autopilotpattern   File: testcases.py    (license) View Source Project 6 votes vote down vote up
def wait_for_service(self, service_name, count=0, timeout=30):
        """
        Polls Consul for the service to become healthy, and optionally
        for a particular `count` of container instances to be healthy.
        """
        while timeout > 0:
            try:
                nodes = self.consul.health.service(service_name, passing=True)[1]
                if nodes:
                    if not count or len(nodes) == count:
                        break
            except (ValueError, IndexError):
                pass
            timeout -= 1
            time.sleep(1)
        else:
            raise WaitTimeoutError("Timeout waiting for {} to be started"
                                   .format(service_name))
        return nodes 
Example 8
Project: znappy   Author: eBayClassifiedsGroup   File: keyvalue.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, dc=None, **kwargs):
        self._consul = consul.Consul(**kwargs)
        self.dc      = None
        self.node    = None 
Example 9
Project: mongodb   Author: autopilotpattern   File: manage.py    (license) View Source Project 5 votes vote down vote up
def mark_as_primary(hostname):
    """ Write flag to Consul to mark this node as primary """
    session_id = get_session()
    if not mark_with_session(PRIMARY_KEY, hostname, session_id):
        log.error('Tried to mark node primary but primary exists, '
                  'exiting for retry on next check.')
        sys.exit(1) 
Example 10
Project: DevOps   Author: YoLoveLife   File: consul_kv.py    (license) View Source Project 5 votes vote down vote up
def get_consul_api(module, token=None):
    return consul.Consul(host=module.params.get('host'),
                         port=module.params.get('port'),
                         scheme=module.params.get('scheme'),
                         verify=module.params.get('validate_certs'),
                         token=module.params.get('token')) 
Example 11
Project: DevOps   Author: YoLoveLife   File: consul_session.py    (license) View Source Project 5 votes vote down vote up
def get_consul_api(module):
    return consul.Consul(host=module.params.get('host'),
                         port=module.params.get('port')) 
Example 12
Project: DevOps   Author: YoLoveLife   File: consul_acl.py    (license) View Source Project 5 votes vote down vote up
def get_consul_api(module, token=None):
    if not token:
        token = module.params.get('token')
    return consul.Consul(host=module.params.get('host'),
                         port=module.params.get('port'),
                         scheme=module.params.get('scheme'),
                         verify=module.params.get('validate_certs'),
                         token=token) 
Example 13
Project: DevOps   Author: YoLoveLife   File: consul.py    (license) View Source Project 5 votes vote down vote up
def get_consul_api(module, token=None):
    return consul.Consul(host=module.params.get('host'),
                         port=module.params.get('port'),
                         scheme=module.params.get('scheme'),
                         verify=module.params.get('validate_certs'),
                         token=module.params.get('token')) 
Example 14
Project: DevOps   Author: YoLoveLife   File: consul_kv.py    (license) View Source Project 5 votes vote down vote up
def run(self, terms, variables=None, **kwargs):

        if not HAS_CONSUL:
            raise AnsibleError('python-consul is required for consul_kv lookup. see http://python-consul.readthedocs.org/en/latest/#installation')

        u = urlparse(self.agent_url)
        consul_api = consul.Consul(host=u.hostname, port=u.port)

        values = []
        try:
            for term in terms:
                params = self.parse_params(term)
                results = consul_api.kv.get(params['key'],
                                            token=params['token'],
                                            index=params['index'],
                                            recurse=params['recurse'])
                if results[1]:
                    # responds with a single or list of result maps
                    if isinstance(results[1], list):
                        for r in results[1]:
                            values.append(r['Value'])
                    else:
                        values.append(results[1]['Value'])
        except Exception as e:
            raise AnsibleError(
                "Error locating '%s' in kv store. Error was %s" % (term, e))

        return values 
Example 15
Project: prom_finagle_exporter   Author: maginetv   File: discovery.py    (license) View Source Project 5 votes vote down vote up
def consul_conn_check(host):
    c = consul.Consul(host=host)
    state = False
    try:
        c.catalog.nodes()
        state = True
    except ConnectionError:
        pass
    finally:
        return state 
Example 16
Project: sea   Author: shanbay   File: consul.py    (license) View Source Project 5 votes vote down vote up
def init_app(self, app):
        opts = app.config.get_namespace('CONSUL_')
        self._client = consul.Consul(**opts) 
Example 17
Project: sengladmin   Author: yufajieluo   File: commConsul.py    (license) View Source Project 5 votes vote down vote up
def connect(self, host, port):
        self.host = host
        self.port = port
        self.save_consul_cert()
        self.consul_conn = consul.Consul(host = host, port = self.port, verify = False, scheme = 'https', cert = self.cert)
        if not self.consul_conn:
            return False
        return True 
Example 18
Project: voltha   Author: opencord   File: config_backend.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, host, port, path_prefix):
        self._consul = Consul(host=host, port=port)
        self.host = host
        self.port = port
        self._path_prefix = path_prefix
        self._cache = {}
        self.retries = 0 
Example 19
Project: voltha   Author: opencord   File: config_backend.py    (license) View Source Project 5 votes vote down vote up
def _redo_consul_connection(self):
        self._consul = Consul(host=self.host, port=self.port)
        self._cache.clear() 
Example 20
Project: voltha   Author: opencord   File: consulhelpers.py    (license) View Source Project 5 votes vote down vote up
def connect_to_consult(consul_endpoint):
    log.debug('getting-service-endpoint', consul=consul_endpoint)

    host = consul_endpoint.split(':')[0].strip()
    port = int(consul_endpoint.split(':')[1].strip())

    return Consul(host=host, port=port) 
Example 21
Project: ansible-provider-docs   Author: alibaba   File: consul_kv.py    (license) View Source Project 5 votes vote down vote up
def run(self, terms, variables=None, **kwargs):

        if not HAS_CONSUL:
            raise AnsibleError('python-consul is required for consul_kv lookup. see http://python-consul.readthedocs.org/en/latest/#installation')

        u = urlparse(self.agent_url)
        consul_api = consul.Consul(host=u.hostname, port=u.port)

        values = []
        try:
            for term in terms:
                params = self.parse_params(term)
                results = consul_api.kv.get(params['key'],
                                            token=params['token'],
                                            index=params['index'],
                                            recurse=params['recurse'])
                if results[1]:
                    # responds with a single or list of result maps
                    if isinstance(results[1], list):
                        for r in results[1]:
                            values.append(r['Value'])
                    else:
                        values.append(results[1]['Value'])
        except Exception as e:
            raise AnsibleError(
                "Error locating '%s' in kv store. Error was %s" % (term, e))

        return values 
Example 22
Project: testing   Author: autopilotpattern   File: testcases.py    (license) View Source Project 5 votes vote down vote up
def consul(self):
        """
        Lazily constructs a Consul client pointing to the first Consul
        instance. We can't configure Consul during `setupClass` because
        we don't necessarily have Consul up and running at that point.
        """
        if not self._consul:
            insp = self.docker_inspect('consul_1')
            ip = insp[0]['NetworkSettings']['IPAddress']
            consul_host = ip if ip else os.environ.get('CONSUL', 'consul')
            self._consul = pyconsul.Consul(host=consul_host)
        return self._consul 
Example 23
Project: testing   Author: autopilotpattern   File: testcases.py    (license) View Source Project 5 votes vote down vote up
def get_service_ips(self, service, ignore_errors=False):
        """
        Gets a list of IPs for a service by checking each of its containers.
        Returns a pair of lists (public, private).
        """
        out = self.compose('ps', '-q', service)
        containers = out.splitlines()
        private_ips = []
        public_ips = []

        for container in containers:
            # we have the "real" name here and not the container-only name
            try:
                public_ip, private_ip = self.get_ips(container)
                if private_ip:
                    private_ips.append(private_ip)
                if public_ip:
                    public_ips.append(public_ip)
            except subprocess.CalledProcessError:
                if not ignore_errors:
                    # sometimes we've stopped an instance or have updated
                    # the service a container reports to Consul so we want
                    # to skip CalledProcessError. In this case the caller
                    # should be comparing the length of the lists returned
                    # vs the expected length.
                    raise

        return public_ips, private_ips 
Example 24
Project: testing   Author: autopilotpattern   File: testcases.py    (license) View Source Project 5 votes vote down vote up
def get_consul_key(self, key):
        """
        Return the Value field for a given Consul key. Handles None
        results safely but lets all other exceptions just bubble up.
        """
        result = self.consul.kv.get(key)
        if result[1]:
            return result[1]['Value']
        return None 
Example 25
Project: testing   Author: autopilotpattern   File: testcases.py    (license) View Source Project 5 votes vote down vote up
def get_service_instances_from_consul(self, service_name):
        """
        Asks Consul for list of containers for a service. Relies on
        the naming convention for services done by ContainerPilot
        which injects the container hostname into the service ID.
        """
        # https://www.consul.io/docs/agent/http/health.html#health_service
        nodes = self.consul.health.service(service_name, passing=True)[1]
        if nodes:
            prefix = '{}-'.format(service_name)
            node_ids = [service['Service']['ID'].replace(prefix, '', 1)
                        for service in nodes]
            return node_ids
        return [] 
Example 26
Project: testing   Author: autopilotpattern   File: testcases.py    (license) View Source Project 5 votes vote down vote up
def wait_for_service_removed(self, service_name, timeout=30):
        """
        Polls Consul for the service to be removed.
        """
        while timeout > 0:
            nodes = self.consul.health.service(service_name, passing=True)[1]
            if not nodes:
                break
            timeout -= 1
            time.sleep(1)
        else:
            raise WaitTimeoutError("Timeout waiting for {} to be removed"
                                   .format(service_name))
        return True 
Example 27
Project: magnet   Author: pymag09   File: consul.py    (license) View Source Project 5 votes vote down vote up
def get_consul_api():
        config = configparser.ConfigParser()
        consumed_files = config.read('%s/.magnet/consul.conf' % str(environ['HOME']))
        if not consumed_files:
            raise PluginConfigNotFound('Config file for consul plugin is missing.')
        host = config['DEFAULT']['host']
        port = config['DEFAULT']['port']
        token = config['DEFAULT']['token']
        scheme = config['DEFAULT']['scheme']
        return consul.Consul(host=host, port=port, token=token, scheme=scheme) 
Example 28
Project: SwallowKeeper   Author: easemob   File: update_nginx_upstream.py    (license) View Source Project 4 votes vote down vote up
def main():

    index_old = None
    consul_services_old = {}

    while True:

        time.sleep(SLEEP_INTERVAL)

        try:
            (index, consul_services) = getConsulServices(index_old)
            #check consul service count to ensure the services are really up running, in case consul was in incorrect and
            # return empty service list and cause nginx upstreams deleted unexpectedly.
            if consul_services and len(consul_services) < MIN_CONSUL_SERVICE_NUM:
                    logging.error("Consul service number is less than %d , please check the system!" % MIN_CONSUL_SERVICE_NUM)
                    continue
        except requests.exceptions.ConnectionError:
            logging.error("Can't not connect to local consul agent, please check it,  will exit program.")
            continue
        except requests.exceptions.RequestException as err:
            logging.error("There is http request error during calling consul agent, will exit program")
            logging.error(err)
            continue
        except Exception as err:
            logging.error(err)
            continue

        logging.info("There are %d service found in consul." % len(consul_services))

        if index_old is None:
            index_old = 'empty'
        logging.info("---new index is " + index + " old index is " + index_old)

        if index != index_old:
            updateNginxUpstream(consul_services_old, consul_services)
            logging.info("old consul services" + str(consul_services_old))
            logging.info("-----------")
            logging.info("new consul services" + str(consul_services))
            index_old = index
            consul_services_old = consul_services
        else:
            logging.info("Nothing changes") 
Example 29
Project: SwallowKeeper   Author: easemob   File: update_nginx_upstream.py    (license) View Source Project 4 votes vote down vote up
def main():

    index_old = None
    consul_services_old = {}

    while True:

        time.sleep(SLEEP_INTERVAL)

        try:
            (index, consul_services) = getConsulServices(index_old)
            #check consul service count to ensure the services are really up running, in case consul was in incorrect and
            # return empty service list and cause nginx upstreams deleted unexpectedly.
            if consul_services and len(consul_services) < MIN_CONSUL_SERVICE_NUM:
                    logging.error("Consul service number is less than %d , please check the system!" % MIN_CONSUL_SERVICE_NUM)
                    continue
        except requests.exceptions.ConnectionError:
            logging.error("Can't not connect to local consul agent, please check it,  will exit program.")
            continue
        except requests.exceptions.RequestException as err:
            logging.error("There is http request error during calling consul agent, will exit program")
            logging.error(err)
            continue
        except Exception as err:
            logging.error(err)
            continue

        logging.info("There are %d service found in consul." % len(consul_services))

        if index_old is None:
            index_old = 'empty'
        logging.info("---new index is " + index + " old index is " + index_old)

        if index != index_old:
            updateNginxUpstream(consul_services_old, consul_services)
            logging.info("old consul services" + str(consul_services_old))
            logging.info("-----------")
            logging.info("new consul services" + str(consul_services))
            index_old = index
            consul_services_old = consul_services
        else:
            logging.info("Nothing changes") 
Example 30
Project: authserver   Author: jdelic   File: _common.py    (license) View Source Project 3 votes vote down vote up
def _handle_client_registration(client: OT, mgr: CMDT, **options: Any) -> bool:
    credentials = {
        "name": client.name,
        "client_id": client.client_id,
        "client_secret": client.client_secret,
    }

    json_str = json.dumps(credentials, indent=4)

    if options["publish_to_stdout"]:
        mgr.stdout.write(json_str)

    if options["publish_to_consulkv"]:
        urp = urlparse(options["consul_url"])
        if ":" in urp.netloc:
            host, port = urp.netloc.split(":", 1)
        else:
            host = urp.netloc
            port = 8500

        try:
            con = consul.Consul(host=host, port=port, scheme=urp.scheme,
                                token=options.get("consul_token", os.getenv("CONSUL_HTTP_TOKEN", None)))
            path = options["publish_to_consulkv"]
            con.kv.put("%s/json" % path, json_str)
            con.kv.put("%s/name" % path, client.name)
            con.kv.put("%s/client_id" % path, client.client_id)
            con.kv.put("%s/client_secret" % path, client.client_secret)
        except ConsulException as e:
            mgr.stderr.write(mgr.style.ERROR("ERROR: Request to Consul failed: %s" % str(e)))
            return False

        mgr.stderr.write(mgr.style.SUCCESS("INFO: Client credentials published to Consul"))

    if options["publish_to_vault"]:
        try:
            cl = settings.VAULT.authenticated_client()  # type: hvac.Client
            cl.write(
                options["publish_to_vault"],
                **credentials
            )
        except VaultCredentialProviderException as e:
            mgr.stderr.write(mgr.style.ERROR("Can't create Vault credentials: %s" % str(e)))
            return False

        mgr.stderr.write(mgr.style.SUCCESS("INFO: Client credentials published to Vault"))

    return True