# ***************************************************************************** # Copyright (c) 2014, 2019 IBM Corporation and other Contributors. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Eclipse Public License v1.0 # which accompanies this distribution, and is available at # http://www.eclipse.org/legal/epl-v10.html # # ***************************************************************************** import sys import os import time import json import socket import ssl import logging import platform from logging.handlers import RotatingFileHandler from paho.mqtt import __version__ as pahoVersion import paho.mqtt.client as paho import threading import pytz from datetime import datetime from wiotp.sdk import __version__ as wiotpVersion from wiotp.sdk.exceptions import MissingMessageEncoderException, ConnectionException from wiotp.sdk.messages import JsonCodec, RawCodec, Utf8Codec class AbstractClient(object): """ The underlying client object utilised for Platform connectivity over MQTT in devices, gateways, and applications. # Parameters domain (string): Domain denoting the instance of IBM Watson IoT Platform to connect to organization (string): IBM Watson IoT Platform organization ID to connect to clientId (string): MQTT clientId for the underlying Paho client username (string): MQTT username for the underlying Paho client password (string): MQTT password for the underlying Paho client port (int): MQTT port for the underlying Paho client to connect using. Defaults to `8883` logHandlers (list<logging.Handler>): Log handlers to configure. Defaults to `None`, which will result in a default log handler being created. cleanStart (string): Defaults to `false`. Although this is a true|false parameter sessionExpiry (string): Defaults to 3600 seconds. Does nothing today (pending MQTT v5) transport (string): Defaults to `tcp` caFile (string): Defaults to None # Attributes client (paho.mqtt.client.Client): Built-in Paho MQTT client handling connectivity for the client. logger (logging.logger): Client logger. """ def __init__( self, domain, organization, clientId, username, password, port=None, transport="tcp", cleanStart=False, sessionExpiry=3600, keepAlive=60, caFile=None, logLevel=logging.INFO, logHandlers=None, ): self.organization = organization self.username = username self.password = password self.address = organization + ".messaging." + domain self.port = port # This is for a future MQTT v5 feature once Paho library is updated self.sessionExpiry = sessionExpiry self.keepAlive = keepAlive self.connectEvent = threading.Event() # If we are disconnected we lose all our active subscriptions. Keep # track of all subscriptions so that we can internally restore all # subscriptions on reconnect # Also, create a lock for gating access to the subscription dictionary # Finally, create an Event that allows us to track the first subscriptions being made self._subscriptions = {} self._subLock = threading.Lock() self.subscriptionsAcknowledged = threading.Event() # Create a map to contain mids for onPublish() callback handling. # and a lock to gate access to the dictionary self._onPublishCallbacks = {} self._messagesLock = threading.Lock() self.clientId = clientId # Configure logging self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.logger.setLevel(logLevel) # Remove any existing log handlers we may have picked up from getLogger() self.logger.handlers = [] if logHandlers: if isinstance(logHandlers, list): # Add all supplied log handlers for handler in logHandlers: self.logger.addHandler(handler) else: # Add the supplied log handler self.logger.addHandler(logHandlers) else: # Generate a default stream handler fhFormatter = logging.Formatter("%(asctime)-25s %(name)-25s " + " %(levelname)-7s %(message)s") ch = logging.StreamHandler() ch.setFormatter(fhFormatter) self.logger.addHandler(ch) # This will be used as a MQTTv5 connect user property once v5 support is added. # For now it's just a useful debug message logged when we connect self.userAgent = "MQTT/3.1.1 (%s %s) Paho/%s (Python) WIoTP/%s (Python)" % ( platform.system(), platform.release(), pahoVersion, wiotpVersion, ) self.client = paho.Client(self.clientId, transport=transport, clean_session=(not cleanStart)) # Normal usage puts the client in an auto-detect mode, where it will try to use # TLS, and fall back to unencrypted mode ONLY if TLS 1.2 is unavailable. # However, we now support explicit override of this by allowing the client to be # configured to use a specific port. # # If there is no specific port set in the configuration then we will auto-negotiate TLS if possible # If the port has been configured to 80 or 1883 we should not try to auto-enable TLS configuration # if the port has been configured to 443 or 8883 we should not auto-fallback to no TLS on 1883 if self.port in [80, 1883]: # Note: We don't seem to support port 80 fallback (anymore?) self.tlsVersion = None self.logger.warning( "Unable to encrypt messages because client configuration has overridden port selection to an insecure port (%s)" % self.port ) elif self.port in [443, 8883]: self.tlsVersion = ssl.PROTOCOL_TLSv1_2 # We allow an exception to raise here if running in an environment where # TLS 1.2 is unavailable because the configuration explicitly requested # to use encrypted connection elif self.port is None: if self.organization == "quickstart": self.tlsVersion = None self.port = 1883 else: try: self.tlsVersion = ssl.PROTOCOL_TLSv1_2 self.port = 8883 except: self.tlsVersion = None self.port = 1883 self.logger.warning( "Unable to encrypt messages because TLSv1.2 is unavailable (MQTT over SSL requires at least Python v2.7.9 or 3.4 and openssl v1.0.1)" ) else: raise Exception("Unsupported value for port override: %s. Supported values are 1883 & 8883." % self.port) # Configure authentication if self.username is not None: # In environments where either ssl is not available, or TLSv1.2 is not available we will fallback to MQTT over TCP if self.tlsVersion is not None: # Path to default CA certificate if none provided if caFile is None: caFile = os.path.dirname(os.path.abspath(__file__)) + "/messaging.pem" self.client.tls_set( ca_certs=caFile, certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ) self.client.username_pw_set(self.username, self.password) # Attach MQTT callbacks self.client.on_log = self._onLog self.client.on_connect = self._onConnect self.client.on_disconnect = self._onDisconnect self.client.on_publish = self._onPublish self.client.on_subscribe = self._onSubscribe # User configurable callback methods self.subscriptionCallback = None # Initialize default message encoders and decoders. self._messageCodecs = {} self.setMessageCodec("json", JsonCodec) self.setMessageCodec("raw", RawCodec) self.setMessageCodec("utf8", Utf8Codec) def getMessageCodec(self, messageFormat): """ Get the Python class that is currently defined as the encoder/decoder for a specified message format. # Arguments messageFormat (string): The message format to retrieve the encoder for # Returns code (class): The python class, or `None` if there is no codec defined for the `messageFormat` """ if messageFormat not in self._messageCodecs: return None return self._messageCodecs[messageFormat] def setMessageCodec(self, messageFormat, codec): """ Set a Python class as the encoder/decoder for a specified message format. # Arguments messageFormat (string): The message format to retreive the encoder for codec (class): The Python class (subclass of `wiotp.common.MessageCodec` to set as the encoder/decoder for `messageFormat` """ self._messageCodecs[messageFormat] = codec def _logAndRaiseException(self, e): """ Logs an exception at log level `critical` before raising it. # Arguments e (Exception): The exception to log/raise """ self.logger.critical(str(e)) raise e def connect(self): """ Connect the client to IBM Watson IoT Platform using the underlying Paho MQTT client # Raises ConnectionException: If there is a problem establishing the connection. """ self.logger.debug( "Connecting... (address = %s, port = %s, clientId = %s, username = %s)" % (self.address, self.port, self.clientId, self.username) ) try: self.connectEvent.clear() self.logger.debug( "Connecting with clientId %s to host %s on port %s with keepAlive set to %s" % (self.clientId, self.address, self.port, self.keepAlive) ) self.logger.debug("User-Agent: %s" % self.userAgent) self.client.connect(self.address, port=self.port, keepalive=self.keepAlive) self.client.loop_start() if not self.connectEvent.wait(timeout=60): self.client.loop_stop() self._logAndRaiseException( ConnectionException( "Operation timed out connecting to IBM Watson IoT Platform: %s" % (self.address) ) ) except socket.error as serr: self.client.loop_stop() self._logAndRaiseException( ConnectionException("Failed to connect to IBM Watson IoT Platform: %s - %s" % (self.address, str(serr))) ) def disconnect(self): """ Disconnect the client from IBM Watson IoT Platform """ # self.logger.info("Closing connection to the IBM Watson IoT Platform") self.client.disconnect() # If we don't call loop_stop() it appears we end up with a zombie thread which continues to process # network traffic, preventing any subsequent attempt to reconnect using connect() self.client.loop_stop() self.logger.info("Closed connection to the IBM Watson IoT Platform") def isConnected(self): return self.connectEvent.isSet() def _onLog(self, mqttc, obj, level, string): """ Called when the client has log information. See [paho.mqtt.python#on_log](https://github.com/eclipse/paho.mqtt.python#on_log) for more information # Parameters mqttc (paho.mqtt.client.Client): The client instance for this callback obj (object): The private user data as set in Client() or user_data_set() level (int): The severity of the message, will be one of `MQTT_LOG_INFO`, `MQTT_LOG_NOTICE`, `MQTT_LOG_WARNING`, `MQTT_LOG_ERR`, and `MQTT_LOG_DEBUG`. string (string): The log message itself """ self.logger.debug("%d %s" % (level, string)) def _onConnect(self, mqttc, userdata, flags, rc): """ Called when the broker responds to our connection request. The value of rc determines success or not: 0: Connection successful 1: Connection refused - incorrect protocol version 2: Connection refused - invalid client identifier 3: Connection refused - server unavailable 4: Connection refused - bad username or password 5: Connection refused - not authorised 6-255: Currently unused. """ if rc == 0: self.connectEvent.set() self.logger.info("Connected successfully: %s" % (self.clientId)) # Restoring previous subscriptions with self._subLock: if len(self._subscriptions) > 0: for subscription in self._subscriptions: # We use the underlying mqttclient subscribe method rather than _subscribe because we are # claiming a lock on the subscriptions list and do not want anything else to modify it, # which that method does (result, mid) = self.client.subscribe(subscription, qos=self._subscriptions[subscription]) if result != paho.MQTT_ERR_SUCCESS: self._logAndRaiseException(ConnectionException("Unable to subscribe to %s" % subscription)) self.logger.debug("Restored %s previous subscriptions" % len(self._subscriptions)) elif rc == 1: self._logAndRaiseException(ConnectionException("Incorrect protocol version")) elif rc == 2: self._logAndRaiseException(ConnectionException("Invalid client identifier")) elif rc == 3: self._logAndRaiseException(ConnectionException("Server unavailable")) elif rc == 4: self._logAndRaiseException( ConnectionException("Bad username or password: (%s, %s)" % (self.username, self.password)) ) elif rc == 5: self._logAndRaiseException( ConnectionException("Not authorized: s (%s, %s, %s)" % (self.clientId, self.username, self.password)) ) else: self._logAndRaiseException(ConnectionException("Unexpected connection failure: %s" % (rc))) def _onDisconnect(self, mqttc, obj, rc): """ Called when the client disconnects from IBM Watson IoT Platform. See [paho.mqtt.python#on_disconnect](https://github.com/eclipse/paho.mqtt.python#on_disconnect) for more information # Parameters mqttc (paho.mqtt.client.Client): The client instance for this callback obj (object): The private user data as set in Client() or user_data_set() rc (int): indicates the disconnection state. If `MQTT_ERR_SUCCESS` (0), the callback was called in response to a `disconnect()` call. If any other value the disconnection was unexpected, such as might be caused by a network error. """ # Clear the event to indicate we're no longer connected self.connectEvent.clear() if rc != 0: self.logger.error("Unexpected disconnect from IBM Watson IoT Platform: %d" % (rc)) else: self.logger.info("Disconnected from the IBM Watson IoT Platform") def _onPublish(self, mqttc, obj, mid): """ Called when a message from the client has been successfully sent to IBM Watson IoT Platform. See [paho.mqtt.python#on_publish](https://github.com/eclipse/paho.mqtt.python#on_publish) for more information # Parameters mqttc (paho.mqtt.client.Client): The client instance for this callback obj (object): The private user data as set in Client() or user_data_set() mid (int): Gives the message id of the successfully published message. """ with self._messagesLock: if mid in self._onPublishCallbacks: midOnPublish = self._onPublishCallbacks.get(mid) del self._onPublishCallbacks[mid] if midOnPublish != None: midOnPublish() else: # record the fact that paho callback has already come through so it can be called inline # with the publish. self._onPublishCallbacks[mid] = None def _onSubscribe(self, mqttc, userdata, mid, grantedQoS): self.subscriptionsAcknowledged.set() self.logger.debug("Subscribe callback: mid: %s qos: %s" % (mid, grantedQoS)) if self.subscriptionCallback: self.subscriptionCallback(mid, grantedQoS) def _subscribe(self, topic, qos=1): if not self.connectEvent.wait(timeout=10): self.logger.warning("Unable to subscribe to %s because client is in disconnected state" % (topic)) return 0 else: (result, mid) = self.client.subscribe(topic, qos=qos) if result == paho.MQTT_ERR_SUCCESS: with self._subLock: self._subscriptions[topic] = qos return mid else: return 0 def _publishEvent(self, topic, event, msgFormat, data, qos=0, onPublish=None): if not self.connectEvent.wait(timeout=10): self.logger.warning("Unable to send event %s because client is is disconnected state", event) return False else: if self.logger.isEnabledFor(logging.DEBUG): # The data object may not be serializable, e.g. if using a custom binary format try: dataString = json.dumps(data) except: dataString = str(data) self.logger.debug("Sending event %s with data %s" % (event, dataString)) # Raise an exception if there is no codec for this msgFormat if self.getMessageCodec(msgFormat) is None: raise MissingMessageEncoderException(msgFormat) payload = self.getMessageCodec(msgFormat).encode(data, datetime.now(pytz.timezone("UTC"))) result = self.client.publish(topic, payload=payload, qos=qos, retain=False) if result[0] == paho.MQTT_ERR_SUCCESS: # Because we are dealing with aync pub/sub model and callbacks it is possible that # the _onPublish() callback for this mid is called before we obtain the lock to place # the mid into the _onPublishCallbacks list. # # _onPublish knows how to handle a scenario where the mid is not present (no nothing) # in this scenario we will need to invoke the callback directly here, because at the time # the callback was invoked the mid was not yet in the list. with self._messagesLock: if result[1] in self._onPublishCallbacks: # Paho callback beat this thread so call callback inline now del self._onPublishCallbacks[result[1]] if onPublish is not None: onPublish() else: # This thread beat paho callback so set up for call later self._onPublishCallbacks[result[1]] = onPublish return True else: return False