Python process messages

def create_process_job_input_messages(job_ids):
    """Creates messages to process the input for the given jobs

    :param job_ids: The job IDs
    :type job_ids: :func:`list`
    :return: The list of messages
    :rtype: :func:`list`
    """

    messages = []

    for job_id in job_ids:
        message = ProcessJobInput()
        message.job_id = job_id
        messages.append(message)

    return messages 

def process_mqtt_messages(gateway, client, stop_event):
    while not stop_event.is_set():
        try:
            data = client._queue.get()
            if data is None:
                continue
            _LOGGER.debug("data from mqtt: " + format(data))

            sid = data.get("sid", None)
            values = data.get("values", dict())

            gateway.write_to_hub(sid, **values)
            client._queue.task_done()
        except Exception as e:
            _LOGGER.error('Error while sending from mqtt to gateway: ', str(e))
    _LOGGER.info("Stopping MQTT Thread ...") 

def process_gateway_messages(gateway, client, stop_event):
    while not stop_event.is_set():
        try:
            packet = gateway._queue.get()
            if packet is None:
                continue
            _LOGGER.debug("data from queuee: " + format(packet))

            sid = packet.get("sid", None)
            model = packet.get("model", "")
            data = packet.get("data", "")

            if (sid != None and data != ""):
                data_decoded = json.loads(data)
                client.publish(model, sid, data_decoded)
            gateway._queue.task_done()
        except Exception as e:
            _LOGGER.error('Error while sending from gateway to mqtt: ', str(e))
    _LOGGER.info("Stopping Gateway Thread ...") 

def process_messages(self):
        """
        Infinitely reads events off the server. If it is a pause, then it
        pauses the process; otherwise, it passes the message along.
        """
        # Send a message to alert the tracer that we are ready to begin reading
        # messages.
        self.debugger_pipe.put(fmt_msg('reader_started'))
        try:
            for event in get_events_from_socket(self.server_comm):
                if event['e'] == 'pause':
                    self.command_pause()
                else:
                    self.debugger_pipe.put(event)

                    # If we get here, we had a socket error that dropped us
                    # out of get_events(), signal this to the process.
            self.debugger_pipe.put(fmt_msg('disable', 'soft'))
        finally:
            log.info('ServerReader terminating') 

def process_messages(self):
        console_out("monitor started", "MONITOR")
        while self.stop == False:
            try:
                msg_tuple = self.msg_queue.popleft()
                self.consume(msg_tuple[0], msg_tuple[1], msg_tuple[2], msg_tuple[3], msg_tuple[4])
                
                if self.log_messages:
                    self.log_message(f"{msg_tuple[0]}|{msg_tuple[2]}|{msg_tuple[3]}|{msg_tuple[4]}")
            except IndexError:
                time.sleep(1)
            except Exception as e:
                template = "An exception of type {0} occurred. Arguments:{1!r}"
                message = template.format(type(e).__name__, e.args)
                console_out(message, "MONITOR")
                time.sleep(1)
                
        if self.log_messages:
            for msg in self.msg_history:
                self.f.write(msg + "\n")
            self.f.close()
        console_out("Monitor exited", "MONITOR") 

def create_process_condition_messages(condition_ids):
    """Creates messages to process the given conditions

    :param condition_ids: The condition IDs
    :type condition_ids: :func:`list`
    :return: The list of messages
    :rtype: :func:`list`
    """

    messages = []

    for condition_id in condition_ids:
        message = ProcessCondition()
        message.condition_id = condition_id
        messages.append(message)

    return messages 

def create_process_recipe_input_messages(recipe_ids, forced_nodes=None):
    """Creates messages to process the input for the given recipes

    :param recipe_ids: The recipe IDs
    :type recipe_ids: :func:`list`
    :param forced_nodes: Describes the nodes that have been forced to reprocess
    :type forced_nodes: :class:`recipe.diff.forced_nodes.ForcedNodes`
    :return: The list of messages
    :rtype: :func:`list`
    """

    messages = []

    for recipe_id in recipe_ids:
        message = ProcessRecipeInput()
        message.recipe_id = recipe_id
        message.forced_nodes = forced_nodes
        messages.append(message)

    return messages 

def process_messages(self, messages):
        """
        Process vaping messages before the are emitted

        Aggregation is handled here

        **Arguments**

        - messages (`list`): list of vaping messages

        **Returns**

        Result of self.aggregate
        """

        for message in messages:
            if message["data"] and message["data"][0] and message["data"][0].get("ts"):
                message["ts"] = message["data"][0]["ts"]
        return self.aggregate(messages) 

def process_messages(worker):
    while True:
        try:
            msg = sys.stdin.readline().strip()
            if msg:
                parsed_msg = deserialize(msg)
                if parsed_msg.msg_type == MessageTypes.SHUTDOWN:
                    raise WorkershutdownError()
                else:
                    worker.process_message(parsed_msg)
            else:
                raise WorkershutdownError()
        except WorkershutdownError:
            worker.shutdown()
            break
        except Exception as e:  # todo handle other possible exceptions gracefully
            print(traceback.format_exc())
            worker.shutdown()
            break 

def processBackgroundMessages(self):
        while self.bg.hasMessage():
            instruction, message = self.bg.get()
            if instruction == u"exception":
                for line in message:
                    userMessage(line[:-1])
                self.quitNow()
                return
            elif instruction == u"render":
                # It's unlikely that more than one frame would be present in the
                # queue. If/when it happens, only the las/most recent frame
                # matters.
                self.frontFrame = message
            else:
                assert False 

def process_commit_messages(commit_message_filepath):
  with open(commit_message_filepath, "r") as commit_message_file:
    commit_message = commit_message_file.read()
  
  commit_message_funcs = (
    _get_module_level_functions_with_prefix(COMMIT_MESSAGE_FUNCS_PREFIX))
  
  for func in commit_message_funcs:
    commit_message = func(commit_message)
  
  with open(commit_message_filepath, "w") as commit_message_file:
    commit_message_file.write(commit_message) 

def processMessages(self):
        """Process all the queued messages."""
        if self.user is None:
            # User is not authenticated yet, don't process messages. Once the
            # user is authenticated this method will be called to process the
            # queued messages.
            return []

        # Process all the messages in the queue.
        handledMessages = []
        while len(self.messages) > 0:
            message = self.messages.popleft()
            handledMessages.append(message)
            msg_type = self.getMessageField(message, "type")
            if msg_type is None:
                return handledMessages
            if msg_type not in (MSG_TYPE.REQUEST, MSG_TYPE.PING):
                # Only support request messages from the client.
                self.loseConnection(
                    STATUSES.PROTOCOL_ERROR, "Invalid message type."
                )
                return handledMessages
            if self.handleRequest(message, msg_type) is None:
                # Handling of request has failed, stop processing the messages
                # in the queue because the connection will be lost.
                return handledMessages
        return handledMessages 

def process_messages(self, obj):
        if isinstance(obj, list) and obj:
            if obj[0] == MessageEncoder.message_key:
                if len(obj) == 3:
                    # Compatibility with previously-encoded messages
                    return Message(*obj[1:])
                if obj[1]:
                    obj[3] = mark_safe(obj[3])
                return Message(*obj[2:])
            return [self.process_messages(item) for item in obj]
        if isinstance(obj, dict):
            return {key: self.process_messages(value)
                    for key, value in six.iteritems(obj)}
        return obj 

def processPendingIncomingMessages(self, jid, participantJid = None):
        conversationIdentifier = (jid, participantJid)
        if conversationIdentifier in self.pendingIncomingMessages:
            for messageNode in self.pendingIncomingMessages[conversationIdentifier]:
                self.onMessage(messageNode)

            del self.pendingIncomingMessages[conversationIdentifier]

    ##### handling received data ##### 

def process_messages(self, count_arr, recorder=None):
        
        if recorder: recorder.start()
        
        status = MPI.Status()
        
        s = self.comm.Iprobe(source=MPI.ANY_SOURCE, tag=700, status=status)
        
        # if self.test: print '%d probed, got %s' % (self.rank,s)
        
        while s:
            
            src_rank=status.source
            
            self.comm.Recv(buf=count_arr, source=src_rank, tag=700, status=status)
            
            self.gpucomm, src_gpurank, self_gpurank = self.get_gpucomm_with(src_rank)
            
            if self.test: print('%d merging with %d' % (self.rank, src_rank))
            
            self._merge_params_from(src_gpurank, src_rank)
            
            s = self.comm.Iprobe(source=MPI.ANY_SOURCE, tag=700, status=status)
            
            if self.test: print('%d probed again, got %s' % (self.rank,s))
            
        if recorder: recorder.end('comm') 

def process_messages(self, sleep_time=0):
        callbacks = self.process_messages_callbacks.get(threading.get_ident(), None)
        if callbacks is not None:
            for func in callbacks:
                func()

        self.on_process_messages(sleep_time) 

def add_process_messages_callback(self, func ):
        tid = threading.get_ident()
        callbacks = self.process_messages_callbacks.get(tid, None)
        if callbacks is None:
            callbacks = []
            self.process_messages_callbacks[tid] = callbacks

        callbacks.append ( func ) 

def process_messages(management_source, management_data, machine, object_queue):
    now = django.utils.timezone.now()
    for message_item in management_data.get('messages', []):
        object_queue['messages'].append(
            Message(
                machine=machine,
                management_source=management_source,
                text=message_item.get('text'),
                message_type=message_item.get('message_type'),
                date=message_item.get('date', now)))

    return object_queue 

def process_all_messages(messages):
  if 'error' in messages:
    print "Error = ", error
    raise Exception("Error in Response")

  if 'data' not in messages:
    return
  while len(messages['data']) > 0:
    for chat in messages['data']:
      if len(chat['to']['data']) != 2:
        #Ignore Group and self messages
        continue
      friend_name = chat['to']['data'][1]['name'].encode('utf-8')
      if friend_name == OWNER_NAME:
        friend_name = chat['to']['data'][0]['name'].encode('utf-8')
      success = False

      while not success:
        try:
          stdout.write("\rProcessing Chat With : %s                     " % friend_name)
          stdout.flush()
          process_texts(chat['comments'], friend_name)
          success = True
        except OAuthError:
          stdout.write("\nCall Limit Exceeded ! Sleeping for 10 min before retrying !!")
          stdout.flush()
          no_of_messages[friend_name] = 0
          total_chat_length[friend_name] = 0
          stdout.write('\n')
          for i in range(600):
            stdout.write("\rSleeing.......%d" % i)
            stdout.flush()
            time.sleep(1)
          stdout.write("Woke Up! Retrying !!")

    if 'paging' in messages and 'next' in messages['paging']:
      request_str = messages['paging']['next'].replace('https://graph.facebook.com/', '')
      request_str = request_str.replace('limit=25', 'limit=400')
      messages = graph.get(request_str)
    else:
      mesages['data'] = [] 

def process_all_outstanding_messages(self, topic=config.pubsub_receive_topic):
        """
        Process all of the messages currently found within the given pubsub topic.
        :param topic: The topic to retrieve messages for.
        :return: None
        """
        self.__reset_message_processing_state()
        for message in self.connector.receive_all_messages(topic=topic):
            self.__process_message(message)
        self.__handle_processed_messages()
        self.__emit_responses() 

def process_messages(self):
        # TODO(sheeprine): Code kept to handle threading and asynchronous
        # reloading
        # pending_reload = self._rating_endpoint.get_reload_list()
        # pending_states = self._rating_endpoint.get_module_state()
        pass 

def process_messages(self, topic, msg):
        """

        :param topic: topic name of onu.
        :param msg: actual test result dict
        :return: None
        """
        result_frame = {}
        event_name = topic.split(':')[-1]
        onu_device_id = topic.split(':')[-2]
        frame = msg['rx-response']
        for key, value in (frame.fields['omci_message'].fields).iteritems():
            result_frame[key] = long(value)
        self.publish_metrics(result_frame, event_name, onu_device_id) 

def process_messages(self, msgs):
        for received_message in msgs:
            pubsub_message = received_message.get('message')
            log.debug("processing %s" % received_message.get('ackId'))
            if pubsub_message:
                ack_ids = []
                # Process messages
                # Note the design here is to run a single task at a time
                # print base64.urlsafe_b64decode(
                # str(pubsub_message.get('data')))
                # Get the message's ack ID
                cmd_retcode = self.run_task(received_message)
                # TODO if cmd_retcode == 0, the cmd exited clean
                # the retry logic could get complex and is left as an exercise
                ack_ids.append(received_message.get('ackId'))
                # in this case - should ack per message instead of batch
                # as want to make sure task is acked after completion, as
                # nothing else will extend
                # Create a POST body for the acknowledge request
                ack_body = {'ackIds': ack_ids}
                if ack_ids:
                    # Acknowledge the message.
                    log.debug("acking %s" % ack_ids)
                    ack_resp = self.client.projects().subscriptions().acknowledge(
                        subscription=self.sub['name'],
                        body=ack_body).execute() 

def process_messages(self, obj):
        if isinstance(obj, list) and obj:
            if obj[0] == MessageEncoder.message_key:
                if len(obj) == 3:
                    # Compatibility with previously-encoded messages
                    return Message(*obj[1:])
                if obj[1]:
                    obj[3] = mark_safe(obj[3])
                return Message(*obj[2:])
            return [self.process_messages(item) for item in obj]
        if isinstance(obj, dict):
            return {key: self.process_messages(value)
                    for key, value in obj.items()}
        return obj 

def processIncomingMessages(self):
        try:
            while not self.isShuttingDown:
                message = self.proxyStdOut.readline().rstrip()

                if message != "":
                    self.messagePumpQueue.put((time.time(), message))

                if message == PipeTransport.DisconnectMessage:
                    return
        finally:
            logging.info("PipeTransport closing stdOut to proxy (fd = %s)", self.proxyStdOutFileDescriptor)
            self.proxyStdOut.close() 

def process_new_messages(self, new_messages):
        self._append_pre_next_step_handler()
        self.__notify_update(new_messages)
        self._notify_command_handlers(self.message_handlers, new_messages)
        self._notify_message_subscribers(new_messages)
        self._notify_message_next_handler(new_messages) 

def process_new_messages(self, new_messages):
        self._notify_next_handlers(new_messages)
        self._notify_reply_handlers(new_messages)
        self.__notify_update(new_messages)
        self._notify_command_handlers(self.message_handlers, new_messages) 

def process_messages(self, data):
		while data:
			msg = int(data[0])
			fn = self.message_map.get(msg)
			if fn:
				data = fn(data)
			else:
				print("msg %d unhandled" % msg)
				data = b''
		return data 

def process_messages(self, queues, worker, static_queues):
        # type: (list, Worker, list) -> None
        for queue in queues:
            try:
                messages = self.poll_messages(queue)
                logger.debug('[django-eb-sqs] Polled {} messages'.format(len(messages)))

                self._send_signal(MESSAGES_RECEIVED, messages=messages)

                msg_entries = []
                for msg in messages:
                    self._execute_user_code(lambda: self._process_message(msg, worker))
                    msg_entries.append({
                            'Id': msg.message_id,
                            'ReceiptHandle': msg.receipt_handle
                    })

                self._send_signal(MESSAGES_PROCESSED, messages=messages)

                self.delete_messages(queue, msg_entries)

                self._send_signal(MESSAGES_DELETED, messages=messages)
            except ClientError as exc:
                error_code = exc.response.get('Error', {}).get('Code', None)
                if error_code == 'AWS.SimpleQueueService.NonExistentQueue' and queue not in static_queues:
                    logger.debug('[django-eb-sqs] Queue was already deleted {}: {}'.format(queue.url, exc), exc_info=1)
                else:
                    logger.warning('[django-eb-sqs] Error polling queue {}: {}'.format(queue.url, exc), exc_info=1)
            except Exception as exc:
                logger.warning('[django-eb-sqs] Error polling queue {}: {}'.format(queue.url, exc), exc_info=1) 

def process_last_messages(self) -> List[Tuple[int, Exception]]:
        """
        Raises:
            FSError
            FSBackendOfflineError
            FSSharingNotAllowedError
        """
        errors = []
        # Concurrent message processing is totally pointless
        async with self._process_messages_lock:
            user_manifest = self.get_user_manifest()
            initial_last_processed_message = user_manifest.last_processed_message
            try:
                rep = await self.backend_cmds.message_get(offset=initial_last_processed_message)

            except BackendNotAvailable as exc:
                raise FSBackendOfflineError(str(exc)) from exc

            except BackendConnectionError as exc:
                raise FSError(f"Cannot retrieve user messages: {exc}") from exc

            if rep["status"] != "ok":
                raise FSError(f"Cannot retrieve user messages: {rep}")

            new_last_processed_message = initial_last_processed_message
            for msg in rep["messages"]:
                try:
                    await self._process_message(msg["sender"], msg["timestamp"], msg["body"])
                    new_last_processed_message = msg["count"]

                except FSBackendOfflineError:
                    raise

                except FSError as exc:
                    logger.warning(
                        "Invalid message", reason=exc, sender=msg["sender"], count=msg["count"]
                    )
                    errors.append((msg["count"], exc))

            # Update message offset in user manifest
            async with self._update_user_manifest_lock:
                user_manifest = self.get_user_manifest()
                if user_manifest.last_processed_message < new_last_processed_message:
                    user_manifest = user_manifest.evolve_and_mark_updated(
                        last_processed_message=new_last_processed_message
                    )
                    await self.set_user_manifest(user_manifest)
                    self.event_bus.send(CoreEvent.FS_ENTRY_UPDATED, id=self.user_manifest_id)

        return errors 

def process_messages(
    client: pyrogram.client.client.Client,
    chat_id: str,
    last_read_message_id: int,
    media_types: List[str],
    file_formats: dict,
) -> int:
    """Download media from Telegram.

    Parameters
    ----------
    client: pyrogram.client.client.Client
        Client to interact with Telegram APIs.
    chat_id: string
        Id of the chat to download media from.
    last_read_message_id: integer
        Id of last message read from the conversational thread.
    media_types: list
        List of strings of media types to be downloaded.
        Ex : `["audio", "photo"]`
        Supported formats:
            * audio
            * document
            * photo
            * video
            * voice
    file_formats: dict
        Dictionary containing the list of file_formats
        to be downloaded for `audio`, `document` & `video`
        media types
    Returns
    -------
    integer
        last_message_id
    """
    message_ids = await asyncio.gather(
        *[
            download_media(client, message, media_types, file_formats)
            async for message in client.iter_history(
                chat_id, offset_id=last_read_message_id, reverse=True
            )
        ]
    )

    last_message_id = max(message_ids, default=last_read_message_id)
    return last_message_id 

def processMessages(self, msgs):
        log = []
        for msg in msgs:
            CCemail = None
            to_email = None
            header_data = msg["payload"]["headers"]
            for data in header_data:
                if "To" == data["name"]:
                    to_email = data["value"]
                    if '<' in to_email:
                        start = to_email.find('<')
                        end = to_email.find('>')
                        to_email = to_email[start + 1: end]
                if "From" == data["name"]:
                    email_id = data["value"]
                    if '<' in email_id:
                        start = email_id.find('<')
                        end = email_id.find('>')
                        email_id = email_id[start + 1: end]
                if "Received" == data["name"]:
                    timestamp = parsedate_to_datetime(data["value"].split(';', 1)[-1]).astimezone(
                        pytz.timezone("Asia/Calcutta"))
                if "Cc" == data["name"]:
                    CCemail = data["value"]
                    if '<' in CCemail:
                        start = CCemail.find('<')
                        end = CCemail.find('>')
                        CCemail = CCemail[start + 1: end]
            MsgB64 = ""
            try:
                if "parts" in msg["payload"]:
                    MsgB64 = msg["payload"]['parts'][0]['body']['data'].replace("-", "+").replace("_", "/")
                else:
                    MsgB64 = msg["payload"]['body']['data'].replace("-", "+").replace("_", "/")
            except Exception as e:
                errorObj = Errors.objects.create(module='status', errorContent=e, timestamp=datetime.now())
                errorObj.save()

            Msg = base64.b64decode(bytes(MsgB64, 'UTF-8')).decode('UTF-8')
            Msg = "<br />".join(Msg.split("\r\n"))
            Msg = Msg.split("wrote:<br /><br />>")[0]
            Msg = Msg.rsplit("On ", 1)[0]
            Msg = Msg.split("-- <br />You received this message")[0]

            log.append({
                "CCemail": CCemail,
                "to": to_email,
                'email': email_id,
                'date': self.date,
                'timestamp': timestamp.isoformat(),
                'message': Msg
            })

        return log 

def processMessages( self, data ):
        """
        Acts on extracted protocol messages based on header flags.

        After the incoming `data' is decrypted and authenticated, this method
        processes the received data based on the header flags.  Payload is
        written to the local application, new tickets are stored, or keys are
        added to the replay table.
        """

        if (data is None) or (len(data) == 0):
            return

        # Try to extract protocol messages from the encrypted blurb.
        msgs  = self.protoMsg.extract(data, self.recvCrypter, self.recvHMAC)
        if (msgs is None) or (len(msgs) == 0):
            return

        for msg in msgs:
            # Forward data to the application.
            if msg.flags == const.FLAG_PAYLOAD:
                self.circuit.upstream.write(msg.payload)

            # Store newly received ticket.
            elif self.weAreClient and (msg.flags == const.FLAG_NEW_TICKET):
                assert len(msg.payload) == (const.TICKET_LENGTH +
                                            const.MASTER_KEY_LENGTH)
                peer = self.circuit.downstream.transport.getPeer()
                ticket.storeNewTicket(msg.payload[0:const.MASTER_KEY_LENGTH],
                                      msg.payload[const.MASTER_KEY_LENGTH:
                                                  const.MASTER_KEY_LENGTH +
                                                  const.TICKET_LENGTH], peer)

            # Use the PRNG seed to generate the same probability distributions
            # as the server.  That's where the polymorphism comes from.
            elif self.weAreClient and (msg.flags == const.FLAG_PRNG_SEED):
                assert len(msg.payload) == const.PRNG_SEED_LENGTH
                log.debug("Obtained PRNG seed.")
                prng = random.Random(msg.payload)
                pktDist = probdist.new(lambda: prng.randint(const.HDR_LENGTH,
                                                            const.MTU),
                                       seed=msg.payload)
                self.pktMorpher = packetmorpher.new(pktDist)
                self.iatMorpher = probdist.new(lambda: prng.random() %
                                               const.MAX_PACKET_DELAY,
                                               seed=msg.payload)

            else:
                log.warning("Invalid message flags: %d." % msg.flags) 

def process_basestation_messages(home, f):
    count = 0
    range_histo = BinHisto(110, 0, 440000)
    polar_range_histo = PolarHisto(120, 100, 0, 400000)
    polar_elev_histo = PolarHisto(120, 400, -15.0, 90.0)

    try: range_histo.read('range.csv')
    except: traceback.print_exc()

    try: polar_range_histo.read('polar_range.csv')
    except: traceback.print_exc()

    try: polar_elev_histo.read('polar_elev.csv')
    except: traceback.print_exc()

    last_save = last_reset = time.time()

    c = csv.reader(f, delimiter=',')
    for row in c:
        if row[0] != 'MSG': continue
        if row[1] != '3': continue

        try:
            icao = row[4]
            ts = row[8] + ' ' + row[9]
            alt = ft_to_m(float(row[11]))
            lat = float(row[14])
            lng = float(row[15])
        except:
            continue

        r,b,e = range_bearing_elevation(home, (lat,lng,alt))
        if r > ABSOLUTE_MAXIMUM_RANGE:
            # bad data
            continue

        range_histo.add(icao, r)
        polar_range_histo.add(icao, b, r)
        polar_elev_histo.add(icao, b, e)

        now = time.time()
        if (now - last_save) > 30.0:
            range_histo.write('range.csv')
            polar_range_histo.write('polar_range.csv')
            polar_elev_histo.write('polar_elev.csv')
            last_save = now

        if (now - last_reset) > 5.0:
            range_histo.reset_icao_history()
            polar_range_histo.reset_icao_history()
            polar_elev_histo.reset_icao_history()
            last_reset = now 

def process_messages(self, messages: Dict[int, MessageTuple], ignore_unknown_message_types: bool = False):
        """Performs message processing.

        :param messages: indexed by message id dict with messages data
        :param ignore_unknown_message_types: whether to silence exceptions

        :raises UnknownMessageTypeError:

        """
        send = self.send
        exception_handling = self._exception_handling

        messages = list(messages.values())

        with self.before_after_send_handling(messages=messages):

            for message, dispatches in messages:

                try:
                    message_cls = message.get_type()
                    compile_message = partial(message_cls.compile, message=message, messenger=self)

                except UnknownMessageTypeError:
                    if ignore_unknown_message_types:
                        continue
                    raise

                message_type_cache = None

                for dispatch in dispatches:

                    if dispatch.message_cache:
                        continue

                    # Create actual message text for further usage.
                    with exception_handling(dispatches=[dispatch]):
                        if message_type_cache is None and not message_cls.has_dynamic_context:
                            # If a message class doesn't depend upon a dispatch data for message compilation,
                            # we'd compile a message just once.
                            message_type_cache = compile_message(dispatch=dispatch)

                        dispatch.message_cache = message_type_cache or compile_message(dispatch=dispatch)

                with exception_handling(dispatches=dispatches):
                    # Batch send to cover wider messenger scenarios.
                    send(message_cls, message, dispatches) 

def process_messages(msg):
    """
    Process messages and send validation status.

    Processing involves:
    1. Downloading, verifying, extracting, and preparing report files for processing.
    2. Line item processing each report file in the payload (downloaded from step 1).
    3. Check if all reports have been processed for the manifest and if so, kick off
       the celery worker task to summarize.
    4. Send payload validation status to ingress service.

    Args:
        msg (ConsumerRecord) - Message from kafka hccm topic.

    Returns:
        None

    """
    process_complete = False
    status, report_metas = handle_message(msg)

    value = json.loads(msg.value().decode("utf-8"))
    request_id = value.get("request_id", "no_request_id")
    if report_metas:
        for report_meta in report_metas:
            report_meta["process_complete"] = process_report(request_id, report_meta)
            LOG.info(f"Processing: {report_meta.get('current_file')} complete.")
        process_complete = report_metas_complete(report_metas)
        summary_task_id = summarize_manifest(report_meta)
        if summary_task_id:
            LOG.info(f"Summarization celery uuid: {summary_task_id}")
        conversion_task_id = convert_manifest_to_parquet(request_id=request_id, report_meta=report_meta)
        if conversion_task_id:
            LOG.info(f"Conversion of CSV to Parquet uuid: {conversion_task_id}")
    if status:
        if report_metas:
            file_list = [meta.get("current_file") for meta in report_metas]
            files_string = ",".join(map(str, file_list))
            LOG.info(f"Sending Ingress Service confirmation for: {files_string}")
        else:
            LOG.info(f"Sending Ingress Service confirmation for: {value}")
        send_confirmation(value["request_id"], status)

    return process_complete 

def process_messages(self, max_messages=10000):
        """Process all messages ready in the subscription channels.

        This reads messages from the subscription channels and calls the
        appropriate handlers until there are no messages left.

        Args:
            max_messages: The maximum number of messages to process before
                returning.
        """
        subscribe_clients = [self.primary_subscribe_client]
        for subscribe_client in subscribe_clients:
            for _ in range(max_messages):
                message = None
                try:
                    message = subscribe_client.get_message()
                except redis.exceptions.ConnectionError:
                    pass
                if message is None:
                    # Continue on to the next subscribe client.
                    break

                # Parse the message.
                pattern = message["pattern"]
                channel = message["channel"]
                data = message["data"]

                # Determine the appropriate message handler.
                if pattern == ray.gcs_utils.XRAY_HEARTBEAT_BATCH_PATTERN:
                    # Similar functionality as raylet info channel
                    message_handler = self.xray_heartbeat_batch_handler
                elif pattern == ray.gcs_utils.XRAY_JOB_PATTERN:
                    # Handles driver death.
                    message_handler = self.xray_job_notification_handler
                elif (channel ==
                      ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL):
                    message_handler = self.autoscaler_resource_request_handler
                else:
                    assert False, "This code should be unreachable."

                # Call the handler.
                message_handler(channel, data)