Python elasticsearch.exceptions.ElasticsearchException() Examples

def get_dashboard(self, rule, db_name):
        """ Download dashboard which matches use_kibana_dashboard from Elasticsearch. """
        es = elasticsearch_client(rule)
        if not db_name:
            raise EAException("use_kibana_dashboard undefined")
        query = {'query': {'term': {'_id': db_name}}}
        try:
            # TODO use doc_type = _doc
            res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard'])
        except ElasticsearchException as e:
            raise EAException("Error querying for dashboard: %s" % (e)).with_traceback(sys.exc_info()[2])

        if res['hits']['hits']:
            return json.loads(res['hits']['hits'][0]['_source']['dashboard'])
        else:
            raise EAException("Could not find dashboard named %s" % (db_name)) 

def get_aggregated_matches(self, _id):
        """ Removes and returns all matches from writeback_es that have aggregate_id == _id """

        # XXX if there are more than self.max_aggregation matches, you have big alerts and we will leave entries in ES.
        query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}}
        matches = []
        try:
            if self.writeback_es.is_atleastsixtwo():
                res = self.writeback_es.search(index=self.writeback_index, body=query,
                                               size=self.max_aggregation)
            else:
                res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert',
                                                          body=query, size=self.max_aggregation)
            for match in res['hits']['hits']:
                matches.append(match['_source'])
                if self.writeback_es.is_atleastsixtwo():
                    self.writeback_es.delete(index=self.writeback_index, id=match['_id'])
                else:
                    self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=match['_id'])
        except (KeyError, ElasticsearchException) as e:
            self.handle_error("Error fetching aggregated matches: %s" % (e), {'id': _id})
        return matches 

def find_pending_aggregate_alert(self, rule, aggregation_key_value=None):
        query = {'filter': {'bool': {'must': [{'term': {'rule_name': rule['name']}},
                                              {'range': {'alert_time': {'gt': ts_now()}}},
                                              {'term': {'alert_sent': 'false'}}],
                                     'must_not': [{'exists': {'field': 'aggregate_id'}}]}}}
        if aggregation_key_value:
            query['filter']['bool']['must'].append({'term': {'aggregation_key': aggregation_key_value}})
        if self.writeback_es.is_atleastfive():
            query = {'query': {'bool': query}}
        query['sort'] = {'alert_time': {'order': 'desc'}}
        try:
            if self.writeback_es.is_atleastsixtwo():
                res = self.writeback_es.search(index=self.writeback_index, body=query, size=1)
            else:
                res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', body=query, size=1)
            if len(res['hits']['hits']) == 0:
                return None
        except (KeyError, ElasticsearchException) as e:
            self.handle_error("Error searching for pending aggregated matches: %s" % (e), {'rule_name': rule['name']})
            return None

        return res['hits']['hits'][0] 

def get_index_start(self, index, timestamp_field='@timestamp'):
        """ Query for one result sorted by timestamp to find the beginning of the index.

        :param index: The index of which to find the earliest event.
        :return: Timestamp of the earliest event.
        """
        query = {'sort': {timestamp_field: {'order': 'asc'}}}
        try:
            if self.thread_data.current_es.is_atleastsixsix():
                res = self.thread_data.current_es.search(index=index, size=1, body=query,
                                                         _source_includes=[timestamp_field], ignore_unavailable=True)
            else:
                res = self.thread_data.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field],
                                                         ignore_unavailable=True)
        except ElasticsearchException as e:
            self.handle_error("Elasticsearch query error: %s" % (e), {'index': index, 'query': query})
            return '1969-12-30T00:00:00Z'
        if len(res['hits']['hits']) == 0:
            # Index is completely empty, return a date before the epoch
            return '1969-12-30T00:00:00Z'
        return res['hits']['hits'][0][timestamp_field] 

def _es_call(self, cmd, *args, **kwargs):
        try:
            return getattr(self._es, cmd)(*args, **kwargs)
        except ElasticsearchException as e:
            self._log_error(e)
            return None 

def search(self, **kwargs):  # pylint: disable=arguments-differ
        """ this will definitely fail """
        raise exceptions.ElasticsearchException("This search operation failed") 

def test_search_failure(self):
        """ the search operation should fail """
        with self.assertRaises(exceptions.ElasticsearchException):
            self.searcher.search("abc test") 

def test_index_failure_bulk(self):
        """ the index operation should fail """
        with patch('search.elastic.bulk', return_value=[0, [exceptions.ElasticsearchException()]]):
            with self.assertRaises(exceptions.ElasticsearchException):
                self.searcher.index("test_doc", [{"name": "abc test"}]) 

def log_indexing_error(cls, indexing_errors):
        """ Logs indexing errors and raises a general ElasticSearch Exception"""
        indexing_errors_log = []
        for indexing_error in indexing_errors:
            indexing_errors_log.append(str(indexing_error))
        raise exceptions.ElasticsearchException(', '.join(indexing_errors_log)) 

def test_perform_request_error(self, mock_response):
        mock_response.return_value.status = 500
        connection = BotoHttpConnection(aws_access_key_id='access_key', aws_secret_access_key='secret')
        with self.assertRaises(ElasticsearchException):
            with patch('elasticsearch.connection.base.logger.debug') as mock_logger:
                connection.perform_request('get', 'http://example.com')
                self.assertGreater(mock_logger.call_count, 0) 

def load(self):
        """Loads the trained entity resolution model from disk."""
        try:
            if self._use_text_rel:
                scoped_index_name = get_scoped_index_name(
                    self._app_namespace, self._es_index_name
                )
                if not self._es_client.indices.exists(index=scoped_index_name):
                    self.fit()
            else:
                self.fit()

        except EsConnectionError as e:
            logger.error(
                "Unable to connect to Elasticsearch: %s details: %s", e.error, e.info
            )
            raise EntityResolverConnectionError(es_host=self._es_client.transport.hosts)
        except TransportError as e:
            logger.error(
                "Unexpected error occurred when sending requests to Elasticsearch: %s "
                "Status code: %s details: %s",
                e.error,
                e.status_code,
                e.info,
            )
            raise EntityResolverError
        except ElasticsearchException:
            raise EntityResolverError 

def test_perform_request_error(self, mock_response):
        mock_response.return_value.status = 500
        connection = AwsHttpConnection(aws_access_key_id='access_key', aws_secret_access_key='secret')
        with self.assertRaises(ElasticsearchException):
            with patch('elasticsearch.connection.base.logger.debug') as mock_logger:
                connection.perform_request('get', 'http://example.com')
                self.assertGreater(mock_logger.call_count, 0) 

def healthcheck(request):
    index = request.registry.settings["elasticsearch_index"]
    try:
        status = request.es.cluster.health(index=index)["status"]
    except exceptions.ElasticsearchException as exc:
        raise FailedHealthcheck("elasticsearch exception") from exc
    if status not in ("yellow", "green"):
        raise FailedHealthcheck("cluster status was {!r}".format(status))
    return {"status": "ok", "version": bouncer_version} 

def send(self, logentry):
        try:
            logentry.save()
        except ElasticsearchException as ex:
            logger.exception("ElasticsearchLogsProducer error sending log to Elasticsearch: %s", ex)
            raise LogSendException(
                "ElasticsearchLogsProducer error sending log to Elasticsearch: %s" % ex
            )
        except Exception as e:
            logger.exception(
                "ElasticsearchLogsProducer exception sending log to Elasticsearch: %s", e
            )
            raise LogSendException(
                "ElasticsearchLogsProducer exception sending log to Elasticsearch: %s" % e
            ) 

def _create_index_if_missing(self, name):
        try:
            if not self._es.indices.exists(name):
                self._es.indices.create(name)
        except ElasticsearchException as e:
            self._log_error(e) 

def find_recent_pending_alerts(self, time_limit):
        """ Queries writeback_es to find alerts that did not send
        and are newer than time_limit """

        # XXX only fetches 1000 results. If limit is reached, next loop will catch them
        # unless there is constantly more than 1000 alerts to send.

        # Fetch recent, unsent alerts that aren't part of an aggregate, earlier alerts first.
        inner_query = {'query_string': {'query': '!_exists_:aggregate_id AND alert_sent:false'}}
        time_filter = {'range': {'alert_time': {'from': dt_to_ts(ts_now() - time_limit),
                                                'to': dt_to_ts(ts_now())}}}
        sort = {'sort': {'alert_time': {'order': 'asc'}}}
        if self.writeback_es.is_atleastfive():
            query = {'query': {'bool': {'must': inner_query, 'filter': time_filter}}}
        else:
            query = {'query': inner_query, 'filter': time_filter}
        query.update(sort)
        try:
            if self.writeback_es.is_atleastsixtwo():
                res = self.writeback_es.search(index=self.writeback_index, body=query, size=1000)
            else:
                res = self.writeback_es.deprecated_search(index=self.writeback_index,
                                                          doc_type='elastalert', body=query, size=1000)
            if res['hits']['hits']:
                return res['hits']['hits']
        except ElasticsearchException as e:
            logging.exception("Error finding recent pending alerts: %s %s" % (e, query))
        return [] 

def get_starttime(self, rule):
        """ Query ES for the last time we ran this rule.

        :param rule: The rule configuration.
        :return: A timestamp or None.
        """
        sort = {'sort': {'@timestamp': {'order': 'desc'}}}
        query = {'filter': {'term': {'rule_name': '%s' % (rule['name'])}}}
        if self.writeback_es.is_atleastfive():
            query = {'query': {'bool': query}}
        query.update(sort)

        try:
            doc_type = 'elastalert_status'
            index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type)
            if self.writeback_es.is_atleastsixtwo():
                if self.writeback_es.is_atleastsixsix():
                    res = self.writeback_es.search(index=index, size=1, body=query,
                                                   _source_includes=['endtime', 'rule_name'])
                else:
                    res = self.writeback_es.search(index=index, size=1, body=query,
                                                   _source_include=['endtime', 'rule_name'])
            else:
                res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type,
                                                          size=1, body=query, _source_include=['endtime', 'rule_name'])
            if res['hits']['hits']:
                endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime'])

                if ts_now() - endtime < self.old_query_limit:
                    return endtime
                else:
                    elastalert_logger.info("Found expired previous run for %s at %s" % (rule['name'], endtime))
                    return None
        except (ElasticsearchException, KeyError) as e:
            self.handle_error('Error querying for last run: %s' % (e), {'rule': rule['name']}) 

def get_hits_count(self, rule, starttime, endtime, index):
        """ Query Elasticsearch for the count of results and returns a list of timestamps
        equal to the endtime. This allows the results to be passed to rules which expect
        an object for each hit.

        :param rule: The rule configuration dictionary.
        :param starttime: The earliest time to query.
        :param endtime: The latest time to query.
        :return: A dictionary mapping timestamps to number of hits for that time period.
        """
        query = self.get_query(
            rule['filter'],
            starttime,
            endtime,
            timestamp_field=rule['timestamp_field'],
            sort=False,
            to_ts_func=rule['dt_to_ts'],
            five=rule['five']
        )

        try:
            res = self.thread_data.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
            self.handle_error('Error running count query: %s' % (e), {'rule': rule['name'], 'query': query})
            return None

        self.thread_data.num_hits += res['count']
        lt = rule.get('use_local_time')
        elastalert_logger.info(
            "Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count'])
        )
        return {endtime: res['count']} 

def search(self, unit):
        counter = {}
        res = []
        language = unit.store.translation_project.language.code
        index_name = INDEX_PREFIX + language.lower()
        es_res = self._es_call(
            "search",
            index=index_name,
            body={
                "query": {
                    "match": {"source": {"query": unit.source, "fuzziness": "AUTO"}}
                }
            },
        )

        if es_res is None:
            # ElasticsearchException - eg ConnectionError.
            return []
        elif es_res == "":
            # There seems to be an issue with urllib where an empty string is
            # returned
            logger.error(
                "Elasticsearch search (%s:%s) returned an empty " "string: %s",
                self._settings["HOST"],
                self._settings["PORT"],
                unit,
            )
            return []

        hits = filter_hits_by_distance(
            es_res["hits"]["hits"],
            unit.source,
            min_similarity=self._settings.get("MIN_SIMILARITY", DEFAULT_MIN_SIMILARITY),
        )
        for hit in hits:
            if self._is_valuable_hit(unit, hit):
                body = hit["_source"]
                translation_pair = body["source"] + body["target"]
                if translation_pair not in counter:
                    counter[translation_pair] = 1
                    res.append(
                        {
                            "unit_id": hit["_id"],
                            "source": body["source"],
                            "target": body["target"],
                            "project": body["project"],
                            "path": body["path"],
                            "username": body["username"],
                            "fullname": body["fullname"],
                            "email_md5": body["email_md5"],
                            "mtime": body.get("mtime", None),
                            "score": hit["_score"],
                        }
                    )
                else:
                    counter[translation_pair] += 1

        for item in res:
            item["count"] = counter[item["source"] + item["target"]]

        return res 

def is_silenced(self, rule_name):
        """ Checks if rule_name is currently silenced. Returns false on exception. """
        if rule_name in self.silence_cache:
            if ts_now() < self.silence_cache[rule_name][0]:
                return True

        if self.debug:
            return False
        query = {'term': {'rule_name': rule_name}}
        sort = {'sort': {'until': {'order': 'desc'}}}
        if self.writeback_es.is_atleastfive():
            query = {'query': query}
        else:
            query = {'filter': query}
        query.update(sort)

        try:
            doc_type = 'silence'
            index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type)
            if self.writeback_es.is_atleastsixtwo():
                if self.writeback_es.is_atleastsixsix():
                    res = self.writeback_es.search(index=index, size=1, body=query,
                                                   _source_includes=['until', 'exponent'])
                else:
                    res = self.writeback_es.search(index=index, size=1, body=query,
                                                   _source_include=['until', 'exponent'])
            else:
                res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type,
                                                          size=1, body=query, _source_include=['until', 'exponent'])
        except ElasticsearchException as e:
            self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name})

            return False
        if res['hits']['hits']:
            until_ts = res['hits']['hits'][0]['_source']['until']
            exponent = res['hits']['hits'][0]['_source'].get('exponent', 0)
            if rule_name not in list(self.silence_cache.keys()):
                self.silence_cache[rule_name] = (ts_to_dt(until_ts), exponent)
            else:
                self.silence_cache[rule_name] = (ts_to_dt(until_ts), self.silence_cache[rule_name][1])
            if ts_now() < ts_to_dt(until_ts):
                return True
        return False 

def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_size=None):
        rule_filter = copy.copy(rule['filter'])
        base_query = self.get_query(
            rule_filter,
            starttime,
            endtime,
            timestamp_field=rule['timestamp_field'],
            sort=False,
            to_ts_func=rule['dt_to_ts'],
            five=rule['five']
        )
        if term_size is None:
            term_size = rule.get('terms_size', 50)
        query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field'])
        try:
            if not rule['five']:
                res = self.thread_data.current_es.deprecated_search(
                    index=index,
                    doc_type=rule.get('doc_type'),
                    body=query,
                    search_type='count',
                    ignore_unavailable=True
                )
            else:
                res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'),
                                                                    body=query, size=0, ignore_unavailable=True)
        except ElasticsearchException as e:
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
            self.handle_error('Error running query: %s' % (e), {'rule': rule['name']})
            return None
        if 'aggregations' not in res:
            return {}
        if not rule['five']:
            payload = res['aggregations']['filtered']
        else:
            payload = res['aggregations']

        if self.thread_data.current_es.is_atleastseven():
            self.thread_data.num_hits += res['hits']['total']['value']
        else:
            self.thread_data.num_hits += res['hits']['total']

        return {endtime: payload}