Python json.decoder.JSONDecodeError() Examples

The following are code examples for showing how to use json.decoder.JSONDecodeError(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: twstock   Author: mlouielu   File: stock.py    MIT License 6 votes vote down vote up
def fetch(self, year: int, month: int, sid: str, retry: int=5):
        params = {'date': '%d%02d01' % (year, month), 'stockNo': sid}
        for retry_i in range(retry):
            r = requests.get(self.REPORT_URL, params=params,
                             proxies=get_proxies())
            try:
                data = r.json()
            except JSONDecodeError:
                continue
            else:
                break
        else:
            # Fail in all retries
            data = {'stat': '', 'data': []}

        if data['stat'] == 'OK':
            data['data'] = self.purify(data)
        else:
            data['data'] = []
        return data 
Example 2
Project: twstock   Author: mlouielu   File: stock.py    MIT License 6 votes vote down vote up
def fetch(self, year: int, month: int, sid: str, retry: int=5):
        params = {'d': '%d/%d' % (year - 1911, month), 'stkno': sid}
        for retry_i in range(retry):
            r = requests.get(self.REPORT_URL, params=params,
                             proxies=get_proxies())
            try:
                data = r.json()
            except JSONDecodeError:
                continue
            else:
                break
        else:
            # Fail in all retries
            data = {'aaData': []}

        data['data'] = []
        if data['aaData']:
            data['data'] = self.purify(data)
        return data 
Example 3
Project: cb4   Author: joint-online-judge   File: training.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _parse_dag_json(dag):
  try:
    dag = json.decode(dag)
  except decoder.JSONDecodeError:
    raise error.ValidationError('dag') from None
  if not isinstance(dag, list):
    raise error.ValidationError('dag')
  new_dag = []
  try:
    for node in dag:
      if any(k not in node for k in ['_id', 'require_nids', 'pids']):
        raise error.ValidationError('dag')
      new_node = {'_id': int(node['_id']),
                  'title': str(node.get('title', '')),
                  'require_nids': misc.dedupe(map(int, node['require_nids'])),
                  'pids': misc.dedupe(map(document.convert_doc_id, node['pids']))}
      new_dag.append(new_node)
  except ValueError:
    raise error.ValidationError('dag') from None
  return new_dag 
Example 4
Project: vj4   Author: vijos   File: training.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _parse_dag_json(dag):
  try:
    dag = json.decode(dag)
  except decoder.JSONDecodeError:
    raise error.ValidationError('dag') from None
  if not isinstance(dag, list):
    raise error.ValidationError('dag')
  new_dag = []
  try:
    for node in dag:
      if any(k not in node for k in ['_id', 'require_nids', 'pids']):
        raise error.ValidationError('dag')
      new_node = {'_id': int(node['_id']),
                  'title': str(node.get('title', '')),
                  'require_nids': misc.dedupe(map(int, node['require_nids'])),
                  'pids': misc.dedupe(map(document.convert_doc_id, node['pids']))}
      new_dag.append(new_node)
  except ValueError:
    raise error.ValidationError('dag') from None
  return new_dag 
Example 5
Project: ihe-tf-sync   Author: flrt   File: helpers.py    MIT License 6 votes vote down vote up
def load_json(filename):
    """
    Lecture d'un fichier de configuration au format JSON
    Produit un dictionnaire python

    :param config_filename: nom du fichier
    :return: données lues dans 1 dictionnaire
    """
    data = {}

    if not os.path.exists(filename):
        return data
    else:
        with io.open(filename, "r", encoding="utf-8") as fin:
            try:
                udata = fin.read()
                data = json.loads(udata.encode("utf-8"))
            except TypeError as msg_e:
                sys.stderr.write("load_json_config typeError :%s" % msg_e)
            except JSONDecodeError as msg_j:
                sys.stderr.write("load_json_config JSONdecode :%s" % msg_j)
    return data 
Example 6
Project: pyontutils   Author: tgbugs   File: ttlfmt.py    MIT License 6 votes vote down vote up
def parse(source, format_guess, outpath, graph=None, infmt=None, graph_class=GRAPHCLASS):
    graph = graph_class() if graph is None else graph
    errors = []
    if infmt:
        format_guess = infmt

    for format in (format_guess, *(f for f in formats if f != format_guess)):
        # TODO we don't need to reset just saved parsed streams to the point where they fail?
        if type(source) == TextIOWrapper:  # stdin can't reset
            src = source.read()
            source = StringIO(src)
        try:
            graph.parse(source=source, format=format)
            a = next(iter(graph))
            return graph, outpath
        except (StopIteration, BadSyntax, JSONDecodeError) as e:
            print('PARSING FAILED', format, source)
            if infmt:  # or format_guess != None:
                raise e
            errors.append(e)
            if type(source) == StringIO:
                source.seek(0)
    raise BadSyntax(str(errors)) from errors[0] 
Example 7
Project: koku   Author: project-koku   File: view.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def get_queryset(self):
        """Get a queryset.

        Restricts the returned Sources to the associated account,
        by filtering against a `account_id` in the request.
        """
        queryset = Sources.objects.none()
        auth_header = self.request.headers.get('X-Rh-Identity')
        if auth_header:
            try:
                decoded_rh_auth = b64decode(auth_header)
                json_rh_auth = json_loads(decoded_rh_auth)
                account_id = json_rh_auth.get('identity', {}).get('account_number')
                queryset = Sources.objects.filter(account_id=account_id)
            except Sources.DoesNotExist:
                LOG.error('No sources found for account id %s.', account_id)
            except (binascii.Error, JSONDecodeError) as error:
                LOG.error(f'Error decoding authentication header: {str(error)}')

        return queryset 
Example 8
Project: datadog-unix-agent   Author: DataDog   File: test.py    Apache License 2.0 6 votes vote down vote up
def lint_py(ctx, targets=None):
    args = "--rcfile={} --reports=y".format(get_repo_path(PYLINT_RC))
    files = get_matching(get_repo_path(), patterns=[r".*\.py$"], exclude_patterns=LINT_SKIP_PATTERNS)

    stdout, _ = epylint.py_run("{target} {args}".format(target=" ".join(files), args=args), return_std=True)

    try:
        msgs = json.load(stdout)
        for msg in msgs:
            if msg['type'].lower() == 'error':
                print(colored(json.dumps(msg, sort_keys=True, indent=4), "red"))
            else:
                print(colored(json.dumps(msg, sort_keys=True, indent=4), "green"))
        else:
            print(colored("Nice! No lint errors!", "green"))
    except JSONDecodeError:
        print(colored("Whoopsie Daisy! There was an issue linting your code!", "red")) 
Example 9
Project: spotifyMe   Author: Chippasaur   File: authentication.py    MIT License 6 votes vote down vote up
def Spotify_auth(client_id, client_secret):
    """Required to use Spotify's API start with this always must pass a client ID & client secret"""
    try:
        token = util.prompt_for_user_token("Alvin Chung",
                                       client_id=client_id,
                                       client_secret=client_secret,
                                       redirect_uri="https://example.com/callback/")
    except (AttributeError, JSONDecodeError):
        os.remove(f".cache-Alvin Chung")
        token = util.prompt_for_user_token("Alvin Chung",
                                       client_id=client_id,
                                       client_secret=client_secret,
                                       redirect_uri="https://example.com/callback/")


    spotify = spotipy.Spotify(auth=token)
    return spotify 
Example 10
Project: numerapi   Author: uuazed   File: utils.py    MIT License 6 votes vote down vote up
def post_with_err_handling(url, body, headers, timeout=None):
    try:
        r = requests.post(url, json=body, headers=headers, timeout=timeout)
        r.raise_for_status()
    except requests.exceptions.HTTPError as e:
        logger.error("Http Error: {}".format(e))
    except requests.exceptions.ConnectionError as e:
        logger.error("Error Connecting: {}".format(e))
    except requests.exceptions.Timeout as e:
        logger.error("Timeout Error: {}".format(e))
    except requests.exceptions.RequestException as e:
        logger.error("Oops, something went wrong: {}".format(e))

    try:
        return r.json()
    except UnboundLocalError as e:
        # `r` isn't available, probably because the try/except above failed
        pass
    except JSONDecodeError as e:
        logger.error("Did not receive a valid JSON: {}".format(e))
        return {} 
Example 11
Project: webssh   Author: huashengdun   File: handler.py    MIT License 6 votes vote down vote up
def on_message(self, message):
        logging.debug('{!r} from {}:{}'.format(message, *self.src_addr))
        worker = self.worker_ref()
        try:
            msg = json.loads(message)
        except JSONDecodeError:
            return

        if not isinstance(msg, dict):
            return

        resize = msg.get('resize')
        if resize and len(resize) == 2:
            try:
                worker.chan.resize_pty(*resize)
            except (TypeError, struct.error, paramiko.SSHException):
                pass

        data = msg.get('data')
        if data and isinstance(data, UnicodeType):
            worker.data_to_dst.append(data)
            worker.on_write() 
Example 12
Project: Discord-Selfbot   Author: appu1232   File: dataIO.py    GNU General Public License v3.0 6 votes vote down vote up
def save_json(self, filename, data):
        """Atomically save a JSON file given a filename and a dictionary."""
        path, ext = splitext(filename)
        tmp_file = "{}.{}.tmp".format(path, randint(1000, 9999))
        with open(tmp_file, 'w', encoding='utf-8') as f:
            dump(data, f, indent=4,sort_keys=True,separators=(',',' : '))
        try:
            with open(tmp_file, 'r', encoding='utf-8') as f:
                data = load(f)
        except decoder.JSONDecodeError:
            print("Attempted to write file {} but JSON "
                                  "integrity check on tmp file has failed. "
                                  "The original file is unaltered."
                                  "".format(filename))
            return False
        except Exception as e:
            print('A issue has occured saving ' + filename + '.\n'
                  'Traceback:\n'
                  '{0} {1}'.format(str(e), e.args))
            return False

        replace(tmp_file, filename)
        return True 
Example 13
Project: release_maker   Author: R0gnar   File: jira_api.py    MIT License 6 votes vote down vote up
def get_issues(self, jql):
        url = JIRA_API_URL + 'search'
        query = urlencode({
            'jql': jql,
            'maxResults': 100000
        })

        result = make_http_request(url + '?' + query, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            data = json.loads(result)
            issues = data['issues']
        except (JSONDecodeError, KeyError):
            raise JiraError
        else:
            return issues 
Example 14
Project: panda-harvester   Author: PanDAWMS   File: spec_base.py    Apache License 2.0 6 votes vote down vote up
def pack(self, values, slim=False):
        if hasattr(values, '_asdict'):
            values = values._asdict()
        for attr in self.attributes:
            if slim and attr in self.skipAttrsToSlim:
                val = None
            else:
                val = values[attr]
                if attr in self.serializedAttrs and val is not None:
                    try:
                        val = json.loads(val, object_hook=as_python_object)
                    except JSONDecodeError:
                        pass
            object.__setattr__(self, attr, val)

    # set blob attribute 
Example 15
Project: hdyndns   Author: decentral1se   File: requests.py    GNU General Public License v3.0 6 votes vote down vote up
def get(url, headers=None, load_json=True):
    """Run a HTTP GET on some web resource."""
    headers = headers or {}
    try:
        request = Request(url, headers=headers, method='GET')
        response = (
            urlopen(request, timeout=REQUEST_TIMEOUT).read().decode('utf-8')
        )
        if not load_json:
            return response
        try:
            return loads(response)
        except JSONDecodeError:
            message = 'Unable to JSON load {}'.format(response)
            logger.critical(message)
            exit(EXIT_CODE_1_BAD)
    except (URLError, ValueError, timeout) as exception:
        message = 'Unable to GET against {}: {}'.format(url, str(exception))
        logger.critical(message)
        exit(EXIT_CODE_1_BAD) 
Example 16
Project: segments   Author: cldf   File: profile.py    Apache License 2.0 6 votes vote down vote up
def from_file(cls, fname, form=None):
        """
        Read an orthography profile from a metadata file or a default tab-separated profile file.
        """
        try:
            tg = TableGroup.from_file(fname)
            opfname = None
        except JSONDecodeError:
            tg = TableGroup.fromvalue(cls.MD)
            opfname = fname
        if len(tg.tables) != 1:
            raise ValueError('profile description must contain exactly one table')
        metadata = tg.common_props
        metadata.update(fname=Path(fname), form=form)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            res = cls(
                *[{k: None if (k != cls.GRAPHEME_COL and v == cls.NULL) else v
                   for k, v in d.items()}
                  for d in tg.tables[0].iterdicts(fname=opfname)],
                **metadata)
        return res 
Example 17
Project: Price-monitor   Author: qqxx6661   File: crawler_selenium.py    GNU General Public License v3.0 6 votes vote down vote up
def get_huihui_item(self, item_id):
        huihui_info_dict = {"max_price": None, "min_price": None}
        url = 'https://zhushou.huihui.cn/productSense?phu=https://item.jd.com/' + item_id + '.html'
        try:
            self.chrome.get(url)
            url_text = self.chrome.find_element_by_tag_name('body').text
            info = json.loads(url_text)
            huihui_info_dict = {"max_price": info['max'], "min_price": info['min']}
            logging.info(huihui_info_dict)
        except decoder.JSONDecodeError as e:
            logging.warning('Crawl failure: {}'.format(e))
        except NoSuchElementException as e:
            logging.warning('Crawl failure: {}'.format(e))
        except TimeoutException as e:
            logging.warning('Crawl failure: {}'.format(e))
        self.chrome.quit()
        return huihui_info_dict 
Example 18
Project: anipy   Author: twissell-   File: utils.py    MIT License 5 votes vote down vote up
def response_to_dic(response):
    try:
        return json.loads(response.data.decode(_ENCODING))
    except JSONDecodeError as e:
        logger.error('There was an error decoding the response.', exc_info=True)
        return {'error': 'There was an error decoding the response.'} 
Example 19
Project: django-boost   Author: ChanTsune   File: validators.py    MIT License 5 votes vote down vote up
def __call__(self, value):
        try:
            if isinstance(value, str):
                json.loads(value)
                return
            raise TypeError
        except JSONDecodeError:
            raise ValidationError(self.message, code=self.code) 
Example 20
Project: ransomwaretracker.abuse.ch   Author: Ultimate-Hosts-Blacklist   File: update.py    MIT License 5 votes vote down vote up
def from_json(cls, data):
            """
            Convert a JSON formated string into a dictionary.

            Argument:
                data: str
                    A JSON formeted string to convert to dict format.
            """

            try:
                return loads(data)

            except decoder.JSONDecodeError:
                return {} 
Example 21
Project: lando-api   Author: mozilla-conduit   File: treestatus.py    Mozilla Public License 2.0 5 votes vote down vote up
def request(self, method, url_path, **kwargs):
        """Return the response of a request to Tree Status API.

        Args:
            method: HTTP method to use for request.
            url_path: Path to be appended to api url for request.

        Returns:
            JSON decoded response from TreeStatus

        Raises:
            TreeStatusException:
                Base exception class for other exceptions.
            TreeStatusError:
                If the API returns an error response.
            TreeStatusCommunicationException:
                If there is an error communicating with the API.
        """

        try:
            response = self.session.request(method, self.url + url_path, **kwargs)
            data = response.json()
        except requests.RequestException as exc:
            raise TreeStatusCommunicationException(
                "An error occurred when communicating with Tree Status"
            ) from exc
        except JSONDecodeError as exc:
            raise TreeStatusCommunicationException(
                "Tree Status response could not be decoded as JSON"
            ) from exc

        TreeStatusError.raise_if_error(response, data)
        return data 
Example 22
Project: haipproxy   Author: SpiderClub   File: httpbin.py    MIT License 5 votes vote down vote up
def is_transparent(self, response):
        """filter transparent ip resources"""
        if not response.body_as_unicode():
            return True
        try:
            ip = json.loads(response.body_as_unicode()).get('origin')
            if self.origin_ip in ip:
                return True
        except (AttributeError, JSONDecodeError):
            return True

        return False 
Example 23
Project: stac-validator   Author: sparkgeo   File: stac_validator.py    Apache License 2.0 5 votes vote down vote up
def fetch_and_parse_file(self, input_path):
        """
        Fetch and parse STAC file.
        :param input_path: STAC file to get and read
        :return: content or error message
        """

        err_message = {}
        data = None

        try:
            if self.is_valid_url(input_path):
                logger.info("Loading STAC from URL")
                resp = requests.get(input_path)
                data = resp.json()
            else:
                with open(input_path) as f:
                    logger.info("Loading STAC from filesystem")
                    data = json.load(f)

        except JSONDecodeError as e:
            logger.exception("JSON Decode Error")
            err_message["valid_stac"] = False
            err_message["error_type"] = "InvalidJSON"
            err_message["error_message"] = f"{input_path} is not Valid JSON"

        except FileNotFoundError as e:
            logger.exception("STAC File Not Found")
            err_message["valid_stac"] = False
            err_message["error_type"] = "FileNotFoundError"
            err_message["error_message"] = f"{input_path} cannot be found"

        return data, err_message 
Example 24
Project: butian-src-domains   Author: m4yfly   File: __init__.py    GNU General Public License v3.0 5 votes vote down vote up
def update_domains(update_all=False):
    logger.debug('start to update domains')
    try:
        update_butian_src_domains(update_all)
    except RequestException:
        logger.error('update failed with bad network, please retry')
    except JSONDecodeError:
        logger.error('update failed with json decode error, please retry')
    except Exception:
        logger.error('unexpect error occured, please retry')
    logger.debug('finish update domains') 
Example 25
Project: insightconnect-plugins   Author: rapid7   File: action.py    MIT License 5 votes vote down vote up
def run(self, params={}):
        self.request = copy(self.connection.request)
        self.request.url, self.request.method = self.request.url + "/intelligence", "GET"

        # Pagination flag and results placeholder
        self.continue_paging, self.results = True, list()
        # Update the request with the supplied IP address, page size, and offset
        self.request.params.update({
            "md5": params["hash"],
            "limit": 1000,
            "offset": 0
        })

        while self.continue_paging:
            response = self.connection.session.send(self.request.prepare(), verify=self.request.verify)
            if response.status_code not in range(200, 299):
                raise PluginException(cause="Received %d HTTP status code from ThreatStream." % response.status_code,
                                      assistance="Please verify your ThreatStream server status and try again. "
                                                 "If the issue persists please contact support. "
                                                 "Server response was: %s" % response.text)

            try:
                response_data = response.json()
            except JSONDecodeError:
                raise PluginException(preset=PluginException.Preset.INVALID_JSON, data=response.text)

            try:
                # Check pagination indicator. A "null" value means no more pages.
                if not response_data["meta"]["next"]:
                    self.continue_paging = False
            except KeyError:
                raise PluginException(cause='The output did not contain expected keys.',
                                      assistance='Contact support for help.',
                                      data=response_data)

            self.request.params["offset"] += 1000
            self.results.extend(response_data["objects"])

        self.results = komand.helper.clean(self.results)

        return {"results": self.results} 
Example 26
Project: gadakeco-ml   Author: YueNing   File: config.py    MIT License 5 votes vote down vote up
def loadConfig():
    global _values
    try:
        with open(_filePath, "r") as file:
            _values = json.load(file)
        resolveComplete()
    except (FileNotFoundError, JSONDecodeError):
        resetConfig()
        saveConfig() 
Example 27
Project: koku   Author: project-koku   File: storage.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def create_provider_event(source_id, auth_header, offset):
    """
    Create a Sources database object.

    Args:
        source_id (Integer) - Platform-Sources identifier
        auth_header (String) - HTTP Authentication Header
        offset (Integer) - Kafka offset

    Returns:
        None

    """
    try:
        decoded_rh_auth = b64decode(auth_header)
        json_rh_auth = json_loads(decoded_rh_auth)
        account_id = json_rh_auth.get('identity', {}).get('account_number')
    except (binascii.Error, JSONDecodeError) as error:
        LOG.error(str(error))
        return

    try:
        Sources.objects.get(source_id=source_id)
        LOG.debug(f'Source ID {str(source_id)} already exists.')
    except Sources.DoesNotExist:
        new_event = Sources(source_id=source_id, auth_header=auth_header,
                            offset=offset, account_id=account_id)
        new_event.save() 
Example 28
Project: windyquery   Author: bluerelay   File: model.py    MIT License 5 votes vote down vote up
def set(cls, instance, record):
        record = dict(record)
        for col in cls.columns:
            if col.name in record:
                val = record[col.name]
                if col.type == 'jsonb' and isinstance(val, str):
                    try:
                        val = json.loads(val)
                    except JSONDecodeError:
                        pass
                setattr(instance, col.name, val)
        return instance 
Example 29
Project: indy-plenum   Author: hyperledger   File: test_pool_file_raises_descriptive_error.py    Apache License 2.0 5 votes vote down vote up
def getAllTxn(self, frm: int = None, to: int = None):
        raise JSONDecodeError('', '', 0) 
Example 30
Project: learning_python   Author: lxl0928   File: spider.py    MIT License 5 votes vote down vote up
def parse_page_index(text):
    try:
        data = json.loads(text)
        if data and 'data' in data.keys():
            for item in data.get('data'):
                yield item.get('article_url')
    except JSONDecodeError:
        pass 
Example 31
Project: CindicatorArbitrageBot   Author: Cindicator   File: data_loader.py    GNU General Public License v3.0 5 votes vote down vote up
def loader(coin, exchange, logger):
    """
    Retrieve data from exchange and return best bid and ask prices on specific coin.
    
    Args:
        :param coin: <string> coin name (on exchange)
        :param exchange: <string> exchange name
        :param logger: <logging.Logger> crawler's logger
        
    Returns:
        :return: <dict> with structure like: {'ask': best ask price, 'bid': best bid price}
            ({'ask': None, 'bid': None} if an exception was raised while getting data from url)
        
    """
    result = {'ask': None, 'bid': None}
    try:
        url = URL_MAP[exchange](coin)
        async with aiohttp.ClientSession() as session:
            response = await session.get(url)
            data = json.loads(await response.text())
            result = PARSE_MAP[exchange](data, coin)
    except TimeoutError as time_out_e:
        logger.warning('Parse/GET exception in {} exchange for {} coin: {}'
                       '(concurrent.futures._base.TimeoutError)'.format(exchange, coin, str(time_out_e)))
    except JSONDecodeError as json_e:
        logger.warning('Parse/GET exception in {} exchange for {} coin: {}\n'
                       'response: {}'.format(exchange, coin, str(json_e), await response.status))
    except Exception as e:
        logger.warning('Parse/GET exception in {} exchange for {} coin: {}\n'
                       '{}'.format(exchange, coin, str(e), format_exc()))
    finally:
        return result 
Example 32
Project: CindicatorArbitrageBot   Author: Cindicator   File: data_loader.py    GNU General Public License v3.0 5 votes vote down vote up
def exchange_loader(coins, exchange, logger):
    """
    Retrieve data from exchange and return best bid and ask prices on exchange coins.
    
    Args:
        :param coins: <list> of <string> coin name (in db)
        :param exchange: <string> exchange name
        :param logger: <logging.Logger> crawler's logger
        
    Returns:
        :return: <dict> where keys is coins names and value is <dict> with structure like:
            {'ask': best ask price, 'bid': best bid price} ({'ask': None, 'bid': None} if
            an exception was raised while getting data from url)
        
    """
    result = {coin: {'ask': None, 'bid': None} for coin in coins}
    try:
        url = URL_MAP[exchange](None)
        async with aiohttp.ClientSession() as session:
                response = await session.get(url)
                data = json.loads(await response.text())
                for coin in coins:
                    try:
                        result[coin] = PARSE_MAP[exchange](data, coin)
                    except Exception as e:
                        logger.warning('Parse data exception in {} exchange for {} coin: {}\n'
                                       '{}'.format(exchange, coin, str(e), format_exc()))
    except TimeoutError as time_out_e:
        logger.warning('Parse/GET exception in {} exchange: {}'
                       '(concurrent.futures._base.TimeoutError)'.format(exchange, str(time_out_e)))
    except JSONDecodeError as json_e:
        logger.warning('Parse/GET exception in {} exchange: {}\n'
                       'response: {}'.format(exchange, str(json_e), await response.status))
    except Exception as e:
        logger.warning('Parse/GET exception in {} exchange: {}\n'
                       '{}'.format(exchange, str(e), format_exc()))
    finally:
        return result 
Example 33
Project: cameleon_at_sysctl.org   Author: Ultimate-Hosts-Blacklist   File: update.py    MIT License 5 votes vote down vote up
def from_json(cls, data):
            """
            Convert a JSON formated string into a dictionary.

            Argument:
                data: str
                    A JSON formeted string to convert to dict format.
            """

            try:
                return loads(data)

            except decoder.JSONDecodeError:
                return {} 
Example 34
Project: ClasseVivaAPI   Author: MarcoBuster   File: session.py    MIT License 5 votes vote down vote up
def _request(self, *path, use_api_schema=True):
        if not self.logged_in:
            raise NotLoggedInError()

        url = self.rest_api_url + '/' + ('students' + '/' + self.id) if use_api_schema else ''
        for x in path:
            url += '/' + quote_plus(x)

        r = requests.get(
            url=url,
            headers={
                "User-Agent": "zorro/1.0",
                "Z-Dev-Apikey": "+zorro+",
                "Z-Auth-Token": self.token,
                "Content-Type": "application/json",
            },
        )

        try:
            r = r.json()
            if r.get('error'):
                if 'auth token expired' in r['error']:
                    self.login()

            r = requests.get(
                url=url,
                headers={
                    "User-Agent": "zorro/1.0",
                    "Z-Dev-Apikey": "+zorro+",
                    "Z-Auth-Token": self.token,
                    "Content-Type": "application/json",
                },
            )
            try:
                return r.json()
            except JSONDecodeError:
                return r.text

        except JSONDecodeError:
            return r.text 
Example 35
Project: aio-powerview-api   Author: sander76   File: test_aiorequest.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def test_get_invalid_json(self):
        """Test get with invalid Json."""

        async def go():
            await self.start_fake_server()
            return await self.request.get(make_url('invalid_json'))

        with self.assertRaises(JSONDecodeError):
            ret = self.loop.run_until_complete(go()) 
Example 36
Project: django-crmify   Author: MeanPug   File: insightly.py    Apache License 2.0 5 votes vote down vote up
def make_request(self, path, method='GET', **args):
        headers = {
            'Content-Type': 'application/json',
            'Authorization': 'Basic ' + str(base64.b64encode(self.api_key.encode('ascii')), encoding='ascii')
        }

        resp = requests.request(method, self.get_url(path), headers=headers, **args)

        if not resp.ok:
            resp.raise_for_status()

        try:
            return resp.json()
        except JSONDecodeError:
            return resp.content 
Example 37
Project: django-crmify   Author: MeanPug   File: insightly.py    Apache License 2.0 5 votes vote down vote up
def make_request(self, path, method='GET', **args):
        headers = {
            'Content-Type': 'application/json',
            'Authorization': 'Basic ' + str(base64.b64encode(self.api_key.encode('ascii')), encoding='ascii')
        }

        resp = requests.request(method, self.get_url(path), headers=headers, **args)

        if not resp.ok:
            resp.raise_for_status()

        try:
            return resp.json()
        except JSONDecodeError:
            return resp.content 
Example 38
Project: snagsby-py   Author: roverdotcom   File: sources.py    MIT License 5 votes vote down vote up
def get_raw_data(self):
        response = self.get_sm_response()
        if 'SecretString' in response:
            try:
                return json.loads(response['SecretString'])
            except JSONDecodeError:
                return {}
        else:
            logger.debug('Response for key {}{} does not contain SecretString'.format(
                self.url.netloc,
                self.url.path,
            ))
        return {} 
Example 39
Project: Discord-Selfbot   Author: appu1232   File: dataIO.py    GNU General Public License v3.0 5 votes vote down vote up
def append_json(self, filename, data):
        """Append a value to a JSON file."""
        try:
            with open(filename, 'r', encoding='utf-8') as f:
                file = load(f)
        except Exception as e:
            print('A issue has occured loading ' + filename + '.\n'
                  'Traceback:\n'
                  '{0} {1}'.format(str(e), e.args))
            return False
        try:
            file.append(data)
        except Exception as e:
            print('A issue has occured updating ' + filename + '.\n'
                  'Traceback:\n'
                  '{0} {1}'.format(str(e), e.args))
            return False
        path, ext = splitext(filename)
        tmp_file = "{}.{}.tmp".format(path, randint(1000, 9999))
        with open(tmp_file, 'w', encoding='utf-8') as f:
            dump(file, f, indent=4,sort_keys=True,separators=(',',' : '))
        try:
            with open(tmp_file, 'r', encoding='utf-8') as f:
                data = load(f)
        except decoder.JSONDecodeError:
            print("Attempted to write file {} but JSON "
                                  "integrity check on tmp file has failed. "
                                  "The original file is unaltered."
                                  "".format(filename))
            return False
        except Exception as e:
            print('A issue has occured saving ' + filename + '.\n'
                  'Traceback:\n'
                  '{0} {1}'.format(str(e), e.args))
            return False

        replace(tmp_file, filename)
        return True 
Example 40
Project: Discord-Selfbot   Author: appu1232   File: dataIO.py    GNU General Public License v3.0 5 votes vote down vote up
def is_valid_json(self, filename):
        """Verify that a JSON file exists and is readable. Take in a filename and return a boolean."""
        try:
            with open(filename, 'r', encoding='utf-8') as f:
                data = load(f)
            return True
        except (FileNotFoundError, decoder.JSONDecodeError):
            return False
        except Exception as e:
            print('A issue has occured validating ' + filename + '.\n'
                  'Traceback:\n'
                  '{0} {1}'.format(str(e), e.args))
            return False 
Example 41
Project: pipenv   Author: pypa   File: cli.py    MIT License 5 votes vote down vote up
def review(full_report, bare, file):
    if full_report and bare:
        click.secho("Can't choose both --bare and --full-report/--short-report", fg="red")
        sys.exit(-1)

    try:
        input_vulns = read_vulnerabilities(file)
    except JSONDecodeError:
        click.secho("Not a valid JSON file", fg="red")
        sys.exit(-1)

    vulns = safety.review(input_vulns)
    output_report = report(vulns=vulns, full=full_report, bare_report=bare)
    click.secho(output_report, nl=False if bare and not vulns else True) 
Example 42
Project: momoapi-python   Author: sparkplug   File: client.py    MIT License 5 votes vote down vote up
def interpret_response(self, resp):
        rcode = resp.status_code
        rheaders = resp.headers
        print(resp)

        try:
            rbody = resp.json()
        except JSONDecodeError:
            rbody = resp.text
            resp = Response(rbody, rcode, rheaders)

        if not (200 <= rcode < 300):
            self.handle_error_response(rbody, rcode, resp.text, rheaders)

        return resp 
Example 43
Project: Varken   Author: Boerderij   File: helpers.py    MIT License 5 votes vote down vote up
def connection_handler(session, request, verify, as_is_reply=False):
    air = as_is_reply
    s = session
    r = request
    v = verify
    return_json = False

    disable_warnings(InsecureRequestWarning)

    try:
        get = s.send(r, verify=v)
        if get.status_code == 401:
            if 'NoSiteContext' in str(get.content):
                logger.info('Your Site is incorrect for %s', r.url)
            elif 'LoginRequired' in str(get.content):
                logger.info('Your login credentials are incorrect for %s', r.url)
            else:
                logger.info('Your api key is incorrect for %s', r.url)
        elif get.status_code == 404:
            logger.info('This url doesnt even resolve: %s', r.url)
        elif get.status_code == 200:
            try:
                return_json = get.json()
            except JSONDecodeError:
                logger.error('No JSON response. Response is: %s', get.text)
        if air:
            return get
    except InvalidSchema:
        logger.error("You added http(s):// in the config file. Don't do that.")
    except SSLError as e:
        logger.error('Either your host is unreachable or you have an SSL issue. : %s', e)
    except ConnectionError as e:
        logger.error('Cannot resolve the url/ip/port. Check connectivity. Error: %s', e)
    except ChunkedEncodingError as e:
        logger.error('Broken connection during request... oops? Error: %s', e)

    return return_json 
Example 44
Project: ZeroDot1_CoinBlockerLists_browser   Author: Ultimate-Hosts-Blacklist   File: update.py    MIT License 5 votes vote down vote up
def from_json(cls, data):
            """
            Convert a JSON formated string into a dictionary.

            Argument:
                data: str
                    A JSON formeted string to convert to dict format.
            """

            try:
                return loads(data)

            except decoder.JSONDecodeError:
                return {} 
Example 45
Project: spotlight   Author: j178   File: weibo.py    Do What The F*ck You Want To Public License 5 votes vote down vote up
def load_token(self, filename=WEIBO_TOKEN_FILE):
        try:
            with open(WEIBO_TOKEN_FILE, 'r') as f:
                log.debug('Loading token from file %s' % filename)
                token = json.load(f)
        except (FileNotFoundError, JSONDecodeError):
            log.error('Token file not exist or file content has been destroyed')
            raise TokenNotExistError
        self.session.token = OAuth2Token(token) 
Example 46
Project: emg-toolkit   Author: EBI-Metagenomics   File: metadata.py    Apache License 2.0 5 votes vote down vote up
def fetch_metadata(self):
        resp = requests.get(
            sample_url().format(**{'accession': self.accession})
        )
        try:
            resp = resp.json()
        except JSONDecodeError:
            return

        _accessions = {
            r['run_accession']:
                {
                    'sample_accession': r['secondary_sample_accession'],
                    'read_depth': r['depth']
                }
            for r in resp
        }

        meta_csv = dict()
        _sample = None
        _meta = None
        for (run, sample) in _accessions.items():
            if sample != _sample:
                _meta = self.get_metadata(sample['sample_accession'])
            meta_csv[run] = _meta
            meta_csv[run]['Sample'] = sample['sample_accession']
            meta_csv[run]['Read depth'] = sample['read_depth']
            _sample = sample['sample_accession']
        return meta_csv 
Example 47
Project: agir.lafranceinsoumise.fr   Author: lafranceinsoumise   File: filters.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def to_python(self, value):
        if value in self.empty_values:
            return None

        try:
            obj = json.loads(value)
        except JSONDecodeError:
            raise DjangoValidationError(
                self.default_error_messages["invalid_json"], code="invalid_json"
            )

        if set(obj) != {"max_distance", "coordinates"}:
            raise DjangoValidationError(
                self.default_error_messages["invalid_fields"], code="invalid_fields"
            )

        max_distance = obj["max_distance"]
        coordinates = obj["coordinates"]

        try:
            max_distance = float(max_distance)
        except ValueError:
            raise DjangoValidationError(
                self.default_error_messages["invalid_max_distance"],
                code="invalid_max_distance",
            )

        if not check_coordinates(coordinates):
            raise DjangoValidationError(
                self.default_error_messages["invalid_coordinates"],
                code="invalid_coordinates",
            )

        return Point(*coordinates), Distance(m=max_distance) 
Example 48
Project: agir.lafranceinsoumise.fr   Author: lafranceinsoumise   File: filters.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def filter_queryset(self, request, queryset, view):
        if not "order_by_distance_to" in request.query_params:
            return queryset

        try:
            coordinates = json.loads(request.query_params["order_by_distance_to"])
        except JSONDecodeError:
            raise DRFValidationError(detail=self.error_message)

        if not check_coordinates(coordinates):
            raise DRFValidationError(detail=self.error_message)

        return queryset.annotate(
            distance=DistanceFunction("coordinates", Point(coordinates, srid=4326))
        ).order_by("distance") 
Example 49
Project: stubalyzer   Author: kialo   File: analyze_unit_test.py    MIT License 5 votes vote down vote up
def test_wrong_json(self, exists_mock: Mock, read_mock: Mock) -> None:
        with pytest.raises(JSONDecodeError, match=r".*double quotes.*"):
            assert setup_expected_mismatches("a_file") 
Example 50
Project: release_maker   Author: R0gnar   File: bitbucket_api.py    MIT License 5 votes vote down vote up
def get_projects(self):
        url = BITBUCKET_API_URL + 'projects?limit=1000'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise BitbucketConnectionError

        try:
            data = json.loads(result)
            projects = data['values']
        except (JSONDecodeError, KeyError):
            raise BitbucketError
        else:
            return projects 
Example 51
Project: release_maker   Author: R0gnar   File: bitbucket_api.py    MIT License 5 votes vote down vote up
def get_project_repos(self, project):
        url = BITBUCKET_API_URL + 'projects/' + project + '/repos?limit=1000'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise BitbucketConnectionError

        try:
            data = json.loads(result)
            repositories = data['values']
        except (JSONDecodeError, KeyError):
            raise BitbucketError
        else:
            return repositories 
Example 52
Project: release_maker   Author: R0gnar   File: bitbucket_api.py    MIT License 5 votes vote down vote up
def get_commits(self, project, repo, params):
        query = urlencode(params)
        url = BITBUCKET_API_URL + 'projects/' + project + '/repos/' + repo + '/commits?' + query
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise BitbucketConnectionError

        try:
            data = json.loads(result)
            commits = data['values']
        except (JSONDecodeError, KeyError):
            raise BitbucketError
        else:
            return commits 
Example 53
Project: release_maker   Author: R0gnar   File: jira_api.py    MIT License 5 votes vote down vote up
def get_my_permissions(self):
        url = JIRA_API_URL + 'mypermissions'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            data = json.loads(result)
            permissions = data['permissions']
        except (JSONDecodeError, KeyError, TypeError):
            raise JiraError
        else:
            return permissions 
Example 54
Project: release_maker   Author: R0gnar   File: jira_api.py    MIT License 5 votes vote down vote up
def get_projects(self):
        url = JIRA_API_URL + 'project'
        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            return json.loads(result)
        except JSONDecodeError:
            raise JiraError 
Example 55
Project: release_maker   Author: R0gnar   File: jira_api.py    MIT License 5 votes vote down vote up
def get_issue(self, key):
        url = JIRA_API_URL + 'issue/' + key

        result = make_http_request(url, headers=self.get_headers())
        if not result:
            raise JiraConnectionError

        try:
            issue = json.loads(result)
        except JSONDecodeError:
            raise JiraError
        else:
            return issue 
Example 56
Project: ldap-admin-be   Author: lambdaspace   File: crud.py    MIT License 5 votes vote down vote up
def objects_create(request):
    try:
        body = await request.json()
        success = request.app['ldap'].add(
            body['dn'],
            [request.match_info['object_class']],
            body['attributes']
        )
    except (KeyError, JSONDecodeError, LDAPObjectClassError):
        raise web.HTTPBadRequest()
    if not success:
        raise web.HTTPBadRequest(text=str(request.app['ldap'].result))
    return web.HTTPCreated() 
Example 57
Project: panda-harvester   Author: PanDAWMS   File: spec_base.py    Apache License 2.0 5 votes vote down vote up
def set_blob_attribute(self, key, val):
        try:
            val = json.loads(val, object_hook=as_python_object)
            object.__setattr__(self, key, val)
        except JSONDecodeError:
            pass

    # return column names for INSERT 
Example 58
Project: panda-harvester   Author: PanDAWMS   File: queue_config_mapper.py    Apache License 2.0 5 votes vote down vote up
def _load_config_from_file():
        mainLog = _make_logger(method_name='QueueConfigMapper._load_config_from_file')
        # define config file path
        if os.path.isabs(harvester_config.qconf.configFile):
            confFilePath = harvester_config.qconf.configFile
        else:
            # check if in PANDA_HOME
            confFilePath = None
            if 'PANDA_HOME' in os.environ:
                confFilePath = os.path.join(os.environ['PANDA_HOME'],
                                            'etc/panda',
                                            harvester_config.qconf.configFile)
                if not os.path.exists(confFilePath):
                    confFilePath = None
            # look into /etc/panda
            if confFilePath is None:
                confFilePath = os.path.join('/etc/panda',
                                            harvester_config.qconf.configFile)
        # load from config file
        try:
            with open(confFilePath) as f:
                queueConfigJson = json.load(f)
        except OSError as e:
            mainLog.error('Cannot read file: {0} ; {1}'.format(confFilePath, e))
            return None
        except JSONDecodeError as e:
            mainLog.error('Invalid JSON in file: {0} ; {1}'.format(confFilePath, e))
            return None
        return queueConfigJson

    # get resolver module 
Example 59
Project: VKChatFakeTyping   Author: kirpichik   File: typer.py    MIT License 5 votes vote down vote up
def checkAllies(self):
      resp = None
      try:
         resp = request(getChatUsers % (self.token, self.chatId)).json()
      except JSONDecodeError:
         log("JSONDecodeError at line: %d(getting chat users). UserID: %d" % (getLineNumber(), self.userId))
         return
      if "response" not in resp:
         log(self.userId, "can't check chat allies!")
         return
      resp = resp["response"]

      # Проверяем отсутствующих
      self.locker.acquire(blocking=True)
      toAdd = []
      for ally in self.allies:
         if (ally not in resp) and (ally != self.userId):
            toAdd += [ally]
      self.locker.release()

      # Добавляем отсутствующих
      for add in toAdd:
         resp = None
         try:
            resp = request(addChatUser % (self.token, self.chatId, add)).json()
         except JSONDecodeError:
            log("JSONDecodeError at line: %d(readding ally). UserID: %d" % (getLineNumber(), self.userId))
            return

         # Неудалось добавить от имени этого пользователя
         if not (("response" in resp) and (resp["response"] == 1)):
            log(self.userId, "can't add chat allies!")
            return

         log(add, "readded!")


   # Если режим набора активирован, устанавливает статус "набирает сообщение" 
Example 60
Project: VKChatFakeTyping   Author: kirpichik   File: typer.py    MIT License 5 votes vote down vote up
def typing(self):
      if needTyping:
         resp = None
         try:
            resp = request(setActivity % (self.token, self.chatId + 2000000000)).json()
         except JSONDecodeError:
            log("JSONDecodeError at line: %d(start typing). UserID: %d" % (getLineNumber(), self.userId))
            return
         if ("response" in resp) and (resp["response"] == 1):
            log(self.userId, "is typing...")
         else:
            log(self.userId, "isn't typing!") 
Example 61
Project: ava-ce   Author: indeedsecurity   File: vector.py    Apache License 2.0 5 votes vote down vote up
def _parse_post_data(self, vector, num, name):
        """
        Parse post data to verify it can be converted into an object by the auditors. This prevents conversion errors
        for each check and vector combination during the scan.
        :param vector: vector dictionary
        :param num: index number
        :param name: file name
        :return: True or False
        """
        headers = vector['headers']
        post_data = vector['data']

        # check data
        # content-type checked in self._check_content_type()
        if not post_data:
            return True

        # check json
        if headers['Content-Type'].startswith(HTTP.CONTENT_TYPE.JSON):
            try:
                json.loads(post_data)
            except JSONDecodeError:
                logger.warning("'postData' is not valid JSON for vector #%d in '%s'. Ignoring.", num, name)
                return False

        # check multipart
        if headers['Content-Type'].startswith(HTTP.CONTENT_TYPE.MULTIPART):
            try:
                MultipartDecoder(post_data.encode(), headers['Content-Type'])
            except (ImproperBodyPartContentException, AttributeError):
                logger.warning("'postData' is not valid multipart data for vector #%d in '%s'. Ignoring.", num, name)
                return False

        # default
        return True 
Example 62
Project: ava-ce   Author: indeedsecurity   File: vector.py    Apache License 2.0 5 votes vote down vote up
def parse(self, configs):
        """
        Parses a list of HAR files. Vectors from each HAR file are combined and returned as a list.
        :param configs: AVA configs
        :return: list of vectors as dictionaries
        """
        combined = []

        # parse each vector file in list
        for name in self._sources:
            logger.debug("Reading vectors from '%s'.", name)

            try:
                # load HAR as json
                with open(name) as f:
                    har = json.load(f)

                # get requests
                requests = [entry['request'] for entry in har['log']['entries']]

                # parse requests
                vectors = self._parse_requests(requests, configs, name)

                # combine
                combined.extend(vectors)
            except JSONDecodeError as e:
                raise InvalidFormatException(e)
            except KeyError as e:
                raise InvalidFormatException("{} missing in '{}'".format(e, name))

        return combined 
Example 63
Project: ava-ce   Author: indeedsecurity   File: json.py    Apache License 2.0 5 votes vote down vote up
def _parse_json(self, json_string):
        """
        Parse JSON string and return object.
        :param json_string: JSON as string
        :return: JSON as object
        """
        # parse
        try:
            loaded = json.loads(json_string)
        except JSONDecodeError:
            raise InvalidFormatException("Unable to parse JSON")

        return loaded 
Example 64
Project: ava-ce   Author: indeedsecurity   File: test_readers_vector.py    Apache License 2.0 5 votes vote down vote up
def test_parse_negative(self, reader, requests, mocker):
        configs = {'cookies': {}, 'headers': {}, 'parameters': {}, 'value': "avascan",
                   'excludes': ["/admin"], 'domain': "example.com"}

        har = {
            'log': {
                'entries': [
                    {
                        'request': requests[0]
                    }
                ]
            }
        }

        # missing har request
        mocker.patch("builtins.open")
        mocker.patch("json.load", return_value=har)
        del har['log']['entries'][0]['request']
        with pytest.raises(InvalidFormatException):
            reader.parse(configs)

        # invalid har log
        mocker.patch("json.load", return_value={'log': {'creator': {}}})
        with pytest.raises(InvalidFormatException):
            reader.parse(configs)

        # invalid json
        mocker.patch("json.load", side_effect=JSONDecodeError("", "", 0))
        with pytest.raises(InvalidFormatException):
            reader.parse(configs) 
Example 65
Project: python-esppy   Author: sassoftware   File: rest.py    Apache License 2.0 4 votes vote down vote up
def _get(self, path=None, format='xml', raw=False, **kwargs):
        '''
        GET the specified path

        Parameters
        ----------
        path : string, optional
            Absolute or relative URL
        format : string, optional
            Output format: 'xml' or 'json'
        raw : boolean, optional
            Should the raw content be returned rather than an Element?
        **kwargs : keyword parameters, optional
            URL parameters

        Returns
        -------
        dict

        '''
        self._verify_session()

        if path and (path.startswith('http:') or path.startswith('https:')):
            url = path
        elif path:
            url = urllib.parse.urljoin(self.url, path)
        else:
            url = self.url

        if format == 'json':
            try:
                out = self.session.get(url, **kwargs)
                out = out.json()
                return out
            except JSONDecodeError:
                sys.stderr.write('%s\n' % out.content.decode('utf-8'))
                raise

        url, kwargs = self._insert_params(url, **kwargs)

        if get_option('debug.requests'):
            sys.stderr.write('GET %s\n' % url)

        if get_option('debug.request_bodies') and kwargs.get('data'):
            sys.stderr.write('%s\n' % kwargs['data'])

        content = self._error_check(self.session.get(url,
                                                     **kwargs)).content.decode('utf-8')

        if get_option('debug.responses'):
            sys.stderr.write('%s\n' % content)

        if raw:
            return content

        return ET.fromstring(content) 
Example 66
Project: django-boost   Author: ChanTsune   File: auto_translate.py    MIT License 4 votes vote down vote up
def handle_app_config(self, app_config, **options):
        try:
            import polib
            import requests
        except ImportError:
            self.stderr.write(
                'auto_transrate command required `polib` and `requests`.'
                '\nPlease `pip install polib requests`.')
            return
        app_path = app_config.path
        existing_locales = os.listdir(os.path.join(app_path, 'locale'))

        interval = options['interval']
        failed = {}
        exclude_locals = options['exclude_locale'] + [options['source']]
        exclude_id = options['exclude_id'] + [""]
        locales = [l for l in existing_locales if l not in exclude_locals]

        if options['make']:
            call_command('makemessages', locale=existing_locales)
        if not options['t']:
            return
        for lng in locales:
            po_file_path = os.path.join(app_path, 'locale',
                                        lng, 'LC_MESSAGES', 'django.po')

            lng = self.convert_lng_code(lng)
            po = polib.pofile(po_file_path)
            for i, entry in enumerate(po):
                if entry.msgid in exclude_id:
                    continue
                if entry.msgstr != "" and not options['overwrite']:
                    continue
                text = parse.quote(entry.msgid)
                request_url = APP_URL + '?text={}&source={}&target={}'.format(
                    text, options['source'], lng)
                response = requests.get(request_url)
                try:
                    if response.status_code == 200:
                        result = response.json()
                        self.stdout.write(entry.msgid, ending='')
                        self.stdout.write(' -> ', ending='')
                        self.stdout.write(result['text'])
                        entry.msgstr = result['text']
                        po[i] = entry
                    else:
                        failed[lng] = response.status_code
                except JSONDecodeError:
                    failed[lng] = response.status_code
                sleep(interval)
            po.save(po_file_path)
        self.stdout.write('failed :')
        self.stdout.write(str(failed)) 
Example 67
Project: lando-api   Author: mozilla-conduit   File: phabricator.py    Mozilla Public License 2.0 4 votes vote down vote up
def call_conduit(self, method, **kwargs):
        """Return the result of an RPC call to a conduit method.

        Args:
            **kwargs: Every method parameter is passed as a keyword argument.

        Returns:
            The 'result' key of the conduit method's response or None if
            the 'result' key doesn't exist.

        Raises:
            PhabricatorAPIException:
                if conduit returns an error response.
            requests.exceptions.RequestException:
                if there is a request exception while communicating
                with the conduit API.
        """
        if "__conduit__" not in kwargs:
            kwargs["__conduit__"] = {"token": self.api_token}

        data = {"output": "json", "params": json.dumps(kwargs)}

        extra_data = {
            "params": kwargs.copy(),
            "method": method,
            "api_url": self.api_url,
        }
        del extra_data["params"]["__conduit__"]  # Sanitize the api token.
        logger.debug("call to conduit", extra=extra_data)

        try:
            response = self.session.get(self.api_url + method, data=data).json()
        except requests.RequestException as exc:
            raise PhabricatorCommunicationException(
                "An error occurred when communicating with Phabricator"
            ) from exc
        except JSONDecodeError as exc:
            raise PhabricatorCommunicationException(
                "Phabricator response could not be decoded as JSON"
            ) from exc

        PhabricatorAPIException.raise_if_error(response)
        return response.get("result") 
Example 68
Project: nucypher   Author: nucypher   File: lawful.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def make_web_controller(drone_enrico, crash_on_error: bool = False):

        app_name = bytes(drone_enrico.stamp).hex()[:6]
        controller = WebController(app_name=app_name,
                                   character_controller=drone_enrico.controller,
                                   crash_on_error=crash_on_error)

        drone_enrico.controller = controller

        # Register Flask Decorator
        enrico_control = controller.make_control_transport()

        #
        # Character Control HTTP Endpoints
        #

        @enrico_control.route('/encrypt_message', methods=['POST'])
        def encrypt_message():
            """
            Character control endpoint for encrypting data for a policy and
            receiving the messagekit (and signature) to give to Bob.
            """
            try:
                request_data = json.loads(request.data)
                message = request_data['message']
            except (KeyError, JSONDecodeError) as e:
                return Response(str(e), status=400)

            # Encrypt
            message_kit, signature = drone_enrico.encrypt_message(bytes(message, encoding='utf-8'))

            response_data = {
                'result': {
                    'message_kit': b64encode(message_kit.to_bytes()).decode(),   # FIXME
                    'signature': b64encode(bytes(signature)).decode(),
                },
                'version': str(nucypher.__version__)
            }

            return Response(json.dumps(response_data), status=200)

        return controller 
Example 69
Project: rucio   Author: rucio   File: fts3.py    Apache License 2.0 4 votes vote down vote up
def query_latest(self, state, last_nhours=1):
        """
        Query the latest status transfers status in FTS3 via JSON.

        :param state: Transfer state as a string or a dictionary.
        :returns: Transfer status information as a dictionary.
        """

        jobs = None

        try:
            whoami = requests.get('%s/whoami' % (self.external_host),
                                  verify=self.verify,
                                  cert=self.cert,
                                  headers={'Content-Type': 'application/json'})
            if whoami and whoami.status_code == 200:
                delegation_id = whoami.json()['delegation_id']
            else:
                raise Exception('Could not retrieve delegation id: %s', whoami.content)
            state_string = ','.join(state)
            jobs = requests.get('%s/jobs?dlg_id=%s&state_in=%s&time_window=%s' % (self.external_host,
                                                                                  delegation_id,
                                                                                  state_string,
                                                                                  last_nhours),
                                verify=self.verify,
                                cert=self.cert,
                                headers={'Content-Type': 'application/json'})
        except ReadTimeout as error:
            raise TransferToolTimeout(error)
        except JSONDecodeError as error:
            raise TransferToolWrongAnswer(error)
        except Exception:
            logging.warn('Could not query latest terminal states from %s', self.external_host)

        if jobs and (jobs.status_code == 200 or jobs.status_code == 207):
            record_counter('transfertool.fts3.%s.query_latest.success' % self.__extract_host(self.external_host))
            try:
                jobs_json = jobs.json()
                return jobs_json
            except ReadTimeout as error:
                raise TransferToolTimeout(error)
            except JSONDecodeError as error:
                raise TransferToolWrongAnswer(error)
            except Exception as error:
                logging.error("Failed to parse the jobs status %s" % (str(error)))

        record_counter('transfertool.fts3.%s.query.failure' % self.__extract_host(self.external_host)) 
Example 70
Project: rucio   Author: rucio   File: fts3.py    Apache License 2.0 4 votes vote down vote up
def bulk_query(self, transfer_ids, timeout=None):
        """
        Query the status of a bulk of transfers in FTS3 via JSON.

        :param transfer_ids: FTS transfer identifiers as a list.
        :returns: Transfer status information as a dictionary.
        """

        jobs = None

        if not isinstance(transfer_ids, list):
            transfer_ids = [transfer_ids]

        responses = {}
        fts_session = requests.Session()
        xfer_ids = ','.join(transfer_ids)
        jobs = fts_session.get('%s/jobs/%s?files=file_state,dest_surl,finish_time,start_time,reason,source_surl,file_metadata' % (self.external_host, xfer_ids),
                               verify=self.verify,
                               cert=self.cert,
                               headers={'Content-Type': 'application/json'},
                               timeout=timeout)

        if jobs is None:
            record_counter('transfertool.fts3.%s.bulk_query.failure' % self.__extract_host(self.external_host))
            for transfer_id in transfer_ids:
                responses[transfer_id] = Exception('Transfer information returns None: %s' % jobs)
        elif jobs.status_code == 200 or jobs.status_code == 207:
            try:
                record_counter('transfertool.fts3.%s.bulk_query.success' % self.__extract_host(self.external_host))
                jobs_response = jobs.json()
                responses = self.__bulk_query_responses(jobs_response)
            except ReadTimeout as error:
                raise TransferToolTimeout(error)
            except JSONDecodeError as error:
                raise TransferToolWrongAnswer(error)
            except Exception as error:
                raise Exception("Failed to parse the job response: %s, error: %s" % (str(jobs), str(error)))
        else:
            record_counter('transfertool.fts3.%s.bulk_query.failure' % self.__extract_host(self.external_host))
            for transfer_id in transfer_ids:
                responses[transfer_id] = Exception('Could not retrieve transfer information: %s', jobs.content)

        return responses 
Example 71
Project: rucio   Author: rucio   File: fts3.py    Apache License 2.0 4 votes vote down vote up
def __get_transfer_baseid_voname(self):
        """
        Get transfer VO name from the external host.

        :returns base id as a string and VO name as a string.
        """
        result = (None, None)
        try:
            key = 'voname: %s' % self.external_host
            result = REGION_SHORT.get(key)
            if isinstance(result, NoValue):
                logging.debug("Refresh transfer baseid and voname for %s", self.external_host)

                get_result = None
                try:
                    get_result = requests.get('%s/whoami' % self.external_host,
                                              verify=self.verify,
                                              cert=self.cert,
                                              headers={'Content-Type': 'application/json'},
                                              timeout=5)
                except ReadTimeout as error:
                    raise TransferToolTimeout(error)
                except JSONDecodeError as error:
                    raise TransferToolWrongAnswer(error)
                except Exception as error:
                    logging.warn('Could not get baseid and voname from %s - %s' % (self.external_host, str(error)))

                if get_result and get_result.status_code == 200:
                    baseid = str(get_result.json()['base_id'])
                    voname = str(get_result.json()['vos'][0])
                    result = (baseid, voname)

                    REGION_SHORT.set(key, result)

                    logging.debug("Get baseid %s and voname %s from %s", baseid, voname, self.external_host)
                else:
                    logging.warn("Failed to get baseid and voname from %s, error: %s", self.external_host, get_result.text if get_result is not None else get_result)
                    result = (None, None)
        except Exception as error:
            logging.warning("Failed to get baseid and voname from %s: %s" % (self.external_host, str(error)))
            result = (None, None)
        return result 
Example 72
Project: insightconnect-plugins   Author: rapid7   File: action.py    MIT License 4 votes vote down vote up
def run(self, params={}):
        # Copy and update the base request to avoid mutating the original
        self.request = copy(self.connection.request)
        self.request.url, self.request.method = self.request.url + "/intelligence", "GET"

        # Pagination flag and results placeholder
        self.continue_paging, self.results = True, list()
        # Update the request with the supplied domain, page size, and offset
        self.request.params.update({
            "url": "{url}".format(url=params.get("url")),
            "limit": 1000,
            "offset": 0
        })

        while self.continue_paging:
            response = self.connection.session.send(self.request.prepare(), verify=self.request.verify)

            if response.status_code not in range(200, 299):
                raise PluginException(cause="Received %d HTTP status code from ThreatStream." % response.status_code,
                                      assistance="Please verify your ThreatStream server status and try again. "
                                                 "If the issue persists please contact support. "
                                                 "Server response was: %s" % response.text)

            try:
                response_data = response.json()
            except JSONDecodeError:
                raise PluginException(preset=PluginException.Preset.INVALID_JSON, data=response.text)

            try:
                # Check pagination indicator. A "null" value means no more pages.
                if not response_data["meta"]["next"]:
                    self.continue_paging = False
            except KeyError:
                raise PluginException(cause='The output did not contain expected keys.',
                                      assistance='Contact support for help.',
                                      data=response_data)

            self.request.params["offset"] += 1000
            self.results.extend(response_data["objects"])

        self.results = komand.helper.clean(self.results)
        return {"results": self.results} 
Example 73
Project: insightconnect-plugins   Author: rapid7   File: action.py    MIT License 4 votes vote down vote up
def run(self, params={}):
        # Copy and update the base request to avoid mutating the original
        self.request = copy(self.connection.request)
        self.request.url, self.request.method = self.request.url + "/intelligence", "GET"

        # Pagination flag and results placeholder
        self.continue_paging, self.results = True, list()

        # Update the request with the supplied IP address, page size, and offset
        self.request.params.update({
            "ip": params.get("ip_address"),
            "limit": 1000,
            "offset": 0
        })

        while self.continue_paging:
            response = self.connection.session.send(self.request.prepare(), verify=self.request.verify)

            if response.status_code not in range(200, 299):
                raise PluginException(cause="Received %d HTTP status code from ThreatStream." % response.status_code,
                                      assistance="Please verify your ThreatStream server status and try again. "
                                                 "If the issue persists please contact support. "
                                                 "Server response was: %s" % response.text)

            try:
                response_data = response.json()
            except JSONDecodeError:
                raise PluginException(preset=PluginException.Preset.INVALID_JSON, data=response.text)

            # Check pagination indicator. A "null" value means no more pages.
            try:
                # Check pagination indicator. A "null" value means no more pages.
                if not response_data["meta"]["next"]:
                    self.continue_paging = False
            except KeyError:
                raise PluginException(cause='The output did not contain expected keys.',
                                      assistance='Contact support for help.',
                                      data=response_data)

            self.request.params["offset"] += 1000
            self.results.extend(response_data["objects"])

        self.results = komand.helper.clean(self.results)
        return {"results": self.results} 
Example 74
Project: gif-for-cli   Author: google   File: utils.py    Apache License 2.0 4 votes vote down vote up
def process_input_source(input_source, api_key):
    if input_source.strip().startswith('https://tenor.com/view/'):
        gif_id = input_source.rsplit('-', 1)[-1]
        if gif_id.isdigit():
            input_source = gif_id
        else:
            raise Exception('Bad GIF URL.')

    is_url = input_source.startswith(('http://', 'https://'))

    if not os.path.exists(input_source) and not is_url:
        # get from Tenor GIF API
        params = {'key': api_key}
        if input_source.isdigit():
            endpoint = 'gifs'
            params.update({'ids': input_source})
        elif input_source == '':
            endpoint = 'trending'
            params.update({'limit': 1})
        else:
            endpoint = 'search'
            params.update({'limit': 1, 'q': input_source})

        resp = requests.get(
            'https://api.tenor.com/v1/{}'.format(endpoint),
            params=params
        )

        try:
            resp_json = resp.json()
        except JSONDecodeError:
            raise Exception('A server error occurred.')

        if 'error' in resp_json:
            raise Exception('An error occurred: {}'.format(resp_json['error']))

        results = resp_json.get('results')

        if not results:
            raise Exception('Could not find GIF.')

        input_source = results[0]['media'][0]['mp4']['url']
    return input_source 
Example 75
Project: aries-cloudagent-python   Author: hyperledger   File: routes.py    Apache License 2.0 4 votes vote down vote up
def credential_exchange_store(request: web.BaseRequest):
    """
    Request handler for storing a credential request.

    Args:
        request: aiohttp request object

    Returns:
        The credential request details.

    """

    context = request.app["request_context"]
    outbound_handler = request.app["outbound_message_router"]

    try:
        body = await request.json() or {}
        credential_id = body.get("credential_id")
    except JSONDecodeError:
        credential_id = None

    credential_exchange_id = request.match_info["id"]
    credential_exchange_record = await CredentialExchange.retrieve_by_id(
        context, credential_exchange_id
    )
    connection_id = credential_exchange_record.connection_id

    assert (
        credential_exchange_record.state == CredentialExchange.STATE_CREDENTIAL_RECEIVED
    )

    credential_manager = CredentialManager(context)

    try:
        connection_record = await ConnectionRecord.retrieve_by_id(
            context, connection_id
        )
    except StorageNotFoundError:
        raise web.HTTPBadRequest()

    if not connection_record.is_ready:
        raise web.HTTPForbidden()

    (
        credential_exchange_record,
        credential_stored_message,
    ) = await credential_manager.store_credential(
        credential_exchange_record, credential_id
    )

    await outbound_handler(credential_stored_message, connection_id=connection_id)
    return web.json_response(credential_exchange_record.serialize()) 
Example 76
Project: bert-etl   Author: jbcurtin   File: shortcuts.py    MIT License 4 votes vote down vote up
def load_invoke_args(invoke_args: typing.List[str]) -> typing.List[typing.Dict[str, typing.Any]]:
    loaded: typing.List[typing.Dict[str, typing.Any]] = []
    for invoke_item in invoke_args:
        if isinstance(invoke_item, dict):
            # yaml
            loaded.append(invoke_item)

        elif invoke_item.endswith('.json'):
            with open(invoke_item, 'r') as stream:
                loaded.append(json.loads(stream.read()))

        elif invoke_item.endswith('.yaml') or invoke_item.endswith('.yml'):
            with open(invoke_item, 'r') as stream:
                loaded.append(yaml.load(stream.read(), Loader=yaml.FullLoader))

        else:
            try:
                loaded.extend(_load_invoke_args_module(invoke_item))
            except ImportError:
                pass
            else:
                continue

            try:
                datum = json.loads(invoke_item)
                if isinstance(datum, list):
                    loaded.extend(datum)

                elif isinstance(datum, dict):
                    loaded.append(datum)

                else:
                    raise bert_exceptions.BertConfigError(f'Invalid Json Type. Only List, Dict are supported.')
            except JSONDecodeError:
                pass

            else:
                continue

            raise bert_exceptions.BertConfigError(f'Unable to load invoke_arg[{invoke_item}]')

    return loaded 
Example 77
Project: Discord.FM   Author: FasterSpeeding   File: fm.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def get_cached(
            self,
            params: dict,
            url: str = None,
            cool_down: int = 300,
            item: str = "item"):
        url = (url or self.BASE_URL)
        params = {str(key): str(value) for key, value in params.items()}
        get = self.s.prepare_request(Request("GET", url, params=params))
        url = get.url
        if url not in self.cache or not self.cache[url].expired_check():
            try:
                r = self.s.send(get)
            except requestCError as e:
                self.log.warning(e)
                raise CommandError("Last.FM isn't available right now.")

            if r.status_code == 200:
                if cool_down is not None:
                    self.cache[url] = self.cached_object(
                        exists=True,
                        expire=time() + cool_down,
                        data=r.json(),
                    )
                return r.json()

            if r.status_code == 404:
                self.cache[url] = self.cached_object(
                    exists=False,
                    expire=time() + 1800,
                    error=f"404 - {item} doesn't exist.",
                )
                raise fmEntryNotFound(self.cache[url].error)

            self.log.warning(f"Last.FM threw error {r.status_code}: {r.text}")
            if bot.config.exception_webhooks:
                exception_webhooks(
                    self.client,
                    bot.config.exception_webhooks,
                    content=(f"Last.FM threw error {r.status_code}: "
                             f"```{redact(r.text)[:1950]}```"),
                )

            try:
                message = ": " + r.json().get("message", "")
            except JSONDecodeError:
                message = "."
            else:
                message = redact(message)
            raise fmEntryNotFound(f"{r.status_code} - Last.fm threw "
                                  f"unexpected HTTP status code{message}")

        if self.cache[url].validity_check():
            return self.cache[url].data

        raise fmEntryNotFound(self.cache[url].error) 
Example 78
Project: minio-py   Author: minio   File: api.py    Apache License 2.0 4 votes vote down vote up
def listen_bucket_notification(self, bucket_name, prefix='', suffix='',
                                   events=['s3:ObjectCreated:*',
                                           's3:ObjectRemoved:*',
                                           's3:ObjectAccessed:*']):
        """
        Yeilds new event notifications on a bucket, caller should iterate
        to read new notifications.

        NOTE: Notification is retried in case of `JSONDecodeError` otherwise
        the function raises an exception.

        :param bucket_name: Bucket name to listen event notifications from.
        :param prefix: Object key prefix to filter notifications for.
        :param suffix: Object key suffix to filter notifications for.
        :param events: Enables notifications for specific event types.
             of events.
        """
        is_valid_bucket_name(bucket_name)

        # If someone explicitly set prefix to None convert it to empty string.
        if prefix is None:
            prefix = ''

        # If someone explicitly set suffix to None convert it to empty string.
        if suffix is None:
            suffix = ''

        url_components = urlsplit(self._endpoint_url)
        if url_components.hostname == 's3.amazonaws.com':
            raise InvalidArgumentError(
                'Listening for event notifications on a bucket is a MinIO '
                'specific extension to bucket notification API. It is not '
                'supported by Amazon S3')

        query = {
            'prefix': prefix,
            'suffix': suffix,
            'events': events,
        }
        while True:
            response = self._url_open('GET', bucket_name=bucket_name,
                                      query=query, preload_content=False)
            try:
                for line in response.stream():
                    if line.strip():
                        if hasattr(line, 'decode'):
                            line = line.decode('utf-8')
                        event = json.loads(line)
                        if event['Records'] is not None:
                            yield event
            except JSONDecodeError:
                response.close()
                continue 
Example 79
Project: paasta   Author: Yelp   File: test_secret_tools.py    Apache License 2.0 4 votes vote down vote up
def test_get_hmac_for_secret():
    with mock.patch(
        "paasta_tools.secret_tools.open", autospec=False
    ) as mock_open, mock.patch(
        "json.load", autospec=True
    ) as mock_json_load, mock.patch(
        "paasta_tools.secret_tools.get_secret_name_from_ref", autospec=True
    ) as mock_get_secret_name_from_ref:
        mock_json_load.return_value = {
            "environments": {"dev": {"signature": "notArealHMAC"}}
        }
        mock_get_secret_name_from_ref.return_value = "secretsquirrel"

        ret = get_hmac_for_secret(
            "SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev"
        )
        mock_get_secret_name_from_ref.assert_called_with("SECRET(secretsquirrel)")
        mock_open.assert_called_with(
            "/nail/blah/service-name/secrets/secretsquirrel.json", "r"
        )
        assert ret == "notArealHMAC"

        ret = get_hmac_for_secret(
            "SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev-what"
        )
        assert ret is None

        mock_open.side_effect = IOError
        ret = get_hmac_for_secret(
            "SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev"
        )
        assert ret is None

        ret = get_hmac_for_secret(
            "SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev"
        )
        assert ret is None

        mock_open.side_effect = None
        mock_json_load.side_effect = JSONDecodeError("", "", 1)
        ret = get_hmac_for_secret(
            "SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev"
        )
        assert ret is None 
Example 80
Project: ArgoScuolaNext-Python   Author: hearot   File: session.py    MIT License 4 votes vote down vote up
def __init__(self, school_code: str, username: str, password: str, version: str = _argo_version):
        """
        Initialize the object and login the user.

        :param school_code: The ministerial school code
        :type school_code: str
        :param username: The user's username
        :type username: str
        :param password: The user's password
        :type password: str
        :param version: The version of the ArgoScuolaNext REST API
        :type version: str
        """
        global _argo_version

        login_request = requests.get(
            url=_rest_api_endpoint + "login",
            headers={
                "x-key-app": _argo_key,
                "x-version": version,
                "x-produttore-software": _app_company,
                "x-app-code": _app_code,
                "user-agent": _user_agent,
                "x-cod-min": school_code,
                "x-user-id": username,
                "x-pwd": password
            },
            params={
                "_dc": round(time.time() * 1000)
            }
        )

        if login_request.status_code != requests.codes.ok:
            try:
                version = _version_regex.search(login_request.json()['value']).group(0)
                self.information = Session(school_code, username, password, version).information
                _argo_version = version
            except (AttributeError, JSONDecodeError, KeyError):
                raise AuthenticationFailedError("Bad credentials or request.")
        else:
            self.information = self.get_information(school_code,
                                                    login_request.json()['token'],
                                                    version)[0]
        self.logged_in = True
        self.version = version