Python json.keys() Examples

The following are 30 code examples for showing how to use json.keys(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module json , or try the search function .

Example 1
Project: bbj   Author: bbj-dev   File: server.py    License: MIT License 6 votes vote down vote up
def validate(json, args):
    """
    Ensure the json object contains all the keys needed to satisfy
    its endpoint (and isnt empty)
    """
    if not json:
        raise BBJParameterError(
            "JSON input is empty. This method requires the following "
            "arguments: {}".format(", ".join(args)))

    for arg in args:
        if arg not in json.keys():
            raise BBJParameterError(
                "Required parameter {} is absent from the request. "
                "This method requires the following arguments: {}"
                .format(arg, ", ".join(args))) 
Example 2
Project: histogrammar-python   Author: histogrammar   File: sum.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sum"], ["name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json["entries"], "Sum.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Sum.name")

            if json["sum"] in ("nan", "inf", "-inf") or isinstance(json["sum"], numbers.Real):
                sum = float(json["sum"])
            else:
                raise JsonFormatException(json["sum"], "Sum.sum")

            out = Sum.ed(entries, sum)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Sum") 
Example 3
Project: histogrammar-python   Author: histogrammar   File: minmax.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "min"], ["name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json["entries"], "Minimize.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Minimize.name")

            if json["min"] in ("nan", "inf", "-inf") or isinstance(json["min"], numbers.Real):
                min = float(json["min"])
            else:
                raise JsonFormatException(json["min"], "Minimize.min")

            out = Minimize.ed(entries, min)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Minimize") 
Example 4
Project: histogrammar-python   Author: histogrammar   File: average.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "mean"], ["name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json["entries"], "Average.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Average.name")

            if json["mean"] in ("nan", "inf", "-inf") or isinstance(json["mean"], numbers.Real):
                mean = float(json["mean"])
            else:
                raise JsonFormatException(json["mean"], "Average.mean")

            out = Average.ed(entries, mean)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Average") 
Example 5
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "data"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "Label.entries")

            if isinstance(json["sub:type"], basestring):
                factory = Factory.registered[json["sub:type"]]
            else:
                raise JsonFormatException(json, "Label.sub:type")

            if isinstance(json["data"], dict):
                pairs = dict((k, factory.fromJsonFragment(v, None)) for k, v in json["data"].items())
            else:
                raise JsonFormatException(json, "Label.data")

            return Label.ed(entries, **pairs)

        else:
            raise JsonFormatException(json, "Label") 
Example 6
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "data"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "UntypedLabel.entries")

            if isinstance(json["data"], dict):
                pairs = {}
                for k, v in json["data"].items():
                    if isinstance(v, dict) and hasKeys(v.keys(), ["type", "data"]):
                        factory = Factory.registered[v["type"]]
                        pairs[k] = factory.fromJsonFragment(v["data"], None)

                    else:
                        raise JsonFormatException(k, "UntypedLabel.data {0}".format(v))

            else:
                raise JsonFormatException(json, "UntypedLabel.data")

            return UntypedLabel.ed(entries, **pairs).specialize()

        else:
            raise JsonFormatException(json, "UntypedLabel") 
Example 7
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "data"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "Index.entries")

            if isinstance(json["sub:type"], basestring):
                factory = Factory.registered[json["sub:type"]]
            else:
                raise JsonFormatException(json, "Index.sub:type")

            if isinstance(json["data"], list):
                values = [factory.fromJsonFragment(x, None) for x in json["data"]]
            else:
                raise JsonFormatException(json, "Index.data")

            return Index.ed(entries, *values).specialize()

        else:
            raise JsonFormatException(json, "Index") 
Example 8
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 6 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "data"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "Branch.entries")

            if isinstance(json["data"], list):
                values = []
                for i, x in enumerate(json["data"]):
                    if isinstance(x, dict) and hasKeys(x.keys(), ["type", "data"]):
                        if isinstance(x["type"], basestring):
                            factory = Factory.registered[x["type"]]
                        else:
                            raise JsonFormatException(x, "Branch.data {0} type".format(i))
                        values.append(factory.fromJsonFragment(x["data"], None))

            else:
                raise JsonFormatException(json, "Branch.data")

            return Branch.ed(entries, *values)

        else:
            raise JsonFormatException(json, "Branch") 
Example 9
Project: bbj   Author: bbj-dev   File: server.py    License: MIT License 5 votes vote down vote up
def db_validate(self, args, database, user, **kwargs):
        """
        See also [the Input Validation page](validation.md).

        Requires the arguments `key` and `value`. Returns an object
        with information about the database sanity criteria for
        key. This can be used to validate user input in the client
        before trying to send it to the server.

        If the argument `error` is supplied with a non-nil value,
        the server will return a standard error object on failure
        instead of the special object described below.

        The returned object has two keys:

        {
          "bool": true/false,
          "description": null/"why this value is bad"
        }

        If bool == false, description is a string describing the
        problem. If bool == true, description is null and the
        provided value is safe to use.
        """
        validate(args, ["key", "value"])
        response = dict()
        try:
            db.validate([(args["key"], args["value"])])
            response["bool"] = True
            response["description"] = None
        except BBJException as e:
            if args.get("error"):
                raise
            response["bool"] = False
            response["description"] = e.description
        return response 
Example 10
Project: grimoirelab-elk   Author: chaoss   File: enrich.py    License: GNU General Public License v3.0 5 votes vote down vote up
def __convert_json_to_projects_map(self, json):
        """ Convert JSON format to the projects map format
        map[ds][repository] = project
        If a repository is in several projects assign to leaf
        Check that all JSON data is in the database

        :param json: data with the projects to repositories mapping
        :returns: the repositories to projects mapping per data source
        """
        ds_repo_to_prj = {}

        # Sent the unknown project to the end of the list.
        # This change is needed to avoid assigning repositories to
        # the `Main` project when they exist in the `unknown`
        # section and in other sections too.
        project_names = list(json.keys())
        if UNKNOWN_PROJECT in json:
            project_names.remove(UNKNOWN_PROJECT)
            project_names.append(UNKNOWN_PROJECT)

        for project in project_names:
            for ds in json[project]:
                if ds == "meta":
                    continue  # not a real data source
                if ds not in ds_repo_to_prj:
                    if ds not in ds_repo_to_prj:
                        ds_repo_to_prj[ds] = {}
                for repo in json[project][ds]:
                    repo, _ = self.extract_repo_labels(repo)
                    if repo in ds_repo_to_prj[ds]:
                        if project == ds_repo_to_prj[ds][repo]:
                            logger.debug("Duplicated repo: {} {} {}".format(ds, repo, project))
                        else:
                            if len(project.split(".")) > len(ds_repo_to_prj[ds][repo].split(".")):
                                logger.debug("Changed repo project because we found a leaf: {} leaf vs "
                                             "{} ({}, {})".format(project, ds_repo_to_prj[ds][repo], repo, ds))
                                ds_repo_to_prj[ds][repo] = project
                    else:
                        ds_repo_to_prj[ds][repo] = project
        return ds_repo_to_prj 
Example 11
Project: codex-backend   Author: codexgigassys   File: Functions.py    License: MIT License 5 votes vote down vote up
def clean_tree(s):
    if type(s) == dict:
        for child in s.keys():
            s[child] = clean_tree(s[child])
    elif type(s) == list:
        for index, value in enumerate(s):
            s[index] = clean_tree(value)
    elif type(s) == str or type(s) == unicode:
        if(not is_printable(s)):
            return display_with_hex(s)
        else:
            return s
    elif isinstance(s, (int, long, float)):
        if isinstance(s, (int, long)):
            return str(s) + " (" + str(hex(s)) + ")"
        else:
            return s
    elif isinstance(s, datetime.datetime):
        return str(s)
    elif s is None:
        return s
    else:
        if(is_iterable(s) and not is_printable(s)):
            return display_with_hex(s)
        else:
            return str(s)
    return s

# This function recives a dictionary like
# {"key1": { "something": 1},
#  "key2": { "something": 2}}
# and returns
# [ {"name": "key1", "something": 1 },
#   {"name": "key2", "something": 2 }]
# This is useful for converting the format
# VirusTotal API sends the AV scans into
# something easily searchable by mongo. 
Example 12
Project: codex-backend   Author: codexgigassys   File: Functions.py    License: MIT License 5 votes vote down vote up
def key_dict_clean(json):
    if json is None:
        return None
    array = []
    for key in json.keys():
        tmp_dict = json.get(key)
        tmp_dict["name"] = key
        array.append(tmp_dict)
    return array

# Replace dot with _
# in dictionaries keys
# in order to save them in mongo 
Example 13
Project: codex-backend   Author: codexgigassys   File: Functions.py    License: MIT License 5 votes vote down vote up
def key_list_clean(json):
    if json is None:
        return None
    array = []
    for key in json.keys():
        tmp_dict = {}
        tmp_dict["name"] = key
        tmp_dict["values"] = json.get(key)
        array.append(tmp_dict)
    return array 
Example 14
Project: codex-backend   Author: codexgigassys   File: Functions.py    License: MIT License 5 votes vote down vote up
def cursor_to_dict(f1, retrieve):
    results = []
    for f in f1:
        results.append(f)

    ret = []
    for a in results:
        dic = {}
        for key in retrieve.keys():
            steps = key.split('.')
            partial_res = a
            for step in steps:
                partial_res = partial_res.get(step)
                if partial_res is None:
                    break
                if isinstance(partial_res, list):
                    partial_res = None
                    break

            legend_to_show = key.split('.')[-1]
            if (legend_to_show == "file_id"):
                legend_to_show = "sha1"

            if (legend_to_show == "TimeDateStamp" and partial_res is not None):
                partial_res = time.strftime(
                    "%Y-%m-%d %H:%M:%S", time.gmtime(int(eval(partial_res), 16)))
            if (legend_to_show == "timeDateStamp" and partial_res is not None):
                partial_res = time.strftime(
                    "%Y-%m-%d %H:%M:%S", time.gmtime(partial_res))

            dic[legend_to_show] = partial_res

        ret.append(dic)
    return ret


# ****************TEST_CODE****************** 
Example 15
Project: histogrammar-python   Author: histogrammar   File: fraction.py    License: Apache License 2.0 5 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "numerator", "denominator"], ["name", "sub:name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "Fraction.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Fraction.name")

            if isinstance(json["sub:type"], basestring):
                factory = Factory.registered[json["sub:type"]]
            else:
                raise JsonFormatException(json, "Fraction.type")

            if isinstance(json.get("sub:name", None), basestring):
                subName = json["sub:name"]
            elif json.get("sub:name", None) is None:
                subName = None
            else:
                raise JsonFormatException(json["sub:name"], "Fraction.sub:name")

            numerator = factory.fromJsonFragment(json["numerator"], subName)
            denominator = factory.fromJsonFragment(json["denominator"], subName)

            out = Fraction.ed(entries, numerator, denominator)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Fraction") 
Example 16
Project: histogrammar-python   Author: histogrammar   File: deviate.py    License: Apache License 2.0 5 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "mean", "variance"], ["name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json["entries"], "Deviate.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Deviate.name")

            if json["mean"] in ("nan", "inf", "-inf") or isinstance(json["mean"], numbers.Real):
                mean = float(json["mean"])
            else:
                raise JsonFormatException(json["mean"], "Deviate.mean")

            if json["variance"] in ("nan", "inf", "-inf") or isinstance(json["variance"], numbers.Real):
                variance = float(json["variance"])
            else:
                raise JsonFormatException(json["variance"], "Deviate.variance")

            out = Deviate.ed(entries, mean, variance)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Deviate") 
Example 17
Project: histogrammar-python   Author: histogrammar   File: select.py    License: Apache License 2.0 5 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "data"], ["name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "Select.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Select.name")

            if isinstance(json["sub:type"], basestring):
                factory = Factory.registered[json["sub:type"]]
            else:
                raise JsonFormatException(json, "Select.type")

            cut = factory.fromJsonFragment(json["data"], None)

            out = Select.ed(entries, cut)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Select") 
Example 18
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, **pairs):
        """Create a Label that is capable of being filled and added.

        Parameters:
            pairs (list of str, :doc:`Container <histogrammar.defs.Container>` pairs): the collection of aggregators to fill.

        Other Parameters:
            entries (float): the number of entries, initially 0.0.
        """
        if not all(isinstance(k, basestring) and isinstance(v, Container) for k, v in pairs.items()):
            raise TypeError("pairs ({0}) must be a dict from strings to Containers".format(pairs))
        if any(not isinstance(x, basestring) for x in pairs.keys()):
            raise ValueError("all Label keys must be strings")
        if len(pairs) < 1:
            raise ValueError("at least one pair required")

        contentType = list(pairs.values())[0].name
        if any(x.name != contentType for x in pairs.values()):
            raise ContainerException("all Label values must have the same type")
        if contentType == "Bag":
            rangeType = list(pairs.values())[0].range
            if any(x.range != rangeType for x in pairs.values()):
                raise ContainerException("all Label values must have the same type")

        self.entries = 0.0
        self.pairs = pairs

        super(Label, self).__init__()
        self.specialize() 
Example 19
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def keySet(self):
        """Set of keys among the ``pairs``."""
        return set(self.pairs.keys()) 
Example 20
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def __add__(self, other):
        if isinstance(other, Label):
            if self.keySet != other.keySet:
                raise ContainerException("cannot add Labels because keys differ:\n    {0}\n    {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys))))

            out = Label(**dict((k, self(k) + other(k)) for k in self.keys))
            out.entries = self.entries + other.entries
            return out.specialize()

        else:
            raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) 
Example 21
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def __iadd__(self, other):
        if isinstance(other, Label):
            if self.keySet != other.keySet:
                raise ContainerException("cannot add Labels because keys differ:\n    {0}\n    {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys))))
            self.entries += other.entries
            for k in self.keys:
                v = self(k)
                v += other(k)
            return self
        else:
            raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) 
Example 22
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def keys(self):
        """Iterable over the keys of the ``pairs``."""
        return self.pairs.keys() 
Example 23
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def __add__(self, other):
        if isinstance(other, UntypedLabel):
            if self.keySet != other.keySet:
                raise ContainerException("cannot add UntypedLabels because keys differ:\n    {0}\n    {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys))))

            out = UntypedLabel(**dict((k, self(k) + other(k)) for k in self.keys))
            out.entries = self.entries + other.entries
            return out.specialize()

        else:
            raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) 
Example 24
Project: histogrammar-python   Author: histogrammar   File: collection.py    License: Apache License 2.0 5 votes vote down vote up
def __iadd__(self, other):
        if isinstance(other, UntypedLabel):
            if self.keySet != other.keySet:
                raise ContainerException("cannot add UntypedLabels because keys differ:\n    {0}\n    {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys))))
            self.entries += other.entries
            for k in self.keys:
                v = self(k)
                v += other(k)
            return self
        else:
            raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) 
Example 25
Project: NeuronBlocks   Author: microsoft   File: ModelConf.py    License: MIT License 5 votes vote down vote up
def load_data(cls, obj, json, key_prefix_desc=''):
        if cls.__name__ in json.keys():
            json = json[cls.__name__]
        for key in cls.__dict__.keys():
            if not hasattr(cls.__dict__[key], 'load_data'):
                continue
            item = cls.__dict__[key]
            obj = item.load_data(obj, json, cls.concat_key_desc(key_prefix_desc, item.__name__))
        return obj 
Example 26
Project: NeuronBlocks   Author: microsoft   File: ModelConf.py    License: MIT License 5 votes vote down vote up
def configurate_architecture(self):
        self.input_types = self.architecture[0]['conf']
        
        # extra feature
        feature_all = set([_.lower() for _ in self.input_types.keys()])
        formal_feature = set(['word', 'char'])
        extra_feature_num = feature_all - formal_feature
        self.extra_feature = len(extra_feature_num) != 0
        if self.extra_feature:
            if self.DBC2SBC:
                logging.warning("Detect the extra feature %s, set the DBC2sbc is False." % ''.join(list(extra_feature_num)))
            if self.unicode_fix:
                logging.warning("Detect the extra feature %s, set the unicode_fix is False." % ''.join(list(extra_feature_num)))
            if self.remove_stopwords:
                logging.warning("Detect the extra feature %s, set the remove_stopwords is False." % ''.join(list(extra_feature_num)))

        # output layer
        self.output_layer_id = []
        for single_layer in self.architecture:
            if 'output_layer_flag' in single_layer and single_layer['output_layer_flag']:
                self.output_layer_id.append(single_layer['layer_id'])

        # check CNN layer & change min sentence length
        cnn_rele_layers = ['Conv', 'ConvPooling']
        self.min_sentence_len = 0
        for layer_index, single_layer in enumerate(self.architecture):
            if layer_index == 0:
                continue
            if sum([_ == single_layer['layer'] for _ in cnn_rele_layers]):
                # get window_size conf: type maybe int or list
                for single_conf, single_conf_value in single_layer['conf'].items():
                    if 'window' in single_conf.lower():
                        self.min_sentence_len = max(self.min_sentence_len, np.max(np.array([single_conf_value])))
                        break 
Example 27
Project: grimoirelab-elk   Author: chaoss   File: enrich.py    License: GNU General Public License v3.0 4 votes vote down vote up
def __compare_projects_map(self, db, json):
        # Compare the projects coming from db and from a json file in eclipse
        ds_map_db = {}
        ds_map_json = {
            "git": "scm",
            "pipermail": "mls",
            "gerrit": "scr",
            "bugzilla": "its"
        }
        for ds in ds_map_json:
            ds_map_db[ds_map_json[ds]] = ds

        db_projects = []
        dss = db.keys()

        # Check that all db data is in the JSON file
        for ds in dss:
            for repository in db[ds]:
                # A repository could be in more than one project. But we get only one.
                project = db[ds][repository]
                if project not in db_projects:
                    db_projects.append(project)
                if project not in json:
                    logger.error("Project not found in JSON {}".format(project))
                    raise NotFoundError("Project not found in JSON {}".format(project))
                else:
                    if ds == 'mls':
                        repo_mls = repository.split("/")[-1]
                        repo_mls = repo_mls.replace(".mbox", "")
                        repository = 'https://dev.eclipse.org/mailman/listinfo/' + repo_mls
                    if ds_map_db[ds] not in json[project]:
                        logger.error("db repository not found in json {}".format(repository))
                    elif repository not in json[project][ds_map_db[ds]]:
                        logger.error("db repository not found in json {}".format(repository))

        for project in json.keys():
            if project not in db_projects:
                logger.debug("JSON project {} not found in db".format(project))

        # Check that all JSON data is in the database
        for project in json:
            for ds in json[project]:
                if ds not in ds_map_json:
                    # meta
                    continue
                for repo in json[project][ds]:
                    if ds == 'pipermail':
                        repo_mls = repo.split("/")[-1]
                        repo = "/mnt/mailman_archives/%s.mbox/%s.mbox" % (repo_mls, repo_mls)
                    if repo in db[ds_map_json[ds]]:
                        # print("Found ", repo, ds)
                        pass
                    else:
                        logger.debug("Not found repository in db {} {}".format(repo, ds))

        logger.debug("Number of db projects: {}".format(db_projects))
        logger.debug("Number of json projects: {} (>={})".format(json.keys(), db_projects)) 
Example 28
Project: histogrammar-python   Author: histogrammar   File: centrallybin.py    License: Apache License 2.0 4 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "bins:type", "bins", "nanflow:type", "nanflow"], ["name", "bins:name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "CentrallyBin.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "CentrallyBin.name")

            if isinstance(json["bins:type"], basestring):
                factory = Factory.registered[json["bins:type"]]
            else:
                raise JsonFormatException(json, "CentrallyBin.bins:type")
            if isinstance(json.get("bins:name", None), basestring):
                binsName = json["bins:name"]
            elif json.get("bins:name", None) is None:
                binsName = None
            else:
                raise JsonFormatException(json["bins:name"], "CentrallyBin.bins:name")
            if isinstance(json["bins"], list):
                bins = []
                for i, binpair in enumerate(json["bins"]):
                    if isinstance(binpair, dict) and hasKeys(binpair.keys(), ["center", "data"]):
                        if binpair["center"] in ("nan", "inf", "-inf") or isinstance(binpair["center"], numbers.Real):
                            center = float(binpair["center"])
                        else:
                            JsonFormatException(binpair["center"], "CentrallyBin.bins {0} center".format(i))
                        
                        bins.append((center, factory.fromJsonFragment(binpair["data"], binsName)))

                    else:
                        raise JsonFormatException(binpair, "CentrallyBin.bins {0}".format(i))

            if isinstance(json["nanflow:type"], basestring):
                nanflowFactory = Factory.registered[json["nanflow:type"]]
            else:
                raise JsonFormatException(json, "CentrallyBin.nanflow:type")
            nanflow = nanflowFactory.fromJsonFragment(json["nanflow"], None)

            out = CentrallyBin.ed(entries, bins, nanflow)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "CentrallyBin") 
Example 29
Project: histogrammar-python   Author: histogrammar   File: stack.py    License: Apache License 2.0 4 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "bins:type", "bins", "nanflow:type", "nanflow"], ["name", "bins:name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = float(json["entries"])
            else:
                raise JsonFormatException(json, "Stack.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Stack.name")

            if isinstance(json["bins:type"], basestring):
                factory = Factory.registered[json["bins:type"]]
            else:
                raise JsonFormatException(json, "Stack.bins:type")

            if isinstance(json.get("bins:name", None), basestring):
                dataName = json["bins:name"]
            elif json.get("bins:name", None) is None:
                dataName = None
            else:
                raise JsonFormatException(json["bins:name"], "Stack.bins:name")

            if isinstance(json["nanflow:type"], basestring):
                nanflowFactory = Factory.registered[json["nanflow:type"]]
            else:
                raise JsonFormatException(json, "Stack.nanflow:type")
            nanflow = nanflowFactory.fromJsonFragment(json["nanflow"], None)

            if isinstance(json["bins"], list):
                bins = []
                for i, elementPair in enumerate(json["bins"]):
                    if isinstance(elementPair, dict) and hasKeys(elementPair.keys(), ["atleast", "data"]):
                        if elementPair["atleast"] not in ("nan", "inf", "-inf") and not isinstance(elementPair["atleast"], numbers.Real):
                            raise JsonFormatException(json, "Stack.bins {0} atleast".format(i))

                        bins.append((float(elementPair["atleast"]), factory.fromJsonFragment(elementPair["data"], dataName)))

                    else:
                        raise JsonFormatException(json, "Stack.bins {0}".format(i))

                out = Stack.ed(entries, bins, nanflow)
                out.quantity.name = nameFromParent if name is None else name
                return out.specialize()

            else:
                raise JsonFormatException(json, "Stack.bins")

        else:
            raise JsonFormatException(json, "Stack") 
Example 30
Project: histogrammar-python   Author: histogrammar   File: bag.py    License: Apache License 2.0 4 votes vote down vote up
def fromJsonFragment(json, nameFromParent):
        if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "values", "range"], ["name"]):
            if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real):
                entries = json["entries"]
            else:
                raise JsonFormatException(json["entries"], "Bag.entries")

            if isinstance(json.get("name", None), basestring):
                name = json["name"]
            elif json.get("name", None) is None:
                name = None
            else:
                raise JsonFormatException(json["name"], "Bag.name")

            if json["values"] is None:
                values = None

            elif json["values"] is None or isinstance(json["values"], list):
                values = {}
                for i, nv in enumerate(json["values"]):
                    if isinstance(nv, dict) and hasKeys(nv.keys(), ["w", "v"]):
                        if nv["w"] in ("nan", "inf", "-inf") or isinstance(nv["w"], numbers.Real):
                            n = float(nv["w"])
                        else:
                            raise JsonFormatException(nv["w"], "Bag.values {0} n".format(i))

                        if nv["v"] in ("nan", "inf", "-inf") or isinstance(nv["v"], numbers.Real):
                            v = floatOrNan(nv["v"])
                        elif isinstance(nv["v"], basestring):
                            v = nv["v"]
                        elif isinstance(nv["v"], (list, tuple)):
                            for j, d in enumerate(nv["v"]):
                                if d not in ("nan", "inf", "-inf") and not isinstance(d, numbers.Real):
                                    raise JsonFormatException(d, "Bag.values {0} v {1}".format(i, j))
                            v = tuple(map(floatOrNan, nv["v"]))
                        else:
                            raise JsonFormatException(nv["v"], "Bag.values {0} v".format(i))

                        values[v] = n

                    else:
                        raise JsonFormatException(nv, "Bag.values {0}".format(i))

            elif json["values"] is None:
                values = None

            else:
                raise JsonFormatException(json["values"], "Bag.values")

            if isinstance(json["range"], basestring):
                range = json["range"]
            else:
                raise JsonFormatException(json["range"], "Bag.range")

            out = Bag.ed(entries, values, range)
            out.quantity.name = nameFromParent if name is None else name
            return out.specialize()

        else:
            raise JsonFormatException(json, "Bag")