Python json.keys() Examples
The following are 30
code examples of json.keys().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
json
, or try the search function
.

Example #1
Source File: server.py From bbj with MIT License | 6 votes |
def validate(json, args): """ Ensure the json object contains all the keys needed to satisfy its endpoint (and isnt empty) """ if not json: raise BBJParameterError( "JSON input is empty. This method requires the following " "arguments: {}".format(", ".join(args))) for arg in args: if arg not in json.keys(): raise BBJParameterError( "Required parameter {} is absent from the request. " "This method requires the following arguments: {}" .format(arg, ", ".join(args)))
Example #2
Source File: collection.py From histogrammar-python with Apache License 2.0 | 6 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "data"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "Label.entries") if isinstance(json["sub:type"], basestring): factory = Factory.registered[json["sub:type"]] else: raise JsonFormatException(json, "Label.sub:type") if isinstance(json["data"], dict): pairs = dict((k, factory.fromJsonFragment(v, None)) for k, v in json["data"].items()) else: raise JsonFormatException(json, "Label.data") return Label.ed(entries, **pairs) else: raise JsonFormatException(json, "Label")
Example #3
Source File: collection.py From histogrammar-python with Apache License 2.0 | 6 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "data"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "UntypedLabel.entries") if isinstance(json["data"], dict): pairs = {} for k, v in json["data"].items(): if isinstance(v, dict) and hasKeys(v.keys(), ["type", "data"]): factory = Factory.registered[v["type"]] pairs[k] = factory.fromJsonFragment(v["data"], None) else: raise JsonFormatException(k, "UntypedLabel.data {0}".format(v)) else: raise JsonFormatException(json, "UntypedLabel.data") return UntypedLabel.ed(entries, **pairs).specialize() else: raise JsonFormatException(json, "UntypedLabel")
Example #4
Source File: collection.py From histogrammar-python with Apache License 2.0 | 6 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "data"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "Index.entries") if isinstance(json["sub:type"], basestring): factory = Factory.registered[json["sub:type"]] else: raise JsonFormatException(json, "Index.sub:type") if isinstance(json["data"], list): values = [factory.fromJsonFragment(x, None) for x in json["data"]] else: raise JsonFormatException(json, "Index.data") return Index.ed(entries, *values).specialize() else: raise JsonFormatException(json, "Index")
Example #5
Source File: collection.py From histogrammar-python with Apache License 2.0 | 6 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "data"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "Branch.entries") if isinstance(json["data"], list): values = [] for i, x in enumerate(json["data"]): if isinstance(x, dict) and hasKeys(x.keys(), ["type", "data"]): if isinstance(x["type"], basestring): factory = Factory.registered[x["type"]] else: raise JsonFormatException(x, "Branch.data {0} type".format(i)) values.append(factory.fromJsonFragment(x["data"], None)) else: raise JsonFormatException(json, "Branch.data") return Branch.ed(entries, *values) else: raise JsonFormatException(json, "Branch")
Example #6
Source File: server.py From bbj with MIT License | 5 votes |
def db_validate(self, args, database, user, **kwargs): """ See also [the Input Validation page](validation.md). Requires the arguments `key` and `value`. Returns an object with information about the database sanity criteria for key. This can be used to validate user input in the client before trying to send it to the server. If the argument `error` is supplied with a non-nil value, the server will return a standard error object on failure instead of the special object described below. The returned object has two keys: { "bool": true/false, "description": null/"why this value is bad" } If bool == false, description is a string describing the problem. If bool == true, description is null and the provided value is safe to use. """ validate(args, ["key", "value"]) response = dict() try: db.validate([(args["key"], args["value"])]) response["bool"] = True response["description"] = None except BBJException as e: if args.get("error"): raise response["bool"] = False response["description"] = e.description return response
Example #7
Source File: enrich.py From grimoirelab-elk with GNU General Public License v3.0 | 5 votes |
def __convert_json_to_projects_map(self, json): """ Convert JSON format to the projects map format map[ds][repository] = project If a repository is in several projects assign to leaf Check that all JSON data is in the database :param json: data with the projects to repositories mapping :returns: the repositories to projects mapping per data source """ ds_repo_to_prj = {} # Sent the unknown project to the end of the list. # This change is needed to avoid assigning repositories to # the `Main` project when they exist in the `unknown` # section and in other sections too. project_names = list(json.keys()) if UNKNOWN_PROJECT in json: project_names.remove(UNKNOWN_PROJECT) project_names.append(UNKNOWN_PROJECT) for project in project_names: for ds in json[project]: if ds == "meta": continue # not a real data source if ds not in ds_repo_to_prj: if ds not in ds_repo_to_prj: ds_repo_to_prj[ds] = {} for repo in json[project][ds]: repo, _ = self.extract_repo_labels(repo) if repo in ds_repo_to_prj[ds]: if project == ds_repo_to_prj[ds][repo]: logger.debug("Duplicated repo: {} {} {}".format(ds, repo, project)) else: if len(project.split(".")) > len(ds_repo_to_prj[ds][repo].split(".")): logger.debug("Changed repo project because we found a leaf: {} leaf vs " "{} ({}, {})".format(project, ds_repo_to_prj[ds][repo], repo, ds)) ds_repo_to_prj[ds][repo] = project else: ds_repo_to_prj[ds][repo] = project return ds_repo_to_prj
Example #8
Source File: Functions.py From codex-backend with MIT License | 5 votes |
def clean_tree(s): if type(s) == dict: for child in s.keys(): s[child] = clean_tree(s[child]) elif type(s) == list: for index, value in enumerate(s): s[index] = clean_tree(value) elif type(s) == str or type(s) == unicode: if(not is_printable(s)): return display_with_hex(s) else: return s elif isinstance(s, (int, long, float)): if isinstance(s, (int, long)): return str(s) + " (" + str(hex(s)) + ")" else: return s elif isinstance(s, datetime.datetime): return str(s) elif s is None: return s else: if(is_iterable(s) and not is_printable(s)): return display_with_hex(s) else: return str(s) return s # This function recives a dictionary like # {"key1": { "something": 1}, # "key2": { "something": 2}} # and returns # [ {"name": "key1", "something": 1 }, # {"name": "key2", "something": 2 }] # This is useful for converting the format # VirusTotal API sends the AV scans into # something easily searchable by mongo.
Example #9
Source File: Functions.py From codex-backend with MIT License | 5 votes |
def key_dict_clean(json): if json is None: return None array = [] for key in json.keys(): tmp_dict = json.get(key) tmp_dict["name"] = key array.append(tmp_dict) return array # Replace dot with _ # in dictionaries keys # in order to save them in mongo
Example #10
Source File: Functions.py From codex-backend with MIT License | 5 votes |
def key_list_clean(json): if json is None: return None array = [] for key in json.keys(): tmp_dict = {} tmp_dict["name"] = key tmp_dict["values"] = json.get(key) array.append(tmp_dict) return array
Example #11
Source File: Functions.py From codex-backend with MIT License | 5 votes |
def cursor_to_dict(f1, retrieve): results = [] for f in f1: results.append(f) ret = [] for a in results: dic = {} for key in retrieve.keys(): steps = key.split('.') partial_res = a for step in steps: partial_res = partial_res.get(step) if partial_res is None: break if isinstance(partial_res, list): partial_res = None break legend_to_show = key.split('.')[-1] if (legend_to_show == "file_id"): legend_to_show = "sha1" if (legend_to_show == "TimeDateStamp" and partial_res is not None): partial_res = time.strftime( "%Y-%m-%d %H:%M:%S", time.gmtime(int(eval(partial_res), 16))) if (legend_to_show == "timeDateStamp" and partial_res is not None): partial_res = time.strftime( "%Y-%m-%d %H:%M:%S", time.gmtime(partial_res)) dic[legend_to_show] = partial_res ret.append(dic) return ret # ****************TEST_CODE******************
Example #12
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def __init__(self, **pairs): """Create a Label that is capable of being filled and added. Parameters: pairs (list of str, :doc:`Container <histogrammar.defs.Container>` pairs): the collection of aggregators to fill. Other Parameters: entries (float): the number of entries, initially 0.0. """ if not all(isinstance(k, basestring) and isinstance(v, Container) for k, v in pairs.items()): raise TypeError("pairs ({0}) must be a dict from strings to Containers".format(pairs)) if any(not isinstance(x, basestring) for x in pairs.keys()): raise ValueError("all Label keys must be strings") if len(pairs) < 1: raise ValueError("at least one pair required") contentType = list(pairs.values())[0].name if any(x.name != contentType for x in pairs.values()): raise ContainerException("all Label values must have the same type") if contentType == "Bag": rangeType = list(pairs.values())[0].range if any(x.range != rangeType for x in pairs.values()): raise ContainerException("all Label values must have the same type") self.entries = 0.0 self.pairs = pairs super(Label, self).__init__() self.specialize()
Example #13
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def keySet(self): """Set of keys among the ``pairs``.""" return set(self.pairs.keys())
Example #14
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def __add__(self, other): if isinstance(other, Label): if self.keySet != other.keySet: raise ContainerException("cannot add Labels because keys differ:\n {0}\n {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys)))) out = Label(**dict((k, self(k) + other(k)) for k in self.keys)) out.entries = self.entries + other.entries return out.specialize() else: raise ContainerException("cannot add {0} and {1}".format(self.name, other.name))
Example #15
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def __iadd__(self, other): if isinstance(other, Label): if self.keySet != other.keySet: raise ContainerException("cannot add Labels because keys differ:\n {0}\n {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys)))) self.entries += other.entries for k in self.keys: v = self(k) v += other(k) return self else: raise ContainerException("cannot add {0} and {1}".format(self.name, other.name))
Example #16
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def keys(self): """Iterable over the keys of the ``pairs``.""" return self.pairs.keys()
Example #17
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def __add__(self, other): if isinstance(other, UntypedLabel): if self.keySet != other.keySet: raise ContainerException("cannot add UntypedLabels because keys differ:\n {0}\n {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys)))) out = UntypedLabel(**dict((k, self(k) + other(k)) for k in self.keys)) out.entries = self.entries + other.entries return out.specialize() else: raise ContainerException("cannot add {0} and {1}".format(self.name, other.name))
Example #18
Source File: collection.py From histogrammar-python with Apache License 2.0 | 5 votes |
def __iadd__(self, other): if isinstance(other, UntypedLabel): if self.keySet != other.keySet: raise ContainerException("cannot add UntypedLabels because keys differ:\n {0}\n {1}".format(", ".join(sorted(self.keys)), ", ".join(sorted(other.keys)))) self.entries += other.entries for k in self.keys: v = self(k) v += other(k) return self else: raise ContainerException("cannot add {0} and {1}".format(self.name, other.name))
Example #19
Source File: ModelConf.py From NeuronBlocks with MIT License | 5 votes |
def load_data(cls, obj, json, key_prefix_desc=''): if cls.__name__ in json.keys(): json = json[cls.__name__] for key in cls.__dict__.keys(): if not hasattr(cls.__dict__[key], 'load_data'): continue item = cls.__dict__[key] obj = item.load_data(obj, json, cls.concat_key_desc(key_prefix_desc, item.__name__)) return obj
Example #20
Source File: ModelConf.py From NeuronBlocks with MIT License | 5 votes |
def configurate_architecture(self): self.input_types = self.architecture[0]['conf'] # extra feature feature_all = set([_.lower() for _ in self.input_types.keys()]) formal_feature = set(['word', 'char']) extra_feature_num = feature_all - formal_feature self.extra_feature = len(extra_feature_num) != 0 if self.extra_feature: if self.DBC2SBC: logging.warning("Detect the extra feature %s, set the DBC2sbc is False." % ''.join(list(extra_feature_num))) if self.unicode_fix: logging.warning("Detect the extra feature %s, set the unicode_fix is False." % ''.join(list(extra_feature_num))) if self.remove_stopwords: logging.warning("Detect the extra feature %s, set the remove_stopwords is False." % ''.join(list(extra_feature_num))) # output layer self.output_layer_id = [] for single_layer in self.architecture: if 'output_layer_flag' in single_layer and single_layer['output_layer_flag']: self.output_layer_id.append(single_layer['layer_id']) # check CNN layer & change min sentence length cnn_rele_layers = ['Conv', 'ConvPooling'] self.min_sentence_len = 0 for layer_index, single_layer in enumerate(self.architecture): if layer_index == 0: continue if sum([_ == single_layer['layer'] for _ in cnn_rele_layers]): # get window_size conf: type maybe int or list for single_conf, single_conf_value in single_layer['conf'].items(): if 'window' in single_conf.lower(): self.min_sentence_len = max(self.min_sentence_len, np.max(np.array([single_conf_value]))) break
Example #21
Source File: enrich.py From grimoirelab-elk with GNU General Public License v3.0 | 4 votes |
def __compare_projects_map(self, db, json): # Compare the projects coming from db and from a json file in eclipse ds_map_db = {} ds_map_json = { "git": "scm", "pipermail": "mls", "gerrit": "scr", "bugzilla": "its" } for ds in ds_map_json: ds_map_db[ds_map_json[ds]] = ds db_projects = [] dss = db.keys() # Check that all db data is in the JSON file for ds in dss: for repository in db[ds]: # A repository could be in more than one project. But we get only one. project = db[ds][repository] if project not in db_projects: db_projects.append(project) if project not in json: logger.error("Project not found in JSON {}".format(project)) raise NotFoundError("Project not found in JSON {}".format(project)) else: if ds == 'mls': repo_mls = repository.split("/")[-1] repo_mls = repo_mls.replace(".mbox", "") repository = 'https://dev.eclipse.org/mailman/listinfo/' + repo_mls if ds_map_db[ds] not in json[project]: logger.error("db repository not found in json {}".format(repository)) elif repository not in json[project][ds_map_db[ds]]: logger.error("db repository not found in json {}".format(repository)) for project in json.keys(): if project not in db_projects: logger.debug("JSON project {} not found in db".format(project)) # Check that all JSON data is in the database for project in json: for ds in json[project]: if ds not in ds_map_json: # meta continue for repo in json[project][ds]: if ds == 'pipermail': repo_mls = repo.split("/")[-1] repo = "/mnt/mailman_archives/%s.mbox/%s.mbox" % (repo_mls, repo_mls) if repo in db[ds_map_json[ds]]: # print("Found ", repo, ds) pass else: logger.debug("Not found repository in db {} {}".format(repo, ds)) logger.debug("Number of db projects: {}".format(db_projects)) logger.debug("Number of json projects: {} (>={})".format(json.keys(), db_projects))
Example #22
Source File: centrallybin.py From histogrammar-python with Apache License 2.0 | 4 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "bins:type", "bins", "nanflow:type", "nanflow"], ["name", "bins:name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "CentrallyBin.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "CentrallyBin.name") if isinstance(json["bins:type"], basestring): factory = Factory.registered[json["bins:type"]] else: raise JsonFormatException(json, "CentrallyBin.bins:type") if isinstance(json.get("bins:name", None), basestring): binsName = json["bins:name"] elif json.get("bins:name", None) is None: binsName = None else: raise JsonFormatException(json["bins:name"], "CentrallyBin.bins:name") if isinstance(json["bins"], list): bins = [] for i, binpair in enumerate(json["bins"]): if isinstance(binpair, dict) and hasKeys(binpair.keys(), ["center", "data"]): if binpair["center"] in ("nan", "inf", "-inf") or isinstance(binpair["center"], numbers.Real): center = float(binpair["center"]) else: JsonFormatException(binpair["center"], "CentrallyBin.bins {0} center".format(i)) bins.append((center, factory.fromJsonFragment(binpair["data"], binsName))) else: raise JsonFormatException(binpair, "CentrallyBin.bins {0}".format(i)) if isinstance(json["nanflow:type"], basestring): nanflowFactory = Factory.registered[json["nanflow:type"]] else: raise JsonFormatException(json, "CentrallyBin.nanflow:type") nanflow = nanflowFactory.fromJsonFragment(json["nanflow"], None) out = CentrallyBin.ed(entries, bins, nanflow) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "CentrallyBin")
Example #23
Source File: stack.py From histogrammar-python with Apache License 2.0 | 4 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "bins:type", "bins", "nanflow:type", "nanflow"], ["name", "bins:name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "Stack.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Stack.name") if isinstance(json["bins:type"], basestring): factory = Factory.registered[json["bins:type"]] else: raise JsonFormatException(json, "Stack.bins:type") if isinstance(json.get("bins:name", None), basestring): dataName = json["bins:name"] elif json.get("bins:name", None) is None: dataName = None else: raise JsonFormatException(json["bins:name"], "Stack.bins:name") if isinstance(json["nanflow:type"], basestring): nanflowFactory = Factory.registered[json["nanflow:type"]] else: raise JsonFormatException(json, "Stack.nanflow:type") nanflow = nanflowFactory.fromJsonFragment(json["nanflow"], None) if isinstance(json["bins"], list): bins = [] for i, elementPair in enumerate(json["bins"]): if isinstance(elementPair, dict) and hasKeys(elementPair.keys(), ["atleast", "data"]): if elementPair["atleast"] not in ("nan", "inf", "-inf") and not isinstance(elementPair["atleast"], numbers.Real): raise JsonFormatException(json, "Stack.bins {0} atleast".format(i)) bins.append((float(elementPair["atleast"]), factory.fromJsonFragment(elementPair["data"], dataName))) else: raise JsonFormatException(json, "Stack.bins {0}".format(i)) out = Stack.ed(entries, bins, nanflow) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Stack.bins") else: raise JsonFormatException(json, "Stack")
Example #24
Source File: minmax.py From histogrammar-python with Apache License 2.0 | 4 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "min"], ["name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json["entries"], "Minimize.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Minimize.name") if json["min"] in ("nan", "inf", "-inf") or isinstance(json["min"], numbers.Real): min = float(json["min"]) else: raise JsonFormatException(json["min"], "Minimize.min") out = Minimize.ed(entries, min) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Minimize")
Example #25
Source File: select.py From histogrammar-python with Apache License 2.0 | 4 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "data"], ["name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "Select.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Select.name") if isinstance(json["sub:type"], basestring): factory = Factory.registered[json["sub:type"]] else: raise JsonFormatException(json, "Select.type") cut = factory.fromJsonFragment(json["data"], None) out = Select.ed(entries, cut) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Select")
Example #26
Source File: bag.py From histogrammar-python with Apache License 2.0 | 4 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "values", "range"], ["name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = json["entries"] else: raise JsonFormatException(json["entries"], "Bag.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Bag.name") if json["values"] is None: values = None elif json["values"] is None or isinstance(json["values"], list): values = {} for i, nv in enumerate(json["values"]): if isinstance(nv, dict) and hasKeys(nv.keys(), ["w", "v"]): if nv["w"] in ("nan", "inf", "-inf") or isinstance(nv["w"], numbers.Real): n = float(nv["w"]) else: raise JsonFormatException(nv["w"], "Bag.values {0} n".format(i)) if nv["v"] in ("nan", "inf", "-inf") or isinstance(nv["v"], numbers.Real): v = floatOrNan(nv["v"]) elif isinstance(nv["v"], basestring): v = nv["v"] elif isinstance(nv["v"], (list, tuple)): for j, d in enumerate(nv["v"]): if d not in ("nan", "inf", "-inf") and not isinstance(d, numbers.Real): raise JsonFormatException(d, "Bag.values {0} v {1}".format(i, j)) v = tuple(map(floatOrNan, nv["v"])) else: raise JsonFormatException(nv["v"], "Bag.values {0} v".format(i)) values[v] = n else: raise JsonFormatException(nv, "Bag.values {0}".format(i)) elif json["values"] is None: values = None else: raise JsonFormatException(json["values"], "Bag.values") if isinstance(json["range"], basestring): range = json["range"] else: raise JsonFormatException(json["range"], "Bag.range") out = Bag.ed(entries, values, range) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Bag")
Example #27
Source File: ModelConf.py From NeuronBlocks with MIT License | 4 votes |
def configurate_training_params(self): # optimizer if self.phase == 'train': if self.optimizer is None: self.raise_configuration_error('training_params.optimizer') if 'name' not in self.optimizer.keys(): self.raise_configuration_error('training_params.optimizer.name') self.optimizer_name = self.optimizer['name'] if 'params' not in self.optimizer.keys(): self.raise_configuration_error('training_params.optimizer.params') self.optimizer_params = self.optimizer['params'] if hasattr(self.params, 'learning_rate') and self.params.learning_rate: self.optimizer_params['lr'] = self.params.learning_rate # batch size self.batch_size_each_gpu = self.batch_size # the batch_size in conf file is the batch_size on each GPU if hasattr(self.params, 'batch_size') and self.params.batch_size: self.batch_size_each_gpu = self.params.batch_size if self.batch_size_each_gpu is None: self.raise_configuration_error('training_params.batch_size') self.batch_size_total = self.batch_size_each_gpu if torch.cuda.device_count() > 1: self.batch_size_total = torch.cuda.device_count() * self.batch_size_each_gpu self.batch_num_to_show_results = self.batch_num_to_show_results // torch.cuda.device_count() if hasattr(self.params, 'max_epoch') and self.params.max_epoch: self.max_epoch = self.params.max_epoch if self.valid_times_per_epoch is not None: logging.info("configuration[training_params][valid_times_per_epoch] is deprecated, please use configuration[training_params][steps_per_validation] instead") # sequence length if self.fixed_lengths: self.max_lengths = None if ProblemTypes[self.problem_type] == ProblemTypes.sequence_tagging: self.fixed_lengths = None self.max_lengths = None # text preprocessing self.__text_preprocessing = self.text_preprocessing self.DBC2SBC = True if 'DBC2SBC' in self.__text_preprocessing else False self.unicode_fix = True if 'unicode_fix' in self.__text_preprocessing else False self.remove_stopwords = True if 'remove_stopwords' in self.__text_preprocessing else False # tokenzier if self.tokenizer is None: self.tokenizer = 'jieba' if self.language == 'chinese' else 'nltk' # GPU/CPU if self.phase != 'cache': if torch.cuda.is_available() and torch.cuda.device_count() > 0 and self.use_gpu: logging.info("Activating GPU mode, there are %d GPUs available" % torch.cuda.device_count()) else: self.use_gpu = False logging.info("Activating CPU mode")
Example #28
Source File: sum.py From histogrammar-python with Apache License 2.0 | 3 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sum"], ["name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json["entries"], "Sum.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Sum.name") if json["sum"] in ("nan", "inf", "-inf") or isinstance(json["sum"], numbers.Real): sum = float(json["sum"]) else: raise JsonFormatException(json["sum"], "Sum.sum") out = Sum.ed(entries, sum) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Sum")
Example #29
Source File: average.py From histogrammar-python with Apache License 2.0 | 3 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "mean"], ["name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json["entries"], "Average.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Average.name") if json["mean"] in ("nan", "inf", "-inf") or isinstance(json["mean"], numbers.Real): mean = float(json["mean"]) else: raise JsonFormatException(json["mean"], "Average.mean") out = Average.ed(entries, mean) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Average")
Example #30
Source File: fraction.py From histogrammar-python with Apache License 2.0 | 2 votes |
def fromJsonFragment(json, nameFromParent): if isinstance(json, dict) and hasKeys(json.keys(), ["entries", "sub:type", "numerator", "denominator"], ["name", "sub:name"]): if json["entries"] in ("nan", "inf", "-inf") or isinstance(json["entries"], numbers.Real): entries = float(json["entries"]) else: raise JsonFormatException(json, "Fraction.entries") if isinstance(json.get("name", None), basestring): name = json["name"] elif json.get("name", None) is None: name = None else: raise JsonFormatException(json["name"], "Fraction.name") if isinstance(json["sub:type"], basestring): factory = Factory.registered[json["sub:type"]] else: raise JsonFormatException(json, "Fraction.type") if isinstance(json.get("sub:name", None), basestring): subName = json["sub:name"] elif json.get("sub:name", None) is None: subName = None else: raise JsonFormatException(json["sub:name"], "Fraction.sub:name") numerator = factory.fromJsonFragment(json["numerator"], subName) denominator = factory.fromJsonFragment(json["denominator"], subName) out = Fraction.ed(entries, numerator, denominator) out.quantity.name = nameFromParent if name is None else name return out.specialize() else: raise JsonFormatException(json, "Fraction")