Python simplejson.load() Examples

The following are code examples for showing how to use simplejson.load(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: sunlightapi   Author: sunlightlabs   File: utils.py    GNU General Public License v3.0 6 votes vote down vote up
def _query_boundary_server(**params):
    BOUNDARY_SERVER_URL = 'http://ec2-184-73-61-66.compute-1.amazonaws.com/boundaries/cd/?'
    url = BOUNDARY_SERVER_URL + urllib.urlencode(params)
    data = urllib.urlopen(url)
    result = json.load(data)

    objs = []
    for zobj in result.get('objects', []):
        if '(at Large)' in zobj['name']:
            state = zobj['name'][0:2]
            number = '0'
        elif 'Congressional District' in zobj['name']:
            state, number = zobj['name'].split(' Congressional District ')
        else:
            state, number = zobj['name'].split(' ', 1)
            if number.startswith('0'):
                number = number[1:]

        objs.append({'district': {'state': state, 'number': number}})
    return objs 
Example 2
Project: autopyfactory   Author: PanDAWMS   File: configloader.py    Apache License 2.0 6 votes vote down vote up
def _loadQueueData(self, queue):
                '''
                
                '''
                queueDataUrl = 'http://pandaserver.cern.ch:25080/cache/schedconfig/%s.factory.json' % queue
                try:
                        handle = urlopen(queueDataUrl)
                        jsonData = json.load(handle, 'utf-8')
                        handle.close()
                        self.log.debug('JSON returned: %s' % jsonData)
                        factoryData = {}
                        # json always gives back unicode strings (eh?) - convert unicode to utf-8
                        for k, v in jsonData.iteritems():
                                if isinstance(k, unicode):
                                        k = k.encode('utf-8')
                                if isinstance(v, unicode):
                                        v = v.encode('utf-8')
                                factoryData[k] = v
                        self.log.debug('Converted to: %s' % factoryData)
                except ValueError, err:
                        self.log.error('%s for queue %s, downloading from %s' % (err, queue, queueDataUrl))
                        return None 
Example 3
Project: sqlitebiter   Author: thombashi   File: sqlitebiter.py    MIT License 6 votes vote down vote up
def configure(ctx):
    """
    Configure the following application settings:

    (1) Default encoding to load files.
    (2) HTTP/HTTPS proxy server URI (for url sub-command).

    Configurations are written to '~/.sqlitebiter'.
    You can remove these settings by deleting '~/.sqlitebiter'.
    """

    logger = make_logger("{:s} file".format(PROGRAM_NAME), ctx.obj[Context.LOG_LEVEL])

    logger.debug("{} configuration file existence: {}".format(PROGRAM_NAME, app_config_mgr.exists))

    sys.exit(app_config_mgr.configure()) 
Example 4
Project: devops-starter   Author: lesmono   File: ec2.py    MIT License 6 votes vote down vote up
def get_host_info(self):
        ''' Get variables about a specific host '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if self.args.host not in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if self.args.host not in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        (region, instance_id) = self.index[self.args.host]

        instance = self.get_instance(region, instance_id)
        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 
Example 5
Project: cloud-blueprints   Author: haproxytech   File: ec2.py    GNU General Public License v2.0 6 votes vote down vote up
def get_host_info(self):
        ''' Get variables about a specific host '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if self.args.host not in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if self.args.host not in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        (region, instance_id) = self.index[self.args.host]

        instance = self.get_instance(region, instance_id)
        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 
Example 6
Project: cloud-blueprints   Author: haproxytech   File: ec2.py    GNU General Public License v2.0 6 votes vote down vote up
def get_host_info(self):
        ''' Get variables about a specific host '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if self.args.host not in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if self.args.host not in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        (region, instance_id) = self.index[self.args.host]

        instance = self.get_instance(region, instance_id)
        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 
Example 7
Project: FUTU_Stop_Loss   Author: BigtoC   File: tool.py    MIT License 6 votes vote down vote up
def main():
    if len(sys.argv) == 1:
        infile = sys.stdin
        outfile = sys.stdout
    elif len(sys.argv) == 2:
        infile = open(sys.argv[1], 'r')
        outfile = sys.stdout
    elif len(sys.argv) == 3:
        infile = open(sys.argv[1], 'r')
        outfile = open(sys.argv[2], 'w')
    else:
        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
    with infile:
        try:
            obj = json.load(infile,
                            object_pairs_hook=json.OrderedDict,
                            use_decimal=True)
        except ValueError:
            raise SystemExit(sys.exc_info()[1])
    with outfile:
        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
        outfile.write('\n') 
Example 8
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_decode.py    MIT License 6 votes vote down vote up
def test_object_pairs_hook(self):
        s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
        p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
             ("qrt", 5), ("pad", 6), ("hoy", 7)]
        self.assertEqual(json.loads(s), eval(s))
        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
        self.assertEqual(json.load(StringIO(s),
                                   object_pairs_hook=lambda x: x), p)
        od = json.loads(s, object_pairs_hook=OrderedDict)
        self.assertEqual(od, OrderedDict(p))
        self.assertEqual(type(od), OrderedDict)
        # the object_pairs_hook takes priority over the object_hook
        self.assertEqual(json.loads(s,
                                    object_pairs_hook=OrderedDict,
                                    object_hook=lambda x: None),
                         OrderedDict(p)) 
Example 9
Project: tra-tracking-on-gae   Author: elliot79313   File: tool.py    MIT License 6 votes vote down vote up
def main():
    if len(sys.argv) == 1:
        infile = sys.stdin
        outfile = sys.stdout
    elif len(sys.argv) == 2:
        infile = open(sys.argv[1], 'r')
        outfile = sys.stdout
    elif len(sys.argv) == 3:
        infile = open(sys.argv[1], 'r')
        outfile = open(sys.argv[2], 'w')
    else:
        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
    with infile:
        try:
            obj = json.load(infile,
                            object_pairs_hook=json.OrderedDict,
                            use_decimal=True)
        except ValueError:
            raise SystemExit(sys.exc_info()[1])
    with outfile:
        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
        outfile.write('\n') 
Example 10
Project: tra-tracking-on-gae   Author: elliot79313   File: test_decode.py    MIT License 6 votes vote down vote up
def test_object_pairs_hook(self):
        s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
        p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
             ("qrt", 5), ("pad", 6), ("hoy", 7)]
        self.assertEqual(json.loads(s), eval(s))
        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
        self.assertEqual(json.load(StringIO(s),
                                   object_pairs_hook=lambda x: x), p)
        od = json.loads(s, object_pairs_hook=OrderedDict)
        self.assertEqual(od, OrderedDict(p))
        self.assertEqual(type(od), OrderedDict)
        # the object_pairs_hook takes priority over the object_hook
        self.assertEqual(json.loads(s,
                                    object_pairs_hook=OrderedDict,
                                    object_hook=lambda x: None),
                         OrderedDict(p)) 
Example 11
Project: afl-utils   Author: rc0r   File: afl_multicore.py    Apache License 2.0 6 votes vote down vote up
def read_config(config_file):
    config_file = os.path.abspath(os.path.expanduser(config_file))

    if not os.path.isfile(config_file):
        print_err("Config file not found!")
        sys.exit(1)

    with open(config_file, "r") as f:
        config = json.load(f)

        if "session" not in config:
            config["session"] = "SESSION"

        if "fuzzer" not in config:
            config["fuzzer"] = "afl-fuzz"

        return config 
Example 12
Project: glusterfs-linode   Author: pathbreak   File: gluster_perf.py    MIT License 6 votes vote down vote up
def create_cluster(name, datacenter):
    
    test_cluster = load_cluster(name)
    if test_cluster is not None:
        print('Cluster %s already exists. Use a different name or load this one instead of creating.' % (name))
        return None
        
    cluster = collections.OrderedDict()
    cluster['name'] = name
    cluster['dc'] = datacenter
    cluster['servers'] = []
    cluster['clients'] = []
    
    save_cluster(cluster)
        
    return cluster 
Example 13
Project: glusterfs-linode   Author: pathbreak   File: cluster_plan.py    MIT License 6 votes vote down vote up
def load_from_json(self, filepath):
        
        try:
            with open(filepath, 'r') as f:
                # object_pairs_hook and object_hook ensure that data is retained
                # in the same order as the file. Keeps it readable when written back.
                self.plan = json.load(f, 
                    object_pairs_hook = collections.OrderedDict, 
                    object_hook = collections.OrderedDict)
            
            # TODO If version is not current schema version, the file
            # should be upgraded
                
            validated = self.validate()
            
            if not validated:
                logger.error_msg("%s is not a valid JSON cluster plan file" % (filepath))
                return False
                
            return True
            
        except JSONDecodeError as e:
            logger.error_msg(str(e))
            return False 
Example 14
Project: ableton-live-webapi   Author: valsteen   File: tool.py    The Unlicense 6 votes vote down vote up
def main():
    if len(sys.argv) == 1:
        infile = sys.stdin
        outfile = sys.stdout
    elif len(sys.argv) == 2:
        infile = open(sys.argv[1], 'r')
        outfile = sys.stdout
    elif len(sys.argv) == 3:
        infile = open(sys.argv[1], 'r')
        outfile = open(sys.argv[2], 'w')
    else:
        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
    with infile:
        try:
            obj = json.load(infile,
                            object_pairs_hook=json.OrderedDict,
                            use_decimal=True)
        except ValueError:
            raise SystemExit(sys.exc_info()[1])
    with outfile:
        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
        outfile.write('\n') 
Example 15
Project: ableton-live-webapi   Author: valsteen   File: test_decode.py    The Unlicense 6 votes vote down vote up
def test_object_pairs_hook(self):
        s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
        p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
             ("qrt", 5), ("pad", 6), ("hoy", 7)]
        self.assertEqual(json.loads(s), eval(s))
        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
        self.assertEqual(json.load(StringIO(s),
                                   object_pairs_hook=lambda x: x), p)
        od = json.loads(s, object_pairs_hook=OrderedDict)
        self.assertEqual(od, OrderedDict(p))
        self.assertEqual(type(od), OrderedDict)
        # the object_pairs_hook takes priority over the object_hook
        self.assertEqual(json.loads(s,
                                    object_pairs_hook=OrderedDict,
                                    object_hook=lambda x: None),
                         OrderedDict(p)) 
Example 16
Project: kodi-addons   Author: onepas   File: tool.py    GNU General Public License v2.0 6 votes vote down vote up
def main():
    if len(sys.argv) == 1:
        infile = sys.stdin
        outfile = sys.stdout
    elif len(sys.argv) == 2:
        infile = open(sys.argv[1], 'r')
        outfile = sys.stdout
    elif len(sys.argv) == 3:
        infile = open(sys.argv[1], 'r')
        outfile = open(sys.argv[2], 'w')
    else:
        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
    with infile:
        try:
            obj = json.load(infile,
                            object_pairs_hook=json.OrderedDict,
                            use_decimal=True)
        except ValueError:
            raise SystemExit(sys.exc_info()[1])
    with outfile:
        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
        outfile.write('\n') 
Example 17
Project: kodi-addons   Author: onepas   File: tool.py    GNU General Public License v2.0 6 votes vote down vote up
def main():
    if len(sys.argv) == 1:
        infile = sys.stdin
        outfile = sys.stdout
    elif len(sys.argv) == 2:
        infile = open(sys.argv[1], 'r')
        outfile = sys.stdout
    elif len(sys.argv) == 3:
        infile = open(sys.argv[1], 'r')
        outfile = open(sys.argv[2], 'w')
    else:
        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
    with infile:
        try:
            obj = json.load(infile,
                            object_pairs_hook=json.OrderedDict,
                            use_decimal=True)
        except ValueError:
            raise SystemExit(sys.exc_info()[1])
    with outfile:
        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
        outfile.write('\n') 
Example 18
Project: kodi-addons   Author: onepas   File: tool.py    GNU General Public License v2.0 6 votes vote down vote up
def main():
    if len(sys.argv) == 1:
        infile = sys.stdin
        outfile = sys.stdout
    elif len(sys.argv) == 2:
        infile = open(sys.argv[1], 'r')
        outfile = sys.stdout
    elif len(sys.argv) == 3:
        infile = open(sys.argv[1], 'r')
        outfile = open(sys.argv[2], 'w')
    else:
        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
    with infile:
        try:
            obj = json.load(infile,
                            object_pairs_hook=json.OrderedDict,
                            use_decimal=True)
        except ValueError:
            raise SystemExit(sys.exc_info()[1])
    with outfile:
        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
        outfile.write('\n') 
Example 19
Project: ansible-rails-toolkit   Author: jorgemanrubia   File: ec2.py    MIT License 6 votes vote down vote up
def get_host_info(self):
        ''' Get variables about a specific host '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if self.args.host not in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if self.args.host not in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        (region, instance_id) = self.index[self.args.host]

        instance = self.get_instance(region, instance_id)
        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 
Example 20
Project: ansible-rails-toolkit   Author: jorgemanrubia   File: ec2.py    MIT License 6 votes vote down vote up
def get_host_info(self):
        ''' Get variables about a specific host '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if self.args.host not in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if self.args.host not in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        (region, instance_id) = self.index[self.args.host]

        instance = self.get_instance(region, instance_id)
        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 
Example 21
Project: openhatch   Author: campbe13   File: mimeparse_test.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def run_tests():
    json_object = json.load(open("testdata.json"))

    suite = unittest.TestSuite()
    add_tests(suite, json_object, "parse_media_range", test_parse_media_range)
    add_tests(suite, json_object, "quality", test_quality)
    add_tests(suite, json_object, "best_match", test_best_match)
    add_tests(suite, json_object, "parse_mime_type", test_parse_mime_type)

    test_runner = unittest.TextTestRunner(verbosity=1)
    test_runner.run(suite) 
Example 22
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def GetAllImageData(dataDir=None):
  if dataDir is None:
    dataDir = utils.GetDataDir()
  dataFile = os.path.join(dataDir, 'image_data.json')
  data = json.load(open(dataFile))
  return [utils.ParseImageData(image) for image in data] 
Example 23
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def GetAllRegionDescriptions(dataDir=None):
  if dataDir is None:
    dataDir = utils.GetDataDir()
  dataFile = os.path.join(dataDir, 'region_descriptions.json')
  imageData = GetAllImageData(dataDir)
  imageMap = {}
  for d in imageData:
    imageMap[d.id] = d
  images = json.load(open(dataFile))
  output = []
  for image in images:
    output.append(utils.ParseRegionDescriptions(image['regions'], imageMap[image['id']]))
  return output 
Example 24
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def GetAllQAs(dataDir=None):
  if dataDir is None:
    dataDir = utils.GetDataDir()
  dataFile = os.path.join(dataDir, 'question_answers.json')
  imageData = GetAllImageData(dataDir)
  imageMap = {}
  for d in imageData:
    imageMap[d.id] = d
  images = json.load(open(dataFile))
  output = []
  for image in images:
    output.append(utils.ParseQA(image['qas'], imageMap))
  return output 
Example 25
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def GetSceneGraph(image_id, images=None, imageDataDir='by-id/'):
  if images is None:
    images = utils.GetDataDir()
  imageDataDir = utils.GetDataDir() + '/' + imageDataDir

  if type(images) is str:
    images = ListToDict(GetAllImageData(images))

  fname = str(image_id) + '.json'
  image = images[image_id]
  data = json.load(open(imageDataDir + fname, 'r'))

  scene_graph = ParseGraphLocal(data, image)
  return scene_graph 
Example 26
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def SaveSceneGraphsById(dataDir=None, imageDataDir='by-id/'):
  if dataDir is None:
    dataDir = utils.GetDataDir()
  imageDataDir = dataDir + '/' + imageDataDir

  if not os.path.exists(imageDataDir): os.mkdir(imageDataDir)

  all_data = json.load(open(os.path.join(dataDir,'scene_graphs.json')))
  for sg_data in all_data:
    img_fname = str(sg_data['image_id']) + '.json'
    with open(os.path.join(imageDataDir, img_fname), 'w') as f:
      json.dump(sg_data, f)

  del all_data
  gc.collect()  # clear memory 
Example 27
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def GetSceneGraphsModified(startIndex=0, endIndex=-1,
                           dataDir='data/', imageDataDir='data/by-id/',
                           minRels=1, maxRels=100,
                           oword_fname='data/pk/obj_words.pk', rword_fname='data/pk/rel_words.pk',
                           ofilter_fname='data/pk/obj_counts.pk', rfilter_fname='data/pk/rel_counts.pk'):
  images = ListToDict(GetAllImageData(dataDir))
  scene_graphs = []

  img_fnames = os.listdir(imageDataDir)
  if (endIndex < 1): endIndex = len(img_fnames)

  import pickle
  obj_words = pickle.load(open(oword_fname,'r'))
  rel_words = pickle.load(open(rword_fname,'r'))
  obj_filter = pickle.load(open(ofilter_fname,'r'))
  rel_filter = pickle.load(open(rfilter_fname,'r'))

  for fname in img_fnames[startIndex : endIndex]:
    image_id = int(fname.split('.')[0])
    scene_graph = GetSceneGraph(image_id, images, imageDataDir)
    n_rels = len(scene_graph.relationships)
    if (minRels <= n_rels <= maxRels):
      scene_graphs += fix_words(scene_graph, obj_words, rel_words, obj_filter, rel_filter)

  return scene_graphs


# ----------------------------------------------------------------------------------------------------------------------
# For loading VRD dataset
# ---------------------------------------------------------------------------------------------------------------------- 
Example 28
Project: px2graph_lab   Author: roytseng-tw   File: local.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def GetSceneGraphsVRD(json_file='data/vrd/json/test.json'):
  scene_graphs = []
  with open(json_file,'r') as f:
    D = simplejson.load(f)

  scene_graphs = [ParseGraphVRD(d) for d in D]
  return scene_graphs 
Example 29
Project: sqlitebiter   Author: thombashi   File: sqlitebiter.py    MIT License 5 votes vote down vote up
def load_convert_config(logger, config_filepath, subcommand):
    import simplejson as json
    import io

    if not config_filepath:
        return {}

    if not os.path.isfile(config_filepath):
        logger.debug("{} not found".format(config_filepath))
        return {}

    with io.open(config_filepath, encoding="utf-8") as f:
        configs = json.load(f)

    return configs.get(subcommand) 
Example 30
Project: devops-starter   Author: lesmono   File: ec2.py    MIT License 5 votes vote down vote up
def load_index_from_cache(self):
        ''' Reads the index from the cache file sets self.index '''

        with open(self.cache_path_index, 'rb') as f:
            self.index = json.load(f) 
Example 31
Project: sbrick-controller   Author: mwinters-stuff   File: main.py    Apache License 2.0 5 votes vote down vote up
def read_config(self):
        fp = open(self.configFile)
        try:
            self.config = json.load(fp)
        finally:
            fp.close()

    # noinspection PyUnusedLocal 
Example 32
Project: nova   Author: ZhanHan   File: xenhost.py    Apache License 2.0 5 votes vote down vote up
def _get_config_dict():
    """Returns a dict containing the key/values in the config file.
    If the file doesn't exist, it is created, and an empty dict
    is returned.
    """
    try:
        conf_file = file(config_file_path)
        config_dct = json.load(conf_file)
        conf_file.close()
    except IOError:
        # File doesn't exist
        config_dct = {}
        # Create the file
        _write_config_dict(config_dct)
    return config_dct 
Example 33
Project: celerystalk   Author: sethsec   File: report.py    MIT License 5 votes vote down vote up
def aquatone_parse_paths():

    output_dir = lib.db.get_output_dir_for_workspace(lib.db.get_current_workspace()[0][0])[0][0]
    aquatone_session_file = os.path.join(output_dir, 'celerystalkReports/aquatone/aquatone_session.json')
    aquatone_report_dir = os.path.join(output_dir, 'celerystalkReports/aquatone/')
    workspace = lib.db.get_current_workspace()[0][0]

    good_sections = ["url", "screenshotPath"]

    with open(aquatone_session_file, 'r') as aquatone_file:
        aquatone_file_json = simplejson.load(aquatone_file)

        for key,value in aquatone_file_json['pages'].iteritems():
            #print(page)
            path = value['url']
            screenshot_path = value['screenshotPath']
            filename = os.path.join(aquatone_report_dir, screenshot_path)
            lib.db.update_path_with_filename(path, filename, workspace)



    #with open(aquatone_report_file) as file:
    #    soup = BeautifulSoup(file, 'html.parser')

    # for site in soup.find_all(class_='page card mb-3'):
    #     print(site.find(class_='card-title'))
    #     path = site.find(class_='card-title').split('card-title">')[1].split('<')[0]
    #
    #     print(site.find(class_='screenshot'))
    #     filename = site.find(class_='screenshot').split('src="')[1].split('"')[0]
    #     filename = os.path.join(aquatone_report_dir,filename)
    #     lib.db.update_path_with_filename(path,filename,workspace) 
Example 34
Project: pennai   Author: EpistasisLab   File: loadInitialDatasets.py    GNU General Public License v3.0 5 votes vote down vote up
def main():
	'''
	Attempt to load the inital datasets using the user directory and lab host defined in environmental variables

	Also add an additional file log handler to '../target/log/loadInitialDatasets.log'
	'''
	meta_features_all = []
	parser = argparse.ArgumentParser(description="Reads or creates 'DATASET_metadata.json' file given a dataset", add_help=False)
	#parser.add_argument('DIRECTORY', type=str, help='Direcory to get get datafiles from')    

	args = parser.parse_args()

	# set up the file logger
	logpath = os.path.join(os.environ['PROJECT_ROOT'], "target/logs")
	if not os.path.exists(logpath):
		os.makedirs(logpath)

	formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
	fhandler = logging.FileHandler(os.path.join(logpath, 'loadInitialDatasets.log'))
	fhandler.setFormatter(formatter)
	logger.addHandler(fhandler)

	print("logpath: " + logpath)
	print(os.path.join(logpath, 'loadInitialDatasets.log'))

	apiPath = 'http://' + os.environ['LAB_HOST'] + ':' + os.environ['LAB_PORT']
	directory = os.environ['STARTUP_DATASET_PATH']

	registerDatafiles(directory, apiPath) 
Example 35
Project: lib-esrijson   Author: geoadmin   File: codec.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def load(fp,
         cls=json.JSONDecoder,
         parse_constant=_enforce_strict_numbers,
         object_hook=esrijson.base.EsriJSON.to_instance,
         **kwargs):
    return json.load(fp,
                     cls=cls, object_hook=object_hook,
                     parse_constant=parse_constant,
                     **kwargs) 
Example 36
Project: ecl_ekf_analysis   Author: Auterion   File: test_golden_flight_logs.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def compare_analysis_to_golden(log_filename: str, log_file_path: str) -> None:
    """
    runs an analysis and compares the result file to the golden result (uses approximate value
    for floating points)
    :param log_filename:
    :param log_file_path:
    :return:
    """

    golden_result_filename = os.path.join(
        log_file_path, '{:s}_golden.json'.format(os.path.splitext(log_filename)[0]))

    with TemporaryDirectory() as tmp_dir:

        tmp_log_filename = os.path.join(tmp_dir, os.path.basename(log_filename))

        os.system('cp {:s} {:s}'.format(log_filename, tmp_log_filename))

        assert os.path.exists(tmp_log_filename), '{:s} does not exist.'.format(tmp_log_filename)

        process_logdata_ekf(tmp_log_filename, plot=False)

        analysis_result_filename = os.path.join(
            tmp_dir, '{:s}.json'.format(os.path.splitext(tmp_log_filename)[0]))

        assert os.path.exists(analysis_result_filename), '{:s} does not exist.'.format(
            analysis_result_filename)

        with open(analysis_result_filename, 'r') as file:
            analysis_results = json.load(file)

        with open(golden_result_filename, 'r') as file:
            golden_results = json.load(file)

        print('comparing analysis to golden results')

        for local_analysis_check, golden_analysis_check in zip(analysis_results, golden_results):
            print('{:s}'.format(local_analysis_check['type']))
            compare_check_analysis_result_ground_truth(
                local_analysis_check, golden_analysis_check,
                os.path.splitext(os.path.basename(tmp_log_filename))[0]) 
Example 37
Project: cloud-blueprints   Author: haproxytech   File: ec2.py    GNU General Public License v2.0 5 votes vote down vote up
def load_index_from_cache(self):
        ''' Reads the index from the cache file sets self.index '''

        with open(self.cache_path_index, 'rb') as f:
            self.index = json.load(f) 
Example 38
Project: cloud-blueprints   Author: haproxytech   File: ec2.py    GNU General Public License v2.0 5 votes vote down vote up
def load_index_from_cache(self):
        ''' Reads the index from the cache file sets self.index '''

        with open(self.cache_path_index, 'rb') as f:
            self.index = json.load(f) 
Example 39
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_unicode.py    MIT License 5 votes vote down vote up
def test_strip_bom(self):
        content = u"\u3053\u3093\u306b\u3061\u308f"
        json_doc = codecs.BOM_UTF8 + b(json.dumps(content))
        self.assertEqual(json.load(BytesIO(json_doc)), content)
        for doc in json_doc, json_doc.decode('utf8'):
            self.assertEqual(json.loads(doc), content) 
Example 40
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_decimal.py    MIT License 5 votes vote down vote up
def loads(self, s, **kw):
        sio = StringIO(s)
        res = json.loads(s, **kw)
        self.assertEqual(res, json.load(sio, **kw))
        return res 
Example 41
Project: QAUniBonn   Author: jtrillos   File: app.py    Apache License 2.0 5 votes vote down vote up
def ranking(question):
  with open('data/templates.json', 'r') as templates_json:
    try:
      templatesDict = json.load(templates_json)
      templateVectorMatrix = np.array([])
      n=0
      for template in templatesDict :
        n=n+1
        vector = np.array(template['vec_representation'])
        if templateVectorMatrix.size > 0:
          if templateVectorMatrix.size==1:
            templateVectorMatrix = np.stack((templateVectorMatrix,vector))
          else:
            templateVectorMatrix = np.vstack((templateVectorMatrix,vector))
        else:
          templateVectorMatrix = np.hstack((templateVectorMatrix,vector))

      sims=cosine_similarity(getQuestionVector(question).reshape(1,-1),templateVectorMatrix)
      sims_index = np.argsort(sims)[0][::-1][:n]

      for i in range(n):
        templatesDict[i]['ranking'] = sims[0][i]

      # sort temaplates by ranking
      templatesDict = sorted(templatesDict, key=sort_by_ranking, reverse=True)
      #print("sims_index computed")
      #print(str(sims_index))

      return json.dumps(templatesDict)
    except ValueError:
      print("error")
      return {'err' : 'No templates found'} 
Example 42
Project: QAUniBonn   Author: jtrillos   File: app.py    Apache License 2.0 5 votes vote down vote up
def addtemplate():
  question = request.json['question']
  sparl_query= request.json['sparql']
  vec = getQuestionVector(question).tolist()

  with open('data/templates.json','r') as templates_json:
    try:
      templatesDict = json.load(templates_json)
    except ValueError:
      print("no json could be read")
      templatesDict=[]

  ranking = -1;

  for template in templatesDict:
    print(type(template))
    try:
      ranking = template['ranking'] if(template['ranking'] > ranking) else ranking
    except TypeError:
      ranking = ranking

  ranking = ranking + 1

  new_entry = {
    'ranking' : ranking,
    'question' : question,
    'vec_representation' : vec,
    'sparql_query' : sparl_query
  }
  templatesDict.append(new_entry)

  with open('data/templates.json', 'w+') as templates_json:
    json.dump(templatesDict,templates_json)

  return "Template saved" + str(new_entry) 
Example 43
Project: plugin.program.blstfusion   Author: harrygg   File: bsc.py    GNU General Public License v3.0 5 votes vote down vote up
def __restore_data(self):
    with open(os.path.join(self.__path, '', 'data.dat'), 'r') as f:
      self.__js = json.load(f) 
Example 44
Project: xen_ansible_inventory   Author: mikesimos   File: xen_inventory.py    GNU General Public License v2.0 5 votes vote down vote up
def cached_inventory(self, cache_path=None, cache_ttl=3600, refresh=False):
        """
        Wrapper method implementing caching functionality over list_inventory.
        :param str cache_path: A path for caching inventory list data. Quite a necessity for large environments.
        :param int cache_ttl: Integer Inventory list data cache Time To Live. Expiration period.
        :param boolean refresh: Setting this True, triggers a cache refresh. Fresh data is fetched.
        :return:
        Returns an Ansible pluggable dynamic inventory, as a Python json serializable dictionary.
        """
        if refresh:
            return self.list_and_save(cache_path)
        else:
            if os.path.isfile(cache_path) and time() - os.stat(cache_path).st_mtime < cache_ttl:
                try:
                    with open(cache_path) as f:
                        data = load(f)
                        return data
                except (ValueError, IOError):
                    return self.list_and_save(cache_path)
            else:
                if not os.path.exists(os.path.dirname(cache_path)):
                    try:
                        if cache_path:
                            os.makedirs(os.path.dirname(cache_path))
                        else:
                            raise OSError("cache_path not defined: {}".format(cache_path))
                    # handle race condition
                    except OSError as exc:
                        if exc.errno == errno.EACCES:
                            print("{}".format(str(exc)))
                            exit(1)
                        elif exc.errno != errno.EEXIST:
                            raise
                return self.list_and_save(cache_path) 
Example 45
Project: canvas-api-python   Author: dkloz   File: file_utilities.py    MIT License 5 votes vote down vote up
def load_pickle(filename):
    print '--> Loading ', filename, ' with pickle was ',
    sys.stdout.flush()
    t = time.time()
    with open(filename, 'rb') as gfp:
        r = pickle.load(gfp)
    print time.time() - t
    return r 
Example 46
Project: canvas-api-python   Author: dkloz   File: file_utilities.py    MIT License 5 votes vote down vote up
def load_array(filename):
    print '--> Loading ', filename, ' with pickle was ',
    sys.stdout.flush()
    t = time.time()
    r = np.load(filename)
    print time.time() - t
    return r 
Example 47
Project: canvas-api-python   Author: dkloz   File: file_utilities.py    MIT License 5 votes vote down vote up
def load_json(filename):
    print '--> Loading ', filename, ' with json was ',
    sys.stdout.flush()
    t = time.time()
    with open(filename, 'r') as fp:
        data = json.load(fp)

    print time.time() - t
    return data 
Example 48
Project: diamond-next   Author: diamond-next   File: puppetdb.py    MIT License 5 votes vote down vote up
def fetch_metrics(self, url):
        try:
            url = "http://%s:%s/%s" % (
                self.config['host'], int(self.config['port']), url)
            response = urllib2.urlopen(url)
        except Exception as e:
            self.log.error('Couldn\'t connect to puppetdb: %s -> %s', url, e)
            return {}
        return json.load(response) 
Example 49
Project: diamond-next   Author: diamond-next   File: dseopscenter.py    MIT License 5 votes vote down vote up
def _get_schema(self):
        time_now = int(datetime.datetime.utcnow().strftime('%s'))
        if ((self.column_families is None or
             (time_now - self.last_schema_sync_time < 3600))):
            return False
        url = 'http://%s:%i/%s/keyspaces' % (self.config['host'],
                                             int(self.config['port']),
                                             self.config['cluster_id'])
        try:
            response = urllib2.urlopen(url)
        except Exception as err:
            self.log.error('%s: %s', url, err)
            return False

        try:
            result = json.load(response)
            column_families = []
            for ks in result:
                i = []
                for cf in result[ks]['column_families']:
                    i.append("%s.%s" % (ks, cf))
                column_families.append(i)
            self.column_families = ','.join(sum(column_families, []))
            self.log.debug('DseOpsCenterCollector columnfamilies = %s',
                           self.column_families)
            self.last_schema_sync_time = time_now
            return True

        except (TypeError, ValueError):
            self.log.error(
                "Unable to parse response from opscenter as a json object")
            return False 
Example 50
Project: diamond-next   Author: diamond-next   File: dseopscenter.py    MIT License 5 votes vote down vote up
def _get(self, start, end, step=60):
        self._get_schema()
        url = ('http://%s:%i/%s/new-metrics?node_group=%s&columnfamilies=%s'
               '&metrics=%s&start=%i&end=%i&step=%i%s') % (
            self.config['host'],
            int(self.config['port']),
            self.config['cluster_id'],
            self.config['node_group'],
            self.column_families,
            self.config['metrics'],
            start, end, step,
            self.config['default_tail_opts'])

        try:
            response = urllib2.urlopen(url)
        except Exception as err:
            self.log.error('%s: %s', url, err)
            return False

        self.log.debug('DseOpsCenterCollector metrics url = %s', url)

        try:
            return json.load(response)
        except (TypeError, ValueError):
            self.log.error(
                "Unable to parse response from opscenter as a json object")
            return False