Python json.loads() Examples

The following are code examples for showing how to use json.loads(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: incubator-spot   Author: apache   File: proxy_oa.py    Apache License 2.0 7 votes vote down vote up
def _get_suspicious_details(self):
        uri_list = []
        iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if os.path.isfile(iana_conf_file):
            iana_config  = json.loads(open(iana_conf_file).read())
            proxy_iana = IanaTransform(iana_config["IANA"])

        for conn in self._proxy_scores:
            clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
            fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
            date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
            if len(date) == 3:
                year=date[0]
                month=date[1].zfill(2)
                day=date[2].zfill(2)
                hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
                self._get_proxy_details(fulluri,clientip,year,month,day,hh,proxy_iana) 
Example 2
Project: incubator-spot   Author: apache   File: dns_oa.py    Apache License 2.0 6 votes vote down vote up
def _get_suspicious_details(self):

        iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if os.path.isfile(iana_conf_file):
            iana_config  = json.loads(open(iana_conf_file).read())
            dns_iana = IanaTransform(iana_config["IANA"])
        
        for conn in self._dns_scores:

            timestamp = conn[self._conf["dns_score_fields"]["unix_tstamp"]]
            full_date = datetime.datetime.utcfromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')

            date = full_date.split(" ")[0].split("-")
            # get date parameters.
            yr = date[0]
            mn = date[1]
            dy = date[2]
            time = full_date.split(" ")[1].split(":")
            hh = int(time[0])

            dns_qry_name = conn[self._conf["dns_score_fields"]["dns_qry_name"]]
            self._get_dns_details(dns_qry_name,yr,mn,dy,hh,dns_iana) 
Example 3
Project: incubator-spot   Author: apache   File: gti.py    Apache License 2.0 6 votes vote down vote up
def _call_gti(self, command, num_values):
        try:
            response_json = check_output(command, shell=True)
            result_dict = json.loads(response_json[0:len(response_json) - 1])
            responses = result_dict['a']
            return responses

        except CalledProcessError as e:
            self._logger.error("Error calling McAfee GTI client in gti module: " + e.output)
            error_resp = [{self.REP_KEY: self.DEFAULT_REP}] * num_values
            return error_resp

        except ValueError as e:
            self._logger.error("Error reading JSON response in gti module: " + e.message)
            error_resp = [{self.REP_KEY: self.DEFAULT_REP}] * num_values
            return error_resp 
Example 4
Project: Coulomb   Author: DynamoDS   File: data_files_to_sessions.py    MIT License 6 votes vote down vote up
def flush(session_maps):
    lns_count = 0
    for sessions_lst in session_maps.values():
        lns_count += len(sessions_lst)

    log ("Flushing lines/sessions: " + str(lns_count) + " / " + str(len(session_maps.keys())))

    # Verify that the folders exist
    for session_id in sorted(session_maps.keys()):
        log ("Flushing session: " + session_id)
        sessionPath = ensure_have_session(session_id)
        log ("Session path: " + sessionPath)
            
        o = gzip.open(sessionPath, 'ab')
        for ln in session_maps[session_id]:
            assert (json.loads(ln)["SessionID"] == session_id)
            o.write(ln)
        o.flush()
        log ("Flushing complete for: " + session_id)

    log ("Flushing complete. Total sessions:\t" + str(len(sessionIDSet)) + "\tTotal new sessions:\t" + str(len(newSessionIDSet))) 
Example 5
Project: AboveTustin   Author: kevinabrandon   File: flightdata.py    MIT License 6 votes vote down vote up
def refresh(self):
        try:
            #open the data url
            self.req = urlopen(self.data_url)

            #read data from the url
            self.raw_data = self.req.read()

            #load in the json
            self.json_data = json.loads(self.raw_data.decode())

            #get time from json
            self.time = datetime.fromtimestamp(self.parser.time(self.json_data))

            #load all the aircarft
            self.aircraft = self.parser.aircraft_data(self.json_data, self.time)

        except Exception:
            print("exception in FlightData.refresh():")
            traceback.print_exc() 
Example 6
Project: mycode   Author: gmraabe   File: inventory.py    GNU General Public License v3.0 6 votes vote down vote up
def search_entry(search_str):    # function for searching inventory
    with open(filename, 'r') as myfile:            # open file with implied close (with)
        myDict = json.loads(myfile.read())           # read file into a dictionary
    print('The Location for ' + search_str + ' is: ' + myDict.get(search_str, "not found"))
    if type(myDict.get(search_str))() is None:     # check to see if no entry is found
        input('Press enter to continue')             # if no entry then only option is to return to Main Menu
    else:                                          # if an entry is found
        while True:                                  # loop until a valid selection is made
            print('Enter 1 to delete ' + search_str + ' or 2 to return to Main Menu')
            selection = input()                        # get input from user
            if selection == '1':                         # check if user wishes to delete the found search item
                myDict.pop(search_str)                     # remove search string from dictionary
                with open(filename, 'w') as myfile:        # open file with implied close (with)
                    myfile.write(json.dumps(myDict))         # write dictionary to file
                print(search_str + ' is deleted')          # print confirmation message
                input('Press enter to continue')           # wait for user before continuing
                break
            elif selection == '2':                     # if user wishes to exit to main menu
                break
            else:                                      # if anything else then loop
                print('Invalid selection') 
Example 7
Project: mycode   Author: gmraabe   File: iss_tracking.py    GNU General Public License v3.0 6 votes vote down vote up
def isspass(userlat, userlon):
    passiss = 'http://api.open-notify.org/iss-pass.json'
    passiss = passiss + '?lat=' + str(userlat) + '&lon=' + str(userlon)
    response = urllib.request.urlopen(passiss)
    result = json.loads(response.read())
    # print(result) ## uncomment to see the downloaded result

    over = result['response'][0]['risetime']

    style = ('Arial', 6, 'bold')
    mylocation.write(time.ctime(over), font=style)

    print('The next five passes over ' + str(yellowlat) + ' ' + str(yellowlon))
    print('Pass 1 = ' + time.ctime(result['response'][0]['risetime']))
    print('Pass 2 = ' + time.ctime(result['response'][1]['risetime']))
    print('Pass 3 = ' + time.ctime(result['response'][2]['risetime']))
    print('Pass 4 = ' + time.ctime(result['response'][3]['risetime']))
    print('Pass 5 = ' + time.ctime(result['response'][4]['risetime']))



## Get user location 
Example 8
Project: god-eye   Author: PiScale   File: test_ping_plugin.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def test_get_result(self):

        @asyncio.coroutine
        def go():
            queue = asyncio.Queue(loop=self.loop)
            ping = Ping(self.loop, queue)
            with aiohttp.ClientSession(loop=self.loop) as client:
                yield from ping(client, 'http://httpbin.org/get')
            cor_result = yield from queue.get()
            result = yield from cor_result
            result = json.loads(result)
            self.assertIsInstance(result, dict)
            self.assertEqual(result['url'], 'http://httpbin.org/get')

        self.loop.run_until_complete(go()) 
Example 9
Project: parsechain   Author: Suor   File: chains.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def ld(node):
        text = C.css('script[type="application/ld+json"]').inner_text(node)
        try:
            return json.loads(text)
        except ValueError as e:
            try:
                # Try parsing non-strict
                import demjson
                return demjson.decode(text)
            except:
                raise e  # reraise first one

    # Select 
Example 10
Project: parsechain   Author: Suor   File: response.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def json(self):
        return json.loads(self.body) 
Example 11
Project: BERT-Classification-Tutorial   Author: Socialbird-AILab   File: modeling_test.py    Apache License 2.0 5 votes vote down vote up
def test_config_to_json_string(self):
        config = modeling.BertConfig(vocab_size=99, hidden_size=37)
        obj = json.loads(config.to_json_string())
        self.assertEqual(obj["vocab_size"], 99)
        self.assertEqual(obj["hidden_size"], 37) 
Example 12
Project: BERT-Classification-Tutorial   Author: Socialbird-AILab   File: modeling.py    Apache License 2.0 5 votes vote down vote up
def from_json_file(cls, json_file):
        """Constructs a `BertConfig` from a json file of parameters."""
        with tf.gfile.GFile(json_file, "r") as reader:
            text = reader.read()
        return cls.from_dict(json.loads(text)) 
Example 13
Project: apistar-msgpack   Author: juancarlospaco   File: apistar_msgpack.py    GNU General Public License v3.0 5 votes vote down vote up
def parse(self, body: http.Body) -> typing.Any:
        if not body:
            raise exceptions.BadRequest(detail=f'Empty MessagePack: {body}.')
        try:
            if has_msgpack:
                data_from_msgpack = loads(msgpack.unpackb(body))
            else:
                data_from_msgpack = loads(umsgpack.unpackb(body))
        except UnicodeEncodeError as error:
            raise exceptions.BadRequest(
                detail=f'Invalid Unicode UTF-8 on MessagePack error: {error}.')
        except ValueError as error:
            raise exceptions.BadRequest(
                detail=f'Invalid Keys or Values on MessagePack error: {error}')
        except JSONDecodeError as error:
            raise exceptions.BadRequest(detail=f'Invalid MessagePack: {error}')
        except Exception as error:
            raise exceptions.BadRequest(
                detail=f'{self} Unknown Exception: {error}, parsing {body}.')
        else:
            return data_from_msgpack 
Example 14
Project: navitia_client   Author: leonardbinet   File: client.py    MIT License 5 votes vote down vote up
def _extract_nbr_results(self, response):
        """
        Out of a request response, finds total number of results
        """
        parsed = json.loads(response.text)
        try:
            nbr_results = parsed["pagination"]["total_result"]
            return nbr_results
        except KeyError:
            # No pagination in page
            print("WARNING: not able to extract pagination out of first request.")
            return False 
Example 15
Project: navitia_client   Author: leonardbinet   File: parser.py    MIT License 5 votes vote down vote up
def parse_requests(self):
        # First operation, to parse requests text into python dictionnaries
        for page, value in self.results.items():
            # Only add if answer was good
            if value.status_code == 200:
                try:
                    self.parsed[page] = json.loads(value.text)
                except ValueError:
                    print("JSON decoding error.")
                    self.parsing_errors[page] = "JSON decoding error" 
Example 16
Project: navitia_client   Author: leonardbinet   File: parser.py    MIT License 5 votes vote down vote up
def extract_nbr_expected_items(self):
        if self.results[0].status_code != 200:
            return None
        # Parse first request answer.
        parsed = json.loads(self.results[0].text)
        # Extract pagination part.
        pagination = parsed["pagination"]
        # Extract total_result
        self.nbr_expected_items = pagination["total_result"] 
Example 17
Project: factotum   Author: Denubis   File: settings.py    GNU General Public License v3.0 5 votes vote down vote up
def configAuthenticate(username, password):
	FACTORIOPATH = getFactorioPath()

	url = "https://auth.factorio.com/api-login"
	params = {'username': username, 'password': password, 'apiVersion': 2}


	if not os.path.isfile("%s/bin/x64/factorio" % (FACTORIOPATH) ):
		print("Could not find factorio at %s" % (FACTORIOPATH))
		sys.exit(1)


	print("Fetching token for %s" %  (username))
	myResponse = requests.post(url,data=params, verify=True)
	if(myResponse.ok):

	    jData = json.loads(myResponse.text)
	    print("Writing %s to settings.json" % (jData[0]))
	    
	else:
	  # If response code is not ok (200), print the resulting http error code with description
	    myResponse.raise_for_status()
	    sys.exit(1)
	

	try:
		with codecs.open(getSettingsFile(), 'r', encoding='utf-8') as settings_file:
			settingsJson = json.load(settings_file)
			settingsJson['token'] = jData[0]
			settingsJson['username'] = username
				


		with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'w', encoding='utf-8') as settings_file:
			json.dump(settingsJson, settings_file, indent=4)
	except Exception as e:
		print(e)
		print("Help! Can't deal with the settings file!") 
Example 18
Project: fs_image   Author: facebookincubator   File: update_package_db.py    MIT License 5 votes vote down vote up
def _parse_updates(
    description: str,
    items: List[Tuple[Package, Tag, str]],
) -> ExplicitUpdates:
    updates = {}
    for package, tag, opts_str in items:
        opts = json.loads(opts_str)
        stored_opts = updates.setdefault(package, {}).setdefault(tag, opts)
        if stored_opts is not opts:  # `!=` would permit duplicates
            # This detects conflicts only within a single update type,
            # `_get_updated_db` detects conflicts between types.
            raise RuntimeError(
                f'Conflicting "{description}" updates for {package} / {tag}: '
                f'{opts} ({id(opts)}) is not {stored_opts} ({id(stored_opts)}.'
            )
    return updates 
Example 19
Project: fs_image   Author: facebookincubator   File: pluggable.py    MIT License 5 votes vote down vote up
def from_json(cls, json_cfg: str) -> 'Pluggable':
        'Uniform parsing for Storage configs e.g. on the command-line.'
        cfg = json.loads(json_cfg)
        cfg['kind']  # KeyError if not set, or if not a dict
        return cls.make(**cfg) 
Example 20
Project: fs_image   Author: facebookincubator   File: repo_server.py    MIT License 5 votes vote down vote up
def add_snapshot_db_objs(db):
    location_to_obj = {}
    for repo, build_timestamp, metadata_xml in db.execute('''
    SELECT "repo", "build_timestamp", "metadata_xml" FROM "repomd"
    ''').fetchall():
        set_new_key(
            location_to_obj,
            os.path.join(repo, 'repodata/repomd.xml'),
            {
                'size': len(metadata_xml),
                'build_timestamp': build_timestamp,
                'content_bytes': metadata_xml.encode(),
            }
        )
    for table in ['repodata', 'rpm']:
        for (
            repo, path, build_timestamp, checksum, error, error_json, size,
            storage_id,
        ) in db.execute(f'''
        SELECT
            "repo", "path", "build_timestamp", "checksum", "error",
            "error_json", "size", "storage_id"
        FROM "{table}"
        ''').fetchall():
            obj = {
                'checksum': checksum,
                'size': size,
                'build_timestamp': build_timestamp,
            }
            # `storage_id` is populated in the DB table for `mutable_rpm`
            # errors, but we don't want to serve up those files.
            if storage_id and not error and not error_json:
                obj['storage_id'] = storage_id
            elif error and error_json:
                obj['error'] = {'error': error, **json.loads(error_json)}
            else:  # pragma: no cover
                raise AssertionError(f'{storage_id} {error} {error_json}')
            set_new_key(location_to_obj, os.path.join(repo, path), obj)
    return location_to_obj 
Example 21
Project: fs_image   Author: facebookincubator   File: test_layer_mount_config.py    MIT License 5 votes vote down vote up
def test_config_merging(self):
        out = StringIO()
        main(StringIO('{"runtime_source": "meow"}'), out, '//layer:path')
        self.assertEqual({
            'runtime_source': 'meow',
            'is_directory': True,
            'build_source': {'source': '//layer:path', 'type': 'layer'},
        }, json.loads(out.getvalue())) 
Example 22
Project: fs_image   Author: facebookincubator   File: mount.py    MIT License 5 votes vote down vote up
def build(self, subvol: Subvol, layer_opts: LayerOpts):
        mount_dir = os.path.join(META_MOUNTS_DIR, self.mountpoint, MOUNT_MARKER)
        for name, data in (
            # NB: Not exporting self.mountpoint since it's implicit in the path.
            ('is_directory', self.is_directory),
            ('build_source', self.build_source._asdict()),
            ('runtime_source', json.loads(self.runtime_source)),
        ):
            procfs_serde.serialize(data, subvol, os.path.join(mount_dir, name))
        source_path = self.build_source.to_path(
            target_to_path=layer_opts.target_to_path,
            subvolumes_dir=layer_opts.subvolumes_dir,
        )
        # Support mounting directories and non-directories...  This check
        # follows symlinks for the mount source, which seems correct.
        is_dir = os.path.isdir(source_path)
        assert is_dir == self.is_directory, self
        if is_dir:
            subvol.run_as_root([
                'mkdir', '--mode=0755', subvol.path(self.mountpoint),
            ])
        else:  # Regular files, device nodes, FIFOs, you name it.
            # `touch` lacks a `--mode` argument, but the mode of this
            # mountpoint will be shadowed anyway, so let it be whatever.
            subvol.run_as_root(['touch', subvol.path(self.mountpoint)])
        ro_rbind_mount(source_path, subvol, self.mountpoint) 
Example 23
Project: paws   Author: funkybob   File: rpc.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def post(self, request):
        action = request.headers.get('X-Rpc-Action')
        if action:
            method = getattr(self, action, None)
            if is_rpc_method(method):
                if request.body:
                    data = json.loads(request.body)
                else:
                    data = {}
                log.info("Method: %r %r", action, data)
                return method(request, **data)
        return response(status=404) 
Example 24
Project: odorik   Author: nijel   File: __init__.py    GNU General Public License v3.0 5 votes vote down vote up
def get_json(self, path, args=None):
        """JSON parser on top of get."""
        result = json.loads(self.get(path, args))
        if isinstance(result, dict) and 'errors' in result:
            raise OdorikException(result['errors'])
        return result 
Example 25
Project: odorik   Author: nijel   File: test_main.py    GNU General Public License v3.0 5 votes vote down vote up
def test_version_json(self):
        """Test version printing."""
        output = execute(['--format', 'json', 'version'], True)
        values = json.loads(output)
        self.assertEqual({'version': odorik.__version__}, values) 
Example 26
Project: odorik   Author: nijel   File: test_main.py    GNU General Public License v3.0 5 votes vote down vote up
def test_data_list_json(self):
        """Test getting data list."""
        register_uris()
        output = execute(
            ['--format', 'json', 'mobile-data', '--list'],
            True
        )
        values = json.loads(output)
        self.assertEqual(len(values), 1) 
Example 27
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    MIT License 5 votes vote down vote up
def select_score_item(self, course_id, score_id):
        """select one scoreitem and do check"""
        params = {'courseId': course_id, 
                  'scoId': score_id, 
                  'firstLoad': 'true'}
        r = self.http.post(self.apis['select_resourse'], params, json_ret=False)
        try:
            location = float(json.loads(r)['location'])
        except:
            location = 0.1
        select_check_api = self.apis['select_check']
        api = select_check_api % (course_id, score_id)
        r = self.http.post(api, json_ret=False)
        return location 
Example 28
Project: http2mqtt   Author: chris-gunawardena   File: chip_neopixels.py    MIT License 5 votes vote down vote up
def on_message(client, userdata, msg):
    print(msg.topic+" "+str(msg.qos)+"====" + str(msg.payload))
    try:
      root = json.loads(str(msg.payload.decode('utf-8')))
      light_state = root["body"]["result"]["parameters"]["light-state"]; # "on"
      if msg.topic == "/lights" :
        if light_state == "on" :
          print("ON")
          on()
        elif light_state == "off" :
          print("OFF")
          off()
        else:
          off()
          print("NO MATCH")
    except:  # includes simplejson.decoder.JSONDecodeError
        print ('Decoding JSON failed') 
Example 29
Project: autolims   Author: scottbecker   File: test_autoprotocol_interpreter.py    MIT License 5 votes vote down vote up
def test_pipette_operations(self):
    
        #same as https://secure.transcriptic.com/becker-lab/p19aqhcbep8ea/runs/r19uvbk55tb54
        with open(os.path.join(os.path.dirname(__file__),'data','pipette_operations.json')) as f:
            protocol = json.loads(f.read())             
    
        run = Run.objects.create(title='Pipette Operation Run',
                                 test_mode=False,
                                 protocol=protocol,
                                 project = self.project,
                                 owner=self.user)
        assert isinstance(run, Run)
    
        execute_run(run)
    
        self.assertEqual(run.containers.count(),1)   
        self.assertEqual(run.instructions.count(),2)    
    
        test_plate = run.containers.get(label='test plate')
        
        volumes = [Decimal('745'),Decimal('85'),Decimal('20'),Decimal('20'),Decimal('30')]
        
        self.assertEqual(test_plate.aliquots.count(),5)
        
        for aq in test_plate.aliquots.all():
            assert isinstance(aq, Aliquot)
            self.assertEqual(Decimal(aq.volume_ul),volumes[aq.well_idx]) 
Example 30
Project: autolims   Author: scottbecker   File: test_run.py    MIT License 5 votes vote down vote up
def test_run_setup_all_new_containers(self):
        
        #same as https://secure.transcriptic.com/becker-lab/p19aqhcbep8ea/runs/r19u4jkqxhbt8
        with open(os.path.join(os.path.dirname(__file__),'data','oligosynthesis.json')) as f:
            protocol = json.loads(f.read())
    
        run = Run.objects.create(title='Oligosynthesis Run',
                                 test_mode=False,
                                 protocol=protocol,
                                 project = self.project,
                                 owner=self.user)   
        assert isinstance(run,Run)
            
            
        #check that instructions have been created and they aren't executed
        
        self.assertEqual(run.instructions.count(),6)
        
        self.assertEqual(run.containers.count(),2) 
Example 31
Project: autolims   Author: scottbecker   File: test_run.py    MIT License 5 votes vote down vote up
def test_run_setup_existing_containers(self):
        """ Check that Runs can be created that reference existing containers"""
        
        #create existing containers to be referenced
        
        existing_container = Container.objects.create(container_type_id = 'micro-1.5',
                                                   label = 'My Container',
                                                   test_mode = False,
                                                   storage_condition = Temperature.cold_80.name,
                                                   status = 'available',
                                                   organization = self.org
                                                   ) 
        
        
        
        #same as https://secure.transcriptic.com/becker-lab/p19aqhcbep8ea/runs/r19uqqkmr5u8f
        with open(os.path.join(os.path.dirname(__file__),'data','pellet_bacteria.json')) as f:
            protocol = json.loads(f.read())        
        
        #update the protocol to reference the correct id post import
        
        protocol['refs']['bacteria_tube']['id'] = existing_container.id
        
        
        run = Run.objects.create(title='Pellet Bacteria Run',
                         test_mode=False,
                         protocol=protocol,
                         project = self.project,
                         owner=self.user)   
        assert isinstance(run,Run)
    
    
        #check that refs is updated correctly
        
        self.assertEqual(run.instructions.count(),13)
    
        self.assertEqual(run.containers.count(),4) 
Example 32
Project: autolims   Author: scottbecker   File: custom_connection.py    MIT License 5 votes vote down vote up
def from_file(path):
        """Loads connection from file"""
        with open(expanduser(path), 'r') as f:
            cfg = json.loads(f.read())
            return CustomConnection(**cfg)    
        
        
    #custom version that prevents ssl verification 
Example 33
Project: incubator-spot   Author: apache   File: worker.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self, db_name, hdfs_app_path, kafka_consumer, conf_type):

        # get logger instance.
        self._logger = Util.get_logger('SPOT.INGEST.WRK.FLOW')

        self._db_name = db_name
        self._hdfs_app_path = hdfs_app_path

        # read proxy configuration.
        self._script_path = os.path.dirname(os.path.abspath(__file__))
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._conf = conf["pipelines"][conf_type]
        self._id = "spot-{0}-worker".format(conf_type)

        self._process_opt = self._conf['process_opt']
        self._local_staging = self._conf['local_staging']
        self.kafka_consumer = kafka_consumer

        # self._cursor = hive_engine.create_connection()
        self._cursor = hive_engine 
Example 34
Project: incubator-spot   Author: apache   File: collector.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self, hdfs_app_path, kafkaproducer, conf_type):

        # getting parameters.
        self._logger = logging.getLogger('SPOT.INGEST.FLOW')
        self._hdfs_app_path = hdfs_app_path
        self._producer = kafkaproducer

        # get script path
        self._script_path = os.path.dirname(os.path.abspath(__file__))

        # read flow configuration.
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._conf = conf["pipelines"][conf_type]

        # set configuration.
        self._collector_path = self._conf['collector_path']        
        self._dsource = 'flow'
        self._hdfs_root_path = "{0}/{1}".format(hdfs_app_path, self._dsource)

        self._supported_files = self._conf['supported_files']

        # create collector watcher
        self._watcher = FileWatcher(self._collector_path,self._supported_files)
        
        # Multiprocessing. 
        self._processes = conf["collector_processes"]
        self._ingestion_interval = conf["ingestion_interval"]
        self._pool = Pool(processes=self._processes)
        # TODO: review re-use of hdfs.client
        self._hdfs_client = hdfs.get_client() 
Example 35
Project: incubator-spot   Author: apache   File: worker.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self,db_name,hdfs_app_path,kafka_consumer,conf_type,processes):
        
        # get logger instance.
        self._logger = Util.get_logger('SPOT.INGEST.WRK.PROXY')

        self._db_name = db_name
        self._hdfs_app_path = hdfs_app_path
        self._kafka_consumer = kafka_consumer

        # read proxy configuration.
        self._script_path = os.path.dirname(os.path.abspath(__file__))
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._spark_conf  = conf["spark-streaming"]
        self._conf = conf["pipelines"][conf_type]
        self._processes = processes 
Example 36
Project: incubator-spot   Author: apache   File: collector.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self,hdfs_app_path,kafka_topic,conf_type):
        
        # getting parameters.
        self._logger = logging.getLogger('SPOT.INGEST.PROXY')
        self._hdfs_app_path = hdfs_app_path
        self._kafka_topic= kafka_topic

        # get script path
        self._script_path = os.path.dirname(os.path.abspath(__file__))

        # read proxy configuration.
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._message_size = conf["kafka"]["message_size"]
        self._conf = conf["pipelines"][conf_type]

        # get collector path.
        self._collector_path = self._conf['collector_path']

        #get supported files
        self._supported_files = self._conf['supported_files']

        # create collector watcher
        self._watcher = FileWatcher(self._collector_path,self._supported_files)

        # Multiprocessing. 
        self._processes = conf["collector_processes"]
        self._ingestion_interval = conf["ingestion_interval"]
        self._pool = Pool(processes=self._processes) 
Example 37
Project: incubator-spot   Author: apache   File: collector.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self, hdfs_app_path, kafkaproducer, conf_type):

        # getting parameters.
        self._logger = logging.getLogger('SPOT.INGEST.DNS')
        self._hdfs_app_path = hdfs_app_path
        self._producer = kafkaproducer

        # get script path
        self._script_path = os.path.dirname(os.path.abspath(__file__))

        # read dns configuration.
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._conf = conf["pipelines"][conf_type]

        # set configuration.
        self._collector_path = self._conf['collector_path']
        self._dsource = 'dns'
        self._hdfs_root_path = "{0}/{1}".format(hdfs_app_path, self._dsource)

        # set configuration.
        self._pkt_num = self._conf['pkt_num']
        self._pcap_split_staging = self._conf['pcap_split_staging']
        self._supported_files = self._conf['supported_files']

        # create collector watcher
        self._watcher = FileWatcher(self._collector_path, self._supported_files)

        # Multiprocessing.
        self._processes = conf["collector_processes"]
        self._ingestion_interval = conf["ingestion_interval"]
        self._pool = Pool(processes=self._processes)
        # TODO: review re-use of hdfs.client
        self._hdfs_client = hdfs.get_client() 
Example 38
Project: incubator-spot   Author: apache   File: graphql.py    Apache License 2.0 5 votes vote down vote up
def send_query(self):
        assert(self.url is not None)
        assert(type(self.url) is str)
        assert(self.query is not None)
        assert(type(self.query) is str)

        data = {
            'query': self.query
        }

        if self.variables is not None and type(self.variables) is dict:
            data['variables'] = self.variables

        encoded_data = json.dumps(data).encode('utf-8')

        http = urllib3.PoolManager()

        response = http.request(
            'POST',
            self.url,
            body=encoded_data,
            headers={
                'Accept': 'application/json',
                'Content-type': 'application/json'
            }
        )

        try:
            return json.loads(response.data.decode('utf-8'))
        except:
            return {
                'errors': [
                    {
                        'status': response.status,
                        'message': 'Failed to contact GraphQL endpoint. Is "{}" the correct URL?'.format(self.url)
                    }
                ]
            } 
Example 39
Project: incubator-spot   Author: apache   File: flow.py    Apache License 2.0 5 votes vote down vote up
def sc_geo(ip,date):

    app_path = Configuration.spot()
    file_name = "globe-{0}.json".format(ip.replace(".","_"))
    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
    .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))

    if HDFSClient.file_exists(hdfs_path,file_name):
        return json.loads(HDFSClient.get_file("{0}/{1}" \
        .format(hdfs_path,file_name)))
    else:
        return {} 
Example 40
Project: incubator-spot   Author: apache   File: flow.py    Apache License 2.0 5 votes vote down vote up
def impact_analysis(ip,date):

    app_path = Configuration.spot()
    file_name = "stats-{0}.json".format(ip.replace(".","_"))
    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
    .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))

    if HDFSClient.file_exists(hdfs_path,file_name):
        return json.loads(HDFSClient.get_file("{0}/{1}" \
        .format(hdfs_path,file_name)))
    else:
        return {} 
Example 41
Project: incubator-spot   Author: apache   File: proxy.py    Apache License 2.0 5 votes vote down vote up
def incident_progression(date,uri):

    app_path = Configuration.spot()
    hdfs_path = "{0}/proxy/oa/storyboard/{1}/{2}/{3}".format(app_path,\
        date.year,date.month,date.day)

    hash_name = md5.new(str(uri)).hexdigest()
    file_name = "incident-progression-{0}.json".format(hash_name)

    if HDFSClient.file_exists(hdfs_path,file_name):
        return json.loads(HDFSClient.get_file("{0}/{1}"\
        .format(hdfs_path,file_name)))
    else:
        return {} 
Example 42
Project: incubator-spot   Author: apache   File: flow_oa.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self,date,limit,logger): 

        # get logger if exists. if not, create new instance.
        self._logger = logging.getLogger('OA.Flow') if logger else Util.get_logger('OA.Flow',create_file=False)

        # initialize required parameters.
        self._scrtip_path = os.path.dirname(os.path.abspath(__file__))
        self._date = date
        self._table_name = "flow"
        self._flow_results = []
        self._limit = limit
        self._data_path = None
        self._ipynb_path = None
        self._ingest_summary_path = None
        self._flow_scores = []
        self._results_delimiter = '\t'
        

        # get app configuration.
        self._spot_conf = Util.get_spot_conf()

        # # get scores fields conf
        conf_file = "{0}/flow_conf.json".format(self._scrtip_path)
        self._conf = json.loads(open (conf_file).read(),object_pairs_hook=OrderedDict)

        # initialize data engine
        self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '') 
Example 43
Project: incubator-spot   Author: apache   File: proxy_oa.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self,date,limit,logger):

        # get logger if exists. if not, create new instance.
        self._logger = logging.getLogger('OA.PROXY') if logger else Util.get_logger('OA.PROXY',create_file=False)

        # initialize required parameters.
        self._scrtip_path = os.path.dirname(os.path.abspath(__file__))
        self._date = date
        self._table_name = "proxy"
        self._proxy_results = []
        self._limit = limit
        self._data_path = None
        self._ipynb_path = None
        self._ingest_summary_path = None
        self._proxy_scores = []
        self._proxy_scores_headers = []
        self._proxy_extra_columns = []
        self._results_delimiter = '\t'

        # get app configuration.
        self._spot_conf = Util.get_spot_conf()

        # get scores fields conf
        conf_file = "{0}/proxy_conf.json".format(self._scrtip_path)
        self._conf = json.loads(open (conf_file).read(),object_pairs_hook=OrderedDict)

        # initialize data engine
        self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '') 
Example 44
Project: incubator-spot   Author: apache   File: proxy_oa.py    Apache License 2.0 5 votes vote down vote up
def _add_network_context(self):

        nc_conf_file = "{0}/components/nc/nc_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if os.path.isfile(nc_conf_file):
            nc_conf = json.loads(open(nc_conf_file).read())["NC"]
            proxy_nc = NetworkContext(nc_conf,self._logger)
            ip_dst_index = self._conf["proxy_score_fields"]["clientip"]
            self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ] 
        else:
            self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ] 
Example 45
Project: incubator-spot   Author: apache   File: dns_oa.py    Apache License 2.0 5 votes vote down vote up
def _initialize_members(self,date,limit,logger):
        
        # get logger if exists. if not, create new instance.
        self._logger = logging.getLogger('OA.DNS') if logger else Util.get_logger('OA.DNS',create_file=False)

        # initialize required parameters.
        self._scrtip_path = os.path.dirname(os.path.abspath(__file__))
        self._date = date
        self._table_name = "dns"
        self._dns_results = []
        self._limit = limit
        self._data_path = None
        self._ipynb_path = None
        self._ingest_summary_path = None
        self._dns_scores = []
        self._dns_scores_headers = []
        self._results_delimiter = '\t'
        self._details_limit = 250

        # get app configuration.
        self._spot_conf = Util.get_spot_conf()

        # get scores fields conf
        conf_file = "{0}/dns_conf.json".format(self._scrtip_path)
        self._conf = json.loads(open (conf_file).read(),object_pairs_hook=OrderedDict)

        # initialize data engine
        self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '') 
Example 46
Project: incubator-spot   Author: apache   File: dns_oa.py    Apache License 2.0 5 votes vote down vote up
def _add_iana(self):
        iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if os.path.isfile(iana_conf_file):
            iana_config  = json.loads(open(iana_conf_file).read())
            dns_iana = IanaTransform(iana_config["IANA"])

            dns_qry_class_index = self._conf["dns_results_fields"]["dns_qry_class"]
            dns_qry_type_index = self._conf["dns_results_fields"]["dns_qry_type"]
            dns_qry_rcode_index = self._conf["dns_results_fields"]["dns_qry_rcode"]
            self._dns_scores = [ conn + [ dns_iana.get_name(conn[dns_qry_class_index],"dns_qry_class")] + [dns_iana.get_name(conn[dns_qry_type_index],"dns_qry_type")] + [dns_iana.get_name(conn[dns_qry_rcode_index],"dns_qry_rcode")] for conn in self._dns_scores ]
            
        else:            
            self._dns_scores = [ conn + ["","",""] for conn in self._dns_scores ] 
Example 47
Project: incubator-spot   Author: apache   File: dns_oa.py    Apache License 2.0 5 votes vote down vote up
def _get_dns_details(self,dns_qry_name,year,month,day,hh,dns_iana):
        value_string = ""
        query_to_load =("""
            SELECT unix_tstamp,frame_len,ip_dst,ip_src,dns_qry_name,dns_qry_class,dns_qry_type,dns_qry_rcode,dns_a,h as hh
            FROM {0}.{1} WHERE y={2} AND m={3} AND d={4} AND dns_qry_name LIKE '%{5}%' AND h={6} LIMIT {7};
        """).format(self._db,self._table_name,year,month,day,dns_qry_name,hh,self._details_limit)

        try:
             dns_details = impala.execute_query(query_to_load)
        except:
            self._logger.info("WARNING. Details couldn't be retreived for {0}, skipping this step".format(dns_qry_name))
        else:
        # add IANA to results.
            update_rows = []
            if dns_iana:
                self._logger.info("Adding IANA translation to details results")

                dns_details = [ conn + (dns_iana.get_name(str(conn[5]),"dns_qry_class"),dns_iana.get_name(str(conn[6]),"dns_qry_type"),dns_iana.get_name(str(conn[7]),"dns_qry_rcode")) for conn in dns_details ]
            else:
                self._logger.info("WARNING: NO IANA configured.")
                dns_details = [ conn + ("","","") for conn in dns_details ]

            nc_conf_file = "{0}/components/nc/nc_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
            if os.path.isfile(nc_conf_file):
                nc_conf = json.loads(open(nc_conf_file).read())["NC"]
                dns_nc = NetworkContext(nc_conf,self._logger)
                dns_details = [ conn + (dns_nc.get_nc(conn[2]),) for conn in dns_details ]
            else:
                dns_details = [ conn + (0,) for conn in dns_details ]

            for row in dns_details:
                value_string += str(tuple(item for item in row)) + ","

            if value_string != "":
                
                query_to_insert=("""
                    INSERT INTO {0}.dns_edge PARTITION (y={1}, m={2}, d={3}) VALUES ({4});
                """).format(self._db,year, month, day,  value_string[:-1])

                impala.execute_query(query_to_insert) 
Example 48
Project: incubator-spot   Author: apache   File: data.py    Apache License 2.0 5 votes vote down vote up
def _initialize_engine(self,db, pipeline):

        # read engine configuration.
        data_conf_file = "{0}/engine.json".format(os.path.dirname(os.path.abspath(__file__)))

        self._logger.info("Reading data component configuration: {0}".format(data_conf_file))
        self._engine_conf = json.loads(open (data_conf_file).read())       
        self._engine_name = self._engine_conf["oa_data_engine"]

        # import configured data engine.
        self._logger.info("Initializating {0} instance".format(self._engine_name))        
        module = __import__("components.data.{0}".format(self._engine_name),fromlist=['Engine'])

        # start data engine with configuration.
        self._engine = module.Engine(db,self._engine_conf[self._engine_name], pipeline) 
Example 49
Project: Coulomb   Author: DynamoDS   File: features_JSON.py    MIT License 5 votes vote down vote up
def getVersion(b64decodedData):
    json_map = json.loads(b64decodedData)
    if not json_map.has_key("View"):
        return None
    return json.loads(b64decodedData)["View"]["Dynamo"]["Version"] 
Example 50
Project: Coulomb   Author: DynamoDS   File: features_JSON.py    MIT License 5 votes vote down vote up
def getVersion(b64decodedData):
    json_map = json.loads(b64decodedData)
    if not "View" in json_map.keys():
        return None
    return json_map["View"]["Dynamo"]["Version"]