Python config.get() Examples

def setupDomain(domain, folder=False):
    endpoint = config.get("hsds_endpoint")
    headers = getRequestHeaders(domain=domain)
    req = endpoint + "/"
    rsp = requests.get(req, headers=headers)
    if rsp.status_code == 200:
        return  # already have domain
    if rsp.status_code != 404:
        # something other than "not found"
        raise ValueError(f"Unexpected get domain error: {rsp.status_code}")
    parent_domain = getParentDomain(domain)
    if parent_domain is None:
        raise ValueError(f"Invalid parent domain: {domain}")
    # create parent domain if needed
    setupDomain(parent_domain, folder=True)

    headers = getRequestHeaders(domain=domain)
    body=None
    if folder:
        body = {"folder": True}
        rsp = requests.put(req, data=json.dumps(body), headers=headers)
    else:
        rsp = requests.put(req, headers=headers)
    if rsp.status_code != 201:
        raise ValueError(f"Unexpected put domain error: {rsp.status_code}") 

def getEndpoints():
    docker_machine_ip = config.get("docker_machine_ip")
    req = "{}/nodestate/sn".format(config.get("head_endpoint")) 
    client = globals["client"]
    globals["request_count"] += 1
    async with client.get(req) as rsp:
        if rsp.status == 200:
            rsp_json = await rsp.json()
    nodes = rsp_json["nodes"]
    sn_endpoints = []
    docker_links = checkDockerLink()
    for node in nodes:
        if not node["host"]:
            continue
        if docker_links:
            # when running in docker, use the machine address as host
            host = "hsds_sn_{}".format(node["node_number"])
        elif docker_machine_ip:
            host = docker_machine_ip
        else:
            host = node["host"]
        url = "http://{}:{}".format(host, node["port"])
        sn_endpoints.append(url)
    log.info("{} endpoints".format(len(sn_endpoints)))
    globals["sn_endpoints"] = sn_endpoints 

def load_config(self, config):
        if self._alert_configs is None:
            self._alert_configs = []
        if self._log_configs is None:
            self._log_configs = []
        if self._rules is None:
            self._rules = []
        if self._tests is None:
            self._tests = {}

        self._alert_configs += config.get("alert configs",[])
        self._log_configs += config.get("log configs",[])
        self._rules += config.get("regex_rules",[]) 
        self._tests.update(config.get("tests",{}))
        self._name = config.get("name",self._name)
        self._version = config.get("version",self._version) 

def response(req, resp=None, code=None, message=None):
	level = "INFO"
	if code is None:
		# rsp needs to be set otherwise
		code = resp.status
	if message is None:
		message=resp.reason
	if code > 399:
		if  code < 500:
			level = "WARN"
		else:
			level = "ERROR"
	
	log_level = config.get("log_level")
	if log_level == "INFO" or (log_level == "WARN" and level != "INFO") or (log_level == "ERROR" and level == "ERROR"):
		print("{} RSP> <{}> ({}): {}".format(level, code, message, req.path)) 

def load_module_menus(self):
        global module_menus
        module_menus = {}
        #config.load()
        modules = config.get(['modules'], None)
        for module in modules:
            values = modules[module]
            if module != "launcher" and config.get(["modules", module, "auto_start"], 0) != 1:
                continue

            #version = values["current_version"]
            menu_path = os.path.join(root_path, module, "web_ui", "menu.json")
            if not os.path.isfile(menu_path):
                continue
                
            module_menu = json.load(file(menu_path, 'r'))          
            module_menus[module] = module_menu

        module_menus = sorted(module_menus.iteritems(), key=lambda (k,v): (v['menu_sort_id']))
        #for k,v in self.module_menus:
        #    logging.debug("m:%s id:%d", k, v['menu_sort_id']) 

def _if_rule(self, data, rule, alert_callback):
        if_conditions = rule.get("if")
        then_rules = rule.get("then")
        else_rules = rule.get("else")

        if data is None or if_conditions is None:
            return

        if_condition_met = True
        for if_condition in if_conditions:
            if_condition = self._regexes.get(if_condition, if_condition)
            ignorecase = re.compile(if_condition, re.IGNORECASE | re.MULTILINE | re.DOTALL)
            if ignorecase.search(data) is None:
                if_condition_met = False
                break

        if if_condition_met:
            if then_rules is not None:
                for then_rule in then_rules:
                    self._process(data, then_rule, alert_callback)
        else:
            if else_rules is not None:
                for else_rule in else_rules:
                    self._process(data, else_rule, alert_callback) 

def load_config(self, config):
        if self._alert_configs is None:
            self._alert_configs = {}
        if self._log_configs is None:
            self._log_configs = []
        if self._rules is None:
            self._rules = []
        if self._regexes is None:
            self._regexes = {}
        if self._tests is None:
            self._tests = {}

        self._name = config.get("name",self._name)
        self._version = config.get("version",self._version)

        self._alert_configs.update(config.get("alert configs",[]))
        self._log_configs.extend(config.get("log configs",[]))

        self._regexes.update(config.get("regexes",{}))
        self._rules.extend(config.get("regex_rules",[]))

        self._tests.update(config.get("tests",{})) 

def import_file(filename):
    log.info("import_file: {}".format(filename))
    loop = globals["loop"]
    max_concurrent_tasks = config.get("max_concurrent_tasks")
    tasks = []
    with open(filename, 'r') as fh:
        for line in fh:
            line = line.rstrip()
            #loop.run_until_complete(import_line(line))
            tasks.append(asyncio.ensure_future(import_line(line)))
            if len(tasks) < max_concurrent_tasks:
                continue  # get next line
            # got a batch, move them out!
            loop.run_until_complete(asyncio.gather(*tasks))
            tasks = []
    # finish any remaining tasks
    loop.run_until_complete(asyncio.gather(*tasks))
    globals["files_read"] += 1 

def testReadPoints(self):

        dset_id = config.get("dset111_id")
        print("dset_id:", dset_id)

        # these are the properties of the /g1/g1.1/dset1.1.1. dataset in tall.h5
        dset_json = {"id": dset_id}
        dset_json["root"] = getRootObjId(dset_id)
        dset_json["type"] = {"class": "H5T_INTEGER", "base": "H5T_STD_I32BE"}
        dset_json["shape"] = {"class": "H5S_SIMPLE", "dims": [10, 10], "maxdims": [10, 10]}
        dset_json["layout"] = {"class": "H5D_CHUNKED", "dims": [10, 10]}

        chunk_id = 'c' + dset_id[1:] + "_0_0"

        params = {}
        params["dset_json"] = dset_json
        params["chunk_id"] = chunk_id
        params["bucket"] = config.get("bucket")
        loop = asyncio.get_event_loop()
        app = get_app(loop=loop)
        loop.run_until_complete(self.read_points_test(app, params))

        loop.close() 

def testReadHyperslab(self):

        dset_id = config.get("dset111_id")
        print("dset_id:", dset_id)

        # these are the properties of the /g1/g1.1/dset1.1.1. dataset in tall.h5
        dset_json = {"id": dset_id}
        dset_json["root"] = getRootObjId(dset_id)
        dset_json["type"] = {"class": "H5T_INTEGER", "base": "H5T_STD_I32BE"}
        dset_json["shape"] = {"class": "H5S_SIMPLE", "dims": [10, 10], "maxdims": [10, 10]}
        dset_json["layout"] = {"class": "H5D_CHUNKED", "dims": [10, 10]}

        chunk_id = 'c' + dset_id[1:] + "_0_0"

        params = {}
        params["dset_json"] = dset_json
        params["chunk_id"] = chunk_id
        params["bucket"] = config.get("bucket")
        loop = asyncio.get_event_loop()
        app = get_app(loop=loop)
         
        loop.run_until_complete(self.read_hyperslab_test(app, params))

        loop.close() 

def testInvalidFillValue(self):
        # test Dataset with simple type and fill value that is incompatible with the type
        domain = self.base_domain + "/testInvalidFillValue.h5"
        helper.setupDomain(domain)
        print("testInvalidFillValue", domain)
        headers = helper.getRequestHeaders(domain=domain)

        # get domain
        req = helper.getEndpoint() + '/'
        rsp = requests.get(req, headers=headers)
        rspJson = json.loads(rsp.text)
        self.assertTrue("root" in rspJson)

        fill_value = 'XXXX'  # can't convert to int!
        # create the dataset
        req = self.endpoint + "/datasets"
        payload = {'type': 'H5T_STD_I32LE', 'shape': 10}
        payload['creationProperties'] = {'fillValue': fill_value }
        req = self.endpoint + "/datasets"
        rsp = requests.post(req, data=json.dumps(payload), headers=headers)
        self.assertEqual(rsp.status_code, 400)  # invalid param 

def getRequestHeaders(domain=None, username=None, password=None):
    if username is None:
        username = config.get("user_name")
    if password is None:
        password = config.get("user_password")
    headers = { }
    if domain is not None:
        headers['host'] = domain
    if username and password:
        auth_string = username + ':' + password
        auth_string = auth_string.encode('utf-8')
        auth_string = base64.b64encode(auth_string)
        auth_string = auth_string.decode('utf-8')
        auth_string = "Basic " + auth_string
        headers['Authorization'] = auth_string
    return headers 

def setupDomain(domain):
    endpoint = config.get("hsds_endpoint")
    print("setupdomain: ", domain)
    headers = getRequestHeaders(domain=domain)
    req = endpoint + "/"
    rsp = requests.get(req, headers=headers)
    if rsp.status_code == 200:
        return  # already have domain
    if rsp.status_code != 404:
        # something other than "not found"
        raise ValueError("Unexpected get domain error: {}".format(rsp.status_code))

    parent_domain = getParentDomain(domain)
    if parent_domain is None:
        raise ValueError("Invalid parent domain: {}".format(domain))
    # create parent domain if needed
    setupDomain(parent_domain)  
     
    headers = getRequestHeaders(domain=domain)
    rsp = requests.put(req, headers=headers)
    if rsp.status_code != 201:
        raise ValueError("Unexpected put domain error: {}".format(rsp.status_code)) 

def response(req, resp=None, code=None, message=None):
	level = "INFO"
	if code is None:
		# rsp needs to be set otherwise
		code = resp.status
	if message is None:
		message=resp.reason
	if code > 399:
		if  code < 500:
			level = "WARN"
		else:
			level = "ERROR"
	
	log_level = config.get("log_level")
	if log_level == "INFO" or (log_level == "WARN" and level != "INFO") or (log_level == "ERROR" and level == "ERROR"):
		print("{} RSP> <{}> ({}): {}".format(level, code, message, req.path)) 

def testGetTopLevelDomain(self):
        domain = "/home"
        print("testGetTopLevelDomain", domain)
        headers = helper.getRequestHeaders(domain=domain)

        req = helper.getEndpoint() + '/'
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 200)
        rspJson = json.loads(rsp.text)
        self.assertFalse("root" in rspJson)  # no root group for folder domain
        self.assertTrue("owner" in rspJson)
        self.assertTrue("hrefs" in rspJson)
        self.assertTrue("class" in rspJson)
        self.assertEqual(rspJson["class"], "folder")
        domain = "test_user1.home"
        headers = helper.getRequestHeaders(domain=domain)

        req = helper.getEndpoint() + '/'
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 200) 

def testBaseDomain(self):
        # make a non-folder domain
        print("testBaseDomain", self.base_domain)
        headers = helper.getRequestHeaders(domain=self.base_domain)

        req = helper.getEndpoint() + '/'
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 200)
        self.assertEqual(rsp.headers['content-type'], 'application/json; charset=utf-8')
        rspJson = json.loads(rsp.text)

        # verify that passing domain as query string works as well
        del headers["X-Hdf-domain"]
        req += "?host=" + self.base_domain
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 200)
        self.assertEqual(rsp.headers['content-type'], 'application/json; charset=utf-8')

        # try using DNS-style domain name
        domain = helper.getDNSDomain(self.base_domain)
        params = { "host": domain }
        rsp = requests.get(req, params=params, headers=headers)
        self.assertEqual(rsp.status_code, 200)
        self.assertEqual(rsp.headers['content-type'], 'application/json; charset=utf-8') 

def _alert_rule(self, data, rule, alert_callback):
        alert_actions = rule.get("alert")
        for alert_action in alert_actions:
            if alert_callback:
                alert_callback(alert_action) 

def getTestDomainName(name):
    now = time.time()
    dt = datetime.fromtimestamp(now, pytz.utc)
    domain = "{:04d}{:02d}{:02d}T{:02d}{:02d}{:02d}_{:06d}Z".format(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond)  
    domain += '.' 
    domain += name.lower()
    domain += '.'
    domain += config.get('user_name')
    domain += '.home'
    return domain 

def verifyDomain(domain):
    """ create domain if it doesn't already exist
    """
    params = {"host": domain}
    headers = getRequestHeaders()
    client = globals["client"]
    req = getEndpoint() + '/'
    root_id = None
    log.info("GET " + req)
    timeout = config.get("timeout")
    async with client.get(req, headers=headers, timeout=timeout, params=params) as rsp:
        if rsp.status == 200:
            domain_json = await rsp.json()
        else:
            log.info("got status: {}".format(rsp.status))
    if rsp.status == 200:
        root_id = domain_json["root"]
    elif rsp.status == 404:
        # create the domain
        setupDomain(domain)
        async with client.get(req, headers=headers, timeout=timeout, params=params) as rsp:
            if rsp.status == 200:
                domain_json = await rsp.json()
                root_id = domain_json["root"]
            else:
                log.error("got status: {} for GET req: {}".format(rsp.status, req))
                raise HttpProcessingError(code=rsp.status, message="Service error")
    globals["root"] = root_id 

def testGetInfo(self):
        req = self.endpoint + "/info"
        print("req", req)
        rsp = requests.get(req)
        self.assertEqual(rsp.status_code, 200)
        self.assertEqual(rsp.headers['content-type'], 'application/json')
        rsp_json = json.loads(rsp.text)
        helper.validateId(rsp_json["id"])
        self.assertEqual(rsp_json["active_dn_count"], 0)
        self.assertEqual(rsp_json["active_sn_count"], 0)
        self.assertEqual(rsp_json["target_dn_count"], 4)
        self.assertEqual(rsp_json["target_sn_count"], 4)
        self.assertEqual(rsp_json["cluster_state"], "INITIALIZING")
        #print(rsp_json) 

def testGetTopLevelDomains(self):
        print("testGetTopLevelDomains", self.base_domain)

        import os.path as op
        # Either '/' or no domain should get same result
        for domain in (None, '/'):
            headers = helper.getRequestHeaders(domain=domain)
            req = helper.getEndpoint() + '/domains'
            rsp = requests.get(req, headers=headers)
            self.assertEqual(rsp.status_code, 200)
            self.assertEqual(rsp.headers['content-type'], 'application/json; charset=utf-8')
            rspJson = json.loads(rsp.text)
            self.assertTrue("domains" in rspJson)
            domains = rspJson["domains"]

            domain_count = len(domains)
            if domain_count == 0:
                # this should only happen in the very first test run
                print("Expected to find more domains!")
                self.assertTrue(False)
                return

            for item in domains:
                self.assertTrue("name" in item)
                name = item["name"]
                self.assertEqual(name[0], '/')
                self.assertTrue(name[-1] != '/')
                self.assertTrue("owner" in item)
                self.assertTrue("created" in item)
                self.assertTrue("lastModified" in item)
                self.assertFalse("size" in item)
                self.assertTrue("class") in item
                self.assertTrue(item["class"] in ("domain", "folder")) 

def testDNSDomain(self):
        # DNS domain names are in reverse order with dots as seperators...

        dns_domain = helper.getDNSDomain(self.base_domain)
        print("testDNSDomain", dns_domain)
        # verify we can access base domain as via dns name
        headers = helper.getRequestHeaders(domain=dns_domain)

        req = helper.getEndpoint() + '/'
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 200)
        self.assertEqual(rsp.headers['content-type'], 'application/json; charset=utf-8')

        # can't have two consecutive dots'
        domain = 'two.dots..are.bad.' + dns_domain
        req = helper.getEndpoint() + '/'
        headers = helper.getRequestHeaders(domain=domain)
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 400)  # 400 == bad syntax

        # can't have a slash
        domain = 'no/slash.' + dns_domain
        req = helper.getEndpoint() + '/'
        headers = helper.getRequestHeaders(domain=domain)
        rsp = requests.get(req, headers=headers)
        # somehow this is showing up as a 400 in ceph and 404 in S3
        self.assertTrue(rsp.status_code in (400, 404))  # 400 == bad syntax

        # just a dot is no good
        domain = '.'  + dns_domain
        req = helper.getEndpoint() + '/'
        headers = helper.getRequestHeaders(domain=domain)
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 400)  # 400 == bad syntax

        # dot in the front is bad
        domain =  '.dot.in.front.is.bad.' + dns_domain
        req = helper.getEndpoint() + '/'
        headers = helper.getRequestHeaders(domain=domain)
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 400)  # 400 == bad syntax 

def testGetNotFound(self):
        domain =  self.base_domain + "/doesnotexist.h6"
        print("testGetNotFound", domain)
        headers = helper.getRequestHeaders(domain=domain)
        req = helper.getEndpoint() + '/'

        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 404) 

def testCreateFolder(self):
        domain = self.base_domain + "/newfolder"
        print("testCreateFolder", domain)
        headers = helper.getRequestHeaders(domain=domain)
        req = helper.getEndpoint() + '/'
        body = {"folder": True}
        rsp = requests.put(req, data=json.dumps(body), headers=headers)
        self.assertEqual(rsp.status_code, 201)
        rspJson = json.loads(rsp.text)
        for k in ("owner", "acls", "created", "lastModified"):
             self.assertTrue(k in rspJson)
        self.assertFalse("root" in rspJson)  # no root -> folder

        # verify that putting the same domain again fails with a 409 error
        rsp = requests.put(req, data=json.dumps(body), headers=headers)
        self.assertEqual(rsp.status_code, 409)

        # do a get on the new folder
        rsp = requests.get(req, headers=headers)
        self.assertEqual(rsp.status_code, 200)
        rspJson = json.loads(rsp.text)

        self.assertTrue("owner" in rspJson)
        self.assertTrue("class" in rspJson)
        self.assertEqual(rspJson["class"], "folder")

        # try doing a un-authenticated request
        if config.get("test_noauth") and config.get("default_public"):
            headers = helper.getRequestHeaders()
            req = helper.getEndpoint() + "/?host=" + domain
            # do a get on the folder with a query arg for host
            rsp = requests.get(req)
            self.assertEqual(rsp.status_code, 200)
            rspJson = json.loads(rsp.text)
            for k in ("class", "owner"):
                self.assertTrue(k in rspJson)
            self.assertFalse("root" in rspJson) 

def testGetByPath(self):
        domain = helper.getTestDomain("tall.h5")
        print("testGetByPath", domain)
        headers = helper.getRequestHeaders(domain=domain)

        req = helper.getEndpoint() + '/'
        rsp = requests.get(req, headers=headers)
        if rsp.status_code != 200:
            print("WARNING: Failed to get domain: {}. Is test data setup?".format(domain))
            return  # abort rest of test
        domainJson = json.loads(rsp.text)
        self.assertTrue("root" in domainJson)
        root_id = domainJson["root"]

        # Get group at /g1/g1.1 by using h5path
        params = {"h5path": "/g1/g1.1"}
        rsp = requests.get(req, headers=headers, params=params)
        self.assertEqual(rsp.status_code, 200)
        rspJson = json.loads(rsp.text)
        self.assertTrue("id" in rspJson)
        g11id = helper.getUUIDByPath(domain, "/g1/g1.1")
        self.assertEqual(g11id, rspJson["id"])
        self.assertTrue("root" in rspJson)
        self.assertEqual(root_id, rspJson["root"])

        # Get dataset at /g1/g1.1/dset1.1.1 by using relative h5path
        params = {"h5path": "./g1/g1.1/dset1.1.1"}
        rsp = requests.get(req, headers=headers, params=params)
        self.assertEqual(rsp.status_code, 200)
        rspJson = json.loads(rsp.text)
        self.assertTrue("id" in rspJson)
        d111id = helper.getUUIDByPath(domain, "/g1/g1.1/dset1.1.1")
        self.assertEqual(d111id, rspJson["id"])
        self.assertTrue("root" in rspJson)
        self.assertEqual(root_id, rspJson["root"]) 

def process(self, repo_patch, alert_callback=None):
        data = u'\n'.join(repo_patch.diff.additions).encode('utf-8').strip()
        def _alert_callback(alert_action):
            alert_config_key = alert_action.get("alert config")
            alert_config = self._alert_configs.get(alert_config_key)
            if alert_config is None:
                logger.error("Alert config for [%s] is None", alert_config_key);
                return
            if alert_config.get("email"):
                default_email = config.Configuration('config.json').get(('email', 'to'))
                to_email = alert_config.get("email", default_email)
                patch_lines = u'\n'.join(repo_patch.diff.additions).encode('utf-8').strip()
                subject = alert_action.get("subject","Unknown Subject")
                (text, html) = self.create_alert_email(subject, data, repo_patch.repo_commit)
                ea = EmailAlert(Alert(subject=subject, 
                                      message=text.encode('utf-8'), 
                                      message_html=html.encode('utf-8')), 
                                to_email=to_email)
                if (self.test_mode == True):
                    print ea
                else:
                    ea.send()
            else:
                logger.warn("Alert type unknown %s" % (alert_config))

        if alert_callback is None:
            alert_callback = _alert_callback
        #data = repo_patch
        for rule in self._rules:
            self._process(data, rule, alert_callback) 

def main():
    if len(sys.argv) == 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
        printUsage()
        sys.exit(1)
    
    obj_id = sys.argv[-1]
    s3_gateway = config.get("aws_s3_gateway")
    print("aws_s3_gateway: {}".format(s3_gateway))
    region = config.get("aws_region")
    print("region: {}".format(region))
    print("now: {}".format(time.time()))
    conn = boto.s3.connect_to_region(
        region,
        calling_format=boto.s3.connection.OrdinaryCallingFormat()
    )
    
    bucket_name = config.get("bucket_name")
    print("bucket_name: {}".format(bucket_name))
    bucket = conn.get_bucket(bucket_name)

    if obj_id.startswith("d-") or obj_id.startswith("g-") or obj_id.startswith("t-"):
        # add the checksum prefix
        obj_id = getIdHash(obj_id) + '-' + obj_id

    k = Key(bucket)
    k.key = obj_id
    data = k.get_contents_as_string()
    if not isinstance(data, str):
        # Python 3 - convert from bytes to str
        data = data.decode("utf-8")
    json_data = json.loads(data)
    print(json.dumps(json_data, sort_keys=True, indent=4)) 

def getRootUUID(domain, username=None, password=None):
    req = getEndpoint() + "/"
    headers = getRequestHeaders(domain=domain, username=username, password=password)

    rsp = requests.get(req, headers=headers)
    root_uuid= None
    if rsp.status_code == 200:
        rspJson = json.loads(rsp.text)
        root_uuid = rspJson["root"]
    return root_uuid 

def getUUIDByPath(domain, path, username=None, password=None):
    if path[0] != '/':
        raise KeyError("only abs paths") # only abs paths

    parent_uuid = getRootUUID(domain, username=username, password=password)

    if path == '/':
        return parent_uuid

    headers = getRequestHeaders(domain=domain)

    # make a fake tgt_json to represent 'link' to root group
    tgt_json = {'collection': "groups", 'class': "H5L_TYPE_HARD", 'id': parent_uuid }
    tgt_uuid = None

    names = path.split('/')

    for name in names:
        if not name:
            continue
        if parent_uuid is None:
            raise KeyError("not found")

        req = getEndpoint() + "/groups/" + parent_uuid + "/links/" + name
        rsp = requests.get(req, headers=headers)
        if rsp.status_code != 200:
            raise KeyError("not found")
        rsp_json = json.loads(rsp.text)
        tgt_json = rsp_json['link']

        if tgt_json['class'] == 'H5L_TYPE_HARD':
            if tgt_json['collection'] == 'groups':
                parent_uuid = tgt_json['id']
            else:
                parent_uuid = None
            tgt_uuid = tgt_json['id']
        else:
            raise KeyError("non-hard link")
    return tgt_uuid 

def fetch(url):
    async with ClientSession() as session:
        async with session.get(url) as response:
            delay = response.headers.get("DELAY")
            date = response.headers.get("DATE")
            print("{}:{} with delay {}".format(date, response.url, delay))
            return await response.read()