Python shelve.open() Examples

def __init__(self, reduce_memory=False):
        if reduce_memory:
            self.temp_dir = TemporaryDirectory()
            self.working_dir = Path(self.temp_dir.name)
            self.document_shelf_filepath = self.working_dir / 'shelf.db'
            self.document_shelf = shelve.open(str(self.document_shelf_filepath),
                                              flag='n', protocol=-1)
            self.documents = None
        else:
            self.documents = []
            self.document_shelf = None
            self.document_shelf_filepath = None
            self.temp_dir = None
        self.doc_lengths = []
        self.doc_cumsum = None
        self.cumsum_max = None
        self.reduce_memory = reduce_memory 

def openflow_manager(br_name, command, flow_options=None, ovs=None):
    """
    Manager openvswitch flow rules

    :param br_name: name of the bridge
    :param command: manager cmd(add-flow, del-flows, dump-flows..)
    :param flow_options: open flow options
    :param ovs: OpenVSwitch object.
    """
    if ovs is None:
        ovs = __ovs

    if ovs is None or br_name not in ovs.list_br():
        raise OpenflowSwitchError(br_name)

    manager_cmd = "ovs-ofctl %s %s" % (command, br_name)
    if flow_options:
        manager_cmd += " %s" % flow_options
    return process.run(manager_cmd) 

def ibdev_find_pkey_idx(device_name, pkey_value):
    pkey_idx_path = "/sys/class/infiniband/%s/ports/1/pkeys" % (
        device_name)

    for pkey_idx in os.listdir(pkey_idx_path):
        this_pkey_idx_path = os.path.join(pkey_idx_path, pkey_idx)
        with open(this_pkey_idx_path) as f:
            try:
                this_pkey_value = int(f.read().strip(), 0)
            except ValueError:
                continue

            if this_pkey_value & 0x7fff == pkey_value & 0x7fff:
                return pkey_idx

    raise NetworkSetupError('pkey %s not found on device %s' % (
        hex(pkey_value), device_name)) 

def loadSetting(self):
        """加载配置"""
        try:
            with open(self.settingFilePath) as f:
                l = json.load(f)
                
                for setting in l:
                    result, msg = self.createSpread(setting)
                    self.writeLog(msg)
                    
                self.writeLog(u'价差配置加载完成')
        except:
            content = u'价差配置加载出错，原因：' + traceback.format_exc()
            self.writeLog(content)
    
    #---------------------------------------------------------------------- 

def test_03_db(self):
        """
        Load from database created in test_02_db, verify data against params
        """
        # Verify on-disk data matches dummy data just written
        self.zero_counter()
        db = shelve.open(self.db_filename)
        db_keys = list(db.keys())
        self.assertEqual(len(db_keys), self.db_item_count)
        for key in db_keys:
            db_value = eval(db[key], {}, {})
            self.assert_(isinstance(db_value, list))
            self.assert_(len(db_value) > 0)
            self.assert_(isinstance(db_value[0], dict))
            for nic in db_value:
                mac = nic.get('mac')
                if mac:
                    # Another test already checked mac_is_valid behavior
                    self.assert_(utils_net.VirtIface.mac_is_valid(mac))
            self.print_and_inc()
        db.close() 

def loadState():
    """
    Attempt to load the program from a pickled state in the saves directory.
    """
    savePath = pathlib.Path("RedditDataExtractor", "saves")
    if not savePath.exists():
        savePath.mkdir()
    shelf = shelve.open(str(savePath / "settings.db"))
    rddtDataExtractor = None
    try:
        rddtDataExtractor = shelf['rddtDataExtractor']
        userListSettings = shelf['userLists']
        subredditListSettings = shelf['subredditLists']
        rddtDataExtractor.userLists = {}
        rddtDataExtractor.subredditLists = {}
        # Reconstruct the lists because GUI stuff isn't pickleable
        for key, val in userListSettings.items():
            rddtDataExtractor.userLists[key] = ListModel(val, User)
        for key, val in subredditListSettings.items():
            rddtDataExtractor.subredditLists[key] = ListModel(val, Subreddit)
    except KeyError:
        pass
    finally:
        shelf.close()
        return rddtDataExtractor 

def close(self):
        """Cleanup batch job."""
        self.groups_shelf.close()
        self.indicators_shelf.close()
        if self.debug and self.enable_saved_file:
            fqfn = os.path.join(self.tcex.args.tc_temp_path, 'xids-saved')
            if os.path.isfile(fqfn):
                os.remove(fqfn)  # remove previous file to prevent duplicates
            with open(fqfn, 'w') as fh:
                for xid in self.saved_xids:
                    fh.write(f'{xid}\n')
        else:
            # delete saved files
            if os.path.isfile(self.group_shelf_fqfn):
                os.remove(self.group_shelf_fqfn)
            if os.path.isfile(self.group_shelf_fqfn):
                os.remove(self.indicator_shelf_fqfn) 

def _allow_host_pkeys(self):
        device_path = "/sys/class/infiniband/{0}".format(self._ibdev_name)
        num_ports = len(os.listdir(os.path.join(device_path, "ports")))

        for port in xrange(1, num_ports + 1):
            pkeys_path = os.path.join(device_path, "ports", str(port),
                                      "pkeys")
            pkey_idx_path = os.path.join(device_path, "iov", self._dev_addr,
                                         "ports", str(port), "pkey_idx")

            idx = 0
            for pkey_idx in os.listdir(pkeys_path):
                p = os.path.join(pkeys_path, pkey_idx)
                with open(p) as f:
                    try:
                        this_pkey_value = int(f.read().strip(), 0)
                    except ValueError:
                        continue

                    if this_pkey_value:
                        with open(os.path.join(pkey_idx_path, str(idx)), 'w') as f:
                            f.write(pkey_idx)
                        idx+=1 

def textFile( readFromFile ):
   
   # open text file
   try:
      outputFile = open( "print.txt", "w" )
   except IOError:
      print >> sys.stderr, "File could not be opened."
      sys.exit( 1 )

   print >> outputFile, "Account".ljust( 10 ),
   print >> outputFile, "Last Name".ljust( 10 ),
   print >> outputFile, "First Name".ljust( 10 ),
   print >> outputFile, "Balance".rjust( 10 )

   # print shelve values to text file
   for key in readFromFile.keys():
      print >> outputFile, key.ljust( 10 ),
      print >> outputFile, readFromFile[ key ][ 0 ].ljust( 10 ),
      print >> outputFile, readFromFile[ key ][ 1 ].ljust( 10 ),
      print >> outputFile, readFromFile[ key ][ 2 ].rjust( 10 )

   outputFile.close()

# update account balance 

def label_indivs(valuation, lexicon=False):
    """
    Assign individual constants to the individuals in the domain of a ``Valuation``.

    Given a valuation with an entry of the form ``{'rel': {'a': True}}``,
    add a new entry ``{'a': 'a'}``.

    :type valuation: Valuation
    :rtype: Valuation
    """
    # collect all the individuals into a domain
    domain = valuation.domain
    # convert the domain into a sorted list of alphabetic terms
    # use the same string as a label
    pairs = [(e, e) for e in domain]
    if lexicon:
        lex = make_lex(domain)
        with open("chat_pnames.cfg", 'w') as outfile:
            outfile.writelines(lex)
    # read the pairs into the valuation
    valuation.update(pairs)
    return valuation 

def val_dump(rels, db):
    """
    Make a ``Valuation`` from a list of relation metadata bundles and dump to
    persistent database.

    :param rels: bundle of metadata needed for constructing a concept
    :type rels: list of dict
    :param db: name of file to which data is written.
               The suffix '.db' will be automatically appended.
    :type db: string
    """
    concepts = process_bundle(rels).values()
    valuation = make_valuation(concepts, read=True)
    db_out = shelve.open(db, 'n')

    db_out.update(valuation)

    db_out.close() 

def update_self():
    logger.debug('def update_self started')
    logger.info('Update - will restart')
    cfg.update_date = datetime.now().strftime("%d-%m %H:%M:%S")
    prem_config.set('update', 'update_date', cfg.update_date)
    with open(os.path.join(ConfDir, 'settings.cfg'), 'w') as configfile:  # save
        prem_config.write(configfile)
    db.close()
    socketio.stop()
    if os_arg == '--windows':
        subprocess.call(['python', os.path.join(runningdir, 'utils.py'), '--update', '--windows'])
        os._exit(1)
    else:
        subprocess.Popen(['python', os.path.join(runningdir, 'utils.py'), '--update', '--none'],
                         shell=False,
                         close_fds=True)
        os._exit(1)


# noinspection PyProtectedMember 

def _testKeys(self):
	"""Verify that index lookup can find each word in the index file."""
	print "Testing: ", self
	file = open(self.indexFile.file.name, _FILE_OPEN_MODE)
	counter = 0
	while 1:
	    line = file.readline()
	    if line == '': break
	    if line[0] != ' ':
		key = string.replace(line[:string.find(line, ' ')], '_', ' ')
		if (counter % 1000) == 0:
		    print "%s..." % (key,),
		    import sys
		    sys.stdout.flush()
		counter = counter + 1
		self[key]
	file.close()
	print "done." 

def _testKeys(self):
	"""Verify that index lookup can find each word in the index file."""
	print "Testing: ", self
	file = open(self.indexFile.file.name, _FILE_OPEN_MODE)
	counter = 0
	while 1:
	    line = file.readline()
	    if line == '': break
	    if line[0] != ' ':
		key = string.replace(line[:string.find(line, ' ')], '_', ' ')
		if (counter % 1000) == 0:
		    print "%s..." % (key,),
		    import sys
		    sys.stdout.flush()
		counter = counter + 1
		self[key]
	file.close()
	print "done." 

def input_file_to_training_data(args, input_file, epoch, tokenizer, num_files):
    print(input_file)
    with DocumentDatabase(reduce_memory=args.reduce_memory) as docs:
        with open(input_file) as f:
            doc = []
            for line in tqdm(f, desc="Loading Dataset", unit=" lines"):
                line = line.strip()
                if line == "":
                    docs.add_document(doc)
                    doc = []
                else:
                    tokens = tokenizer.tokenize(line)
                    doc.append(tokens)
            if doc:
                docs.add_document(doc)  # If the last doc didn't end on a newline, make sure it still gets added
        if len(docs) <= 1:
            exit("ERROR: No document breaks were found in the input file! These are necessary to allow the script to "
                    "ensure that random NextSentences are not sampled from the same document. Please add blank lines to "
                    "indicate breaks between documents in your input file. If your dataset does not contain multiple "
                    "documents, blank lines can be inserted at any natural boundary, such as the ends of chapters, "
                    "sections or paragraphs.")

        for i in range(args.epochs_to_generate):
            create_training_file(docs, tokenizer, args, epoch + i * num_files) 

def lock_db(self):
        if not hasattr(self, 'lock'):
            self.lock = utils_misc.lock_file(self.db_lockfile)
            if not hasattr(self, 'db'):
                self.db = shelve.open(self.db_filename)
            else:
                raise DbNoLockError
        else:
            raise DbNoLockError 

def groups_shelf(self):
        """Return dictionary of all Groups data."""
        if self._groups_shelf is None:
            self._groups_shelf = shelve.open(self.group_shelf_fqfn, writeback=False)
        return self._groups_shelf 

def indicators_shelf(self):
        """Return dictionary of all Indicator data."""
        if self._indicators_shelf is None:
            self._indicators_shelf = shelve.open(self.indicator_shelf_fqfn, writeback=False)
        return self._indicators_shelf 

def __init__(self):
        """

        """
        self.SHELVE = shelve.open('bast.session', writeback=True)
        Session.__init__(self, self.SHELVE)
        # print(self.SHELVE.) 

def __init__(self, github_obj, repo_full_name, cache_filename):
        """
        Initialize cache and reference github repository issues
        """

        self.github = github_obj
        self.repo_full_name = repo_full_name
        self.shelf = shelve.open(filename=cache_filename,
                                 protocol=self.protocol,
                                 writeback=True)

        # Avoid exceeding rate-limit per hour
        requests = self.github.rate_limiting[1]  # requests per hour
        period = 60.0 * 60.0  # one hour in seconds
        sleeptime = period / requests
        self.pre_fetch_partial = Partial(time.sleep, sleeptime)
        # self.pre_fetch_partial = None # cheat-mode enable (no delays)

        repo_cache_key = 'repo_%s' % self.repo_full_name
        # get_repo called same way throughout instance life
        cache_get_partial = Partial(self.shelf.__getitem__, repo_cache_key)
        cache_set_partial = Partial(self.shelf.__setitem__, repo_cache_key)
        cache_del_partial = Partial(self.shelf.__delitem__, repo_cache_key)
        fetch_partial = Partial(self.github.get_repo,
                                self.repo_full_name)
        # Callable instance retrieves cached or fetched value for key
        self.get_repo = self.cache_class(self.github,
                                         cache_get_partial,
                                         cache_set_partial,
                                         cache_del_partial,
                                         self.pre_fetch_partial,
                                         fetch_partial)
        super(GithubIssuesBase, self).__init__() 

def upload_nzb(filename):
    logger.debug('def upload_nzb started')
    payload = {'apikey': cfg.prem_apikey}
    files = {'src': open(filename, 'rb')}
    logger.debug('Uploading nzb to the cloud: %s', filename)
    r = prem_connection("postfile", "https://www.premiumize.me/api/transfer/create", payload, files)
    if 'failed' not in r:
        response_content = json.loads(r.content)
        if response_content['status'] == "success":
            logger.debug('Upload nzb successful: %s', filename)
            return response_content['id']
        elif response_content[
            'message'] == 'An error occured. Please try again and contact customer service if the problem persists.':
            gevent.sleep(10)
            r = prem_connection("postfile", "https://www.premiumize.me/api/transfer/create", payload, files)
            if 'failed' not in r:
                response_content = json.loads(r.content)
                if response_content['status'] == "success" or response_content[
                    'message'] == 'You already have this job added.':
                    logger.debug('Upload nzb successful: %s', filename)
                    return response_content['id']
        else:
            msg = 'Upload of nzb: %s failed, message: %s' % (filename, response_content['message'])
            logger.error(msg)
            if response_content['message'] == 'You already have this job added.':
                return 'duplicate'
            if cfg.email_enabled:
                email('Upload of nzb failed', msg)
            return 'failed'
    else:
        return 'failed' 

def upload_torrent(torrent):
    logger.debug('def upload_torrent started')
    payload = {'apikey': cfg.prem_apikey}
    files = {'src': open(torrent, 'rb')}
    logger.debug('Uploading torrent to the cloud: %s', torrent)
    r = prem_connection("postfile", "https://www.premiumize.me/api/transfer/create", payload, files)
    if 'failed' not in r:
        response_content = json.loads(r.content)
        if response_content['status'] == "success":
            logger.debug('Upload successful: %s', torrent)
            return response_content['id']
        elif response_content['message'] == \
                'An error occured. Please try again and contact customer service if the problem persists.':
            gevent.sleep(10)
            r = prem_connection("postfile", "https://www.premiumize.me/api/transfer/create", payload, files)
            if 'failed' not in r:
                response_content = json.loads(r.content)
                if response_content['status'] == \
                        "success" or response_content['message'] == 'You already have this job added.':
                    logger.debug('Upload successful: %s', torrent)
                    return response_content['id']
        else:
            msg = 'Upload of torrent: %s failed, message: %s' % (torrent, response_content['message'])
            logger.error(msg)
            if response_content['message'] == 'You already have this job added.':
                return 'duplicate'
            if cfg.email_enabled:
                email('Upload of torrent failed', msg)
            return 'failed'
    else:
        return 'failed' 

def get_host_iface():
    """
    List the nic interface in host.
    :return: a list of the interfaces in host
    :rtype: builtin.list
    """
    proc_net_file = open(PROCFS_NET_PATH, 'r')
    host_iface_info = proc_net_file.read()
    proc_net_file.close()
    return [_.strip() for _ in re.findall("(.*):", host_iface_info)] 

def get_all_ips():
    """
    Get all IPv4 and IPv6 addresses from all interfaces.
    """
    ips = []

    # Get all ipv4 IPs.
    for iface in get_host_iface():
        try:
            ip_addr = get_ip_address_by_interface(iface)
        except NetError:
            pass
        else:
            if ip_addr is not None:
                ip_info = {
                    'iface': iface,
                    'addr': ip_addr,
                    'version': 'ipv4',
                }
                ips.append(IPAddress(info=ip_info))

    # Get all ipv6 IPs.
    if_inet6_fp = open('/proc/net/if_inet6', 'r')
    for line in if_inet6_fp.readlines():
        # ipv6_ip, dev_no, len_prefix, scope, iface_flag, iface
        ipv6_ip, dev_no, _, _, _, iface = line.split()
        ipv6_ip = ":".join([ipv6_ip[i:i + 4] for i in range(0, 32, 4)])
        ip_info = {
            'iface': iface,
            'addr': ipv6_ip,
            'version': 'ipv6',
            'scope': int(dev_no, 16)
        }
        ips.append(IPAddress(info=ip_info))
    if_inet6_fp.close()

    return ips 

def saveContracts(self):
        """保存所有合约对象到硬盘"""
        self.mainEngine.writeLog(u'持久化合约数据')
        f = shelve.open(self.contractFilePath)
        f['data'] = self.contractDict
        f.close()

    # ---------------------------------------------------------------------- 

def open_tap(devname, ifname, queues=1, vnet_hdr=True):
    """
    Open a tap device and returns its file descriptors which are used by
    fds=<fd1:fd2:..> parameter of qemu

    For single queue, only returns one file descriptor, it's used by
    fd=<fd> legacy parameter of qemu

    :param devname: TUN device path
    :param ifname: TAP interface name
    :param queues: Queue number
    :param vnet_hdr: Whether enable the vnet header
    """
    tapfds = []

    for i in range(int(queues)):
        try:
            tapfds.append(str(os.open(devname, os.O_RDWR)))
        except OSError as e:
            raise TAPModuleError(devname, "open", e)

        flags = arch.IFF_TAP | arch.IFF_NO_PI

        if vnet_mq_probe(int(tapfds[i])):
            flags |= arch.IFF_MULTI_QUEUE
        elif (int(queues) > 1):
            raise TAPCreationError(ifname, "Host doesn't support MULTI_QUEUE")

        if vnet_hdr and vnet_hdr_probe(int(tapfds[i])):
            flags |= arch.IFF_VNET_HDR

        ifr = struct.pack("16sh", ifname.encode(), flags)
        try:
            r = fcntl.ioctl(int(tapfds[i]), arch.TUNSETIFF, ifr)
        except IOError as details:
            raise TAPCreationError(ifname, details)

    return ':'.join(tapfds) 

def get_structure(self):
        """
        Get bridge list.
        """
        sysfs_path = "/sys/class/net"
        result = dict()
        for br_iface in os.listdir(sysfs_path):
            br_iface_path = os.path.join(sysfs_path, br_iface)
            try:
                if (not os.path.isdir(br_iface_path) or
                        "bridge" not in os.listdir(br_iface_path)):
                    continue
            except OSError as e:
                if e.errno == errno.ENOENT:
                    continue
                else:
                    raise e

            result[br_iface] = dict()
            # Get stp_state
            stp_state_path = os.path.join(br_iface_path, "bridge", "stp_state")
            with open(stp_state_path, "r") as stp_state_file:
                stp_state = int(stp_state_file.read().strip())
            # Assign with 'yes' or 'no' to keep ABI compatibility
            result[br_iface]["stp"] = "yes" if stp_state else "no"
            # Get ports
            brif_path = os.path.join(br_iface_path, "brif")
            result[br_iface]["iface"] = os.listdir(brif_path)
        return result 

def create_and_open_macvtap(ifname, mode="vepa", queues=1, base_if=None,
                            mac_addr=None):
    """
    Create a new macvtap device, open it, and return the fds

    :param ifname: macvtap interface name
    :param mode:  macvtap type mode ("vepa, bridge,..)
    :param queues: Queue number
    :param base_if: physical interface to create macvtap
    :param mac_addr: macvtap mac address
    """
    o_macvtap = create_macvtap(ifname, mode, base_if, mac_addr)
    return open_macvtap(o_macvtap, queues) 

def open(self):
        device = self.get_device()
        try:
            return os.open(device, os.O_RDWR)
        except OSError as e:
            raise TAPModuleError(device, "open", e) 

def get_stats(self):
        """
        Get the status information of the Interface
        """
        spl_re = re.compile(r"\s+")

        fp = open(PROCFS_NET_PATH)
        # Skip headers
        fp.readline()
        fp.readline()
        while True:
            data = fp.readline()
            if not data:
                return None

            name, stats_str = data.split(":")
            if name.strip() != self.name:
                continue

            stats = [int(a) for a in spl_re.split(stats_str.strip())]
            break

        titles = ["rx_bytes", "rx_packets", "rx_errs", "rx_drop", "rx_fifo",
                  "rx_frame", "rx_compressed", "rx_multicast", "tx_bytes",
                  "tx_packets", "tx_errs", "tx_drop", "tx_fifo", "tx_colls",
                  "tx_carrier", "tx_compressed"]
        return dict(zip(titles, stats))