Python humanize.naturalsize() Examples

The following are 30 code examples of humanize.naturalsize(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module humanize , or try the search function .
Example #1
Source File: detect.py    From yolo2-pytorch with GNU Lesser General Public License v3.0 6 votes vote down vote up
def __init__(self, args, config):
        self.args = args
        self.config = config
        self.cache_dir = utils.get_cache_dir(config)
        self.model_dir = utils.get_model_dir(config)
        self.category = utils.get_category(config, self.cache_dir if os.path.exists(self.cache_dir) else None)
        self.draw_bbox = utils.visualize.DrawBBox(self.category, colors=args.colors, thickness=args.thickness)
        self.anchors = torch.from_numpy(utils.get_anchors(config)).contiguous()
        self.height, self.width = tuple(map(int, config.get('image', 'size').split()))
        self.path, self.step, self.epoch = utils.train.load_model(self.model_dir)
        state_dict = torch.load(self.path, map_location=lambda storage, loc: storage)
        self.dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config, state_dict), self.anchors, len(self.category))
        self.dnn.load_state_dict(state_dict)
        self.inference = model.Inference(config, self.dnn, self.anchors)
        self.inference.eval()
        if torch.cuda.is_available():
            self.inference.cuda()
        logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.inference.state_dict().values())))
        self.cap = self.create_cap()
        self.keys = set(args.keys)
        self.resize = transform.parse_transform(config, config.get('transform', 'resize_test'))
        self.transform_image = transform.get_transform(config, config.get('transform', 'image_test').split())
        self.transform_tensor = transform.get_transform(config, config.get('transform', 'tensor').split()) 
Example #2
Source File: direct_link.py    From BotHub with Apache License 2.0 6 votes vote down vote up
def cm_ru(url: str) -> str:
    """ cloud.mail.ru direct links generator
    Using https://github.com/JrMasterModelBuilder/cmrudl.py"""
    reply = ''
    try:
        link = re.findall(r'\bhttps?://.*cloud\.mail\.ru\S+', url)[0]
    except IndexError:
        reply = "`No cloud.mail.ru links found`\n"
        return reply
    command = f'cmrudl -s {link}'
    result = popen(command).read()
    result = result.splitlines()[-1]
    try:
        data = json.loads(result)
    except json.decoder.JSONDecodeError:
        reply += "`Error: Can't extract the link`\n"
        return reply
    dl_url = data['download']
    name = data['file_name']
    size = naturalsize(int(data['file_size']))
    reply += f'[{name} ({size})]({dl_url})\n'
    return reply 
Example #3
Source File: direct_link.py    From BotHub with Apache License 2.0 6 votes vote down vote up
def mega_dl(url: str) -> str:
    """ MEGA.nz direct links generator
    Using https://github.com/tonikelope/megadown"""
    reply = ''
    try:
        link = re.findall(r'\bhttps?://.*mega.*\.nz\S+', url)[0]
    except IndexError:
        reply = "`No MEGA.nz links found`\n"
        return reply
    command = f'megadown -q -m {link}'
    result = popen(command).read()
    try:
        data = json.loads(result)
        print(data)
    except json.JSONDecodeError:
        reply += "`Error: Can't extract the link`\n"
        return reply
    dl_url = data['url']
    name = data['name']
    size = naturalsize(int(data['file_size']))
    reply += f'[{name} ({size})]({dl_url})\n'
    return reply 
Example #4
Source File: api.py    From 115wangpan with BSD 2-Clause "Simplified" License 6 votes vote down vote up
def __init__(self, api, add_time, file_id, info_hash, last_update,
                 left_time, move, name, peers, percent_done, rate_download,
                 size, status, cid, pid, url, *args, **kwargs):
        self.api = api
        self.cid = cid
        self.name = name
        self.add_time = add_time
        self.file_id = file_id
        self.info_hash = info_hash
        self.last_update = last_update
        self.left_time = left_time
        self.move = move
        self.peers = peers
        self.percent_done = percent_done
        self.rate_download = rate_download
        self.size = size
        self.size_human = humanize.naturalsize(size, binary=True)
        self.status = status
        self.url = url
        self._directory = None
        self._deleted = False
        self._count = -1 
Example #5
Source File: ao.py    From miui-updates-tracker with MIT License 6 votes vote down vote up
def add_rom(codename, link, info):
    update = {}
    file_size = naturalsize(int(get(link, stream=True).headers['Content-Length']))
    file = link.split('/')[-1]
    version = link.split('/')[3]
    android = link.split('_')[-2]
    update.update({"android": android})
    update.update({"codename": codename})
    update.update({"device": info['name']})
    update.update({"download": link})
    update.update({"filename": file})
    update.update({"size": file_size})
    update.update({"md5": "null"})
    update.update({"version": version})
    DATA.append(update)
    with open(f'stable_fastboot/{codename}.yml', 'w', newline='\n') as output:
        yaml.dump(update, output, Dumper=yaml.CDumper) 
Example #6
Source File: meta.py    From modelforge with Apache License 2.0 6 votes vote down vote up
def extract_model_meta(base_meta: dict, extra_meta: dict, model_url: str) -> dict:
    """
    Merge the metadata from the backend and the extra metadata into a dict which is suitable for \
    `index.json`.

    :param base_meta: tree["meta"] :class:`dict` containing data from the backend.
    :param extra_meta: dict containing data from the user, similar to `meta.json`.
    :param model_url: public URL of the model.
    :return: converted dict.
    """
    meta = {"default": {"default": base_meta["uuid"],
                        "description": base_meta["description"],
                        "code": extra_meta["code"]}}
    del base_meta["model"]
    del base_meta["uuid"]
    meta["model"] = base_meta
    meta["model"].update({k: extra_meta[k] for k in ("code", "datasets", "references", "tags",
                                                     "extra")})
    response = requests.get(model_url, stream=True)
    meta["model"]["size"] = humanize.naturalsize(int(response.headers["content-length"]))
    meta["model"]["url"] = model_url
    meta["model"]["created_at"] = format_datetime(meta["model"]["created_at"])
    return meta 
Example #7
Source File: model.py    From modelforge with Apache License 2.0 6 votes vote down vote up
def __str__(self):
        """Format model description as a string."""
        try:
            dump = self.dump()
        except NotImplementedError:
            dump = ""
        except AttributeError:
            return repr(self)
        if dump:
            dump = "\n" + dump
        meta = deepcopy(self.meta)
        meta["created_at"] = format_datetime(meta["created_at"])
        meta["size"] = humanize.naturalsize(self.size)
        try:
            meta["environment"]["packages"] = \
                " ".join("%s==%s" % tuple(p) for p in self.environment["packages"])
        except KeyError:
            pass
        return "%s%s" % (pformat(meta, width=1024), dump) 
Example #8
Source File: playlist.py    From Wavelink with MIT License 6 votes vote down vote up
def info(self, ctx):
        """Retrieve various Node/Server/Player information."""
        player = self.bot.wavelink.get_player(ctx.guild.id)
        node = player.node

        used = humanize.naturalsize(node.stats.memory_used)
        total = humanize.naturalsize(node.stats.memory_allocated)
        free = humanize.naturalsize(node.stats.memory_free)
        cpu = node.stats.cpu_cores

        fmt = f'**WaveLink:** `{wavelink.__version__}`\n\n' \
              f'Connected to `{len(self.bot.wavelink.nodes)}` nodes.\n' \
              f'Best available Node `{self.bot.wavelink.get_best_node().__repr__()}`\n' \
              f'`{len(self.bot.wavelink.players)}` players are distributed on nodes.\n' \
              f'`{node.stats.players}` players are distributed on server.\n' \
              f'`{node.stats.playing_players}` players are playing on server.\n\n' \
              f'Server Memory: `{used}/{total}` | `({free} free)`\n' \
              f'Server CPU: `{cpu}`\n\n' \
              f'Server Uptime: `{datetime.timedelta(milliseconds=node.stats.uptime)}`'
        await ctx.send(fmt) 
Example #9
Source File: receptive_field_analyzer.py    From yolo2-pytorch with GNU Lesser General Public License v3.0 6 votes vote down vote up
def __init__(self, args, config):
        self.args = args
        self.config = config
        self.model_dir = utils.get_model_dir(config)
        self.category = utils.get_category(config)
        self.anchors = torch.from_numpy(utils.get_anchors(config)).contiguous()
        self.dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config), self.anchors, len(self.category))
        self.dnn.eval()
        logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.dnn.state_dict().values())))
        if torch.cuda.is_available():
            self.dnn.cuda()
        self.height, self.width = tuple(map(int, config.get('image', 'size').split()))
        output = self.dnn(torch.autograd.Variable(utils.ensure_device(torch.zeros(1, 3, self.height, self.width)), volatile=True))
        _, _, self.rows, self.cols = output.size()
        self.i, self.j = self.rows // 2, self.cols // 2
        self.output = output[:, :, self.i, self.j]
        dataset = Dataset(self.height, self.width)
        try:
            workers = self.config.getint('data', 'workers')
        except configparser.NoOptionError:
            workers = multiprocessing.cpu_count()
        self.loader = torch.utils.data.DataLoader(dataset, batch_size=self.args.batch_size, num_workers=workers) 
Example #10
Source File: carml_stream.py    From carml with The Unlicense 6 votes vote down vote up
def stream_closed(self, stream, **kw):
        # print("closed", stream, self._active)
        if stream.id not in self._active:
            print(
                "Previously unknown stream to {stream.target_host} died".format(
                    stream=stream,
                )
            )
        else:
            bw = self._active[stream.id]
            print(
                "Stream {stream.id} to {stream.target_host}: {read} read, {written} written in {duration:.1f}s ({read_rate})".format(
                    stream=stream,
                    read=util.colors.green(humanize.naturalsize(bw.bytes_read())),
                    written=util.colors.red(humanize.naturalsize(bw.bytes_written())),
                    read_rate=humanize.naturalsize(sum(bw.rate())) + '/s',
                    duration=bw.duration(),
                )
            ) 
Example #11
Source File: meta.py    From EmoteCollector with GNU Affero General Public License v3.0 6 votes vote down vote up
def about(self, context):
		"""Tells you information about the bot itself."""
		# this command is based off of code provided by Rapptz under the MIT license
		# https://github.com/Rapptz/RoboDanny/blob/f6638d520ea0f559cb2ae28b862c733e1f165970/cogs/stats.py
		# Copyright © 2015 Rapptz

		embed = discord.Embed(description=self.bot.config['description'])

		embed.add_field(name='Latest changes', value=await self._latest_changes(), inline=False)

		owner = self.bot.get_user(self.bot.config.get('primary_owner', self.bot.owner_id))
		embed.set_author(name=str(owner), icon_url=owner.avatar_url)

		embed.add_field(name='Servers', value=await self.bot.cogs['Stats'].guild_count())

		cpu_usage = self.process.cpu_percent() / psutil.cpu_count()
		mem_usage = humanize.naturalsize(self.process.memory_full_info().uss)
		embed.add_field(name='Process', value=f'{mem_usage}\n{cpu_usage:.2f}% CPU')

		embed.add_field(name='Uptime', value=self.bot.cogs['BotBinMisc'].uptime(brief=True))
		embed.set_footer(text='Made with discord.py', icon_url='https://i.imgur.com/5BFecvA.png')

		await context.send(embed=embed) 
Example #12
Source File: cmd_receive.py    From magic-wormhole with MIT License 6 votes vote down vote up
def _handle_file(self, them_d):
        file_data = them_d["file"]
        self.abs_destname = self._decide_destname("file",
                                                  file_data["filename"])
        self.xfersize = file_data["filesize"]
        free = estimate_free_space(self.abs_destname)
        if free is not None and free < self.xfersize:
            self._msg(u"Error: insufficient free space (%sB) for file (%sB)" %
                      (free, self.xfersize))
            raise TransferRejectedError()

        self._msg(u"Receiving file (%s) into: %s" %
                  (naturalsize(self.xfersize),
                   os.path.basename(self.abs_destname)))
        self._ask_permission()
        tmp_destname = self.abs_destname + ".tmp"
        return open(tmp_destname, "wb") 
Example #13
Source File: convert_torch_onnx.py    From yolo2-pytorch with GNU Lesser General Public License v3.0 6 votes vote down vote up
def main():
    args = make_args()
    config = configparser.ConfigParser()
    utils.load_config(config, args.config)
    for cmd in args.modify:
        utils.modify_config(config, cmd)
    with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
        logging.config.dictConfig(yaml.load(f))
    height, width = tuple(map(int, config.get('image', 'size').split()))
    cache_dir = utils.get_cache_dir(config)
    model_dir = utils.get_model_dir(config)
    category = utils.get_category(config, cache_dir if os.path.exists(cache_dir) else None)
    anchors = utils.get_anchors(config)
    anchors = torch.from_numpy(anchors).contiguous()
    path, step, epoch = utils.train.load_model(model_dir)
    state_dict = torch.load(path, map_location=lambda storage, loc: storage)
    dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config, state_dict), anchors, len(category))
    inference = model.Inference(config, dnn, anchors)
    inference.eval()
    logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in inference.state_dict().values())))
    dnn.load_state_dict(state_dict)
    image = torch.autograd.Variable(torch.randn(args.batch_size, 3, height, width), volatile=True)
    path = model_dir + '.onnx'
    logging.info('save ' + path)
    torch.onnx.export(dnn, image, path, export_params=True, verbose=args.verbose) # PyTorch's bug 
Example #14
Source File: model.py    From gridsync with GNU General Public License v3.0 5 votes vote down vote up
def on_nodes_updated(self, num_connected, num_happy):
        if num_connected < num_happy:
            self.grid_status = "Connecting ({}/{} nodes){}".format(
                num_connected,
                num_happy,
                (" via Tor..." if self.gateway.use_tor else "..."),
            )
        elif num_connected >= num_happy:
            self.grid_status = "Connected to {} {}{} {} available".format(
                num_connected,
                "storage " + ("node" if num_connected == 1 else "nodes"),
                (" via Tor;" if self.gateway.use_tor else ";"),
                naturalsize(self.available_space),
            )
        self.gui.main_window.set_current_grid_status()  # TODO: Use pyqtSignal? 
Example #15
Source File: merge-jobs.py    From osm2vectortiles with MIT License 5 votes vote down vote up
def merge_results(rabbitmq_url, merge_target, result_queue_name):
    if not os.path.isfile(merge_target):
        raise ValueError('File {} does not exist'.format(merge_target))

    connection = pika.BlockingConnection(pika.URLParameters(rabbitmq_url))
    channel = connection.channel()
    channel.basic_qos(prefetch_count=3)
    channel.confirm_delivery()
    print('Connect with RabbitMQ server {}'.format(rabbitmq_url))

    def callback(ch, method, properties, body):
        msg = json.loads(body.decode('utf-8'))
        print('Download {}'.format(msg['url']))
        merge_source = download_mbtiles(msg['url'])
        action = functools.partial(merge_mbtiles, merge_source, merge_target)
        diff_size = compare_file_after_action(merge_target, action)

        print("Merge {} from {} into {}".format(
            humanize.naturalsize(diff_size),
            merge_source,
            merge_target
        ))

        os.remove(merge_source)
        ch.basic_ack(delivery_tag=method.delivery_tag)

    channel.basic_consume(callback, queue=result_queue_name)
    try:
        channel.start_consuming()
    except KeyboardInterrupt:
        channel.stop_consuming()

    connection.close() 
Example #16
Source File: base.py    From harpoon with MIT License 5 votes vote down vote up
def log_context_size(self, context, conf):
        context_size = humanize.naturalsize(os.stat(context.name).st_size)
        log.info(
            "Building '%s' in '%s' with %s of context",
            conf.name,
            conf.context.parent_dir,
            context_size,
        ) 
Example #17
Source File: convert_torch_onnx.py    From openpose-pytorch with GNU Lesser General Public License v3.0 5 votes vote down vote up
def main():
    args = make_args()
    config = configparser.ConfigParser()
    utils.load_config(config, args.config)
    for cmd in args.modify:
        utils.modify_config(config, cmd)
    with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
        logging.config.dictConfig(yaml.load(f))
    height, width = tuple(map(int, config.get('image', 'size').split()))
    model_dir = utils.get_model_dir(config)
    _, num_parts = utils.get_dataset_mappers(config)
    limbs_index = utils.get_limbs_index(config)
    path, step, epoch = utils.train.load_model(model_dir)
    state_dict = torch.load(path, map_location=lambda storage, loc: storage)
    config_channels_dnn = model.ConfigChannels(config, state_dict['dnn'])
    dnn = utils.parse_attr(config.get('model', 'dnn'))(config_channels_dnn)
    config_channels_stages = model.ConfigChannels(config, state_dict['stages'], config_channels_dnn.channels)
    channel_dict = model.channel_dict(num_parts, len(limbs_index))
    stages = nn.Sequential(*[utils.parse_attr(s)(config_channels_stages, channel_dict, config_channels_dnn.channels, str(i)) for i, s in enumerate(config.get('model', 'stages').split())])
    dnn.load_state_dict(config_channels_dnn.state_dict)
    stages.load_state_dict(config_channels_stages.state_dict)
    inference = model.Inference(config, dnn, stages)
    inference.eval()
    logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in inference.state_dict().values())))
    image = torch.randn(args.batch_size, 3, height, width)
    path = model_dir + '.onnx'
    logging.info('save ' + path)
    forward = inference.forward
    inference.forward = lambda self, *x: [[output[name] for name in 'parts, limbs'.split(', ')] for output in forward(self, *x)]
    torch.onnx.export(inference, image, path, export_params=True, verbose=args.verbose) 
Example #18
Source File: estimate.py    From openpose-pytorch with GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, args, config):
        self.args = args
        self.config = config
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.cache_dir = utils.get_cache_dir(config)
        self.model_dir = utils.get_model_dir(config)
        _, self.num_parts = utils.get_dataset_mappers(config)
        self.limbs_index = utils.get_limbs_index(config)
        if args.debug is None:
            self.draw_cluster = utils.visualize.DrawCluster(colors=args.colors, thickness=args.thickness)
        else:
            self.draw_feature = utils.visualize.DrawFeature()
            s = re.search('(-?[0-9]+)([a-z]+)(-?[0-9]+)', args.debug)
            stage = int(s.group(1))
            name = s.group(2)
            channel = int(s.group(3))
            self.get_feature = lambda outputs: outputs[stage][name][0][channel]
        self.height, self.width = tuple(map(int, config.get('image', 'size').split()))
        if args.caffe:
            init_net = caffe2_pb2.NetDef()
            with open(os.path.join(self.model_dir, 'init_net.pb'), 'rb') as f:
                init_net.ParseFromString(f.read())
            predict_net = caffe2_pb2.NetDef()
            with open(os.path.join(self.model_dir, 'predict_net.pb'), 'rb') as f:
                predict_net.ParseFromString(f.read())
            p = workspace.Predictor(init_net, predict_net)
            self.inference = lambda tensor: [{'parts': torch.from_numpy(parts), 'limbs': torch.from_numpy(limbs)} for parts, limbs in zip(*[iter(p.run([tensor.detach().cpu().numpy()]))] * 2)]
        else:
            self.step, self.epoch, self.dnn, self.stages = self.load()
            self.inference = model.Inference(config, self.dnn, self.stages)
            self.inference.eval()
            if torch.cuda.is_available():
                self.inference.cuda()
            logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.inference.state_dict().values())))
        self.cap = self.create_cap()
        self.keys = set(args.keys)
        self.resize = transform.parse_transform(config, config.get('transform', 'resize_test'))
        self.transform_image = transform.get_transform(config, config.get('transform', 'image_test').split())
        self.transform_tensor = transform.get_transform(config, config.get('transform', 'tensor').split()) 
Example #19
Source File: visualize.py    From openpose-pytorch with GNU Lesser General Public License v3.0 5 votes vote down vote up
def _draw_node(self, node):
        if hasattr(node, 'variable'):
            tensor = node.variable.data
            name = self.var_name[tensor._cdata]
            label = '\n'.join(map(str, [
                '%d: %s' % (self.index, name),
                list(tensor.size()),
                humanize.naturalsize(tensor.numpy().nbytes),
            ]))
            fillcolor, fontcolor = self._tensor_color(tensor)
            self.dot.node(str(id(node)), label, shape='note', fillcolor=fillcolor, fontcolor=fontcolor)
            self.drawn.add(name)
        else:
            self.dot.node(str(id(node)), '%d: %s' % (self.index, type(node).__name__), fillcolor='white') 
Example #20
Source File: visualize.py    From openpose-pytorch with GNU Lesser General Public License v3.0 5 votes vote down vote up
def _draw_tensor(self, node, tensor):
        name = self.var_name[tensor._cdata]
        label = '\n'.join(map(str, [
            name,
            list(tensor.size()),
            humanize.naturalsize(tensor.numpy().nbytes),
        ]))
        fillcolor, fontcolor = self._tensor_color(tensor)
        self.dot.node(name, label, style='filled, rounded', fillcolor=fillcolor, fontcolor=fontcolor)
        self.dot.edge(name, str(id(node)), style='dashed', arrowhead='none', arrowtail='none') 
Example #21
Source File: merge-jobs.py    From osm2vectortiles with MIT License 5 votes vote down vote up
def download_mbtiles(download_url):
    """Download MBTiles specified in message and return local filepath"""
    merge_source = os.path.basename(download_url)
    urlretrieve(download_url, merge_source)

    if not os.path.isfile(merge_source):
        raise ValueError('File {} does not exist'.format(merge_source))

    merge_source_size = os.path.getsize(merge_source)
    print('Download {} ({})'.format(
        download_url,
        humanize.naturalsize(merge_source_size))
    )
    return merge_source 
Example #22
Source File: utils.py    From coldfront with GNU General Public License v3.0 5 votes vote down vote up
def _humanize_user_quota(self, user_used, user_limit):

        user_quota = {
            'username': self.username,
            'used': humanize.naturalsize(user_used),
            'limit': humanize.naturalsize(user_limit),
            'percent_used': round((user_used / user_limit) * 100)
        }

        return user_quota 
Example #23
Source File: api.py    From 115wangpan with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def __init__(self, torrent, path, size, selected, *args, **kwargs):
        self.torrent = torrent
        self.path = path
        self.size = size
        self.size_human = humanize.naturalsize(size, binary=True)
        self.selected = selected 
Example #24
Source File: api.py    From 115wangpan with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def __init__(self, api, name, size, info_hash, file_count, files=None,
                 *args, **kwargs):
        self.api = api
        self.name = name
        self.size = size
        self.size_human = humanize.naturalsize(size, binary=True)
        self.info_hash = info_hash
        self.file_count = file_count
        self.files = files
        self.submitted = False 
Example #25
Source File: train.py    From openpose-pytorch with GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, args, config):
        self.args = args
        self.config = config
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.model_dir = utils.get_model_dir(config)
        self.cache_dir = utils.get_cache_dir(config)
        _, self.num_parts = utils.get_dataset_mappers(config)
        self.limbs_index = utils.get_limbs_index(config)
        logging.info('use cache directory ' + self.cache_dir)
        logging.info('tensorboard --logdir ' + self.model_dir)
        if args.delete:
            logging.warning('delete model directory: ' + self.model_dir)
            shutil.rmtree(self.model_dir, ignore_errors=True)
        os.makedirs(self.model_dir, exist_ok=True)
        with open(self.model_dir + '.ini', 'w') as f:
            config.write(f)

        self.step, self.epoch, self.dnn, self.stages = self.load()
        self.inference = model.Inference(self.config, self.dnn, self.stages)
        logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.inference.state_dict().values())))
        if self.args.finetune:
            path = os.path.expanduser(os.path.expandvars(self.args.finetune))
            logging.info('finetune from ' + path)
            self.finetune(self.dnn, path)
        self.inference = self.inference.to(self.device)
        self.inference.train()
        self.optimizer = eval(self.config.get('train', 'optimizer'))(filter(lambda p: p.requires_grad, self.inference.parameters()), self.args.learning_rate)

        self.saver = utils.train.Saver(self.model_dir, config.getint('save', 'keep'))
        self.timer_save = utils.train.Timer(config.getfloat('save', 'secs'), False)
        try:
            self.timer_eval = utils.train.Timer(eval(config.get('eval', 'secs')), config.getboolean('eval', 'first'))
        except configparser.NoOptionError:
            self.timer_eval = lambda: False
        self.summary_worker = SummaryWorker(self)
        self.summary_worker.start() 
Example #26
Source File: api.py    From 115wangpan with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def __init__(self, api, fid, cid, name, size, file_type, sha,
                 date_created, thumbnail, pickcode, *args, **kwargs):

        super(File, self).__init__(api, cid, name)

        self.fid = fid
        self.size = size
        self.size_human = humanize.naturalsize(size, binary=True)
        self.file_type = file_type
        self.sha = sha
        self.date_created = date_created
        self.thumbnail = thumbnail
        self.pickcode = pickcode
        self._directory = None
        self._download_url = None 
Example #27
Source File: utils.py    From coldfront with GNU General Public License v3.0 5 votes vote down vote up
def _humanize_group_quota(self, group_user, group_limit):

        group_quota = {
            'used': humanize.naturalsize(group_user),
            'limit': humanize.naturalsize(group_limit),
            'percent_used': round((group_user / group_limit) * 100)
        }

        return group_quota 
Example #28
Source File: fhost.py    From 0x0 with ISC License 5 votes vote down vote up
def store_url(url, addr):
    if is_fhost_url(url):
        return segfault(508)

    h = { "Accept-Encoding" : "identity" }
    r = requests.get(url, stream=True, verify=False, headers=h)

    try:
        r.raise_for_status()
    except requests.exceptions.HTTPError as e:
        return str(e) + "\n"

    if "content-length" in r.headers:
        l = int(r.headers["content-length"])

        if l < app.config["MAX_CONTENT_LENGTH"]:
            def urlfile(**kwargs):
                return type('',(),kwargs)()

            f = urlfile(stream=r.raw, content_type=r.headers["content-type"], filename="")

            return store_file(f, addr)
        else:
            hl = naturalsize(l, binary = True)
            hml = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True)

            return "Remote file too large ({0} > {1}).\n".format(hl, hml), 413
    else:
        return "Could not determine remote file size (no Content-Length in response header; shoot admin).\n", 411 
Example #29
Source File: cmd_receive.py    From magic-wormhole with MIT License 5 votes vote down vote up
def _handle_directory(self, them_d):
        file_data = them_d["directory"]
        zipmode = file_data["mode"]
        if zipmode != "zipfile/deflated":
            self._msg(u"Error: unknown directory-transfer mode '%s'" %
                      (zipmode, ))
            raise RespondError("unknown mode")
        self.abs_destname = self._decide_destname("directory",
                                                  file_data["dirname"])
        self.xfersize = file_data["zipsize"]
        free = estimate_free_space(self.abs_destname)
        if free is not None and free < file_data["numbytes"]:
            self._msg(
                u"Error: insufficient free space (%sB) for directory (%sB)" %
                (free, file_data["numbytes"]))
            raise TransferRejectedError()

        self._msg(u"Receiving directory (%s) into: %s/" %
                  (naturalsize(self.xfersize),
                   os.path.basename(self.abs_destname)))
        self._msg(u"%d files, %s (uncompressed)" %
                  (file_data["numfiles"], naturalsize(file_data["numbytes"])))
        self._ask_permission()
        f = tempfile.SpooledTemporaryFile()
        # workaround for https://bugs.python.org/issue26175 (STF doesn't
        # fully implement IOBase abstract class), which breaks the new
        # zipfile in py3.7.0 that expects .seekable
        if not hasattr(f, "seekable"):
            # AFAICT all the filetypes that STF wraps can seek
            f.seekable = lambda: True
        return f 
Example #30
Source File: api.py    From 115wangpan with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def get_storage_info(self, human=False):
        """
        Get storage info

        :param bool human: whether return human-readable size
        :return: total and used storage
        :rtype: dict

        """
        res = self._req_get_storage_info()
        if human:
            res['total'] = humanize.naturalsize(res['total'], binary=True)
            res['used'] = humanize.naturalsize(res['used'], binary=True)
        return res