Python requests_toolbelt.MultipartEncoderMonitor() Examples

def _multipart_post(self, data):
        encoder = MultipartEncoder(fields=data)
        monitor = MultipartEncoderMonitor(encoder, callback=self._progress_bar)
        r = requests.post(self.file_host_url,
                          data=monitor,
                          headers={'Content-Type': monitor.content_type})
        return r 

def api_upload(service, encData, encMeta, keys):
    '''
       Uploads data to Send.
       Caution! Data is uploaded as given, this function will not encrypt it for you
    '''
    service += 'api/upload'
    files = requests_toolbelt.MultipartEncoder(fields={'file': ('blob', encData, 'application/octet-stream') })
    pbar = progbar(files.len)
    monitor = requests_toolbelt.MultipartEncoderMonitor(files, lambda files: pbar.update(monitor.bytes_read - pbar.n))

    headers = {
        'X-File-Metadata' : unpadded_urlsafe_b64encode(encMeta),
        'Authorization' : 'send-v1 ' + unpadded_urlsafe_b64encode(keys.authKey),
        'Content-type' : monitor.content_type
    }

    r = requests.post(service, data=monitor, headers=headers, stream=True)
    r.raise_for_status()
    pbar.close()

    body_json = r.json()
    secretUrl = body_json['url'] + '#' + unpadded_urlsafe_b64encode(keys.secretKey)
    fileId = body_json['id']
    fileNonce = unpadded_urlsafe_b64decode(r.headers['WWW-Authenticate'].replace('send-v1 ', ''))
    try:
        owner_token = body_json['owner']
    except:
        owner_token = body_json['delete']
    return secretUrl, fileId, fileNonce, owner_token 

def upload_with_progress_bar(data, url, kwargs, label=None, token=None):
    """
    Uses multipart data to show progress of upload to the user.
    Requires request.files['data'].read() instead of request.data on the backend site.
    :param data: path to file to upload
    :param url: target url
    :param kwargs: additional args for the post request
    :param label: label of progress bar
    :param token: token to authorize the request
    :return: response from server
    """
    encoder = MultipartEncoder({'data': ('data', data, 'text/plain')})

    with tqdm(desc=label,
              total=encoder.len,
              disable=not label,
              dynamic_ncols=True,
              unit='B',
              unit_scale=True,
              unit_divisor=1024) as bar:
        multipart_monitor = MultipartEncoderMonitor(encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n))
        r = requests.post(url,
                          data=multipart_monitor,
                          headers={'Content-Type': multipart_monitor.content_type, 'X-Token': token},
                          params=kwargs)

    return r 

def create(self, module, cli_default=None):
        try:
            upload_files, total_file_size = get_files_in_current_directory(file_type='code')
        except OSError:
            sys.exit("Directory contains too many files to upload. If you have data files in the current directory, "
                     "please upload them separately using \"floyd data\" command and remove them from here.\n"
                     "See http://docs.floydhub.com/faqs/job/#i-get-too-many-open-files-error-when-i-run-my-project "
                     "for more details on how to fix this.")

        if total_file_size > self.MAX_UPLOAD_SIZE:
            sys.exit(("Code size too large to sync, please keep it under %s.\n"
                      "If you have data files in the current directory, please upload them "
                      "separately using \"floyd data\" command and remove them from here.\n"
                      "You may find the following documentation useful:\n\n"
                      "\thttps://docs.floydhub.com/guides/create_and_upload_dataset/\n"
                      "\thttps://docs.floydhub.com/guides/data/mounting_data/\n"
                      "\thttps://docs.floydhub.com/guides/floyd_ignore/") % (sizeof_fmt(self.MAX_UPLOAD_SIZE)))

        floyd_logger.info("Creating project run. Total upload size: %s",
                          sizeof_fmt(total_file_size))
        floyd_logger.debug("Creating module. Uploading: %s files",
                           len(upload_files))
        floyd_logger.info("Syncing code ...")

        # Add request data
        args_payload = module.to_dict()
        if cli_default:
            args_payload['cli_default'] = cli_default
        upload_files.append(("json", json.dumps(args_payload)))
        multipart_encoder = MultipartEncoder(
            fields=upload_files
        )

        # Attach progress bar
        progress_callback, bar = create_progress_callback(multipart_encoder)
        multipart_encoder_monitor = MultipartEncoderMonitor(multipart_encoder, progress_callback)

        try:
            response = self.request("POST",
                                    self.url,
                                    data=multipart_encoder_monitor,
                                    headers={"Content-Type": multipart_encoder.content_type},
                                    timeout=3600)
        finally:
            # always make sure we clear the console
            bar.done()
        return response.json().get("id") 

def _upload_small_file(self, file_path, folder_id=-1, callback=None) -> (int, int, bool):
        """绕过格式限制上传不超过 max_size 的文件"""
        if not os.path.isfile(file_path):
            return LanZouCloud.PATH_ERROR, 0, True

        need_delete = False  # 上传完成是否删除
        if not is_name_valid(os.path.basename(file_path)):  # 不允许上传的格式
            file_path = let_me_upload(file_path)  # 添加了报尾的新文件
            need_delete = True

        # 文件已经存在同名文件就删除
        filename = name_format(os.path.basename(file_path))
        file_list = self.get_file_list(folder_id)
        if file_list.find_by_name(filename):
            self.delete(file_list.find_by_name(filename).id)
        logger.debug(f'Upload file {file_path=} to {folder_id=}')

        file = open(file_path, 'rb')
        post_data = {
            "task": "1",
            "folder_id": str(folder_id),
            "id": "WU_FILE_0",
            "name": filename,
            "upload_file": (filename, file, 'application/octet-stream')
        }

        post_data = MultipartEncoder(post_data)
        tmp_header = self._headers.copy()
        tmp_header['Content-Type'] = post_data.content_type

        # MultipartEncoderMonitor 每上传 8129 bytes数据调用一次回调函数，问题根源是 httplib 库
        # issue : https://github.com/requests/toolbelt/issues/75
        # 上传完成后，回调函数会被错误的多调用一次(强迫症受不了)。因此，下面重新封装了回调函数，修改了接受的参数，并阻断了多余的一次调用
        self._upload_finished_flag = False  # 上传完成的标志

        def _call_back(read_monitor):
            if callback is not None:
                if not self._upload_finished_flag:
                    callback(filename, read_monitor.len, read_monitor.bytes_read)
                if read_monitor.len == read_monitor.bytes_read:
                    self._upload_finished_flag = True

        monitor = MultipartEncoderMonitor(post_data, _call_back)
        result = self._post('https://pc.woozooo.com/fileup.php', monitor, headers=tmp_header, timeout=None)
        if not result:  # 网络异常
            logger.debug('Upload file no result')
            return LanZouCloud.NETWORK_ERROR, 0, True
        else:
            result = result.json()
        if result["zt"] != 1:
            logger.debug(f'Upload failed: {result=}')
            return LanZouCloud.FAILED, 0, True  # 上传失败

        file_id = result["text"][0]["id"]
        self.set_passwd(file_id)  # 文件上传后默认关闭提取码
        if need_delete:
            file.close()
            os.remove(file_path)
        return LanZouCloud.SUCCESS, int(file_id), True 

def upload(self,
               url,
               files,
               files_size,
               params=None,
               json_data=None,
               timeout=None,
               headers=None,
               session=None):

        if files_size > settings.WARN_UPLOAD_SIZE:
            logger.warning(
                "You are uploading %s, there's a hard limit of %s.\n"
                "If you have data files in the current directory, "
                "please make sure to add them to .polyaxonignore or "
                "add them directly to your data volume, or upload them "
                "separately using `polyaxon data` command and remove them from here.\n",
                self.format_sizeof(settings.WARN_UPLOAD_SIZE),
                self.format_sizeof(settings.MAX_UPLOAD_SIZE))

        if files_size > settings.MAX_UPLOAD_SIZE:
            raise PolyaxonShouldExitError(
                "Files too large to sync, please keep it under {}.\n"
                "If you have data files in the current directory, "
                "please add them directly to your data volume, or upload them "
                "separately using `polyaxon data` command and remove them from here.\n".format(
                    self.format_sizeof(settings.MAX_UPLOAD_SIZE)))

        files = to_list(files)
        if json_data:
            files.append(('json', json.dumps(json_data)))

        multipart_encoder = MultipartEncoder(
            fields=files
        )
        request_headers = headers or {}
        request_headers.update({"Content-Type": multipart_encoder.content_type})

        # Attach progress bar
        progress_callback, callback_bar = self.create_progress_callback(multipart_encoder)
        multipart_encoder_monitor = MultipartEncoderMonitor(multipart_encoder, progress_callback)

        timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT

        try:
            response = self.put(url=url,
                                params=params,
                                data=multipart_encoder_monitor,
                                headers=request_headers,
                                timeout=timeout,
                                session=session)
        finally:
            # always make sure we clear the console
            callback_bar.done()

        return response