Python botocore.exceptions.IncompleteReadError() Examples

The following are 25 code examples of botocore.exceptions.IncompleteReadError(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module botocore.exceptions , or try the search function .
Example #1
Source File: __init__.py    From aws-builders-fair-projects with Apache License 2.0 6 votes vote down vote up
def _get_object(self, bucket, key, filename, extra_args, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        last_exception = None
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename,
                                           extra_args, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                continue
        raise RetriesExceededError(last_exception) 
Example #2
Source File: __init__.py    From aws-extender with MIT License 6 votes vote down vote up
def _get_object(self, bucket, key, filename, extra_args, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        last_exception = None
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename,
                                           extra_args, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                continue
        raise RetriesExceededError(last_exception) 
Example #3
Source File: __init__.py    From faces with GNU General Public License v2.0 6 votes vote down vote up
def _get_object(self, bucket, key, filename, extra_args, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        last_exception = None
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename,
                                           extra_args, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                continue
        raise RetriesExceededError(last_exception) 
Example #4
Source File: __init__.py    From AWS-Transit-Gateway-Demo-MultiAccount with MIT License 6 votes vote down vote up
def _get_object(self, bucket, key, filename, extra_args, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        last_exception = None
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename,
                                           extra_args, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                continue
        raise RetriesExceededError(last_exception) 
Example #5
Source File: __init__.py    From bash-lambda-layer with MIT License 6 votes vote down vote up
def _get_object(self, bucket, key, filename, extra_args, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        last_exception = None
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename,
                                           extra_args, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                continue
        raise RetriesExceededError(last_exception) 
Example #6
Source File: __init__.py    From mxnet-lambda with Apache License 2.0 6 votes vote down vote up
def _get_object(self, bucket, key, filename, extra_args, callback):
        # precondition: num_download_attempts > 0
        max_attempts = self._config.num_download_attempts
        last_exception = None
        for i in range(max_attempts):
            try:
                return self._do_get_object(bucket, key, filename,
                                           extra_args, callback)
            except (socket.timeout, socket.error,
                    ReadTimeoutError, IncompleteReadError) as e:
                # TODO: we need a way to reset the callback if the
                # download failed.
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                continue
        raise RetriesExceededError(last_exception) 
Example #7
Source File: __init__.py    From mxnet-lambda with Apache License 2.0 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #8
Source File: __init__.py    From aws-builders-fair-projects with Apache License 2.0 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #9
Source File: response.py    From aws-builders-fair-projects with Apache License 2.0 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #10
Source File: __init__.py    From aws-extender with MIT License 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #11
Source File: response.py    From aws-extender with MIT License 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #12
Source File: __init__.py    From AWS-Transit-Gateway-Demo-MultiAccount with MIT License 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #13
Source File: response.py    From AWS-Transit-Gateway-Demo-MultiAccount with MIT License 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #14
Source File: __init__.py    From AWS-Transit-Gateway-Demo-MultiAccount with MIT License 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #15
Source File: response.py    From AWS-Transit-Gateway-Demo-MultiAccount with MIT License 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #16
Source File: response.py    From aiobotocore with Apache License 2.0 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._self_content_length is not None and \
                self._self_amount_read != int(self._self_content_length):
            raise IncompleteReadError(
                actual_bytes=self._self_amount_read,
                expected_bytes=int(self._self_content_length)) 
Example #17
Source File: __init__.py    From bash-lambda-layer with MIT License 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #18
Source File: response.py    From bash-lambda-layer with MIT License 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #19
Source File: response.py    From deepWordBug with Apache License 2.0 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #20
Source File: __init__.py    From faces with GNU General Public License v2.0 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #21
Source File: response.py    From faces with GNU General Public License v2.0 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #22
Source File: __init__.py    From faces with GNU General Public License v2.0 5 votes vote down vote up
def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index) 
Example #23
Source File: response.py    From faces with GNU General Public License v2.0 5 votes vote down vote up
def _verify_content_length(self):
        # See: https://github.com/kennethreitz/requests/issues/1855
        # Basically, our http library doesn't do this for us, so we have
        # to do this ourself.
        if self._content_length is not None and \
                self._amount_read != int(self._content_length):
            raise IncompleteReadError(
                actual_bytes=self._amount_read,
                expected_bytes=int(self._content_length)) 
Example #24
Source File: test_response.py    From aiobotocore with Apache License 2.0 5 votes vote down vote up
def test_streaming_body_with_single_read():
    body = AsyncBytesIO(b'123456789')
    stream = response.StreamingBody(body, content_length=10)
    with pytest.raises(IncompleteReadError):
        await stream.read() 
Example #25
Source File: test_response.py    From aiobotocore with Apache License 2.0 5 votes vote down vote up
def test_streaming_body_with_invalid_length():
    body = AsyncBytesIO(b'123456789')
    stream = response.StreamingBody(body, content_length=10)
    with pytest.raises(IncompleteReadError):
        assert await stream.read(9) == b'123456789'
        # The next read will have nothing returned and raise
        # an IncompleteReadError because we were expectd 10 bytes, not 9.
        await stream.read()