Python google.appengine.ext.blobstore.BlobKey() Examples
The following are 30
code examples of google.appengine.ext.blobstore.BlobKey().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
google.appengine.ext.blobstore
, or try the search function
.
Example #1
Source File: input_readers.py From appengine-mapreduce with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_position, end_position): """Initializes this instance with the given blob key and character range. This BlobstoreInputReader will read from the first record starting after strictly after start_position until the first record ending at or after end_position (exclusive). As an exception, if start_position is 0, then this InputReader starts reading at the first record. Args: blob_key: the BlobKey that this input reader is processing. start_position: the position to start reading at. end_position: a position in the last record to read. """ self._blob_key = blob_key self._blob_reader = blobstore.BlobReader(blob_key, self._BLOB_BUFFER_SIZE, start_position) self._end_position = end_position self._has_iterated = False self._read_before_start = bool(start_position)
Example #2
Source File: input_readers.py From locality-sensitive-hashing with MIT License | 6 votes |
def validate(cls, mapper_spec): """Validates mapper spec and all mapper parameters. Args: mapper_spec: The MapperSpec for this InputReader. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Mapper input reader class mismatch") params = _get_params(mapper_spec) if cls.BLOB_KEY_PARAM not in params: raise BadReaderParamsError("Must specify 'blob_key' for mapper input") blob_key = params[cls.BLOB_KEY_PARAM] blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) if not blob_info: raise BadReaderParamsError("Could not find blobinfo for key %s" % blob_key)
Example #3
Source File: input_readers.py From locality-sensitive-hashing with MIT License | 6 votes |
def __init__(self, blob_key, start_file_index, end_file_index, offset, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipLineInputReader will read from the file with index start_file_index up to but not including the file with index end_file_index. It will return lines starting at offset within file[start_file_index] Args: blob_key: the BlobKey that this input reader is processing. start_file_index: the index of the first file to read within the zip. end_file_index: the index of the first file that will not be read. offset: the byte offset within blob_key.zip[start_file_index] to start reading. The reader will continue to the end of the file. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_file_index = start_file_index self._end_file_index = end_file_index self._initial_offset = offset self._reader = _reader self._zip = None self._entries = None self._filestream = None
Example #4
Source File: input_readers.py From browserscope with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_index, end_index, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipInputReader will read from the file with index start_index up to but not including the file with index end_index. Args: blob_key: the BlobKey that this input reader is processing. start_index: the index of the first file to read. end_index: the index of the first file that will not be read. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_index = start_index self._end_index = end_index self._reader = _reader self._zip = None self._entries = None
Example #5
Source File: input_readers.py From browserscope with Apache License 2.0 | 6 votes |
def validate(cls, mapper_spec): """Validates mapper spec and all mapper parameters. Args: mapper_spec: The MapperSpec for this InputReader. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Mapper input reader class mismatch") params = _get_params(mapper_spec) if cls.BLOB_KEY_PARAM not in params: raise BadReaderParamsError("Must specify 'blob_key' for mapper input") blob_key = params[cls.BLOB_KEY_PARAM] blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) if not blob_info: raise BadReaderParamsError("Could not find blobinfo for key %s" % blob_key)
Example #6
Source File: input_readers.py From browserscope with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_file_index, end_file_index, offset, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipLineInputReader will read from the file with index start_file_index up to but not including the file with index end_file_index. It will return lines starting at offset within file[start_file_index] Args: blob_key: the BlobKey that this input reader is processing. start_file_index: the index of the first file to read within the zip. end_file_index: the index of the first file that will not be read. offset: the byte offset within blob_key.zip[start_file_index] to start reading. The reader will continue to the end of the file. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_file_index = start_file_index self._end_file_index = end_file_index self._initial_offset = offset self._reader = _reader self._zip = None self._entries = None self._filestream = None
Example #7
Source File: input_readers.py From browserscope with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_position, end_position): """Initializes this instance with the given blob key and character range. This BlobstoreInputReader will read from the first record starting after strictly after start_position until the first record ending at or after end_position (exclusive). As an exception, if start_position is 0, then this InputReader starts reading at the first record. Args: blob_key: the BlobKey that this input reader is processing. start_position: the position to start reading at. end_position: a position in the last record to read. """ self._blob_key = blob_key self._blob_reader = blobstore.BlobReader(blob_key, self._BLOB_BUFFER_SIZE, start_position) self._end_position = end_position self._has_iterated = False self._read_before_start = bool(start_position)
Example #8
Source File: input_readers.py From locality-sensitive-hashing with MIT License | 6 votes |
def __init__(self, blob_key, start_position, end_position): """Initializes this instance with the given blob key and character range. This BlobstoreInputReader will read from the first record starting after strictly after start_position until the first record ending at or after end_position (exclusive). As an exception, if start_position is 0, then this InputReader starts reading at the first record. Args: blob_key: the BlobKey that this input reader is processing. start_position: the position to start reading at. end_position: a position in the last record to read. """ self._blob_key = blob_key self._blob_reader = blobstore.BlobReader(blob_key, self._BLOB_BUFFER_SIZE, start_position) self._end_position = end_position self._has_iterated = False self._read_before_start = bool(start_position)
Example #9
Source File: input_readers.py From appengine-mapreduce with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_index, end_index, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipInputReader will read from the file with index start_index up to but not including the file with index end_index. Args: blob_key: the BlobKey that this input reader is processing. start_index: the index of the first file to read. end_index: the index of the first file that will not be read. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_index = start_index self._end_index = end_index self._reader = _reader self._zip = None self._entries = None
Example #10
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_index, end_index, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipInputReader will read from the file with index start_index up to but not including the file with index end_index. Args: blob_key: the BlobKey that this input reader is processing. start_index: the index of the first file to read. end_index: the index of the first file that will not be read. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_index = start_index self._end_index = end_index self._reader = _reader self._zip = None self._entries = None
Example #11
Source File: input_readers.py From appengine-mapreduce with Apache License 2.0 | 6 votes |
def validate(cls, mapper_spec): """Validates mapper spec and all mapper parameters. Args: mapper_spec: The MapperSpec for this InputReader. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Mapper input reader class mismatch") params = _get_params(mapper_spec) if cls.BLOB_KEY_PARAM not in params: raise BadReaderParamsError("Must specify 'blob_key' for mapper input") blob_key = params[cls.BLOB_KEY_PARAM] blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) if not blob_info: raise BadReaderParamsError("Could not find blobinfo for key %s" % blob_key)
Example #12
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_file_index, end_file_index, offset, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipLineInputReader will read from the file with index start_file_index up to but not including the file with index end_file_index. It will return lines starting at offset within file[start_file_index] Args: blob_key: the BlobKey that this input reader is processing. start_file_index: the index of the first file to read within the zip. end_file_index: the index of the first file that will not be read. offset: the byte offset within blob_key.zip[start_file_index] to start reading. The reader will continue to the end of the file. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_file_index = start_file_index self._end_file_index = end_file_index self._initial_offset = offset self._reader = _reader self._zip = None self._entries = None self._filestream = None
Example #13
Source File: input_readers.py From appengine-mapreduce with Apache License 2.0 | 6 votes |
def __init__(self, blob_key, start_file_index, end_file_index, offset, _reader=blobstore.BlobReader): """Initializes this instance with the given blob key and file range. This BlobstoreZipLineInputReader will read from the file with index start_file_index up to but not including the file with index end_file_index. It will return lines starting at offset within file[start_file_index] Args: blob_key: the BlobKey that this input reader is processing. start_file_index: the index of the first file to read within the zip. end_file_index: the index of the first file that will not be read. offset: the byte offset within blob_key.zip[start_file_index] to start reading. The reader will continue to the end of the file. _reader: a callable that returns a file-like object for reading blobs. Used for dependency injection. """ self._blob_key = blob_key self._start_file_index = start_file_index self._end_file_index = end_file_index self._initial_offset = offset self._reader = _reader self._zip = None self._entries = None self._filestream = None
Example #14
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInputTwoBlobsThreeShards(self): """Corner case: Shards don't split nicely so we get too few.""" readers = self.split_input(2, 3) # Note we might be able to make this return 3 with a more clever algorithm. self.assertEqual(2, len(readers)) self.assertEqual("blobstore.BlobKey('blob0'):[0, 3]:0", str(readers[0])) self.assertEqual("blobstore.BlobKey('blob1'):[0, 3]:0", str(readers[1]))
Example #15
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInput(self): """Test that split_input functions as expected.""" mapper_spec = model.MapperSpec.from_json({ "mapper_handler_spec": "FooHandler", "mapper_input_reader": self.READER_NAME, "mapper_params": {"blob_key": ["foo"]}, "mapper_shard_count": 2}) readers = input_readers.BlobstoreZipInputReader.split_input( mapper_spec, self.mockZipReader) self.assertEqual(len(readers), 2) self.assertEqual(str(readers[0]), "blobstore.BlobKey(['foo']):[0, 7]") self.assertEqual(str(readers[1]), "blobstore.BlobKey(['foo']):[7, 10]")
Example #16
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInputThreeBlobsTwoShards(self): """Corner case: More blobs than requested shards.""" readers = self.split_input(3, 2) self.assertEqual(3, len(readers)) self.assertEqual("blobstore.BlobKey('blob0'):[0, 3]:0", str(readers[0])) self.assertEqual("blobstore.BlobKey('blob1'):[0, 3]:0", str(readers[1])) self.assertEqual("blobstore.BlobKey('blob2'):[0, 3]:0", str(readers[2]))
Example #17
Source File: input_readers.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def split_input(cls, mapper_spec): """Returns a list of shard_count input_spec_shards for input_spec. Args: mapper_spec: The mapper specification to split from. Must contain 'blob_keys' parameter with one or more blob keys. Returns: A list of BlobstoreInputReaders corresponding to the specified shards. """ params = _get_params(mapper_spec) blob_keys = params[cls.BLOB_KEYS_PARAM] if isinstance(blob_keys, basestring): # This is a mechanism to allow multiple blob keys (which do not contain # commas) in a single string. It may go away. blob_keys = blob_keys.split(",") blob_sizes = {} for blob_key in blob_keys: blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) blob_sizes[blob_key] = blob_info.size shard_count = min(cls._MAX_SHARD_COUNT, mapper_spec.shard_count) shards_per_blob = shard_count // len(blob_keys) if shards_per_blob == 0: shards_per_blob = 1 chunks = [] for blob_key, blob_size in blob_sizes.items(): blob_chunk_size = blob_size // shards_per_blob for i in xrange(shards_per_blob - 1): chunks.append(BlobstoreLineInputReader.from_json( {cls.BLOB_KEY_PARAM: blob_key, cls.INITIAL_POSITION_PARAM: blob_chunk_size * i, cls.END_POSITION_PARAM: blob_chunk_size * (i + 1)})) chunks.append(BlobstoreLineInputReader.from_json( {cls.BLOB_KEY_PARAM: blob_key, cls.INITIAL_POSITION_PARAM: blob_chunk_size * (shards_per_blob - 1), cls.END_POSITION_PARAM: blob_size})) return chunks
Example #18
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInputTwoBlobsSixShards(self): """Corner case: Shards don't split nicely so we get too few.""" readers = self.split_input(2, 6) # Note we might be able to make this return 6 with a more clever algorithm. self.assertEqual(4, len(readers)) self.assertEqual("blobstore.BlobKey('blob0'):[0, 2]:0", str(readers[0])) self.assertEqual("blobstore.BlobKey('blob0'):[2, 3]:0", str(readers[1])) self.assertEqual("blobstore.BlobKey('blob1'):[0, 2]:0", str(readers[2])) self.assertEqual("blobstore.BlobKey('blob1'):[2, 3]:0", str(readers[3]))
Example #19
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInputTwoBlobsTwoShards(self): """Simple case: Ask for num shards == num blobs.""" readers = self.split_input(2, 2) self.assertEqual(2, len(readers)) self.assertEqual("blobstore.BlobKey('blob0'):[0, 3]:0", str(readers[0])) self.assertEqual("blobstore.BlobKey('blob1'):[0, 3]:0", str(readers[1]))
Example #20
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInputOneBlobFourShards(self): """Corner case: Ask for more shards than we can deliver.""" readers = self.split_input(1, 4) self.assertEqual(2, len(readers)) self.assertEqual("blobstore.BlobKey('blob0'):[0, 2]:0", str(readers[0])) self.assertEqual("blobstore.BlobKey('blob0'):[2, 3]:0", str(readers[1]))
Example #21
Source File: input_readers_test.py From appengine-mapreduce with Apache License 2.0 | 5 votes |
def testSplitInputOneBlob(self): """Simple case: split one blob into two groups.""" readers = self.split_input(1, 2) self.assertEqual(2, len(readers)) self.assertEqual("blobstore.BlobKey('blob0'):[0, 2]:0", str(readers[0])) self.assertEqual("blobstore.BlobKey('blob0'):[2, 3]:0", str(readers[1]))
Example #22
Source File: blob_upload_test.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def test_store_and_build_forward_message_total_size_exceeded(self): """Test upload with all blobs larger than the total allowed size.""" self.generate_blob_key().AndReturn(blobstore.BlobKey('item1')) self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40)) self.mox.ReplayAll() form = FakeForm({ 'field1': FakeForm(name='field1', file=StringIO.StringIO('a'), type='image/png', type_options={'a': 'b', 'x': 'y'}, filename='stuff.png', headers={'h1': 'v1', 'h2': 'v2', }), 'field2': FakeForm(name='field2', file=StringIO.StringIO('longerfile'), type='application/pdf', type_options={}, filename='stuff.pdf', headers={}), }) self.assertRaises(webob.exc.HTTPRequestEntityTooLarge, self.handler.store_and_build_forward_message, form, '================1234==', max_bytes_total=3) self.mox.VerifyAll() blob1 = blobstore.get('item1') self.assertIsNone(blob1)
Example #23
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def __str__(self): """Returns the string representation of this reader. Returns: string blobkey:[start file num, end file num]:current offset. """ return "blobstore.BlobKey(%r):[%d, %d]:%d" % ( self._blob_key, self._start_file_index, self._end_file_index, self._next_offset())
Example #24
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def validate(cls, mapper_spec): """Validates mapper spec and all mapper parameters. Args: mapper_spec: The MapperSpec for this InputReader. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Mapper input reader class mismatch") params = _get_params(mapper_spec) if cls.BLOB_KEYS_PARAM not in params: raise BadReaderParamsError("Must specify 'blob_keys' for mapper input") blob_keys = params[cls.BLOB_KEYS_PARAM] if isinstance(blob_keys, basestring): blob_keys = blob_keys.split(",") if len(blob_keys) > cls._MAX_BLOB_KEYS_COUNT: raise BadReaderParamsError("Too many 'blob_keys' for mapper input") if not blob_keys: raise BadReaderParamsError("No 'blob_keys' specified for mapper input") for blob_key in blob_keys: blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) if not blob_info: raise BadReaderParamsError("Could not find blobinfo for key %s" % blob_key)
Example #25
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def __str__(self): """Returns the string representation of this BlobstoreZipInputReader.""" return "blobstore.BlobKey(%r):[%d, %d]" % ( self._blob_key, self._start_index, self._end_index)
Example #26
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def split_input(cls, mapper_spec): """Returns a list of shard_count input_spec_shards for input_spec. Args: mapper_spec: The mapper specification to split from. Must contain 'blob_keys' parameter with one or more blob keys. Returns: A list of BlobstoreInputReaders corresponding to the specified shards. """ params = _get_params(mapper_spec) blob_keys = params[cls.BLOB_KEYS_PARAM] if isinstance(blob_keys, basestring): blob_keys = blob_keys.split(",") blob_sizes = {} for blob_key in blob_keys: blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) blob_sizes[blob_key] = blob_info.size shard_count = min(cls._MAX_SHARD_COUNT, mapper_spec.shard_count) shards_per_blob = shard_count // len(blob_keys) if shards_per_blob == 0: shards_per_blob = 1 chunks = [] for blob_key, blob_size in blob_sizes.items(): blob_chunk_size = blob_size // shards_per_blob for i in xrange(shards_per_blob - 1): chunks.append(BlobstoreLineInputReader.from_json( {cls.BLOB_KEY_PARAM: blob_key, cls.INITIAL_POSITION_PARAM: blob_chunk_size * i, cls.END_POSITION_PARAM: blob_chunk_size * (i + 1)})) chunks.append(BlobstoreLineInputReader.from_json( {cls.BLOB_KEY_PARAM: blob_key, cls.INITIAL_POSITION_PARAM: blob_chunk_size * (shards_per_blob - 1), cls.END_POSITION_PARAM: blob_size})) return chunks
Example #27
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def validate(cls, mapper_spec): """Validates mapper spec and all mapper parameters. Args: mapper_spec: The MapperSpec for this InputReader. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Mapper input reader class mismatch") params = _get_params(mapper_spec) if cls.BLOB_KEYS_PARAM not in params: raise BadReaderParamsError("Must specify 'blob_keys' for mapper input") blob_keys = params[cls.BLOB_KEYS_PARAM] if isinstance(blob_keys, basestring): blob_keys = blob_keys.split(",") if len(blob_keys) > cls._MAX_BLOB_KEYS_COUNT: raise BadReaderParamsError("Too many 'blob_keys' for mapper input") if not blob_keys: raise BadReaderParamsError("No 'blob_keys' specified for mapper input") for blob_key in blob_keys: blob_info = blobstore.BlobInfo.get(blobstore.BlobKey(blob_key)) if not blob_info: raise BadReaderParamsError("Could not find blobinfo for key %s" % blob_key)
Example #28
Source File: input_readers.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def __str__(self): """Returns the string representation of this BlobstoreLineInputReader.""" return "blobstore.BlobKey(%r):[%d, %d]" % ( self._blob_key, self._blob_reader.tell(), self._end_position)
Example #29
Source File: backup_handler.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def get_blob_key(fname): return blobstore.BlobKey(_MaybeStripBlobstorePrefix(fname))
Example #30
Source File: blob_upload_test.py From python-compat-runtime with Apache License 2.0 | 5 votes |
def test_store_and_build_forward_message_utf8_values(self): """Test store and build message method with UTF-8 values.""" self.generate_blob_key().AndReturn(blobstore.BlobKey('item1')) self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40)) self.mox.ReplayAll() form = FakeForm({ 'field1': FakeForm(name='field1', file=StringIO.StringIO('file1'), type='text/plain', type_options={'a': 'b', 'x': 'y'}, filename='chinese_char_name_\xe6\xb1\x89.txt', headers={'h1': 'v1', 'h2': 'v2', }), }) content_type, content_text = self.handler.store_and_build_forward_message( form, '================1234==') self.mox.VerifyAll() self.assertEqual(EXPECTED_GENERATED_UTF8_CONTENT_TYPE, content_type) self.assertMessageEqual(EXPECTED_GENERATED_UTF8_MIME_MESSAGE, content_text) blob1 = blobstore.get('item1') self.assertEquals(u'chinese_char_name_\u6c49.txt', blob1.filename)