Search by Module
Search by Words
Search Projects
Most Popular
Top Python APIs
Popular Projects
Java
Python
JavaScript
TypeScript
C++
Scala
Blog
Project: petastorm
(GitHub Link)
petastorm-master
.coveragerc
examples
hello_world
petastorm_dataset
tensorflow_hello_world.py
pyspark_hello_world.py
python_hello_world.py
pytorch_hello_world.py
generate_petastorm_dataset.py
__init__.py
tests
test_generate_petastorm_dataset.py
external_dataset
tensorflow_hello_world.py
python_hello_world.py
pytorch_hello_world.py
__init__.py
tests
test_generate_external_dataset.py
generate_external_dataset.py
__init__.py
__init__.py
mnist
tf_example.py
schema.py
pytorch
Dockerfile
generate_petastorm_mnist.py
__init__.py
README.md
tests
test_tf_mnist.py
test_pytorch_mnist.py
__init__.py
conftest.py
pytorch_example.py
spark_dataset_converter
__init__.py
pytorch_converter_example.py
utils.py
tests
test_converter_example.py
tensorflow_converter_example.py
imagenet
generate_petastorm_imagenet.py
schema.py
__init__.py
tests
test_generate_imagenet_dataset.py
readthedocs.yml
petastorm
cache.py
local_disk_cache.py
test_util
reader_mock.py
shuffling_analysis.py
__init__.py
etl
rowgroup_indexing.py
metadata_util.py
dataset_metadata.py
legacy.py
petastorm_generate_metadata.py
rowgroup_indexers.py
__init__.py
gcsfs_helpers
gcsfs_wrapper.py
__init__.py
py_dict_reader_worker.py
selectors.py
codecs.py
pytorch.py
pyarrow_helpers
batching_table_queue.py
__init__.py
tests
test_batch_buffer.py
transform.py
unischema.py
reader_impl
shuffling_buffer.py
pyarrow_serializer.py
pytorch_shuffling_buffer.py
arrow_table_serializer.py
__init__.py
pickle_serializer.py
generator.py
spark
__init__.py
spark_dataset_converter.py
reader.py
fs_utils.py
hdfs
__init__.py
tests
test_hdfs_namenode.py
__init__.py
namenode.py
tf_utils.py
arrow_reader_worker.py
ngram.py
local_disk_arrow_table_cache.py
namedtuple_gt_255_fields.py
__init__.py
utils.py
tests
test_codec_ndarray.py
test_cache.py
test_pytorch_dataloader.py
test_unischema.py
test_codec_scalar.py
test_reading_legacy_datasets.py
test_tf_dataset.py
test_predicates.py
test_pyarrow_serializer.py
test_spark_session_cli.py
tempdir.py
test_disk_cache.py
test_ngram_end_to_end.py
test_copy_dataset.py
test_run_in_subprocess.py
test_tf_autograph.py
test_metadata_read.py
test_codec_compressed_image.py
test_generate_metadata.py
test_pickle_serializer.py
test_ngram.py
test_shuffling_buffer.py
test_dataset_metadata.py
test_fs_utils.py
test_common.py
test_decode_row.py
test_tf_utils.py
test_arrow_table_serializer.py
test_end_to_end.py
data
legacy
0.4.3
partition_key=p_9
part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
.part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
partition_key=p_1
.part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
partition_key=p_6
part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
.part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
partition_key=p_4
.part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
partition_key=p_3
.part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
partition_key=p_0
.part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
partition_key=p_7
part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
.part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
partition_key=p_8
part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
.part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
partition_key=p_5
part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
.part-00001-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
_common_metadata
partition_key=p_2
.part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet.crc
part-00000-b5b6f5cd-e687-428b-825d-fe4ed7cd7548.c000.parquet
._SUCCESS.crc
_SUCCESS
0.5.1
partition_key=p_9
part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_1
part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_6
part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_4
part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_3
part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_0
part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_7
part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_8
part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
partition_key=p_5
part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00001-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
_common_metadata
partition_key=p_2
part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet
.part-00000-89dc59a3-273a-4396-9971-b6b4dd18a451.c000.parquet.crc
._SUCCESS.crc
_SUCCESS
0.7.6
partition_key=p_9
part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_1
part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_6
part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_4
part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_3
part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_0
part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_7
part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_8
part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
partition_key=p_5
part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00001-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
_common_metadata
partition_key=p_2
part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet
.part-00000-81e4bee2-d98b-4615-ae10-b7ffd9b5e7b7.c000.parquet.crc
._SUCCESS.crc
_SUCCESS
0.4.0
partition_key=p_9
part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
.part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
partition_key=p_1
.part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
partition_key=p_6
part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
.part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
partition_key=p_4
.part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
partition_key=p_3
.part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
partition_key=p_0
.part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
partition_key=p_7
part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
.part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
partition_key=p_8
part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
.part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
partition_key=p_5
part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
.part-00001-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
_common_metadata
partition_key=p_2
.part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet.crc
part-00000-0009ca2f-7ec6-4fae-9a7f-77d6d137bc9c.c000.parquet
._SUCCESS.crc
_SUCCESS
0.6.0
partition_key=p_9
.part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_1
.part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_6
.part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_4
.part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_3
.part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_0
.part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_7
.part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_8
.part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
partition_key=p_5
.part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00001-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
_common_metadata
partition_key=p_2
.part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet.crc
part-00000-0d73833d-93e1-4beb-bdae-e8ef0824b7f1.c000.parquet
._SUCCESS.crc
_SUCCESS
0.7.0
partition_key=p_9
part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
.part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
partition_key=p_1
.part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
partition_key=p_6
part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
.part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
partition_key=p_4
.part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
partition_key=p_3
.part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
partition_key=p_0
.part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
partition_key=p_7
part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
.part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
partition_key=p_8
part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
.part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
partition_key=p_5
part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
.part-00001-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
_common_metadata
partition_key=p_2
.part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet.crc
part-00000-b06b0cb8-c928-436a-9ac8-719a772ec325.c000.parquet
._SUCCESS.crc
_SUCCESS
test_parquet_reader.py
generate_dataset_for_legacy_tests.py
__init__.py
test_end_to_end_predicates_impl.py
test_reader.py
test_spark_dataset_converter.py
test_spark_utils.py
test_transform.py
test_benchmark.py
test_pytorch_utils.py
test_weighted_sampling_reader.py
test_reader_mock.py
bootstrap_test_schema_data.py
conftest.py
errors.py
tools
copy_dataset.py
spark_session_cli.py
__init__.py
spark_utils.py
compat.py
predicates.py
workers_pool
thread_pool.py
ventilator.py
dummy_pool.py
process_pool.py
exec_in_new_process.py
__init__.py
worker_base.py
tests
test_ventilator.py
test_workers_pool.py
stub_workers.py
__init__.py
weighted_sampling_reader.py
benchmark
dummy_reader.py
throughput.py
cli.py
__init__.py
LICENSE
.flake8
README.rst
.pylintrc
setup.py
docker
Makefile
run_in_venv.sh
Dockerfile
setup.cfg
.travis.yml
NOTICE
.gitignore
docs
migrating-0.3.0.rst
images
migrating-0.5.0.rst
troubleshoot.rst
release-notes.rst
autodoc
migration_guide_include.rst
Makefile
readme_include.rst
api.rst
development_include.rst
troubleshoot_include.rst
release_notes_include.rst
benchmarks_tutorial_include.rst
index.rst
conf.py
_templates
README
_static
README
custom.css
development.rst
benchmarks_tutorial.rst