Project: flytekit (GitHub Link)

flytekit-master
- .github
  - workflows
    - pythonpublish.yml
- flytekit
  - configuration
    - resources.py
    - sdk.py
    - internal.py
    - creds.py
    - statsd.py
    - aws.py
    - gcp.py
    - platform.py
    - common.py
    - __init__.py
    - auth.py
  - engines
    - flyte
      - engine.py
      - __init__.py
    - common.py
    - __init__.py
    - loader.py
    - unit
      - engine.py
      - mock_stats.py
      - __init__.py
      - unit.config
  - type_engines
    - default
      - flyte.py
      - __init__.py
    - common.py
    - __init__.py
  - sdk
    - spark_types.py
    - test_utils.py
    - exceptions.py
    - tasks.py
    - workflow.py
    - __init__.py
    - types.py
  - models
    - named_entity.py
    - admin
      - workflow.py
      - common.py
      - __init__.py
      - task_execution.py
    - array_job.py
    - presto.py
    - project.py
    - task.py
    - literals.py
    - workflow_closure.py
    - matchable_resource.py
    - node_execution.py
    - dynamic_job.py
    - filters.py
    - common.py
    - __init__.py
    - core
      - identifier.py
      - condition.py
      - workflow.py
      - __init__.py
      - compiler.py
      - errors.py
      - types.py
      - execution.py
    - qubole.py
    - launch_plan.py
    - types.py
    - schedule.py
    - execution.py
    - interface.py
  - common
    - workflow_execution.py
    - nodes.py
    - schedules.py
    - promise.py
    - workflow.py
    - constants.py
    - types
      - impl
        schema.py
        __init__.py
        blobs.py
      - primitives.py
      - proto.py
      - schema.py
      - containers.py
      - __init__.py
      - blobs.py
      - base_sdk_types.py
      - helpers.py
    - sdk_bases.py
    - tasks
      - sdk_dynamic.py
      - generic_spark_task.py
      - pytorch_task.py
      - spark_task.py
      - raw_container.py
      - task.py
      - output.py
      - executions.py
      - presto_task.py
      - __init__.py
      - sidecar_task.py
      - sdk_runnable.py
      - hive_task.py
    - __init__.py
    - utils.py
    - exceptions
      - system.py
      - user.py
      - __init__.py
      - scopes.py
      - base.py
    - core
      - identifier.py
      - __init__.py
    - component_nodes.py
    - launch_plan.py
    - notifications.py
    - mixins
      - launchable.py
      - registerable.py
      - hash.py
      - __init__.py
      - artifact.py
    - interface.py
  - plugins
    - __init__.py
  - clients
    - raw.py
    - friendly.py
    - __init__.py
    - helpers.py
  - __init__.py
  - bin
    - entrypoint.py
    - __init__.py
  - tools
    - module_loader.py
    - subprocess.py
    - __init__.py
    - lazy_loader.py
  - clis
    - flyte_cli
      - example.config
      - __init__.py
      - main.py
    - sdk_in_container
      - serialize.py
      - pyflyte.py
      - basic_auth.py
      - constants.py
      - register.py
      - __init__.py
      - launch_plan.py
    - __init__.py
    - auth
      - credentials.py
      - __init__.py
      - auth.py
      - discovery.py
    - helpers.py
  - interfaces
    - random.py
    - data
      - s3
        __init__.py
        s3proxy.py
      - data_proxy.py
      - common.py
      - __init__.py
      - http
        __init__.py
        http_data_proxy.py
      - gcs
        gcs_proxy.py
        __init__.py
      - local
        local_file_proxy.py
        __init__.py
    - __init__.py
    - stats
      - client.py
      - taggable.py
      - __init__.py
  - contrib
    - sensors
      - impl.py
      - task.py
      - base_sensor.py
      - __init__.py
    - __init__.py
- sample-notebooks
  - image.py
  - raw-container.ipynb
  - notebook.config
  - raw-container-shell.ipynb
- pull_request_template.md
- LICENSE
- setup.py
- setup.cfg
- .travis.yml
- README.md
- tests
  - flytekit
    - common
      - task_definitions.py
      - parameterizers.py
      - configs
        deprecated_local.config
        local.config
      - __init__.py
      - workflows
        raw_edge_detector.py
        simple.py
        dynamic_workflows.py
        hive.py
        sidecar.py
        nested.py
        presto.py
        raw_container.py
        scala_spark.py
        spark.py
        batch.py
        __init__.py
        python.py
        failing_workflows.py
        gpu.py
        notifications.py
    - __init__.py
    - loadtests
      - cp_spark.py
      - cp_orchestrator.py
      - orchestrator.py
      - cp_python.py
      - dynamic_job.py
      - __init__.py
      - python.py
    - unit
      - configuration
        test_internal.py
        test_common.py
        configs
        good.config
        empty.config
        bad.config
        __init__.py
        test_temporary_configuration.py
        test_waterfall.py
        test_resources.py
        conftest.py
      - engines
        flyte
        test_engine.py
        __init__.py
        test_loader.py
        __init__.py
        unit
        __init__.py
      - common_tests
        resources
        protos
        CompiledWorkflowClosure.pb
        OneTaskWFForPromote.pb
        test_notifications.py
        test_schedules.py
        test_nodes.py
        test_launch_plan.py
        types
        impl
        test_blobs.py
        test_schema.py
        __init__.py
        test_blobs.py
        test_schema.py
        test_helpers.py
        test_containers.py
        test_primitives.py
        __init__.py
        test_proto.py
        tasks
        test_task.py
        test_raw_container_task.py
        spark
        test_spark_task.py
        __init__.py
        __init__.py
        test_sdk_runnable.py
        __init__.py
        test_interface.py
        exceptions
        test_base.py
        test_scopes.py
        test_user.py
        __init__.py
        test_system.py
        test_workflow.py
        test_workflow_promote.py
        test_promise.py
        mixins
        test_registerable.py
        __init__.py
        sample_registerable.py
      - type_engines
        default
        __init__.py
        test_flyte_type_engine.py
        __init__.py
      - sdk
        types
        test_blobs.py
        test_schema.py
        test_primitives.py
        __init__.py
        tasks
        test_dynamic_tasks.py
        test_spark_task.py
        test_hive_tasks.py
        test_sidecar_tasks.py
        test_pytorch_task.py
        test_tasks.py
        __init__.py
        __init__.py
        test_workflow.py
        conftest.py
      - cli
        test_flyte_cli.py
        test_helpers.py
        auth
        test_discovery.py
        test_credentials.py
        test_auth.py
        pyflyte
        test_register.py
        test_launch_plans.py
        __init__.py
        test_basic_auth.py
        conftest.py
      - models
        test_qubole.py
        test_named_entity.py
        test_dynamic_job.py
        test_types.py
        admin
        test_common.py
        __init__.py
        test_matchable_resource.py
        test_launch_plan.py
        test_tasks.py
        test_schedule.py
        test_common.py
        test_workflow_closure.py
        __init__.py
        test_interface.py
        test_literals.py
        core
        test_errors.py
        test_types.py
        test_identifier.py
        __init__.py
        test_workflow.py
        test_execution.py
        test_filters.py
        test_dynamic_wfs.py
        test_project.py
        test_execution.py
      - clients
        test_raw.py
        __init__.py
      - tasks
        __init__.py
      - __init__.py
      - bin
        test_python_entrypoint.py
        fake.config
        __init__.py
      - tools
        test_subprocess.py
        test_lazy_loader.py
        __init__.py
        test_module_loader.py
        test_aws.py
      - test_plugins.py
      - interfaces
        test_random.py
        data
        gcs
        test_gcs_proxy.py
        __init__.py
      - use_scenarios
        unit_testing
        test_hive_tasks.py
        test_blobs.py
        __init__.py
        test_schemas.py
        __init__.py
      - contrib
        sensors
        test_task.py
        test_impl.py
        __init__.py
        __init__.py
  - __init__.py
- CODE_OF_CONDUCT.md
- scripts
  - flytekit_install_spark.sh
  - flytekit_spark_entrypoint.sh
  - Readme.rst
  - flytekit_venv
  - flytekit_build_image.sh
- requirements.txt
- CODEOWNERS
- NOTICE
- .gitignore

Flytekit

Codecov

Python Library for easily authoring, testing, deploying, and interacting with Flyte tasks, workflows, and launch plans. To understand more about flyte refer to,

Installation

Flytekit is designed for minimal footprint, and thus some features must be installed as extras.

Base Installation

This is the lightest-weight SDK install. This installation includes everything you need to interact with Flyte.

Modules include:

The full Flyte IDL and an additional model layer for easier extension of the data model.
gRPC client for communicating with the platform.
Implementations for authoring and extending all Flyte entities (including tasks, workflows, and launch plans).

Tools include:

flyte-cli (Command-Line Interface for Interacting with the Flyte Platform)
pyflyte (Command-Line tool for easing the registration of Flyte entities)

pip install flytekit

Plugin Installation

Spark

If @spark_task is to be used, one should install the spark plugin.

pip install flytekit[spark]

Schema

If Types.Schema() is to be used for computations involving large dataframes, one should install the schema extension.

pip install flytekit[schema]

Sidecar

If @sidecar_task is to be used, one should install the sidecar plugin.

pip install flytekit[sidecar]

Pytorch

If @pytorch_task is to be used, one should install the pytorch plugin.

pip install flytekit[pytorch]

Full Installation

To install all or multiple available plugins, one can specify them individually:

pip install flytekit[sidecar,spark,schema]

Or install them with the all directive.

pip install flytekit[all]

Testing

Flytekit is Python 2.7+ compatible, so when feasible, it is recommended to test with both Python 2 and 3.

Unit Testing

Setup (Do Once)

virtualenv ~/.virtualenvs/flytekit
source ~/.virtualenvs/flytekit/bin/activate
python -m pip install -r requirements.txt
python -m pip install -U .[all]

Execute

source ~/.virtualenvs/flytekit/bin/activate
python -m pytest tests/flytekit/unit
shellcheck **/*.sh