scale-master
- .github
  - PULL_REQUEST_TEMPLATE.md
  - ISSUE_TEMPLATE
    - feature_request.md
    - bug_report.md
  - CONTRIBUTING.md
  - workflows
    - gitlab-mirror.yml
- scale
  - scale
    - local_settings_DOCS.py
    - custom_logging.py
    - __init__.py.template
    - local_settings_TRAVIS-CI.py
    - wsgi.py
    - urls.py
    - __init__.py
    - local_settings_docker.py
    - settings.py
    - local_settings_dev.py
    - .gitignore
  - source
    - apps.py
    - configuration
      - source_data_file.py
      - __init__.py
    - test
      - configuration
        test_source_data_file.py
        __init__.py
      - messages
        test_purge_source_file.py
        __init__.py
      - test_models.py
      - __init__.py
      - utils.py
    - messages
      - purge_source_file.py
      - __init__.py
    - serializers.py
    - models.py
    - __init__.py
    - migrations
      - 0003_auto_20170127_1342.py
      - 0002_auto_20170127_1336.py
      - 0001_initial.py
      - __init__.py
  - storage
    - apps.py
    - configuration
      - exceptions.py
      - workspace_configuration.py
      - __init__.py
      - json
        workspace_config_v6.py
        workspace_config_1_0.py
        __init__.py
    - media_type.py
    - container.py
    - exceptions.py
    - fixtures
      - basic_storage_errors.json
      - move_files_job_type.json
      - delete_files_job_type.json
    - brokers
      - exceptions.py
      - factory.py
      - nfs_broker.py
      - s3_broker.py
      - broker.py
      - __init__.py
      - host_broker.py
    - test
      - configuration
        test_workspace_configuration.py
        __init__.py
      - test_delete_files_job.py
      - test_move_files_job.py
      - fixtures
        test_delete_files_job_type.py
        __init__.py
      - brokers
        test_nfs_broker.py
        test_host_broker.py
        __init__.py
        test_s3_broker.py
      - messages
        test_delete_files.py
        test_move_files.py
        __init__.py
      - test_models.py
      - test_geospatial_utils.py
      - __init__.py
      - utils.py
      - test_views.py
      - management
        commands
        __init__.py
        test_scale_move_files.py
        test_scale_delete_files.py
        __init__.py
    - messages
      - move_files.py
      - __init__.py
      - delete_files.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - 0011_auto_20180821_1439.py
      - 0007_auto_20170412_1225.py
      - 0009_auto_20171002_1542.py
      - 0006_auto_20170127_1423.py
      - 0013_purgeresults.py
      - 0004_auto_20170127_1408.py
      - 0015_scalefile_data_type_tags.py
      - 0001_initial.py
      - 0008_auto_20170609_1443.py
      - 0010_auto_20180613_1947.py
      - 0016_populate_data_type_tags.py
      - 0012_auto_20180920_1320.py
      - 0014_purgeresults_force_stop_purge.py
      - __init__.py
      - 0003_auto_20161202_1621.py
      - 0019_optional_description.py
      - 0002_workspace_is_move_enabled.py
      - 0017_remove_scalefile_data_type.py
      - 0005_auto_20170127_1412.py
      - 0018_remove_scalefile_is_operational.py
    - settings.py
    - geospatial_utils.py
    - delete_files_job.py
    - move_files_job.py
    - management
      - commands
        scale_upload_file.py
        scale_move_files.py
        scale_download_file.py
        scale_delete_files.py
        scale_list_files.py
        __init__.py
        scale_move_file.py
        scale_delete_file.py
      - __init__.py
  - trigger
    - handler.py
    - apps.py
    - configuration
      - exceptions.py
      - __init__.py
      - trigger_rule.py
    - test
      - tests.py
      - __init__.py
      - utils.py
    - serializers.py
    - models.py
    - __init__.py
    - migrations
      - 0001_initial.py
      - 0005_auto_20170412_1225.py
      - 0004_auto_20151207_1215.py
      - __init__.py
      - 0003_auto_20151202_1325.py
      - 0002_auto_20151125_1438.py
  - error
    - apps.py
    - exceptions.py
    - fixtures
      - basic_errors.json
    - test
      - test_models.py
      - __init__.py
      - utils.py
      - test_views.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - 0004_error_should_be_retried.py
      - 0002_auto_20151217_1319.py
      - 0006_update_retried.py
      - 0005_auto_20180621_2110.py
      - 0001_initial.py
      - __init__.py
      - 0003_error_is_builtin.py
    - handlers.py
  - shared_resource
    - apps.py
    - __init__.py
    - migrations
      - 0002_auto_20170412_1225.py
      - 0003_auto_20170426_0952.py
      - 0001_initial.py
      - __init__.py
  - timeline
    - apps.py
    - test
      - __init__.py
      - test_views.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - __init__.py
  - cli
    - __init__.py
    - management
      - commands
        migratedata.py
        __init__.py
      - __init__.py
  - scheduler
    - apps.py
    - initialize.py
    - database
      - __init__.py
      - updater.py
    - resources
      - manager.py
      - __init__.py
      - offer.py
      - agent.py
    - threads
      - base_thread.py
      - task_update.py
      - task_handling.py
      - recon.py
      - sync.py
      - __init__.py
      - messaging.py
      - scheduler_status.py
      - schedule.py
    - sync
      - job_type_manager.py
      - workspace_manager.py
      - __init__.py
    - task
      - manager.py
      - __init__.py
    - fixtures
      - scheduler_errors.json
    - test
      - database
        test_updater.py
        __init__.py
      - resources
        __init__.py
        test_manager.py
      - sync
        test_job_type_manager.py
        test_scheduler_manager.py
        __init__.py
        test_workspace_manager.py
      - dependencies
        __init__.py
        test_manager.py
      - test_initialize.py
      - messages
        __init__.py
        test_restart_scheduler.py
      - node
        test_node_class.py
        test_conditions.py
        __init__.py
        test_manager.py
      - scheduling
        test_node.py
        __init__.py
        test_manager.py
      - test_scheduler.py
      - tasks
        services
        __init__.py
        messaging
        test_messaging_service.py
        __init__.py
        __init__.py
        test_manager.py
      - __init__.py
      - test_views.py
    - dependencies
      - manager.py
      - __init__.py
    - messages
      - __init__.py
      - restart_scheduler.py
    - serializers.py
    - models.py
    - views.py
    - node
      - conditions.py
      - node_class.py
      - manager.py
      - __init__.py
      - agent.py
    - scheduling
      - scheduling_node.py
      - manager.py
      - __init__.py
    - urls.py
    - cleanup
      - manager.py
      - node.py
      - __init__.py
    - cluster_utils.py
    - manager.py
    - recon
      - manager.py
      - __init__.py
    - vault
      - manager.py
      - __init__.py
    - tasks
      - services
        __init__.py
        messaging
        message_handler_task.py
        messaging_service.py
        __init__.py
        service.py
      - manager.py
      - system_task.py
      - __init__.py
      - db_update_task.py
    - __init__.py
    - configuration.py
    - migrations
      - 0011_auto_20180517_1559.py
      - 0009_scheduler_debugging_level.py
      - 0002_auto_20151007_1352.py
      - 0006_scheduler_status.py
      - 0013_auto_20181220_2014.py
      - 0001_initial.py
      - 0008_auto_20171103_1334.py
      - 0007_scheduler_num_message_handlers.py
      - 0003_auto_20160201_0846.py
      - 0010_scheduler_resource_level.py
      - __init__.py
      - 0012_remove_scheduler_resource_level.py
      - 0005_scheduler_queue_mode.py
      - 0004_auto_20170209_1616.py
    - admin.py
    - scale_scheduler.py
    - management
      - commands
        load_all_data.py
        scale_db_update.py
        __init__.py
        scale_scheduler.py
      - __init__.py
  - product
    - apps.py
    - configuration
      - product_data_file.py
      - __init__.py
    - test
      - configuration
        __init__.py
        test_product_data_file.py
      - test_models.py
      - __init__.py
      - utils.py
    - serializers.py
    - models.py
    - __init__.py
    - migrations
      - 0004_auto_20170127_1324.py
      - 0005_auto_20170127_1344.py
      - 0001_initial.py
      - 0009_auto_20170301_1130.py
      - 0002_auto_20160622_1344.py
      - __init__.py
      - 0003_auto_20170127_1319.py
      - 0007_fileancestrylink_batch.py
      - 0006_auto_20170127_1348.py
      - 0010_auto_20170727_1349.py
    - types.py
  - environment
    - legacy-cloud-init.sh
    - cent7-init.sh
    - win-init.bat
    - mac-init.sh
    - cloud-init.sh
  - manage.py
  - util
    - apps.py
    - active_warnings.py
    - middleware.py
    - exceptions.py
    - parse.py
    - test
      - test_retry.py
      - test_host.py
      - test_broker.py
      - __init__.py
      - test_aws.py
      - test_parse.py
      - test_command.py
      - test_rest.py
    - database.py
    - validation.py
    - host.py
    - aws.py
    - dcos.py
    - os_helper.py
    - broker.py
    - deprecation.py
    - rest.py
    - __init__.py
    - command.py
    - retry.py
    - file_size.py
    - lock.py
    - environment.py
  - job
    - apps.py
    - configuration
      - mount.py
      - exceptions.py
      - results
        exceptions.py
        job_results.py
        __init__.py
        results_manifest
        results_manifest.py
        results_manifest_1_0.py
        __init__.py
      - interface
        job_interface_1_3.py
        exceptions.py
        job_interface.py
        job_interface_1_0.py
        job_interface_1_2.py
        __init__.py
        job_interface_1_1.py
        scale_file.py
      - data
        job_data.py
        exceptions.py
        data_file.py
        __init__.py
        job_connection.py
      - __init__.py
      - configuration.py
      - json
        job_config_v6.py
        __init__.py
    - job_type_serializers.py
    - exceptions.py
    - fixtures
      - basic_system_job_types.json
      - basic_job_errors.json
    - error
      - mapping.py
      - __init__.py
      - error.py
    - test
      - configuration
        results
        test_job_results.py
        __init__.py
        test_results_manifest.py
        interface
        test_configuration_interface.py
        test_job_interface.py
        test_scale_file.py
        __init__.py
        test_configuration.py
        data
        test_job_connection.py
        test_job_data.py
        __init__.py
        __init__.py
        json
        __init__.py
        test_job_config_v6.py
      - error
        test_mapping.py
        test_error.py
        __init__.py
      - messages
        test_failed_jobs.py
        test_unpublish_jobs.py
        test_pending_jobs.py
        test_job_exe_end.py
        test_process_job_input.py
        test_blocked_jobs.py
        test_completed_jobs.py
        test_running_jobs.py
        test_uncancel_jobs.py
        test_create_jobs.py
        test_cancel_jobs.py
        __init__.py
        test_spawn_delete_files_job.py
        test_cancel_jobs_bulk.py
        test_purge_jobs.py
        test_publish_job.py
      - test_models.py
      - test_clock.py
      - execution
        configuration
        test_configurators.py
        __init__.py
        json
        test_exe_config.py
        __init__.py
        tasks
        test_pre_task.py
        __init__.py
        json
        results
        test_task_results.py
        __init__.py
        __init__.py
        test_post_task.py
        __init__.py
        test_metrics.py
        test_job_exe.py
        test_manager.py
      - tasks
        test_base_task.py
        __init__.py
        test_manager.py
      - __init__.py
      - utils.py
      - seed
        results
        test_job_results.py
        __init__.py
        test_metadata.py
        test_types.py
        test_job_data.py
        __init__.py
        test_manifest.py
      - test_views.py
      - management
        commands
        test_scale_post_steps.py
        __init__.py
        test_scale_pre_steps.py
        __init__.py
    - handlers
      - inputs
        files.py
        file.py
        __init__.py
        property.py
        base_input.py
      - __init__.py
    - messages
      - publish_job.py
      - create_jobs.py
      - cancel_jobs_bulk.py
      - blocked_jobs.py
      - unpublish_jobs.py
      - job_exe_end.py
      - uncancel_jobs.py
      - process_job_input.py
      - pending_jobs.py
      - purge_jobs.py
      - __init__.py
      - failed_jobs.py
      - completed_jobs.py
      - running_jobs.py
      - cancel_jobs.py
      - spawn_delete_files_job.py
    - serializers.py
    - models.py
    - views.py
    - clock.py
    - urls.py
    - data
      - job_data.py
      - exceptions.py
      - __init__.py
      - job_connection.py
      - types.py
    - deprecation.py
    - execution
      - configuration
        input_file.py
        exceptions.py
        workspace.py
        volume.py
        configurators.py
        docker_param.py
        __init__.py
        json
        exe_config_1_0.py
        __init__.py
        exe_config.py
        exe_config_1_1.py
      - container.py
      - exceptions.py
      - metrics.py
      - manager.py
      - tasks
        main_task.py
        pull_task.py
        exe_task.py
        post_task.py
        __init__.py
        cleanup_task.py
        json
        results
        __init__.py
        task_results.py
        __init__.py
        pre_task.py
      - __init__.py
      - job_exe.py
    - tasks
      - health_task.py
      - pull_task.py
      - update.py
      - base_task.py
      - manager.py
      - __init__.py
      - node_task.py
    - __init__.py
    - migrations
      - 0025_auto_20170504_1534.py
      - 0019_taskupdate.py
      - 0052_jobtype_is_published.py
      - 0030_remove_job_configuration.py
      - 0021_auto_20161115_1524.py
      - 0055_jobtype_v5_deprecation.py
      - 0034_auto_20180112_1759.py
      - 0057_auto_20190603_1846.py
      - 0049_auto_20180927_1531.py
      - 0012_auto_20160310_1318.py
      - 0023_jobtype_shared_mem_required.py
      - 0032_job_node.py
      - 0044_jobtypetag_job_type.py
      - 0045_auto_20180830_1812.py
      - 0004_auto_20151020_1059.py
      - 0014_auto_20160317_1208.py
      - 0002_jobexecution_job_metrics.py
      - 0010_auto_20151208_1503.py
      - 0022_jobtype_configuration.py
      - 0037_auto_20180314_1827.py
      - 0011_jobtype_max_scheduled.py
      - 0035_auto_20180122_1444.py
      - 0018_auto_20160804_1402.py
      - 0050_job_configuration.py
      - 0001_initial.py
      - 0015_auto_20160420_1345.py
      - 0017_auto_20160426_1556.py
      - 0042_auto_20180802_1830.py
      - 0026_auto_20170510_1151.py
      - 0013_auto_20160316_1805.py
      - 0016_auto_20160421_1553.py
      - 0036_auto_20180123_2042.py
      - 0003_auto_20151016_0849.py
      - 0020_auto_20161110_1517.py
      - 0048_auto_20180913_1632.py
      - 0043_remove_jobtypetag_job_type.py
      - 0038_auto_20180622_1510.py
      - 0024_auto_20170412_1225.py
      - 0047_jobtype_versionarray.py
      - 0006_auto_20151106_1608.py
      - 0051_auto_20190211_1500.py
      - __init__.py
      - 0008_jobtype_trigger_rule.py
      - 0054_convert_manifest.py
      - 0033_auto_20180105_2113.py
      - 0027_auto_20170615_1652.py
      - 0039_auto_20180622_1827.py
      - 0005_auto_20151030_1402.py
      - 0046_jobtyperevision_docker_image.py
      - 0007_job_job_type_rev.py
      - 0028_jobinputfile.py
      - 0053_jobtype_unmet_resources.py
      - 0040_auto_20180709_1807.py
      - 0009_jobtype_revision_num.py
      - 0029_auto_20170707_1034.py
      - 0056_convert_strike_jobs.py
      - 0031_auto_20170822_1544.py
      - 0041_jobtypetag.py
    - errors.py
    - seed
      - exceptions.py
      - results
        job_results.py
        __init__.py
        outputs_json.py
      - manifest.py
      - metadata.py
      - __init__.py
      - types.py
      - schema
        seed.manifest.schema.json
        seed.metadata.schema.json
    - management
      - commands
        scale_pre_steps.py
        scale_post_steps.py
        scale_clock.py
        __init__.py
      - __init__.py
  - diagnostic
    - apps.py
    - exceptions.py
    - fixtures
      - diagnostic_job_types.json
      - diagnostic_recipe_types.json
      - diagnostic_errors.json
    - test
      - __init__.py
      - test_views.py
    - views.py
    - urls.py
    - __init__.py
    - management
      - commands
        scale_hello.py
        scale_count.py
        scale_roulette.py
        scale_bake.py
        __init__.py
      - __init__.py
  - mesos_api
    - apps.py
    - offers.py
    - unversioned
      - __init__.py
      - agent.py
    - tasks.py
    - api.py
    - __init__.py
    - utils.py
  - node
    - apps.py
    - resources
      - exceptions.py
      - gpu_manager.py
      - __init__.py
      - node_resources.py
      - resource.py
      - json
        resources.py
        __init__.py
    - test
      - resources
        test_gpu_manager.py
        __init__.py
        json
        __init__.py
        test_resources.py
      - test_models.py
      - __init__.py
      - utils.py
      - test_views.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - 0006_node_last_offer_received.py
      - 0002_node_pause_reason.py
      - 0004_auto_20170524_1639.py
      - 0001_initial.py
      - 0003_node_is_paused_errors.py
      - __init__.py
      - 0005_auto_20181220_2014.py
    - admin.py
  - recipe
    - apps.py
    - configuration
      - exceptions.py
      - data
        exceptions.py
        recipe_data.py
        __init__.py
        recipe_connection.py
      - __init__.py
      - configuration.py
      - json
        recipe_config_v6.py
        __init__.py
      - definition
        exceptions.py
        recipe_definition.py
        __init__.py
    - exceptions.py
    - instance
      - recipe.py
      - exceptions.py
      - node.py
      - __init__.py
      - json
        __init__.py
        recipe_v6.py
    - test
      - configuration
        test_configuration.py
        data
        __init__.py
        test_recipe_data.py
        __init__.py
        json
        test_recipe_config_v6.py
        definition
        __init__.py
        test_recipe_definition.py
      - instance
        test_recipe.py
        __init__.py
        json
        __init__.py
        test_recipe_v6.py
      - handlers
        test_graph.py
        test_graph_delta.py
        __init__.py
      - diff
        test_diff.py
        __init__.py
        json
        test_diff_v6.py
        test_forced_nodes_v6.py
        __init__.py
      - messages
        test_update_recipe.py
        test_process_condition.py
        test_create_recipes.py
        test_supersede_recipe_nodes.py
        test_create_conditions.py
        test_update_recipe_definition.py
        test_process_recipe_input.py
        test_purge_recipe.py
        __init__.py
        test_update_recipe_metrics.py
      - test_models.py
      - __init__.py
      - utils.py
      - definition
        test_definition.py
        __init__.py
        json
        test_definition_v6.py
        test_definition_v1.py
        __init__.py
      - test_views.py
    - handlers
      - graph_delta.py
      - graph.py
      - connection.py
      - node.py
      - __init__.py
    - diff
      - exceptions.py
      - node.py
      - diff.py
      - __init__.py
      - json
        forced_nodes_v6.py
        __init__.py
        diff_v6.py
      - forced_nodes.py
    - messages
      - purge_recipe.py
      - process_recipe_input.py
      - create_recipes.py
      - create_conditions.py
      - update_recipe.py
      - process_condition.py
      - update_recipe_metrics.py
      - __init__.py
      - supersede_recipe_nodes.py
      - update_recipe_definition.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - 0027_auto_20181008_1756.py
      - 0029_auto_20181016_1431.py
      - 0020_recipe_batch.py
      - 0019_recipetype_is_system.py
      - 0030_auto_20181024_1607.py
      - 0015_recipefile.py
      - 0016_recipefile_data.py
      - 0011_auto_20160330_1505.py
      - 0008_auto_20160317_1208.py
      - 0007_auto_20160310_1318.py
      - 0005_recipetype_revision_num.py
      - 0028_auto_20181016_1349.py
      - 0024_auto_20180523_2057.py
      - 0004_recipetype_trigger_rule.py
      - 0006_auto_20151125_1404.py
      - 0002_auto_20151106_1409.py
      - 0035_auto_20190126_2340.py
      - 0012_auto_20160330_1659.py
      - 0032_recipe_type_link_tables.py
      - 0001_initial.py
      - 0021_auto_20180219_1916.py
      - 0034_recipe_configuration.py
      - 0036_remove_recipetype_version.py
      - 0033_auto_20181102_0307.py
      - 0025_auto_20180608_0114.py
      - 0013_auto_20160331_1127.py
      - __init__.py
      - 0003_recipe_recipe_type_rev.py
      - 0020_auto_20180119_0055.py
      - 0018_recipefile_recipe_input.py
      - 0031_auto_20181026_1417.py
      - 0027_auto_20181016_1344.py
      - 0026_auto_20180723_1914.py
      - 0009_auto_20160511_2204.py
      - 0023_auto_20180523_1247.py
      - 0014_auto_20160608_1402.py
      - 0017_auto_20170412_1225.py
      - 0010_auto_20160330_1412.py
      - 0037_remove_recipetype_trigger_rule.py
      - 0022_auto_20180307_1617.py
    - seed
      - recipe_data.py
      - recipe_definition.py
      - __init__.py
      - schema
        recipe_definition_2_0.json
      - recipe_connection.py
    - definition
      - exceptions.py
      - connection.py
      - node.py
      - __init__.py
      - json
        definition_v1.py
        __init__.py
        definition_v6.py
      - definition.py
  - pip
    - docs.txt
    - requirements.txt
    - production.txt
  - data
    - apps.py
    - exceptions.py
    - interface
      - exceptions.py
      - __init__.py
      - parameter.py
      - json
        interface_v6.py
        __init__.py
      - interface.py
    - dataset
      - __init__.py
      - dataset.py
      - json
        __init__.py
        dataset_v6.py
    - test
      - interface
        test_parameter.py
        __init__.py
        test_interface.py
        json
        test_interface_v6.py
        __init__.py
      - dataset
        test_dataset.py
        __init__.py
        json
        test_dataset_v6.py
        __init__.py
      - test_models.py
      - data
        test_data.py
        __init__.py
        json
        test_data_v1.py
        test_data_v6.py
        __init__.py
        test_value.py
      - __init__.py
      - utils.py
      - filter
        __init__.py
        json
        test_filter_v6.py
        __init__.py
        test_filter.py
      - test_views.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - data
      - data.py
      - exceptions.py
      - value.py
      - __init__.py
      - json
        data_v1.py
        __init__.py
        data_v6.py
      - data_util.py
    - __init__.py
    - migrations
      - 0001_initial.py
      - __init__.py
    - filter
      - exceptions.py
      - filter.py
      - __init__.py
      - json
        filter_v6.py
        __init__.py
  - accounts
    - apps.py
    - test
      - __init__.py
      - test_views.py
    - templates
      - accounts
        login.html
    - serializers.py
    - views.py
    - urls.py
    - __init__.py
    - static
      - accounts
    - management
      - commands
        __init__.py
        scale_superuser.py
      - __init__.py
  - vault
    - apps.py
    - exceptions.py
    - test
      - __init__.py
      - test_secrets_handler.py
    - secrets_handler.py
    - __init__.py
  - README.md
  - ingest
    - apps.py
    - ingest_event_serializers.py
    - fixtures
      - ingest_errors.json
      - ingest_job_types.json
    - test
      - test_app.py
      - messages
        test_create_ingest_jobs.py
        __init__.py
      - test_models.py
      - triggers
        test_ingest_recipe_handler.py
        __init__.py
      - strike
        configuration
        test_strike_configuration.py
        __init__.py
        json
        __init__.py
        test_configuration_v6.py
        monitors
        test_s3_monitor.py
        test_dir_monitor.py
        __init__.py
        __init__.py
      - __init__.py
      - scan
        configuration
        test_scan_configuration.py
        __init__.py
        scanners
        test_s3_scanner.py
        test_dir_scanner.py
        __init__.py
        __init__.py
      - utils.py
      - test_ingest_job.py
      - test_views.py
    - handlers
      - file_rule.py
      - file_handler.py
      - __init__.py
    - messages
      - create_ingest_jobs.py
      - __init__.py
    - serializers.py
    - models.py
    - views.py
    - ingest_job.py
    - urls.py
    - triggers
      - __init__.py
      - ingest_recipe_handler.py
    - strike
      - configuration
        exceptions.py
        strike_configuration.py
        __init__.py
        json
        configuration_v6.py
        __init__.py
      - monitors
        exceptions.py
        monitor.py
        factory.py
        s3_monitor.py
        __init__.py
        dir_monitor.py
      - __init__.py
    - __init__.py
    - scan
      - configuration
        exceptions.py
        scan_configuration.py
        __init__.py
        json
        configuration_v6.py
        __init__.py
      - scanners
        scanner.py
        exceptions.py
        factory.py
        dir_scanner.py
        __init__.py
        s3_scanner.py
      - __init__.py
    - migrations
      - 0010_auto_20170206_1646.py
      - 0012_scan_file_count.py
      - 0016_ingest_data_type_tags.py
      - 0017_populate_data_type_tags.py
      - 0004_auto_20160711_1058.py
      - 0008_auto_20170127_1332.py
      - 0003_auto_20160711_1044.py
      - 0001_initial.py
      - 0015_ingestevent.py
      - __init__.py
      - 0002_auto_20160414_0937.py
      - 0014_auto_20170412_1225.py
      - 0019_filename_index.py
      - 0007_auto_20170127_1327.py
      - 0006_auto_20161202_1621.py
      - 0009_scan.py
      - 0018_remove_ingest_data_type.py
      - 0013_auto_20170406_1517.py
      - 0005_auto_20160728_1243.py
      - 0011_auto_20170302_2130.py
    - management
      - commands
        scale_scan.py
        scale_ingest.py
        scale_strike.py
        __init__.py
      - __init__.py
  - metrics
    - apps.py
    - daily_metrics.py
    - fixtures
      - metrics_job_types.json
      - metrics_trigger_rules.json
    - test
      - test_models.py
      - __init__.py
      - utils.py
      - test_daily_metrics.py
      - test_views.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - registry.py
    - __init__.py
    - migrations
      - 0003_metricsingest.py
      - 0007_auto_20161013_2333.py
      - 0002_auto_20151007_1352.py
      - 0006_metricserror.py
      - 0009_revert_datetime.py
      - 0001_initial.py
      - 0010_occurred_datetime.py
      - 0005_auto_20160415_1636.py
      - 0004_auto_20151104_1640.py
      - __init__.py
      - 0008_occurred_datetime.py
    - management
      - commands
        __init__.py
        scale_daily_metrics.py
      - __init__.py
  - .gitignore
  - messaging
    - apps.py
    - exceptions.py
    - backends
      - amqp.py
      - sqs.py
      - factory.py
      - __init__.py
      - backend.py
    - test
      - test_apps.py
      - test_backends.py
      - test_messages.py
      - __init__.py
      - test_manager_singleton.py
      - test_manager.py
    - messages
      - fail.py
      - factory.py
      - message.py
      - __init__.py
      - echo.py
      - chain.py
    - manager.py
    - __init__.py
    - management
      - commands
        scale_echo_message.py
        scale_message_handler.py
        __init__.py
        scale_send_message.py
      - __init__.py
  - docs
    - Makefile
    - make.bat
    - development
      - index.rst
    - images
      - algorithm_integration
    - algorithm_integration
      - results_manifest.rst
      - step2.rst
      - step1.rst
      - step4.rst
      - step3.rst
      - index.rst
      - step5.rst
    - build_wiki_docs.py
    - cli
      - overview.rst
      - getting_started.rst
      - index.rst
    - architecture
      - workspaces.rst
      - logs.rst
      - messaging.rst
      - port.rst
      - overview.rst
      - triggers.rst
      - scan.rst
      - django
        index.rst
        .gitignore
      - jobs
        seed.rst
        error_interface.rst
        job_interface.rst
        recipe_definition.rst
        job_configuration.rst
        job_data.rst
        index.rst
        task_results.rst
        recipe_data.rst
        exe_configuration.rst
        batch_definition.rst
        resources.rst
      - strike.rst
      - index.rst
    - install
      - index.rst
    - overview
      - index.rst
    - index.rst
    - conf.py
    - rest
      - index.rst
      - v6
        timeline.yml
        queue.yml
        deprecated.rst
        ingest.rst
        system.yml
        strike.yml
        node.rst
        recipe.yml
        diagnostic.yml
        metrics.rst
        batch.yml
        timeline.rst
        workspace.rst
        recipe.rst
        ingest.yml
        job_type.rst
        job.yml
        workspace.yml
        openapi.yml
        batch.rst
        recipe_type.yml
        scan.yml
        data.rst
        job.rst
        data.yml
        error.yml
        recipe_type.rst
        scheduler.rst
        scan.rst
        job_type.yml
        scheduler.yml
        strike.rst
        queue.rst
        node.yml
        metrics.yml
        diagnostic.rst
        error.rst
        scale_file.yml
        scale_file.rst
        system.rst
    - build_openapi_docs.py
    - _templates
      - layout.html
    - _static
      - images
        architecture
        Scale_Architecture.pptx
      - style.css
    - wiki
      - mkdocs.yml
      - scale.wiki
        setup_wiki_git.sh
  - .dockerignore
  - queue
    - apps.py
    - fixtures
      - queue_trigger_rules.json
    - job_load.py
    - test
      - test_job_load.py
      - messages
        test_requeue_jobs_bulk.py
        test_queued_jobs.py
        __init__.py
        test_requeue_jobs.py
      - test_models.py
      - __init__.py
      - utils.py
      - test_views.py
    - messages
      - queued_jobs.py
      - __init__.py
      - requeue_jobs.py
      - requeue_jobs_bulk.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - 0017_queue_docker_image.py
      - 0013_auto_20170720_1702.py
      - 0010_auto_20170615_1704.py
      - 0008_auto_20160421_1648.py
      - 0012_auto_20170616_1101.py
      - 0006_auto_20160316_1625.py
      - 0001_initial.py
      - 0009_auto_20170412_1225.py
      - 0004_remove_queue_is_job_type_paused.py
      - 0002_jobload.py
      - 0018_queue_docker_image_populate.py
      - 0011_auto_20170616_1052.py
      - 0007_auto_20160421_1643.py
      - 0014_queue.py
      - __init__.py
      - 0016_auto_20180123_2037.py
      - 0005_queue_node_required.py
      - 0003_auto_20151023_1104.py
      - 0015_auto_20170731_1527.py
    - job_exe.py
  - batch
    - apps.py
    - configuration
      - exceptions.py
      - __init__.py
      - configuration.py
      - json
        configuration_v6.py
        __init__.py
    - exceptions.py
    - fixtures
      - batch_job_types.json
    - test
      - configuration
        test_configuration.py
        __init__.py
        json
        __init__.py
        test_configuration_v6.py
      - messages
        test_create_batch_recipes.py
        __init__.py
        test_update_batch_metrics.py
      - test_models.py
      - __init__.py
      - utils.py
      - definition
        test_definition.py
        __init__.py
        json
        test_definition_v6.py
        __init__.py
      - test_views.py
      - management
        commands
        __init__.py
        test_scale_batch_creator.py
        __init__.py
    - messages
      - update_batch_metrics.py
      - __init__.py
      - create_batch_recipes.py
    - serializers.py
    - models.py
    - views.py
    - urls.py
    - __init__.py
    - migrations
      - 0008_auto_20190226_1255.py
      - 0007_auto_20180516_1915.py
      - 0002_auto_20170412_1225.py
      - 0009_auto_20190531_1318.py
      - 0001_initial.py
      - 0006_auto_20180401_0218.py
      - 0003_auto_20170706_1948.py
      - 0005_auto_20180314_1827.py
      - __init__.py
      - 0004_auto_20180122_1444.py
    - definition
      - exceptions.py
      - __init__.py
      - json
        __init__.py
        definition_v6.py
      - definition.py
    - management
      - commands
        scale_batch_creator.py
        __init__.py
      - __init__.py
- hooks
  - build
- web-docs
  - walkthrough
    - images
    - generate-pdf.sh
    - generate-outputs.sh
    - styles
      - walkthrough.css
    - index.adoc
  - images
    - quickstart2.gif
    - gif-source
    - quickstart3.gif
  - javascripts
    - scale.fix.js
  - flow.html
  - fonts
    - glyphicons-halflings-regular.woff
    - glyphicons-halflings-regular.eot
    - glyphicons-halflings-regular.woff2
    - glyphicons-halflings-regular.ttf
    - glyphicons-halflings-regular.svg
  - quickstart.html
  - stylesheets
    - styles.css
    - github-light.css
  - js
    - bootstrap.min.js
    - jquery.js
    - bootstrap.js
  - index.html
  - css
    - bootstrap-overrides.css
    - main.css
    - bootstrap.min.css
    - bootstrap.css
  - docs.html
- scale-ui
  - index.html
- LICENSE
- dockerfiles
  - fluentd
    - fluent.conf
    - README.md
    - scripts
      - entrypoint.sh
      - inject-es-config.py
    - Dockerfile
  - examples
    - ace
      - runOpticks.sh
      - ace.batchwiz
      - oli-vis.wmd
      - ace.wiz
      - job_type.yml
      - centroid.py
      - Dockerfile
    - read-bytes-seed
      - scale-job.py
      - seed.manifest.json
      - README.md
      - Dockerfile
      - .dockerignore
    - landsat-base
      - epel-release-7-5.noarch.rpm
      - Dockerfile
    - landsat-tiles
      - landsat_tiles.sh
      - job_type.yml
      - Dockerfile
    - vash
      - run.sh
      - job_type.yml
      - Dockerfile
    - landsat-parse
      - job_type.yml
      - parse_landsat.sh
      - Dockerfile
    - binary-parse
      - parse_binary.sh
      - job_type.yml
      - Dockerfile
    - test-seed
      - seed.manifest.json
      - README.md
      - test.py
      - Dockerfile
      - .dockerignore
    - tad
      - runOpticks.sh
      - tad.batchwiz
      - job_type.yml
      - centroid.py
      - tad.wiz
      - Dockerfile
    - landsat-ndwi
      - landsat_ndwi.py
      - job_type.yml
      - Dockerfile
    - read-bytes
      - scale-job.py
      - README.md
      - Dockerfile
      - .dockerignore
    - pca
      - pca.batchwiz
      - runOpticks.sh
      - pca.wiz
      - job_type.yml
      - Dockerfile
    - source-metadata
      - seed.manifest.json
      - generate-metadata.sh
      - Dockerfile
  - framework
    - scale
      - app-templates
        fluentd.json
        logstash.json
        db.json
        rabbitmq.json
        ui.json
        webserver.json
        silo.json
        elasticsearch.json
      - entryPoint.sh
      - bootstrap.py
      - scale.conf
      - README.md
      - .gitignore
  - vault
    - Readme.md
    - config.hcl
    - Dockerfile
    - run-vault
- graphics
  - images
    - icons
      - safari-pinned-tab.svg
      - browserconfig.xml
      - favicon.ico
      - manifest.json
    - navigation-logo.psd
  - models
    - models.txt
    - Scale_Base.stl
    - Scale_Model.JPG
    - Scale_Logo.stl
  - scale3-logo.svg
  - PoweredByScale
    - pbscale-small-fill.svg
    - pbscale-large-fill.svg
- travis-build.sh
- .gitattributes
- push-docs.sh
- generate-release.sh
- .travis.yml
- README.md
- build-scale.sh
- tests
  - postman
    - environment.json
    - full-host-test.json
- Dockerfile
- dev-build-push.sh
- .gitlab-ci.yml
- .gitignore
- build-support.sh
- .dockerignore

Scale

Scale is a system that provides management of automated processing on a cluster of machines. It allows users to define jobs, which can be any type of script or algorithm. These jobs run on ingested source data and produce product files. The produced products can be disseminated to appropriate users and/or used to evaluate the producing algorithm in terms of performance and accuracy.

Mesos and Nodes

Scale runs across a cluster of networked machines (called nodes) that process the jobs. Scale utilizes Apache Mesos, a free and open source project, for managing the available resources on the nodes. Mesos informs Scale of available computing resources and Scale schedules jobs to run on those resources.

Ingest

Scale ingests source files using a Scale component called Strike. Strike is a process that monitors an ingest directory into which source data files are being copied. After a new source data file has been ingested, Scale produces and places jobs on the queue depending on the type of the ingested file. Many Strike processes can be run simultaneously, allowing Scale to monitor many different ingest directories.

Jobs

Scale creates jobs based on its known job types. A job type defines key characteristics about an algorithm that Scale needs to know in order to run it (what command to run, the algorithm.s inputs and outputs, etc.) Job types are labeled with versions, allowing Scale to run multiple versions of the same algorithm. Jobs may be created automatically due to an event, such as the ingest of a particular type of source data file, or they may be created manually by a user. Jobs that need to be executed are placed onto and prioritized within a queue before being scheduled onto an available node. When multiple jobs need to be run in a serial or parallel sequence, a recipe can be created that defines the job workflow.

Products

Jobs can produce products as a result of their successful execution. Products may be disseminated to users or used to analyze and improve the algorithms that produced them. Scale allows the creation of different workspaces. A workspace defines a separate location for storing source or product files. When a job is created, it is given a workspace to use for storing its results, allowing a user to control whether the job.s results are available to a wider audience or are restricted to a private workspace for the user's own use.

Scale Dependencies

Scale requires several external components to run as intended. PostgreSQL is used to store all internal system state and must be accessible to both the scheduler and web server processes. Fluentd along with Elasticsearch are used to collect and store all algorithm logs. A message broker is required for in-flight storage of internal Scale messages and must be accessible to all system components. The following versions of these services are required to support Scale:

Elasticsearch 6.6.2
Fluentd 1.4
PostgreSQL 9.4+
PostGIS 2.0+
Message Broker (RabbitMQ 3.6+ or Amazon SQS)

Note: We strongly recommend using managed services for PostgreSQL (AWS RDS), Messaging (AWS SQS) and Elasticsearch (AWS Elasticsearch Service), if available to you. Use of these services in Docker containers should be avoided in all but development environments. Reference the Architecture documentation for additional details on configuring supporting services.

Quick Start

While Scale can be entirely run on a pure Apache Mesos cluster, we strongly recommend using Data Center Operating System (DC/OS). DC/OS provides service discovery, load-balancing and fail-over for Scale, as well as deployment scripts for nearly all imaginable target infrastructures. This stack allows Scale users to focus on use of the framework while minimizing effort spent on deployment and configuration. A complete quick start guide can be found at:

https://ngageoint.github.io/scale/quickstart.html

Algorithm Development

Scale is designed to allow development of recipes and jobs for your domain without having to concern yourself with the complexities of cluster scheduling or data flow management. As long as your processing can be accomplished with discrete inputs on a Linux command line, it can be run in Scale. Simple examples of a complete processing chain can be found within the above quick start or you can refer to our in-depth documentation for step-by-step Scale integration:

https://ngageoint.github.io/scale/docs/algorithm_integration/index.html

Scale Development

If you want to contribute to the actual Scale open source project, we welcome your contributions. There are 2 primary components of Scale:

Scale User Interface: https://github.com/ngageoint/scale/tree/master/scale-ui
Scheduler / Service APIs: https://github.com/ngageoint/scale/tree/master/scale

The links provide specific development environment setup instructions for each individual component.

Build

Scale is tested and built using a combination of Travis CI and Docker Hub. All unit test execution and documentation generation are done using Travis CI. We require that any pull request fully pass unit test checks prior to being merged. Docker Hub builds are saved to x.x.x-snapshot image tags between releases and on release tags are matched to release version.

A new release can be cut using the generate-release.sh shell script from a cloned Scale repository (where numbers refer to MAJOR MINOR PATCH versions respectively):

./generate-release.sh 4 0 0

There is no direct connection between the Travis CI and Docker Hub builds, but both are launched via push to the GitHub repository.

Contributing

Scale was developed at the National Geospatial-Intelligence Agency (NGA). The government has "unlimited rights" and is releasing this software to increase the impact of government investments by providing developers with the opportunity to take things in new directions. The software use, modification, and distribution rights are stipulated within the Apache 2.0 license.

All pull request contributions to this project will be released under the Apache 2.0 or compatible license. Software source code previously released under an open source license and then modified by NGA staff is considered a "joint work" (see 17 USC § 101); it is partially copyrighted, partially public domain, and as a whole is protected by the copyrights of the non-government authors and must be released according to the terms of the original open source license.