Project: cloud-integration (GitHub Link)

cloud-integration-master
- .gitallowed
- spark-cloud-integration
  - src
    - main
      - site
        markdown
        index.md
        integration.md
      - scala
        com
        cloudera
        spark
        cloud
        ObjectStoreOperations.scala
        s3
        LandsatIO.scala
        S3ACommitterConstants.scala
        utils
        IntegrationUtils.scala
        HConf.scala
        ConfigSerDeser.scala
        TimeOperations.scala
        applications
        CommitterCheck.scala
        CloudCp.scala
        GeneralCommitterConstants.scala
        ObjectStoreExample.scala
        org
        apache
        spark
        cloudera
        ParallelizedWithLocalityRDD.scala
        package.scala
        streaming
        cloudera
        CloudInputDStream.scala
  - pom.xml
- pom.xml
- LICENSE
- cloud-examples
  - src
    - main
      - resources
        log4j.properties
      - site
        using.md
        testing.md
      - scala
        com
        cloudera
        spark
        cloud
        s3
        S3AFileGenerator.scala
        S3ADataFrames.scala
        S3AOperations.scala
        RandomIOPolicy.scala
        S3ALineCount.scala
        FailingS3ClientFactory.scala
        S3AConstants.scala
        S3AExampleSetup.scala
        S3ATestSetup.scala
        NormalIOPolicy.scala
        S3AStreaming.scala
        IOPolicy.scala
        SequentialIOPolicy.scala
        utils
        Demo.scala
        ExtraAssertions.scala
        StatisticsTracker.scala
        ForceRecentHadoopVersion.scala
        operations
        CloudStreaming.scala
        CloudFileGenerator.scala
        LineCount.scala
        CloudDataFrames.scala
        examples
        BorisBikeExample.scala
        S3DataFrameExample.scala
        AzureStreamingExample.scala
        adl
        AdlTestSetup.scala
        abfs
        AbfsTestSetup.scala
        common
        CopyCsvFileTrait.scala
        CloudTestIntegration.scala
        CloudSuite.scala
        StoreTestHelper.scala
        CsvDatasourceSupport.scala
        StoreTestOperations.scala
        CloudSuiteTrait.scala
        CloudTestKeys.scala
        azure
        AzureTestSetup.scala
        org
        apache
        hadoop
        fs
        FSHelper.scala
    - test
      - resources
        log4j.properties
        core-site.xml
      - scala
        com
        cloudera
        spark
        cloud
        s3
        S3ASeekReadRandomIOSuite.scala
        S3AFileGeneratorSuite.scala
        S3ASeekReadNormalIOSuite.scala
        TestParquetBinding.scala
        S3AStreamingSuite.scala
        S3ACSVReadSuite.scala
        S3ALineCountWritebackSuite.scala
        S3ADataFrameSuite.scala
        commit
        S3APartitionedCommitterSuite.scala
        AbstractCommitterSuite.scala
        S3ACommitBulkDataSuite.scala
        Events.scala
        S3ACommitDataframeSuite.scala
        S3ACommitterFactorySuite.scala
        S3ALineCountSuite.scala
        S3ABasicIOSuite.scala
        S3ANumbersSuite.scala
        S3DependencyCheckSuite.scala
        S3ANumbersSuiteV2APISuite.scala
        S3ASeekReadSequentialIOSuite.scala
        S3AConsistencySuite.scala
        S3AEncryptionSuite.scala
        BorisBikeSuite.scala
        swift
        SwiftStreamingSuite.scala
        SwiftTestSetup.scala
        SwiftFileGeneratorSuite.scala
        SwiftDataFrameSuite.scala
        SwiftBasicIOSuite.scala
        examples
        S3DataFrameExampleSuite.scala
        adl
        AdlLineCountSuite.scala
        AdlDataFrameSuite.scala
        AdlCSVReadSuite.scala
        AdlSeekReadSuite.scala
        AdlStreamingSuite.scala
        AdlBasicIOSuite.scala
        AdlFileGeneratorSuite.scala
        abfs
        AbfsDataFrameSuite.scala
        AbfsBasicIOSuite.scala
        AbfsCSVReadSuite.scala
        common
        BasicIOTests.scala
        CloudSuiteWithCSVDatasource.scala
        GenericTestUtils.scala
        FileGeneratorTests.scala
        SeekReadTests.scala
        CSVReadTests.scala
        ReadSample.scala
        HadoopVersionSuite.scala
        StreamingTests.scala
        DataFrameTests.scala
        NumbersRddTests.scala
        azure
        AzureLineCountSuite.scala
        AzureSeekReadSuite.scala
        AzureCSVReadSuite.scala
        AzureStreamingSuite.scala
        AzureDataFrameSuite.scala
        AzureBasicIOSuite.scala
        AzureFileGeneratorSuite.scala
        org
        apache
        spark
        sql
        sources
        CloudRelationBasicSuite.scala
        CloudRelationScaleTest.scala
        CloudPartitionTest.scala
        AbstractCloudRelationTest.scala
        HiveTestTrait.scala
        hive
        orc
        cloud
        S3AOrcPartitionSuite.scala
        S3AParquetPartitionSuite.scala
        S3AParquetRelationScaleSuite.scala
        S3AOrcRelationScaleSuite.scala
        S3AParquetRelationSuite.scala
        S3AOrcRelationSuite.scala
        SparkScopeWorkarounds.scala
  - pom.xml
  - README.md
- .travis.yml
- README.md
- .gitignore

Cloud Integration for Apache Spark

The cloud-integration repository provides modules to improve Apache Spark's integration with cloud infrastructures.

Module `spark-cloud-integration`

Classes and Tools to make Spark work better in-cloud

Committer integration with the s3a committers.
Proof of concept cloud-first distcp replacement.
Serialization for Hadoop Configuration: class ConfigSerDeser. Use this to get a configuration into an RDD method
Trait HConf to manipulate the hadoop options in a spark config.
Anything else which turns out to be useful.
Variant of FileInputStream for cloud storage, org.apache.spark.streaming.cloudera.CloudInputDStream

See Spark Cloud Integration

Module `cloud-examples`

This does the packaging/integration tests for Spark and cloud against AWS, Azure and openstack.

These are basic tests of the core functionality of I/O, streaming, and verify that the commmitters work in the presence of inconsistent object storage As well as running as unit tests, they have CLI entry points which can be used for scalable functional testing.

Module `minimal-integration-test`

This is a minimal JAR for integration tests

Usage

spark-submit --class com.cloudera.spark.cloud.integration.Generator \
--master yarn \
--num-executors 2 \
--driver-memory 512m \
--executor-memory 512m \
--executor-cores 1 \
minimal-integration-test-1.0-SNAPSHOT.jar \
adl://example.azuredatalakestore.net/output/dest/1 \
2 2 15