Project: spark-snowflake (GitHub Link)

spark-snowflake-master
- snowflake.travis.json.gpg
- .github
  - docker
    - entrypoint.sh
    - cleanup_docker.sh
    - check_result.sh
    - log4j_driver.properties
    - spark-env.sh
    - build_image.sh
    - log4j_executor.properties
    - Dockerfile
    - docker-compose.yml
  - workflows
    - IntegrationTest.yml
    - ClusterTest.yml
  - scripts
    - run-tests-github.sh
    - decrypt_secret.sh
- src
  - main
    - resources
      - META-INF
        services
        org.apache.spark.sql.sources.DataSourceRegister
    - scala
      - net
        snowflake
        spark
        snowflake
        JsonConverter.scala
        CSVConverter.scala
        ProxyInfo.scala
        TableName.scala
        CloudCredentialsUtils.scala
        SnowflakeJDBCWrapper.scala
        test
        TestHook.scala
        Utils.scala
        streaming
        package.scala
        SnowflakeIngestConnector.scala
        SnowflakeSink.scala
        SnowflakeIngestService.scala
        Conversions.scala
        external
        SparkFunctions.scala
        SparkStreamingFunctions.scala
        FilterPushdown.scala
        SnowflakeWriter.scala
        DefaultSource.scala
        io
        StageWriter.scala
        SFInternalStage.scala
        package.scala
        Enums.scala
        SnowflakeRDD.scala
        SFRecordReader.scala
        SnowflakeResultSetRDD.scala
        StageReader.scala
        CloudStorageOperations.scala
        SnowflakeRelation.scala
        SnowflakeConnectorUtils.scala
        SnowflakeTelemetry.scala
        pushdowns
        SnowflakePlan.scala
        package.scala
        querygeneration
        SnowflakeQuery.scala
        DateStatement.scala
        WindowStatement.scala
        UnaryOp.scala
        StringStatement.scala
        package.scala
        MiscStatement.scala
        BooleanStatement.scala
        QueryBuilder.scala
        BinaryOp.scala
        QueryHelper.scala
        UnsupportedStatement.scala
        BasicStatement.scala
        NumericStatement.scala
        AggregationStatement.scala
        SnowflakeStrategy.scala
        Parameters.scala
  - it
    - resources
      - log4j.properties
      - log4j_file.properties
      - log4j_default.properties
    - scala
      - net
        snowflake
        spark
        snowflake
        IntegrationSuiteBase.scala
        SimpleNewPushdownIntegrationSuite.scala
        ColumnNameCaseSuite.scala
        SnowflakeTelemetryIntegrationSuite.scala
        SFTableNameSuite.scala
        OnErrorSuite.scala
        JDBCSuite.scala
        SecuritySuite.scala
        TruncateTableSuite.scala
        streaming
        StreamingSuite.scala
        SinkUtilsSuite.scala
        ColumnMappingIntegrationSuite.scala
        PushdownEnhancement01.scala
        FullNewPushdownIntegrationSuite.scala
        DataTypesIntegrationSuite.scala
        benchmarks
        PerformanceSuite.scala
        TPCHSuite.scala
        RegDeploymentSuite.scala
        TPCDSSuite.scala
        FilterPushdownIntegrationSuite.scala
        SnowflakeResultSetRDDSuite.scala
        VariantTypeSuite.scala
        IssueSuite.scala
        DecimalIntegrationSuite.scala
        SnowflakeIntegrationSuite.scala
  - test
    - resources
      - snowflake_unload_data.txt
    - scala
      - net
        snowflake
        spark
        snowflake
        ColumnMappingSuite.scala
        ParametersSuite.scala
        UtilsSuite.scala
        CloudCredentialsUtilsSuite.scala
        io
        GcpSuite.scala
        RecordReaderSuite.scala
        DirectOutputCommitter.scala
        MockSF.scala
        ConversionsSuite.scala
        TableNameSuite.scala
        QueryTest.scala
        BaseTest.scala
        TestUtils.scala
        FilterPushdownSuite.scala
    - python
      - unittest.py
      - README.md
- scalastyle-config.xml
- ClusterTest
  - src
    - main
      - scala
        net
        snowflake
        spark
        snowflake
        ClusterTest.scala
        testsuite
        ClusterTestSuiteBase.scala
        BasicReadWriteSuite.scala
        ClusterTestResult.scala
        ClusterTestCheckResult.scala
        TestUtils.scala
        ClusterTestResultBuilder.scala
      - python
        ClusterTest.py
  - run_cluster_test.sh
  - project
    - build.properties
    - plugins.sbt
  - build.sbt
- build
  - sbt
  - sbt-launch-lib.bash
- LICENSE
- whitesource
  - run_whitesource.sh
  - wss-sbt-agent.config
- project
  - build.properties
  - plugins.sbt
- README-TEST.md
- dev
  - merge_pr.py
  - run-tests-travis.sh
- snowflake.travis.json.enc
- .travis.yml
- README.md
- legacy
  - tutorial
    - SparkSnowflakeTutorial.scala
    - README.md
  - OLD-README.md
- snowflake.travis.json.example
- deploy.sh
- .codecov.yml
- build.sbt
- .gitignore
- README-DEV.md

`spark-snowflake`

Snowflake Data Source for Apache Spark.

The main version of spark-snowflake works with Spark 2.4. For use with Spark 2.3 and 2.2, please use tag vx.x.x-spark_2.3 and vx.x.x-spark_2.2.

To use it, provide the dependency for Spark in the form of net.snowflake:spark-snowflake_$SCALA_VERSION:$RELEASE, e.g. net.snowflake:spark-snowflake_2.11:2.2.2. See Spark Packages and Maven Central for more info.

For a version working with Spark 1.5 and 1.6, please use branch-1.x. Artifacts of that version are also available in the Snowflake UI.

For a user manual and more information, see the official documentation.

For developer notes, see README-DEV

Acknowledgments

This project was originally forked from the spark-redshift project. We also occasionally port relevant patches from it. We would like to acknowledge and thank the developers of that project, in particular Josh Rosen.