package com.scylladb.migrator.readers

import com.scylladb.migrator.config.SourceSettings
import org.apache.log4j.LogManager
import org.apache.spark.sql.{ DataFrame, SparkSession }

object Parquet {
  val log = LogManager.getLogger("com.scylladb.migrator.readers.Parquet")

  def readDataFrame(spark: SparkSession, source: SourceSettings.Parquet): SourceDataFrame = {
    source.credentials.foreach { credentials =>
      log.info("Loaded AWS credentials from config file")
      spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", credentials.accessKey)
      spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", credentials.secretKey)
    }

    SourceDataFrame(spark.read.parquet(source.path), None, false)
  }

}