• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• spark_mysql-master
  • src
    • main
      • resources
        • log4j.properties
      • scala
        • utils
          • PropertyUtils.scala
          • MySQLUtils.scala
          • MySQLPoolManager.scala
        • InsertMysqlDemo.scala
  • pom.xml
  • LICENSE
  • comflyelephant.iml
  • README.md
  • target
    • classes
      • utils
        • Mutils$$anonfun$getInsertSql$2.class
        • Mutils$$anonfun$insertOrUpdateDFtoDBUsePool$1$$anonfun$apply$5.class
        • Mutils$$anonfun$insertOrUpdateDFtoDBUsePool$1$$anonfun$apply$5$$anonfun$apply$3.class
        • Mutils$$anonfun$saveDFtoDBUsePool$1$$anonfun$apply$4.class
        • Mutils$$anonfun$1.class
        • Mutils$$anonfun$createTableIfNotExist$1.class
        • MySQLPoolManager$MysqlPool.class
        • Mutils$$anonfun$getUpdateSql$1.class
        • Mutils$$anonfun$saveDFtoDBUsePool$1.class
        • MySQLPoolManager$.class
        • Mutils$.class
        • Mutils$$anonfun$2.class
        • Mutils$$anonfun$insertOrUpdateDFtoDBUsePool$1$$anonfun$apply$5$$anonfun$apply$2.class
        • Mutils$$anonfun$verifyFieldConsistency$1.class
        • Mutils$$anonfun$getInsertSql$3.class
        • Mutils$$anonfun$getInsertSql$1.class
        • PropertyUtils.class
        • Mutils$$anonfun$insertOrUpdateDFtoDBUsePool$1.class
        • Mutils$$anonfun$saveDFtoDBUsePool$1$$anonfun$apply$4$$anonfun$apply$1.class
        • Mutils.class
        • MySQLPoolManager.class
        • PropertyUtils$.class
      • InsertMysqlDemo$CardMember.class
      • log4j.properties
      • SimHash.class
      • InsertMysqlDemo$.class
      • InsertMysqlDemo$CardMember$.class
      • mysql-user.properties
      • InsertMysqlDemo$$typecreator1$1.class
      • Test$.class
      • Test.class
      • InsertMysqlDemo.class
  • conf
    • mysql-user.properties

import java.sql.{Date, Timestamp}

import InsertMysqlDemo.CardMember
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkConf, SparkContext}
import utils.MySQLUtils

/**
  * Created with IntelliJ IDEA.
  * Author: [email protected]
  * Description:DataFrame 中数据存入到MySQL
  * Date: Created in 2018-11-17 12:39
  */
object InsertMysqlDemo {

  case class CardMember(m_id: String, card_type: String, expire: Timestamp, duration: Int, is_sale: Boolean, date: Date, user: Long, salary: Float)

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName(getClass.getSimpleName).set("spark.testing.memory", "3147480000")
    val sparkContext = new SparkContext(conf)
    val hiveContext = new SQLContext(sparkContext)
    import hiveContext.implicits._
    val memberSeq = Seq(
      CardMember("member_2", "月卡", new Timestamp(System.currentTimeMillis()), 31, false, new Date(System.currentTimeMillis()), 123223, 0.32f),
      CardMember("member_1", "季卡", new Timestamp(System.currentTimeMillis()), 93, false, new Date(System.currentTimeMillis()), 124224, 0.362f)
    )
    val memberDF = memberSeq.toDF()
    MySQLUtils.saveDFtoDBCreateTableIfNotExist("member_test", memberDF)
    MySQLUtils.insertOrUpdateDFtoDBUsePool("member_test", memberDF, Array("user", "salary"))
    MySQLUtils.getDFFromMysql(hiveContext, "", null)


    sparkContext.stop()
  }
}