...
| Code Block | ||
|---|---|---|
| ||
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.1.6.10" % "provided" libraryDependencies += "org.apache.spark" %% "spark-sql" % "1.62.1" % "provided" libraryDependencies += "org.apache.spark" %% "spark-hive" % "1.6.1" % "provided" libraryDependencies += "com.databricks" %% "spark-csv" % "1.3.0".0" % "provided" |
assembly Dependency
| Code Block | ||
|---|---|---|
| ||
// In build.sbt import sbt.Keys._ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) |
...
| Code Block | ||
|---|---|---|
| ||
// ConfigurationCreation of SparkContextSparkSession val confsparkSession = new SparkConfSparkSession.builder() .setAppNameappName("example-spark-scala-read-and-write-from-hive") // Creation of SparContext and HiveContext val sc = new SparkContext(conf) val hiveContext = new HiveContext(sc) // Configuring hiveContext to find hive metastore hiveContext.setConf.config("hive.metastore.warehouse.dir", params.hiveHost + "user/hive/warehouse") .enableHiveSupport() .getOrCreate() |
How to write to a Hive table with Spark Scala?
Code example
| Code Block | ||
|---|---|---|
| ||
// ====== Creating a dataframe with 1 partition import hiveContextsparkSession.implicits._ val df = Seq(HelloWorld("helloworld")).toDF().coalesce(1) // ======= Writing files // Writing Dataframe as a Hive table import hiveContextsparkSession.sql sql("DROP TABLE IF EXISTS helloworld") hiveContext.sql("CREATE TABLE helloworld (message STRING)") df.write.mode("overwrite"SaveMode.Overwrite).saveAsTable("helloworld"); logger.info("Writing hive table : OK") |
...
| Code Block | ||
|---|---|---|
| ||
// ======= Reading files
// Reading hive table into a Spark Dataframe
val dfHive = hiveContext.sql("SELECT * from helloworld")
logger.info("Reading hive table : OK")
logger.info(dfHive.show()) |
...