version: spark-3.0.0-preview2-bin-hadoop2.7
The syntax checker objects to the following argument which is what I am
supposed to enter.
.schema(staticSchema)
However when I provide the following argument it works but I don't think
that is correct.
What is the correct argument for this case ?
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.{window,column,desc,col}
object RetailData {
def main(args: Array[String]): Unit = {
// crete spark session
val spark =
SparkSession.builder().master("spark://192.168.0.38:7077").appName("Retail
Data").getOrCreate();
// set spark runtime configuration
spark.conf.set("spark,sql.shuffle.partitions","5")
// create a static frame
val staticDataFrame = spark.read.format("csv")
.option ("header","true")
.option("inferschema","true")
.load("/data/retail-data/by-day/*.csv")
staticDataFrame.createOrReplaceTempView("retail_data")
val staticFrame = staticDataFrame.schema
staticDataFrame
.selectExpr(
"CustomerId","UnitPrice * Quantity as total_cost", "InvoiceDate")
.groupBy(col("CustomerId"), window(col("InvoiceDate"), "1 day"))
.sum("total_cost")
.sort(desc("sum(total_cost)"))
.show(2)
val streamingDataFrame = spark.readStream
.schema(staticDataFrame.schema)
.option("maxFilesPerTrigger", 1)
.load("/data/retail-data/by-day/*.csv")
println(streamingDataFrame.isStreaming)
} // main
} // object
Backbutton.co.uk
¯\_(ツ)_/¯
♡۶Java♡۶RMI ♡۶
Make Use Method {MUM}
makeuse.org
<http://www.backbutton.co.uk>
