Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/14112#discussion_r70504461
--- Diff: mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala ---
@@ -566,26 +565,52 @@ object LocalLDAModel extends
MLReadable[LocalLDAModel] {
}
}
+ private case class Data(
+ vocabSize: Int,
+ topicsMatrix: Matrix,
+ docConcentration: Vector,
+ topicConcentration: Double,
+ gammaShape: Double)
+
private class LocalLDAModelReader extends MLReader[LocalLDAModel] {
private val className = classOf[LocalLDAModel].getName
override def load(path: String): LocalLDAModel = {
+ // Import implicits for Dataset Encoder
+ val sparkSession = super.sparkSession
+ import sparkSession.implicits._
+
val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
val dataPath = new Path(path, "data").toString
val data = sparkSession.read.parquet(dataPath)
- .select("vocabSize", "topicsMatrix", "docConcentration",
"topicConcentration",
- "gammaShape")
- .head()
- val vocabSize = data.getAs[Int](0)
- val topicsMatrix = data.getAs[Matrix](1)
- val docConcentration = data.getAs[Vector](2)
- val topicConcentration = data.getAs[Double](3)
- val gammaShape = data.getAs[Double](4)
+ val vectorConverted = MLUtils.convertVectorColumnsToML(data,
"docConcentration")
+ val Row(vocabSize: Int, topicsMatrix: Matrix, docConcentration:
Vector,
--- End diff --
Does this work? You should need to take the first row
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
