IllegalArgumentException at org.apache.xbean.asm5.ClassReader.<init>(Unknown Source) when training an ALS implementation of spark in scala

Question

I was following this tutorial trying to write a collaborative recommender system using the alternating least squares algorithm in spark. I am using the movie lens dataset which can be found here.

My code is as follows:

case class Rating(userId: Int, movieId: Int, rating: Int, timestamp: Long)


object Main {


  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder()
      .appName("Alternating least squares")
      .master("local[*]")
      .getOrCreate()

    def parseRating(str: String): Rating = {
      val fields = str.split("\\t")
      new Rating(fields(0).toInt, fields(1).toInt, fields(2).toInt, fields(3).toLong)
    }

    import spark.implicits._

    val df: DataFrame = spark.read.text("./ml-100k/u1.base")
      .map(row => parseRating(row.mkString))
      .toDF() //"userId", "movieId", "rating", "timestamp"

    df.show(5, false)


    val Array(train, test) = df.randomSplit(Array(.8, .2))

    val als = new ALS()
      .setMaxIter(5)
      .setRegParam(.01)
      .setUserCol("userId")
      .setItemCol("movieId")
      .setRatingCol("rating")

    val model = als.fit(train) // <-- error here

    }
}

Everything is running fine, until it gets to the line where the als algorithm is supposed to be training. It throws this exception:

Exception in thread "main" java.lang.IllegalArgumentException
    at org.apache.xbean.asm5.ClassReader.<init>(Unknown Source)
    at org.apache.xbean.asm5.ClassReader.<init>(Unknown Source)
    at org.apache.xbean.asm5.ClassReader.<init>(Unknown Source)
    at org.apache.spark.util.ClosureCleaner$.getClassReader(ClosureCleaner.scala:46)
    at org.apache.spark.util.FieldAccessFinder$$anon$3$$anonfun$visitMethodInsn$2.apply(ClosureCleaner.scala:449)
    at org.apache.spark.util.FieldAccessFinder$$anon$3$$anonfun$visitMethodInsn$2.apply(ClosureCleaner.scala:432)
    at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
    at scala.collection.mutable.HashMap$$anon$1$$anonfun$foreach$2.apply(HashMap.scala:134)
    at scala.collection.mutable.HashMap$$anon$1$$anonfun$foreach$2.apply(HashMap.scala:134)
    at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236)
    at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
    at scala.collection.mutable.HashMap$$anon$1.foreach(HashMap.scala:134)
    at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
    at org.apache.spark.util.FieldAccessFinder$$anon$3.visitMethodInsn(ClosureCleaner.scala:432)
    at org.apache.xbean.asm5.ClassReader.a(Unknown Source)
    at org.apache.xbean.asm5.ClassReader.b(Unknown Source)
    at org.apache.xbean.asm5.ClassReader.accept(Unknown Source)
    at org.apache.xbean.asm5.ClassReader.accept(Unknown Source)
    at org.apache.spark.util.ClosureCleaner$$anonfun$org$apache$spark$util$ClosureCleaner$$clean$14.apply(ClosureCleaner.scala:262)
    at org.apache.spark.util.ClosureCleaner$$anonfun$org$apache$spark$util$ClosureCleaner$$clean$14.apply(ClosureCleaner.scala:261)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:261)
    at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:159)
    at org.apache.spark.SparkContext.clean(SparkContext.scala:2299)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:2073)
    at org.apache.spark.rdd.RDD$$anonfun$take$1.apply(RDD.scala:1364)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
    at org.apache.spark.rdd.RDD.take(RDD.scala:1337)
    at org.apache.spark.rdd.RDD$$anonfun$isEmpty$1.apply$mcZ$sp(RDD.scala:1472)
    at org.apache.spark.rdd.RDD$$anonfun$isEmpty$1.apply(RDD.scala:1472)
    at org.apache.spark.rdd.RDD$$anonfun$isEmpty$1.apply(RDD.scala:1472)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
    at org.apache.spark.rdd.RDD.isEmpty(RDD.scala:1471)
    at org.apache.spark.ml.recommendation.ALS$.train(ALS.scala:918)
    at org.apache.spark.ml.recommendation.ALS.fit(ALS.scala:674)
    at Main$.main(Main.scala:58)
    at Main.main(Main.scala)

Can someone please exmplain what I am doing wrong, as I am pretty much following the tutorial line by line.

EDIT: I am using java SDK version 11.0.11 with scala version 2.11.12.

Answer 1

Upon further research (searching stackoverflow for the line org.apache.xbean.asm5.ClassReader.(Unknown Source) from the stack trace) I found multiple answers such as this where in a nutshell it says that the reason for this error is that spark, or mllib from spark only supports java 8. Indeed when I switched to java 8 in my project, the code ran as expected.