I'm trying to execute a fat jar builded with sbt-assembly using spark and deepLearning4J, unfortunately during execution i'm confronted to the Exception in thread "main" java.lang.NoClassDefFoundError: error for many jars. I tried to add jars using the --jars option in spark-submit but when i add a jar, i'm falling on the same error for another class from another dependency.
If i understand well, the FatJar generated by sbt-assembly should prevent this kind of problem because it include all needed jar.
My scala files are into myproject/src/main/scala/xxx/spark/yyy/
Maybe it's due to the merging strategy ?
I join my build.sbt file if it can help !
Thank you in advance.
name := "myproject"
version := "1.0"
scalaVersion := "2.10.4"
val sparkVersion = "1.6.2"
mainClass in assembly := Some("xxx.spark.yyy.Main")
resolvers += Resolver.sojava.lang.NoClassDefFoundErrornatypeRepo("releases")
resolvers += "Spark Packages Repo" at "https://dl.bintray.com/spark-packages/maven"
resolvers += "Akka Snapshot Repository" at "http://repo.akka.io/snapshots/"
resolvers += "Artifactory" at "http://artifacts.kameleoon.net:8081/artifactory/sbt/"
resolvers += "Sbt plugins" at "https://dl.bintray.com/sbt/sbt-plugin-releases"
resolvers += "Sonatype Releases" at "https://oss.sonatype.org/content/repositories/releases/"
resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
"com.datastax.spark" %% "spark-cassandra-connector" % "1.6.0",
"org.apache.spark" %% "spark-mllib" % sparkVersion % "provided",
"org.hibernate" % "hibernate-core" % "4.3.11.Final",
"org.hibernate" % "hibernate-entitymanager" % "4.3.11.Final",
compilerPlugin("org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full),
"org.json" % "json" % "20160810",
"org.joda" % "joda-convert" % "1.2",
"jfree" % "jfreechart" % "1.0.13",
"commons-io" % "commons-io" % "2.4",
"com.google.guava" % "guava" % "20.0",
"jfree" % "jfreechart" % "1.0.13",
"org.bytedeco" % "javacv" % "1.2",
"org.datavec" % "datavec-data-codec" % "0.7.2",
"org.datavec" % "datavec-spark_2.10" % "0.7.2",
"org.datavec" % "datavec-api" % "0.7.2",
"org.deeplearning4j" % "deeplearning4j-core" % "0.7.2",
"org.deeplearning4j" % "deeplearning4j-nn" % "0.7.2",
"org.deeplearning4j" % "dl4j-spark_2.10" % "0.7.2",
"org.jblas" % "jblas" % "1.2.4"
)
assemblyMergeStrategy in assembly := {
case PathList("org", "joda", "time", "base", "BaseDateTime.class") => MergeStrategy.first
case PathList("com", "esotericsoftware", "minlog", "Log.class") => MergeStrategy.first
case PathList("org", "apache", xs @ _*) => MergeStrategy.last
case PathList("com", "google", xs @ _*) => MergeStrategy.last
case PathList("META-INF", xs @ _*) => MergeStrategy.rename
case "about.html" => MergeStrategy.rename
case x => val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
}