5

我正在尝试使用 spark 和 deepLearning4J 执行使用 sbt-assembly 构建的胖 jar，不幸的是，在执行过程中，我遇到了线程“main”java.lang.NoClassDefFoundError 中的异常：许多 jar 的错误。我尝试使用 spark-submit 中的 --jars 选项添加 jar，但是当我添加 jar 时，我遇到了来自另一个依赖项的另一个类的相同错误。

如果我理解得很好，sbt-assembly 生成的 FatJar 应该可以防止这种问题，因为它包含所有需要的 jar。

我的 scala 文件位于myproject/src/main/scala/xxx/spark/yyy/

也许是由于合并策略？

如果有帮助，我会加入我的 build.sbt 文件！

先感谢您。

name := "myproject"

version := "1.0"

scalaVersion := "2.10.4"

val sparkVersion = "1.6.2"

mainClass in assembly := Some("xxx.spark.yyy.Main")

resolvers += Resolver.sojava.lang.NoClassDefFoundErrornatypeRepo("releases")

resolvers += "Spark Packages Repo" at "https://dl.bintray.com/spark-packages/maven"

resolvers += "Akka Snapshot Repository" at "http://repo.akka.io/snapshots/"

resolvers += "Artifactory" at "http://artifacts.kameleoon.net:8081/artifactory/sbt/"

resolvers += "Sbt plugins" at "https://dl.bintray.com/sbt/sbt-plugin-releases"

resolvers += "Sonatype Releases" at "https://oss.sonatype.org/content/repositories/releases/"

resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)


libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
"com.datastax.spark" %% "spark-cassandra-connector" % "1.6.0",
"org.apache.spark"  %% "spark-mllib"  % sparkVersion % "provided",
"org.hibernate" % "hibernate-core" % "4.3.11.Final",
"org.hibernate" % "hibernate-entitymanager" % "4.3.11.Final",
compilerPlugin("org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full),
"org.json" % "json" % "20160810",
"org.joda" % "joda-convert" % "1.2",
"jfree" % "jfreechart" % "1.0.13",
"commons-io" % "commons-io" % "2.4",
"com.google.guava" % "guava" % "20.0",
"jfree" % "jfreechart" % "1.0.13",
"org.bytedeco" % "javacv" % "1.2",
"org.datavec" % "datavec-data-codec" % "0.7.2",
"org.datavec" % "datavec-spark_2.10" % "0.7.2",
"org.datavec" % "datavec-api" % "0.7.2",
"org.deeplearning4j" % "deeplearning4j-core" % "0.7.2",
"org.deeplearning4j" % "deeplearning4j-nn" % "0.7.2",
"org.deeplearning4j" % "dl4j-spark_2.10" % "0.7.2",
"org.jblas" % "jblas" % "1.2.4"
)


assemblyMergeStrategy in assembly := {
    case PathList("org", "joda", "time", "base", "BaseDateTime.class") => MergeStrategy.first
    case PathList("com", "esotericsoftware", "minlog", "Log.class") => MergeStrategy.first
    case PathList("org", "apache", xs @ _*) => MergeStrategy.last
    case PathList("com", "google", xs @ _*) => MergeStrategy.last
    case PathList("META-INF", xs @ _*) => MergeStrategy.rename
    case "about.html" => MergeStrategy.rename
    case x => val oldStrategy = (assemblyMergeStrategy in assembly).value
    oldStrategy(x)
}
4

0 回答 0