0 
class Script (engine: com.thinkbiganalytics.spark.repl.ScriptEngine)    extends com.thinkbiganalytics.spark.repl.Script (engine) {
  override def eval (): Any = {
class Transform (sqlContext: org.apache.spark.sql.SQLContext, sparkContextService: com.thinkbiganalytics.spark.SparkContextService) extends com.thinkbiganalytics.spark.metadata.TransformScript20(sqlContext, sparkContextService) {
override def dataFrame: org.apache.spark.sql.DataFrame = {import org.apache.spark.sql.functions.{concat, lit, concat_ws,collect_list,split,size,col,when}
import com.thinkbiganalytics.kylo.catalog._
var listBuffer = new scala.collection.mutable.ListBuffer[org.apache.spark.sql.DataFrame]()
var kyloClientBuilder = KyloCatalog.builder()
var kyloClient = kyloClientBuilder.build()
var kyloClientReader = kyloClient.read.format("com.thinkbiganalytics.spark.file.metadata")
listBuffer += kyloClientReader.load("file:/opt/kylo_backup/read/test.csv")
val dataFrameList = listBuffer.toList
var unionedFileMetadataDf : org.apache.spark.sql.DataFrame = null
dataFrameList.foreach({ df1 =>
 if(unionedFileMetadataDf == null){
 unionedFileMetadataDf = df1
}
else {
unionedFileMetadataDf = unionedFileMetadataDf.unionAll(df1)
}
})
var fileMetadataDf = unionedFileMetadataDf.select(col("mimeType"),col("encoding"),when(col("properties")("headerCount").isNotNull,col("properties")("headerCount")).otherwise(lit("0")).as("headerCount"),when(col("properties")("delimiter").isNotNull,col("properties")("delimiter")).otherwise(lit("")).as("delimiter"),when(col("properties")("rowTag").isNotNull,col("properties")("rowTag")).otherwise(lit("")).as("rowTag"),col("properties").as("properties"),col("resource"))
var df = fileMetadataDf
df
}
}
new Transform(sqlContext, sparkContextService).run()
  }
  def sparkContextService (): com.thinkbiganalytics.spark.SparkContextService = getValue("sparkContextService")
}
new Script(engine).run()

从 kylo wrangler 运行这个我得到

无法使用 Kylo 目录解析数据源：com.thinkbiganalytics.spark.file.metadata：{} java.lang.ClassNotFoundException：com.thinkbiganalytics.spark.file.metadata

有人可以知道这个 com.thinkbiganalytics.spark.file.metadata 类吗？

4

0 回答 0