apache-spark - 如何查询avro格式的数据集？

Question

这适用于镶木地板

 val sqlDF = spark.sql("SELECT DISTINCT field FROM parquet.`file-path'")

我用 Avro 尝试了同样的方法，但即使我使用com.databricks.spark.avro.

当我执行以下查询时：

val sqlDF = spark.sql("SELECT DISTINCT Source_Product_Classification FROM avro.`file path`")

我得到AnalysisException. 为什么？

org.apache.spark.sql.AnalysisException: Failed to find data source: avro. Please find an Avro package at http://spark.apache.org/third-party-projects.html;; line 1 pos 51
  at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
  at org.apache.spark.sql.execution.datasources.ResolveDataSource$$anonfun$apply$1.applyOrElse(rules.scala:61)
  at org.apache.spark.sql.execution.datasources.ResolveDataSource$$anonfun$apply$1.applyOrElse(rules.scala:38)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$resolveOperators$1.apply(LogicalPlan.scala:61)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$resolveOperators$1.apply(LogicalPlan.scala:61)
  at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperators(LogicalPlan.scala:60)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$1.apply(LogicalPlan.scala:58)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$1.apply(LogicalPlan.scala:58)
  at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:307)
  at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:188)
  at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:305)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperators(LogicalPlan.scala:58)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$1.apply(LogicalPlan.scala:58)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$1.apply(LogicalPlan.scala:58)
  at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:307)
  at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:188)
  at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:305)
  at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperators(LogicalPlan.scala:58)
  at org.apache.spark.sql.execution.datasources.ResolveDataSource.apply(rules.scala:38)
  at org.apache.spark.sql.execution.datasources.ResolveDataSource.apply(rules.scala:37)
  at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85)
  at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82)
  at scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124)
  at scala.collection.immutable.List.foldLeft(List.scala:84)
  at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:82)
  at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:74)
  at scala.collection.immutable.List.foreach(List.scala:381)
  at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:74)
  at org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:69)
  at org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:67)
  at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:50)
  at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:63)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592)

将格式名称更改为com.databricks.spark.avro没有任何区别并且查询失败。

val sqlDF = spark.sql("SELECT DISTINCT Source_Product_Classification FROM com.databricks.spark.avro`file-path`")

org.apache.spark.sql.catalyst.parser.ParseException:
extraneous input '.' expecting {<EOF>, ',', 'SELECT', 'FROM', 'ADD', 'AS', 'ALL', 'DISTINCT', 'WHERE', 'GROUP', 'BY', 'GROUPING', 'SETS', 'CUBE', 'ROLLUP', 'ORDER', 'HAVING', 'LIMIT', 'AT', 'OR', 'AND', 'IN', NOT, 'NO', 'EXISTS', 'BETWEEN', 'LIKE', RLIKE, 'IS', 'NULL', 'TRUE', 'FALSE', 'NULLS', 'ASC', 'DESC', 'FOR', 'INTERVAL', 'CASE', 'WHEN', 'THEN', 'ELSE', 'END', 'JOIN', 'CROSS', 'OUTER', 'INNER', 'LEFT', 'RIGHT', 'FULL', 'NATURAL', 'LATERAL', 'WINDOW', 'OVER', 'PARTITION', 'RANGE', 'ROWS', 'UNBOUNDED', 'PRECEDING', 'FOLLOWING', 'CURRENT', 'FIRST', 'LAST', 'ROW', 'WITH', 'VALUES', 'CREATE', 'TABLE', 'VIEW', 'REPLACE', 'INSERT', 'DELETE', 'INTO', 'DESCRIBE', 'EXPLAIN', 'FORMAT', 'LOGICAL', 'CODEGEN', 'CAST', 'SHOW', 'TABLES', 'COLUMNS', 'COLUMN', 'USE', 'PARTITIONS', 'FUNCTIONS', 'DROP', 'UNION', 'EXCEPT', 'MINUS', 'INTERSECT', 'TO', 'TABLESAMPLE', 'STRATIFY', 'ALTER', 'RENAME', 'ARRAY', 'MAP', 'STRUCT', 'COMMENT', 'SET', 'RESET', 'DATA', 'START', 'TRANSACTION', 'COMMIT', 'ROLLBACK', 'MACRO', 'IF', 'DIV', 'PERCENT', 'BUCKET', 'OUT', 'OF', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'OVERWRITE', 'TRANSFORM', 'REDUCE', 'USING', 'SERDE', 'SERDEPROPERTIES', 'RECORDREADER', 'RECORDWRITER', 'DELIMITED', 'FIELDS', 'TERMINATED', 'COLLECTION', 'ITEMS', 'KEYS', 'ESCAPED', 'LINES', 'SEPARATED', 'FUNCTION', 'EXTENDED', 'REFRESH', 'CLEAR', 'CACHE', 'UNCACHE', 'LAZY', 'FORMATTED', 'GLOBAL', TEMPORARY, 'OPTIONS', 'UNSET', 'TBLPROPERTIES', 'DBPROPERTIES', 'BUCKETS', 'SKEWED', 'STORED', 'DIRECTORIES', 'LOCATION', 'EXCHANGE', 'ARCHIVE', 'UNARCHIVE', 'FILEFORMAT', 'TOUCH', 'COMPACT', 'CONCATENATE', 'CHANGE', 'CASCADE', 'RESTRICT', 'CLUSTERED', 'SORTED', 'PURGE', 'INPUTFORMAT', 'OUTPUTFORMAT', DATABASE, DATABASES, 'DFS', 'TRUNCATE', 'ANALYZE', 'COMPUTE', 'LIST', 'STATISTICS', 'PARTITIONED', 'EXTERNAL', 'DEFINED', 'REVOKE', 'GRANT', 'LOCK', 'UNLOCK', 'MSCK', 'REPAIR', 'RECOVER', 'EXPORT', 'IMPORT', 'LOAD', 'ROLE', 'ROLES', 'COMPACTIONS', 'PRINCIPALS', 'TRANSACTIONS', 'INDEX', 'INDEXES', 'LOCKS', 'OPTION', 'ANTI', 'LOCAL', 'INPATH', 'CURRENT_DATE', 'CURRENT_TIMESTAMP', IDENTIFIER, BACKQUOTED_IDENTIFIER}(line 1, pos 65)

== SQL ==
SELECT DISTINCT Source_Product_Classification FROM com.databricks.spark.avro`/uat/myfile`
-----------------------------------------------------------------^^^

  at org.apache.spark.sql.catalyst.parser.ParseException.withCommand(ParseDriver.scala:197)
  at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parse(ParseDriver.scala:99)
  at org.apache.spark.sql.execution.SparkSqlParser.parse(SparkSqlParser.scala:45)
  at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parsePlan(ParseDriver.scala:53)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592)
  ... 48 elided

score 6 · Accepted Answer

Spark SQL 通过单独的spark-avro模块支持 avro 格式。

用于从 Spark SQL 读取和写入 Avro 数据的库。

请注意，这spark-avro是 Spark 默认不包含的独立模块。

您应该使用加载模块spark-submit --packages，例如

$ bin/spark-shell --packages com.databricks:spark-avro_2.11:3.2.0

请参阅使用 spark-shell 或 spark-submit。

score 4 · Accepted Answer

Jaceks 的回答通常有效，但在我的环境中，由于不明原因，它无法正常工作。并且spark-shell --packages com.databricks:spark-avro_2.11:3.2.0挂了很长时间而没有产生任何结果。

我使用--jars选项解决了这个问题spark-shell

脚步：

1）去https://mvnrepository.com/artifact/com.databricks/spark-avro_2.11/4.0.0 复制jar的链接地址http://central.maven.org/maven2/com/databricks/spark- avro_2.11/4.0.0/spark-avro_2.11-4.0.0.jar

2) wget http://central.maven.org/maven2/com/databricks/spark-avro_2.11/4.0.0/spark-avro_2.11-4.0.0.jar。

3) spark-shell --jars <pathwhere you downloaded jar file>/spark-avro_2.11-4.0.0.jar

4)spark.read.format("com.databricks.spark.avro").load("s3://MYAVROLOCATION.avro")

它被转换为数据框并能够打印它。

在您的情况下，一旦您获得数据框，您就可以在途中执行 sql。

注意：如果您不使用 spark-shell，您可以使用以下 maven 坐标使用 sbt 或 maven 和 spark-avro_2.11-4.0.0.jar 制作 uber jar。

<dependency>
    <groupId>com.databricks</groupId>
    <artifactId>spark-avro_2.11</artifactId>
    <version>4.0.0</version>
</dependency>

注意：Avro 数据源是在病房的 spark 2.4 中引入的。SparkSPARK-24768 有一个内置的 AVRO 数据源实现

这意味着以上所有事情都不再需要了。请参阅spark-release-2-4-0 发行说明

score 1 · Accepted Answer

Spark Avro 集成： 通过使用 Spark，我们可以使用spark-avro模块集成avro格式。spark-avro库最初由 databricks 作为开源库开发。spark-avro模块是外部的，默认情况下不包含在spark-submit或spark-shell中。所以在外部我们需要在提交 Spark 作业时指定。

在下一节中，我将解释如何集成 Spark 和 Avro 数据格式。

Spark 版本 > 2.4 Spark 2.4 版本以后，Spark SQL 提供了对读取和写入 Apache Avro 数据的内置支持。

Maven 依赖： https ://mvnrepository.com/artifact/org.apache.spark/spark-avro

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-avro_2.12</artifactId>
    <version>2.4.5</version>
</dependency>

火花提交：

./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:2.4.5 ...

火花壳：

./bin/spark-shell --packages org.apache.spark:spark-avro_2.12:2.4.5 ...

例子：

SparkAvroWriteExample.scala

import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;

case class Employee( id:Long, name:String, salary:Float, deptId: Int)

object SparkAvroWriteExample {
    def main(args: Array[String]): Unit = {

        val conf = new SparkConf().setIfMissing("spark.master", "local[*]").setAppName("Spark Avro Read Examples")
        val spark = SparkSession.builder().config(conf).getOrCreate();

        val employeeList = List(Employee(1, "Ranga", 10000, 1),
            Employee(2, "Vinod", 1000, 1),
            Employee(3, "Nishanth", 500000, 2),
            Employee(4, "Manoj", 25000, 1),
            Employee(5, "Yashu", 1600, 1),
            Employee(6, "Raja", 50000, 2)
        );

        val employeeDF = spark.createDataFrame(employeeList);
        employeeDF.coalesce(1).write.format("avro").mode("overwrite").save("employees.avro");
     spark.close();
    }
}

SparkAvroReadExample.scala

import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;

object SparkAvroReadExample {
    def main(args: Array[String]): Unit = {

        val conf = new SparkConf().setIfMissing("spark.master", "local[*]").setAppName("Spark Avro Read Examples")
        val spark = SparkSession.builder().config(conf).getOrCreate();

        val employeeDF = spark.read.format("avro").load("employees.avro");
        employeeDF.printSchema();
        employeeDF.foreach(employee => {println(employee);});
        spark.close();
    }
}

Github 链接 https://github.com/rangareddy/ranga-spark-poc/tree/master/spark-2.4/SparkAvro

Spark 版本 < 2.4 在 Spark 版本 < 2.4 中，我们需要明确指定 avro 格式为com.databricks.spark.avro否则我们会得到org.apache.spark.sql.AnalysisException: Failed to find data source: avro。错误。

Maven依赖：

Spark Version   Compatible version of Avro Data Source for Spark
1.2             0.2.0
1.3             1.0.0
1.4+            2.0.1
2.0 - 2.1       3.2.0
2.2 - 2.3       4.0.0

https://mvnrepository.com/artifact/com.databricks/spark-avro

<dependency>
    <groupId>com.databricks</groupId>
    <artifactId>spark-avro_2.11</artifactId>
    <version>4.0.0</version>
</dependency>

火花提交：

./bin/spark-submit --packages com.databricks:spark-avro_2.11:4.0.0 ...

火花壳：

./bin/spark-shell --packages com.databricks:spark-avro_2.11:4.0.0 ...

例子：

SparkAvroWriteExample.scala

import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;

case class Employee( id:Long, name:String, salary:Float, deptId: Int)

object SparkAvroWriteExample {
    def main(args: Array[String]): Unit = {

        val conf = new SparkConf().setIfMissing("spark.master", "local[*]").setAppName("Spark Avro Read Examples")
        val spark = SparkSession.builder().config(conf).getOrCreate();

        val employeeList = List(Employee(1, "Ranga", 10000, 1),
            Employee(2, "Vinod", 1000, 1),
            Employee(3, "Nishanth", 500000, 2),
            Employee(4, "Manoj", 25000, 1),
            Employee(5, "Yashu", 1600, 1),
            Employee(6, "Raja", 50000, 2)
        );

        val employeeDF = spark.createDataFrame(employeeList);
        employeeDF.coalesce(1).write.format("com.databricks.spark.avro").mode("overwrite").save("employees.avro");
        spark.close();
    }
}

SparkAvroReadExample.scala

import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;

object SparkAvroReadExample {
    def main(args: Array[String]): Unit = {

        val conf = new SparkConf().setIfMissing("spark.master", "local[*]").setAppName("Spark Avro Read Examples")
        val spark = SparkSession.builder().config(conf).getOrCreate();

        val employeeDF = spark.read.format("com.databricks.spark.avro").load("employees.avro");
        employeeDF.printSchema();
        employeeDF.foreach(employee => {println(employee);});
        spark.close();
    }
}

Github 链接 https://github.com/rangareddy/ranga-spark-poc/tree/master/spark-2.3/SparkAvro

这就是所有人！

apache-spark - 如何查询avro格式的数据集？

3 回答 3

Related

Reference