1

我正在使用 Cloud Data Fusion 设置管道，并且我想导入自己的自定义插件。有没有简单的导入方法？

我已经尝试使用ImportStudio 部分中的按钮，但它给我的工件带来了一些问题。我还尝试使用按钮添加新实体+并上传 .jar 和 .json 文件，但它不返回任何消息。

但是，在App Fabric日志中实际上遇到了这些错误：

2019-06-13 08:37:15,020 - ERROR [appfabric-executor-30:i.c.c.c.HttpExceptionHandler@70] - Unexpected error: request=PUT /v3/namespaces/default/artifacts/org.myCustom.plugin/versions/1.0-SNAPSHOT/properties user=<null>:
java.lang.NullPointerException: null
    at com.google.common.collect.ImmutableMap.copyOf(ImmutableMap.java:280) ~[com.google.guava.guava-13.0.1.jar:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactMeta.<init>(ArtifactMeta.java:53) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.lambda$updateArtifactProperties$19(ArtifactStore.java:648) ~[na:na]
    at io.cdap.cdap.spi.data.sql.SqlTransactionRunner.run(SqlTransactionRunner.java:74) ~[na:na]
    at io.cdap.cdap.spi.data.sql.RetryingSqlTransactionRunner.run(RetryingSqlTransactionRunner.java:64) ~[na:na]
    at io.cdap.cdap.spi.data.transaction.TransactionRunners.run(TransactionRunners.java:92) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.updateArtifactProperties(ArtifactStore.java:637) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository.writeArtifactProperties(DefaultArtifactRepository.java:289) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository.writeArtifactProperties(AuthorizationArtifactRepository.java:216) ~[na:na]
    at io.cdap.cdap.gateway.handlers.ArtifactHttpHandler.writeProperties(ArtifactHttpHandler.java:341) ~[na:na]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_212]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_212]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_212]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_212]
    at io.cdap.http.internal.HttpMethodInfo.invoke(HttpMethodInfo.java:82) ~[io.cdap.http.netty-http-1.2.0.jar:na]
    at io.cdap.http.internal.HttpDispatcher.channelRead(HttpDispatcher.java:45) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:38) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:353) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.cdap.http.internal.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor.run(NonStickyEventExecutorGroup.java:254) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor$NonNotifyRunnable.run(UnorderedThreadPoolEventExecutor.java:277) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_212]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_212]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]
2019-06-13 08:37:24,504 - DEBUG [appfabric-executor-27:i.c.c.a.g.DefaultProgramRunnerFactory@73] - Using runtime provider io.cdap.cdap.app.runtime.spark.Spark2ProgramRuntimeProvider@444b1b21 for program type Spark
2019-06-13 08:37:24,524 - DEBUG [appfabric-executor-27:i.c.c.a.g.DefaultProgramRunnerFactory@73] - Using runtime provider io.cdap.cdap.app.runtime.spark.Spark2ProgramRuntimeProvider@444b1b21 for program type Spark
2019-06-13 08:37:27,200 - ERROR [appfabric-executor-26:i.c.c.c.HttpExceptionHandler@70] - Unexpected error: request=PUT /v3/namespaces/default/artifacts/org.myCustom.plugin/versions/1.0-SNAPSHOT/properties user=<null>:
java.lang.NullPointerException: null
    at com.google.common.collect.ImmutableMap.copyOf(ImmutableMap.java:280) ~[com.google.guava.guava-13.0.1.jar:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactMeta.<init>(ArtifactMeta.java:53) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.lambda$updateArtifactProperties$19(ArtifactStore.java:648) ~[na:na]
    at io.cdap.cdap.spi.data.sql.SqlTransactionRunner.run(SqlTransactionRunner.java:74) ~[na:na]
    at io.cdap.cdap.spi.data.sql.RetryingSqlTransactionRunner.run(RetryingSqlTransactionRunner.java:64) ~[na:na]
    at io.cdap.cdap.spi.data.transaction.TransactionRunners.run(TransactionRunners.java:92) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.updateArtifactProperties(ArtifactStore.java:637) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository.writeArtifactProperties(DefaultArtifactRepository.java:289) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository.writeArtifactProperties(AuthorizationArtifactRepository.java:216) ~[na:na]
    at io.cdap.cdap.gateway.handlers.ArtifactHttpHandler.writeProperties(ArtifactHttpHandler.java:341) ~[na:na]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_212]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_212]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_212]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_212]
    at io.cdap.http.internal.HttpMethodInfo.invoke(HttpMethodInfo.java:82) ~[io.cdap.http.netty-http-1.2.0.jar:na]
    at io.cdap.http.internal.HttpDispatcher.channelRead(HttpDispatcher.java:45) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:38) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:353) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.cdap.http.internal.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor.run(NonStickyEventExecutorGroup.java:254) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor$NonNotifyRunnable.run(UnorderedThreadPoolEventExecutor.java:277) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_212]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_212]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]
2019-06-13 08:38:43,782 - DEBUG [appfabric-executor-30:i.c.c.a.g.DefaultProgramRunnerFactory@73] - Using runtime provider io.cdap.cdap.app.runtime.spark.Spark2ProgramRuntimeProvider@444b1b21 for program type Spark
2019-06-13 08:38:43,803 - DEBUG [appfabric-executor-30:i.c.c.a.g.DefaultProgramRunnerFactory@73] - Using runtime provider io.cdap.cdap.app.runtime.spark.Spark2ProgramRuntimeProvider@444b1b21 for program type Spark
2019-06-13 08:38:46,441 - ERROR [appfabric-executor-38:i.c.c.c.HttpExceptionHandler@70] - Unexpected error: request=PUT /v3/namespaces/default/artifacts/org.myCustom.plugin/versions/1.0-SNAPSHOT/properties user=<null>:
java.lang.NullPointerException: null
    at com.google.common.collect.ImmutableMap.copyOf(ImmutableMap.java:280) ~[com.google.guava.guava-13.0.1.jar:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactMeta.<init>(ArtifactMeta.java:53) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.lambda$updateArtifactProperties$19(ArtifactStore.java:648) ~[na:na]
    at io.cdap.cdap.spi.data.sql.SqlTransactionRunner.run(SqlTransactionRunner.java:74) ~[na:na]
    at io.cdap.cdap.spi.data.sql.RetryingSqlTransactionRunner.run(RetryingSqlTransactionRunner.java:64) ~[na:na]
    at io.cdap.cdap.spi.data.transaction.TransactionRunners.run(TransactionRunners.java:92) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.updateArtifactProperties(ArtifactStore.java:637) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository.writeArtifactProperties(DefaultArtifactRepository.java:289) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository.writeArtifactProperties(AuthorizationArtifactRepository.java:216) ~[na:na]
    at io.cdap.cdap.gateway.handlers.ArtifactHttpHandler.writeProperties(ArtifactHttpHandler.java:341) ~[na:na]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_212]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_212]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_212]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_212]
    at io.cdap.http.internal.HttpMethodInfo.invoke(HttpMethodInfo.java:82) ~[io.cdap.http.netty-http-1.2.0.jar:na]
    at io.cdap.http.internal.HttpDispatcher.channelRead(HttpDispatcher.java:45) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:38) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:353) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.cdap.http.internal.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor.run(NonStickyEventExecutorGroup.java:254) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor$NonNotifyRunnable.run(UnorderedThreadPoolEventExecutor.java:277) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_212]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_212]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]
2019-06-13 08:40:20,681 - DEBUG [appfabric-executor-36:i.c.c.a.g.DefaultProgramRunnerFactory@73] - Using runtime provider io.cdap.cdap.app.runtime.spark.Spark2ProgramRuntimeProvider@444b1b21 for program type Spark
2019-06-13 08:40:20,703 - DEBUG [appfabric-executor-36:i.c.c.a.g.DefaultProgramRunnerFactory@73] - Using runtime provider io.cdap.cdap.app.runtime.spark.Spark2ProgramRuntimeProvider@444b1b21 for program type Spark
2019-06-13 08:40:22,990 - ERROR [appfabric-executor-29:i.c.c.c.HttpExceptionHandler@70] - Unexpected error: request=PUT /v3/namespaces/default/artifacts/org.myCustom.plugin/versions/1.0-SNAPSHOT/properties user=<null>:
java.lang.NullPointerException: null
    at com.google.common.collect.ImmutableMap.copyOf(ImmutableMap.java:280) ~[com.google.guava.guava-13.0.1.jar:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactMeta.<init>(ArtifactMeta.java:53) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.lambda$updateArtifactProperties$19(ArtifactStore.java:648) ~[na:na]
    at io.cdap.cdap.spi.data.sql.SqlTransactionRunner.run(SqlTransactionRunner.java:74) ~[na:na]
    at io.cdap.cdap.spi.data.sql.RetryingSqlTransactionRunner.run(RetryingSqlTransactionRunner.java:64) ~[na:na]
    at io.cdap.cdap.spi.data.transaction.TransactionRunners.run(TransactionRunners.java:92) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.ArtifactStore.updateArtifactProperties(ArtifactStore.java:637) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository.writeArtifactProperties(DefaultArtifactRepository.java:289) ~[na:na]
    at io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository.writeArtifactProperties(AuthorizationArtifactRepository.java:216) ~[na:na]
    at io.cdap.cdap.gateway.handlers.ArtifactHttpHandler.writeProperties(ArtifactHttpHandler.java:341) ~[na:na]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_212]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_212]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_212]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_212]
    at io.cdap.http.internal.HttpMethodInfo.invoke(HttpMethodInfo.java:82) ~[io.cdap.http.netty-http-1.2.0.jar:na]
    at io.cdap.http.internal.HttpDispatcher.channelRead(HttpDispatcher.java:45) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:38) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:353) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at io.cdap.http.internal.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor.run(NonStickyEventExecutorGroup.java:254) [io.cdap.http.netty-http-1.2.0.jar:na]
    at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor$NonNotifyRunnable.run(UnorderedThreadPoolEventExecutor.java:277) [io.netty.netty-all-4.1.16.Final.jar:4.1.16.Final]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_212]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_212]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_212]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_212]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]

我是 Cloud Data Fusion 的新手，以前从未使用过 CDAP。这些错误有解释吗？

4

3 回答 3

1

您可以导入管道，但如果您已创建插件，则可以通过单击绿色上传插件，'+'然后选择"Plugin">"Upload"您已完成的插件。你能分享你的插件 JSON 吗？

于 2019-06-19T12:42:03.803 回答
0

好的，我也尝试过这种方法。我的 JSON 如下所示：

{
  "parents": [
    "system:cdap-data-pipeline[4.0.0,6.1.0)",
    "system:cdap-data-streams[4.0.0,6.1.0)"
  ],
  "artifact":{
    "name" : "org.myCustom.plugin",
    "version": "1.0-SNAPSHOT",
    "scope": "SYSTEM"
  }
  "properties": {
    "widgets.ErrorCollector-errortransform": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Error Collector Configuration\",\"properties\":[{\"widget-type\":\"textbox\",\"label\":\"Message Field\",\"name\":\"messageField\",\"plugin-function\":{\"method\":\"POST\",\"widget\":\"outputSchema\",\"plugin-method\":\"getSchema\"},\"widget-attributes\":{\"default\":\"errMsg\"}},{\"widget-type\":\"textbox\",\"label\":\"Code Field\",\"name\":\"codeField\",\"widget-attributes\":{\"default\":\"errCode\"}},{\"widget-type\":\"textbox\",\"label\":\"Stage Field\",\"name\":\"stageField\",\"widget-attributes\":{\"default\":\"errStage\"}}]}],\"outputs\":[]}",
    "widgets.FilesetDelete-postaction": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Fileset Delete Configuration\",\"properties\":[{\"widget-type\":\"dataset-selector\",\"label\":\"FileSet Name\",\"name\":\"filesetName\"},{\"widget-type\":\"textbox\",\"label\":\"FileSet directory\",\"name\":\"directory\"},{\"widget-type\":\"textbox\",\"label\":\"Delete Regex\",\"name\":\"deleteRegex\"}]}],\"outputs\":[]}",
    "widgets.FilesetMove-action": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Fileset Delete Configuration\",\"properties\":[{\"widget-type\":\"dataset-selector\",\"label\":\"Source FileSet\",\"name\":\"sourceFileset\"},{\"widget-type\":\"dataset-selector\",\"label\":\"Destination FileSet\",\"name\":\"destinationFileset\"},{\"widget-type\":\"textbox\",\"label\":\"Filter Regex\",\"name\":\"filterRegex\"}]}],\"outputs\":[]}",
    "widgets.StringCase-transform-custom": "{\"metadata\":{\"spec-version\":\"1.0\"},\"artifact\":{\"name\":\"org.myCustom.plugin\",\"version\":\"1.0-SNAPSHOT\",\"scope\":\"SYSTEM\"},\"display-name\":\"My Custom Transformation\",\"icon\":{\"type\":\"builtin|link|inline\",\"arguments\":{\"url\":\"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcT1IRBT7dHXIhCkjmoy8esJsWY2Gv89tuoIbLVEwi16fTb5FbcF\",\"data\":\"data:image/png;base64,...\"}},\"configuration-groups\":[{\"label\":\"String Case Configuration\",\"properties\":[{\"widget-type\":\"csv\",\"label\":\"Fields to upper case\",\"name\":\"upperFields\",\"description\":\"List of fields to upper case.\",\"widget-attributes\":{\"delimiter\":\",\"}},{\"widget-type\":\"csv\",\"label\":\"Fields to lower case\",\"name\":\"lowerFields\",\"description\":\"List of fields to lower case.\",\"widget-attributes\":{\"delimiter\":\",\"}}]}],\"outputs\":[]}",
    "widgets.StringCase-transform": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"String Case Configuration\",\"properties\":[{\"widget-type\":\"csv\",\"label\":\"Fields to upper case\",\"name\":\"upperFields\",\"description\":\"List of fields to upper case.\",\"widget-attributes\":{\"delimiter\":\",\"}},{\"widget-type\":\"csv\",\"label\":\"Fields to lower case\",\"name\":\"lowerFields\",\"description\":\"List of fields to lower case.\",\"widget-attributes\":{\"delimiter\":\",\"}}]}],\"outputs\":[]}",
    "widgets.TextFileSet-batchsink": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Text FileSet Sink Configuration\",\"properties\":[{\"widget-type\":\"dataset-selector\",\"label\":\"FileSet Name\",\"name\":\"fileSetName\"},{\"widget-type\":\"textbox\",\"label\":\"Field separator\",\"name\":\"fieldSeparator\"},{\"widget-type\":\"textbox\",\"label\":\"Field separator\",\"name\":\"outputDir\"}]}],\"outputs\":[]}",
    "widgets.TextFileSet-batchsource": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Text FileSet Source Configuration\",\"properties\":[{\"widget-type\":\"dataset-selector\",\"label\":\"FileSet Name\",\"name\":\"fileSetName\"},{\"widget-type\":\"textbox\",\"label\":\"Input files within the FileSet\",\"name\":\"files\"},{\"widget-type\":\"select\",\"label\":\"Create FileSet if it does not exist\",\"name\":\"createIfNotExists\",\"widget-attributes\":{\"values\":[\"true\",\"false\"],\"default\":\"false\"}},{\"widget-type\":\"select\",\"label\":\"Delete data read on pipeline run success\",\"name\":\"deleteInputOnSuccess\",\"widget-attributes\":{\"values\":[\"true\",\"false\"],\"default\":\"false\"}}]}],\"outputs\":[{\"widget-type\":\"non-editable-schema-editor\",\"schema\":{\"position\":\"long\",\"text\":\"string\"}}]}",
    "widgets.WordCount-batchaggregator": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Word Count Aggregator Configuration\",\"properties\":[{\"widget-type\":\"textbox\",\"label\":\"Field Name\",\"name\":\"field\"}]}],\"outputs\":[{\"widget-type\":\"non-editable-schema-editor\",\"schema\":{\"word\":\"string\",\"count\":\"long\"}}]}",
    "widgets.WordCount-sparkcompute": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Word Count Compute Configuration\",\"properties\":[{\"widget-type\":\"textbox\",\"label\":\"Field Name\",\"name\":\"field\"}]}],\"outputs\":[{\"widget-type\":\"non-editable-schema-editor\",\"schema\":{\"word\":\"string\",\"count\":\"long\"}}]}",
    "widgets.WordCount-sparksink": "{\"metadata\":{\"spec-version\":\"1.0\"},\"configuration-groups\":[{\"label\":\"Word Count Sink Configuration\",\"properties\":[{\"widget-type\":\"textbox\",\"label\":\"Field Name\",\"name\":\"field\"},{\"widget-type\":\"dataset-selector\",\"label\":\"Table Name\",\"name\":\"tableName\"}]}],\"outputs\":[]}",
    "doc.ErrorCollector-errortransform": "# Error Collector\r\n\r\n\r\nDescription\r\n-----------\r\nThe ErrorCollector plugin takes errors emitted from the previous stage and flattens them by adding\r\nthe error message, code, and stage to the record and outputting the result.\r\n\r\nUse Case\r\n--------\r\nThe plugin is used when you want to capture errors emitted from another stage and pass them along\r\nwith all the error information flattened into the record. For example, you may want to connect a sink\r\nto this plugin in order to store and later examine the error records.\r\n\r\nProperties\r\n----------\r\n**messageField:** The name of the error message field to use in the output schema. Defaults to 'errMsg'.\r\nIf this is not specified, the error message will be dropped.\r\n\r\n**codeField:** The name of the error code field to use in the output schema. Defaults to 'errCode'.\r\nIf this is not specified, the error code will be dropped.\r\n\r\n**stageField:** The name of the error stage field to use in the output schema. Defaults to 'errStage'.\r\nIf this is not specified, the error stage will be dropped.\r\n\r\n\r\nExample\r\n-------\r\nThis example adds the error message, error code, and error stage as the 'errMsg', 'errCode', and 'errStage' fields.\r\n\r\n    {\r\n        \"name\": \"ErrorCollector\",\r\n        \"type\": \"errortransform\",\r\n        \"properties\": {\r\n            \"messageField\": \"errMsg\",\r\n            \"codeField\": \"errCode\",\r\n            \"stageField\": \"errStage\"\r\n        }\r\n    }\r\n\r\nFor example, suppose the plugin receives this error record:\r\n\r\n    +============================+\r\n    | field name | type | value  |\r\n    +============================+\r\n    | A          | int  | 10     |\r\n    | B          | int  | 20     |\r\n    +============================+\r\n\r\nwith error code 17, error message 'invalid', from stage 'parser'. It will add the error information\r\nto the record and output:\r\n\r\n    +===============================+\r\n    | field name | type   | value   |\r\n    +===============================+\r\n    | A          | int    | 10      |\r\n    | B          | int    | 20      |\r\n    | errMsg     | string | invalid |\r\n    | errCode    | int    | 17      |\r\n    | errStage   | string | parser  |\r\n    +===============================+\r\n",
    "doc.FilesetDelete-postaction": "# FilesetDelete Post Action\r\n\r\nDescription\r\n-----------\r\n\r\nIf a pipeline run succeeds, deletes files in a FileSet that match a configurable regex.\r\n\r\nUse Case\r\n--------\r\n\r\nThis post action is used if you need to clean up some files after a successful pipeline run.\r\n\r\nProperties\r\n----------\r\n\r\n**filesetName:** The name of the FileSet to delete files from.\r\n\r\n**directory:** The directory in the FileSet to delete files from. Macro enabled.\r\n\r\n**deleteRegex:** Delete files that match this regex.\r\n\r\nExample\r\n-------\r\n\r\nThis example deletes any files that have the '.crc' extension from the 2016-01-01 directory of a FileSet named 'users'.\r\n\r\n    {\r\n        \"name\": \"TextFileSet\",\r\n        \"type\": \"batchsource\",\r\n        \"properties\": {\r\n            \"fileSetName\": \"users\",\r\n            \"directory\": \"2016-01-01\",\r\n            \"deleteRegex\": \".*\\\\.crc\"\r\n        }\r\n    }\r\n",
    "doc.FilesetMove-action": "# FilesetMove Action\r\n\r\nDescription\r\n-----------\r\n\r\nMoves files from one FileSet into another FileSet.\r\n\r\nUse Case\r\n--------\r\n\r\nThis action may be used at the start of a pipeline run to move a subset of files from one FileSet into another\r\nFileSet to process. Or it may be used at the end of a pipeline run to move a subset of files from the output FileSet\r\nto some other location for further processing.\r\n\r\nProperties\r\n----------\r\n\r\n**sourceFileset:** The name of the FileSet to move files from\r\n\r\n**destinationFileSet:** The name of the FileSet to move files to\r\n\r\n**filterRegex:** Filter any files whose name matches this regex.\r\nDefaults to '^\\\\.', which filters any files that begin with a period.\r\n\r\nExample\r\n-------\r\n\r\nThis example moves files from the 'staging' FileSet into the 'input' FileSet.\r\n\r\n    {\r\n        \"name\": \"TextFileSet\",\r\n        \"type\": \"batchsource\",\r\n        \"properties\": {\r\n            \"sourceFileset\": \"staging\",\r\n            \"destinationFileset\": \"input\"\r\n        }\r\n    }\r\n",
    "doc.StringCase-transform": "# String Case Transform\r\n\r\nDescription\r\n-----------\r\n\r\nChanges configured fields to lowercase or uppercase.\r\n\r\nUse Case\r\n--------\r\n\r\nThis transform is used whenever you need to uppercase or lowercase one or more fields.\r\n\r\nProperties\r\n----------\r\n\r\n**lowerFields:** Comma separated list of fields to lowercase.\r\n\r\n**upperFields:** Comma separated list of fields to uppercase.\r\n\r\nExample\r\n-------\r\n\r\nThis example lowercases the 'name' field and uppercases the 'id' field:\r\n\r\n    {\r\n        \"name\": \"StringCase\",\r\n        \"type\": \"transform\",\r\n        \"properties\": {\r\n            \"lowerFields\": \"name\",\r\n            \"upperFields\": \"id\"\r\n        }\r\n    }\r\n",
    "doc.TextFileSet-batchsink": "# Text FileSet Batch Sink\r\n\r\nDescription\r\n-----------\r\n\r\nWrites to a CDAP FileSet in text format. One line is written for each record\r\nsent to the sink. All record fields are joined using a configurable separator.\r\n\r\n\r\nUse Case\r\n--------\r\n\r\nThis source is used whenever you need to write to a FileSet in text format.\r\n\r\nProperties\r\n----------\r\n\r\n**fileSetName:** The name of the FileSet to write to.\r\n\r\n**fieldSeparator:** The separator to join input record fields on. Defaults to ','.\r\n\r\n**outputDir:** The output directory to write to. Macro enabled.\r\n\r\nExample\r\n-------\r\n\r\nThis example writes to a FileSet named 'users', using the '|' character to separate record fields:\r\n\r\n    {\r\n        \"name\": \"TextFileSet\",\r\n        \"type\": \"batchsink\",\r\n        \"properties\": {\r\n            \"fileSetName\": \"users\",\r\n            \"fieldSeparator\": \"|\",\r\n            \"outputDir\": \"${outputDir}\"\r\n        }\r\n    }\r\n\r\nBefore running the pipeline, the 'outputDir' runtime argument must be specified.\r\n",
    "doc.TextFileSet-batchsource": "# Text FileSet Batch Source\r\n\r\nDescription\r\n-----------\r\n\r\nReads from a CDAP FileSet in text format. Outputs records with two fields -- position (long), and text (string).\r\n\r\nUse Case\r\n--------\r\n\r\nThis source is used whenever you need to read from a FileSet in text format.\r\n\r\nProperties\r\n----------\r\n\r\n**fileSetName:** The name of the FileSet to read from.\r\n\r\n**createIfNotExists:** Whether to create the FileSet if it does not exist. Defaults to false.\r\n\r\n**deleteInputOnSuccess:** Whether to delete the data read if the pipeline run succeeded. Defaults to false.\r\n\r\n**files:** A comma separated list of files in the FileSet to read. Macro enabled.\r\n\r\nExample\r\n-------\r\n\r\nThis example reads from a FileSet named 'users' and deletes the data it read if the pipeline run succeeded:\r\n\r\n    {\r\n        \"name\": \"TextFileSet\",\r\n        \"type\": \"batchsource\",\r\n        \"properties\": {\r\n            \"fileSetName\": \"users\",\r\n            \"deleteInputOnSuccess\": \"true\",\r\n            \"files\": \"${inputFiles}\"\r\n        }\r\n    }\r\n\r\nBefore running the pipeline, the 'inputFiles' runtime argument must be specified.\r\n",
    "doc.WordCount-batchaggregator": "# Word Count Batch Aggregator\r\n\r\nDescription\r\n-----------\r\n\r\nFor the configured input string field, counts the number of times each word appears in that field.\r\nRecords output will have two fields -- word (string), and count (long).\r\n\r\nUse Case\r\n--------\r\n\r\nThis plugin is used whenever you want to count the number of times each word appears in a field.\r\n\r\nProperties\r\n----------\r\n\r\n**field:** The name of the string field to count words in.\r\n\r\nExample\r\n-------\r\n\r\nThis example counts the words in the 'text' field:\r\n\r\n    {\r\n        \"name\": \"WordCount\",\r\n        \"type\": \"batchaggregator\",\r\n        \"properties\": {\r\n            \"field\": \"text\"\r\n        }\r\n    }\r\n",
    "doc.WordCount-sparkcompute": "# Word Count Spark Compute\r\n\r\nDescription\r\n-----------\r\n\r\nFor the configured input string field, counts the number of times each word appears in that field.\r\nRecords output will have two fields -- word (string), and count (long).\r\n\r\nUse Case\r\n--------\r\n\r\nThis plugin is used whenever you want to count the number of times each word appears in a field.\r\n\r\nProperties\r\n----------\r\n\r\n**field:** The name of the string field to count words in.\r\n\r\nExample\r\n-------\r\n\r\nThis example counts the words in the 'text' field:\r\n\r\n    {\r\n        \"name\": \"WordCount\",\r\n        \"type\": \"sparkcompute\",\r\n        \"properties\": {\r\n            \"field\": \"text\"\r\n        }\r\n    }\r\n",
    "doc.WordCount-sparksink": "# Word Count Spark Sink\r\n\r\nDescription\r\n-----------\r\n\r\nFor the configured input string field, counts the number of times each word appears in that field.\r\nThe results are written to a CDAP KeyValueTable.\r\n\r\nUse Case\r\n--------\r\n\r\nThis plugin is used whenever you want to count and save the number of times each word appears in a field.\r\n\r\nProperties\r\n----------\r\n\r\n**field:** The name of the string field to count words in.\r\n\r\n**tableName:** The name of KeyValueTable to store the results in.\r\n\r\nExample\r\n-------\r\n\r\nThis example counts the words in the 'text' field and stores the results in the 'wordcounts' KeyValueTable:\r\n\r\n    {\r\n        \"name\": \"WordCount\",\r\n        \"type\": \"sparksink\",\r\n        \"properties\": {\r\n            \"field\": \"text\",\r\n            \"tableName\": \"wordcounts\"\r\n        }\r\n    }\r\n"
  }
}
于 2019-06-20T13:27:17.703 回答
0

当您尝试通过 UI 上传插件时，您收到什么错误消息？另外，您是自己构建插件 JSON 还是通过 CDAP 插件模板之一构建的？

于 2019-07-11T19:37:16.010 回答