1

我有一个通过 CloudFormation 部署的 KA SQL 应用程序。它旨在读取 kinesis 流的数据，然后运行简单的 SQL 计数查询并将数据输出到 Firehose。Firehose 应该将这些数据写入 S3。不幸的是，数据无处可寻。我 89% 确定问题出在 SQL 脚本中，但如果没有日志，就很难查明根本原因。

语境：

  • 通过 CloudFormation API 配置 KA SQL 时不会出现“权限/访问被拒绝”错误。
  • KA SQL 启用了 CloudWatch 日志记录，以前在修复之前显示配置错误。
  • Firehose 具有将数据写入 S3 的正确权限。通过 Firehose UI 测试。
  • 不幸的是，目前我无法访问 Kinesis 分析 UI，因为我缺乏通过 UI 创建应用程序的权限。

怀疑：

  • 位于 CFT 中的 SQL 脚本中的流氓引号是通过对流层生成的
  • 取自 AWS 示例并稍作修改的 SQL 脚本中的错误
  • 某处缺少许可？

主要问题：鉴于有许多活动部件（KA app/Firehose/S3/Kinesis），您通过不同的 CloudWatch 指标和日志流对其进行监控，您如何检查您的 SQL 是否简单损坏？这个问题可能听起来很傻，但想象一下您的 SQL 字符串通过某些工具被翻译并包含到 CFT 模板中。

PS 我附上了 CFT 以防万一一双新鲜的眼睛看到了什么

{
  "Description": "Builds KA SQL application",
  "Resources": {
    "ApplicationOutput": {
      "DependsOn": "KinesisAnalyticsApplication",
      "Properties": {
        "ApplicationName": "sql-poc",
        "Output": {
          "DestinationSchema": {
            "RecordFormatType": "JSON"
          },
          "KinesisFirehoseOutput": {
            "ResourceARN": "arn:aws:firehose:eu-west-1:xxxxxxxxxxx:deliverystream/sql-output"
          },
          "Name": "DESTINATION_SQL_STREAM"
        }
      },
      "Type": "AWS::KinesisAnalyticsV2::ApplicationOutput"
    },
    "CloudWatchLoggingOptionForV2KDA": {
      "DependsOn": "KinesisAnalyticsApplication",
      "Properties": {
        "ApplicationName": "sql-poc",
        "CloudWatchLoggingOption": {
          "LogStreamARN": "arn:aws:logs:eu-west-1:xxxxxxxxxxx:log-group:/aws/kinesisanalytics/my-log-group:kinesis-analytics-log-stream"
        }
      },
      "Type": "AWS::KinesisAnalyticsV2::ApplicationCloudWatchLoggingOption"
    },
    "ErrorOutput": {
      "DependsOn": "KinesisAnalyticsApplication",
      "Properties": {
        "ApplicationName": "sql-poc",
        "Output": {
          "DestinationSchema": {
            "RecordFormatType": "JSON"
          },
          "KinesisFirehoseOutput": {
            "ResourceARN": "arn:aws:firehose:eu-west-1:xxxxxxxxxx:deliverystream/sql-output"
          },
          "Name": "error_stream"
        }
      },
      "Type": "AWS::KinesisAnalyticsV2::ApplicationOutput"
    },
    "KinesisAnalyticsApplication": {
      "Properties": {
        "ApplicationConfiguration": {
          "ApplicationCodeConfiguration": {
            "CodeContent": {
              "TextContent": "CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM\" (\"event\" VARCHAR(256),\"count\" INTEGER);CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM\" SELECT STREAM \"event\", COUNT(*) AS \"count\" FROM \"SOURCE_SQL_STREAM_001\" GROUP BY \"event\", FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);"
            },
            "CodeContentType": "PLAINTEXT"
          },
          "SqlApplicationConfiguration": {
            "Inputs": [
              {
                "InputSchema": {
                  "RecordColumns": [
                    {
                      "Mapping": "$.event",
                      "Name": "event",
                      "SqlType": "VARCHAR(256)"
                    }
                  ],
                  "RecordEncoding": "UTF-8",
                  "RecordFormat": {
                    "MappingParameters": {
                      "JSONMappingParameters": {
                        "RecordRowPath": "$"
                      }
                    },
                    "RecordFormatType": "JSON"
                  }
                },
                "KinesisStreamsInput": {
                  "ResourceARN": "arn:aws:kinesis:eu-west-1:xxxxxxxxxxx:stream/my-input-kinesis"
                },
                "NamePrefix": "SOURCE_SQL_STREAM"
              }
            ]
          }
        },
        "ApplicationDescription": "SQL based PoC",
        "ApplicationName": "sql-poc",
        "RuntimeEnvironment": "SQL-1_0",
        "ServiceExecutionRole": "arn:aws:iam::xxxxxxxxxxxx:role/my-iam-role"
      },
      "Type": "AWS::KinesisAnalyticsV2::Application"
    }
  }
}
4

0 回答 0