0

大家好，我们正在将数据库从本地迁移到 Amazon aurora。我们的数据库大小约为 136GB，而且很少有每个表都有超过数百万条记录。然而，在数百万行的完全加载完成后，大约 200,000 到 300,000 行被迁移。我们不知道我们在哪里下降，因为我们是 DMS 的新手。任何人都知道我们如何迁移确切的行数。

migration type :full load

这是我们的 AWS DMS 任务设置

{
  "TargetMetadata": {
    "TargetSchema": "",
    "SupportLobs": true,
    "FullLobMode": true,
    "LobChunkSize": 64,
    "LimitedSizeLobMode": false,
    "LobMaxSize": 0,
    "LoadMaxFileSize": 0,
    "ParallelLoadThreads": 0,
    "BatchApplyEnabled": false
  },
  "FullLoadSettings": {
    "FullLoadEnabled": true,
    "ApplyChangesEnabled": false,
    "TargetTablePrepMode": "TRUNCATE_BEFORE_LOAD",
    "CreatePkAfterFullLoad": false,
    "StopTaskCachedChangesApplied": false,
    "StopTaskCachedChangesNotApplied": false,
    "ResumeEnabled": false,
    "ResumeMinTableSize": 100000,
    "ResumeOnlyClusteredPKTables": true,
    "MaxFullLoadSubTasks": 15,
    "TransactionConsistencyTimeout": 600,
    "CommitRate": 10000
  },
  "Logging": {
    "EnableLogging": true,
    "LogComponents": [
      {
        "Id": "SOURCE_UNLOAD",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "SOURCE_CAPTURE",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "TARGET_LOAD",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "TARGET_APPLY",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "TASK_MANAGER",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      }
    ],
    "CloudWatchLogGroup": "dms-tasks-krishna-smartdata",
    "CloudWatchLogStream": "dms-task-UERQWLR6AYHYIEKMR3HN2VL7T4"
  },
  "ControlTablesSettings": {
    "historyTimeslotInMinutes": 5,
    "ControlSchema": "",
    "HistoryTimeslotInMinutes": 5,
    "HistoryTableEnabled": true,
    "SuspendedTablesTableEnabled": true,
    "StatusTableEnabled": true
  },
  "StreamBufferSettings": {
    "StreamBufferCount": 3,
    "StreamBufferSizeInMB": 8,
    "CtrlStreamBufferSizeInMB": 5
  },
  "ChangeProcessingDdlHandlingPolicy": {
    "HandleSourceTableDropped": true,
    "HandleSourceTableTruncated": true,
    "HandleSourceTableAltered": true
  },
  "ErrorBehavior": {
    "DataErrorPolicy": "LOG_ERROR",
    "DataTruncationErrorPolicy": "LOG_ERROR",
    "DataErrorEscalationPolicy": "SUSPEND_TABLE",
    "DataErrorEscalationCount": 0,
    "TableErrorPolicy": "SUSPEND_TABLE",
    "TableErrorEscalationPolicy": "STOP_TASK",
    "TableErrorEscalationCount": 0,
    "RecoverableErrorCount": -1,
    "RecoverableErrorInterval": 5,
    "RecoverableErrorThrottling": true,
    "RecoverableErrorThrottlingMax": 1800,
    "ApplyErrorDeletePolicy": "IGNORE_RECORD",
    "ApplyErrorInsertPolicy": "LOG_ERROR",
    "ApplyErrorUpdatePolicy": "LOG_ERROR",
    "ApplyErrorEscalationPolicy": "LOG_ERROR",
    "ApplyErrorEscalationCount": 0,
    "FullLoadIgnoreConflicts": true
  },
  "ChangeProcessingTuning": {
    "BatchApplyPreserveTransaction": true,
    "BatchApplyTimeoutMin": 1,
    "BatchApplyTimeoutMax": 30,
    "BatchApplyMemoryLimit": 500,
    "BatchSplitSize": 0,
    "MinTransactionSize": 1000,
    "CommitTimeout": 1,
    "MemoryLimitTotal": 1024,
    "MemoryKeepTime": 60,
    "StatementCacheSize": 50
  }
}

映射方法：

{
  "rules": [
    {
      "rule-type": "selection",
      "rule-id": "1",
      "rule-name": "1",
      "object-locator": {
        "schema-name": "dbo",
        "table-name": "%"
      },
      "rule-action": "include"
    },
    {
      "rule-type": "transformation",
      "rule-id": "2",
      "rule-name": "2",
      "rule-target": "schema",
      "object-locator": {
        "schema-name": "dbo"
      },
      "rule-action": "rename",
      "value": "smartdata_int"
    }
  ]
}
4

3 回答 3

0

您应该可以选择为每个 DMS 任务设置 CloudWatch 日志。您是否检查过此任务的日志？你有 varchar/text 列 > 32KB 吗？将数据迁移到 redshift 等目标时，这些将被截断，因此请注意，这将计入您的错误计数。

于 2017-05-03T13:24:34.243 回答
0

首先要做的是增加日志级别：

"Logging": {
    "EnableLogging": true,
    "LogComponents": [{
        "Id": "SOURCE_UNLOAD",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "SOURCE_CAPTURE",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "TARGET_LOAD",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "TARGET_APPLY",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "TASK_MANAGER",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    }]
  },

然后，您将能够获得有关发生错误的详细信息。

于 2018-06-05T16:37:25.257 回答
0

开启验证：

https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Validating.html

这将减慢迁移速度，因此您还考虑将其拆分为多个任务并在多个复制实例上运行它们，将规则 1 扩展为多个规则，而不是 '%' 添加满足表子集的条件。

您也可以尝试不同的复制引擎，3.1.1 刚刚发布，在撰写本文时还没有 3.1.1 的发布说明。

https://docs.aws.amazon.com/dms/latest/userguide/CHAP_ReleaseNotes.html

于 2018-09-17T21:39:23.277 回答