0 
{
  "name":"{{name}}",
  "tasks.max": "6", //have 6 partitions for this topic
  "topics": "{{topic}}",
  "connector.class": "io.confluent.connect.s3.S3SinkConnector",
  "key.converter": "io.confluent.connect.avro.AvroConverter",
  "key.converter.schemas.enable": "true",
  "key.converter.schema.registry.url": "xx",
  "key.converter.key.subject.name.strategy": "io.confluent.kafka.serializers.subject.TopicRecordNameStrategy",
  "value.converter": "io.confluent.connect.avro.AvroConverter",
  "value.converter.schemas.enable": "true",
  "value.converter.schema.registry.url": "xx",
  "value.converter.value.subject.name.strategy": "io.confluent.kafka.serializers.subject.TopicRecordNameStrategy",
  "errors.retry.timeout":"600000",
  "errors.log.enable":"true",
  "errors.log.include.messages":"true",



  "schema.compatibility": "BACKWARD",

  "format.class": "io.confluent.connect.s3.format.avro.AvroFormat",
  "flush.size": "100000",
  "rotate.schedule.interval.ms": "3600000",
  "rotate.interval.ms": "3600000",
  "enhanced.avro.schema.support": "true",
  "connect.meta.data": "false",


  "partitioner.class": "{{partitioner}}somepartitioner",
  "partition.duration.ms": "3600000",
  "path.format": "'avro/event=?eventClass?/tenant=?tenant?/date'=YYYY-MM-dd/'hour'=HH",
  "locale": "en",
  "timezone": "UTC",
  "timestamp.extractor": "RecordField",
  "timestamp.field": "{{timestampField}}",



  "storage.class": "io.confluent.connect.s3.storage.S3Storage",


  "s3.bucket.name": "somebucket",
  "s3.region": "region",
  "s3.part.size": "5242880",


  "offset.flush.interval.ms": "1200000"
}

该主题的数量约为 739,180，大小为 1.1Gb

我不完全确定我的配置是否完全正确，如果我可以改进它。我想在两种情况下刷新，每小时或大小达到 5gb。

4

0 回答 0