我目前使用 ETL 工具将一堆 CSV 数据插入 OrientDB。我用于试用的系统配置是 EC2 M3 large(7.5 GiB 内存、2 个 vCPU、32 GB 基于 SSD 的本地实例存储、64 位平台)。
我尝试上传的数据(屏蔽)格式如下:
"101.186.130.130","527225725","233 djfnsdkj","0.119836317542"
"125.143.534.148","112212983","1227 sdfsdfds","0.0465215171983"
"103.149.957.752","112364761","1121 sdfsdfds","0.0938863016658"
"103.190.245.128","785804692","6138 sdfsdfsd","0.117767539364"
该模式包含 2 个节点类和一个边缘类。当我尝试使用 plocal 选项中的 ETL 工具加载数据时,速度仅为大约 2300 行/秒。ETL 配置如下所述:
{
"source": { "file": { "path": "/home/ubuntu/labvolume1/orientdb/bin/0001_part_00" } },
"extractor": { "csv": {"columnsOnFirstLine": false, "columns":["ip:string", "dpcb:string", "address:string", "prob:string"] } },
"transformers": [ { "merge": { "joinFieldName":"ip", "lookup":"IpAddress.ip" } },
{ "field":
{ "fieldName": "addr_key",
"expression": "dpcb.append('_').append(address)"
}
},{ "vertex": { "class": "IpAddress" } },
{ "edge": { "class": "Located",
"joinFieldName": "addr_key",
"lookup": "PhyLocation.loc",
"direction": "out",
"targetVertexFields": { "geo_address": "${input.address}", "dpcb_number": "${input.dpcb}"},
"edgeFields": { "confidence": "${input.prob}" },
"unresolvedLinkAction": "CREATE"
}
}
],
"loader": {
"orientdb": {
"dbURL": "plocal:/home/ubuntu/labvolume1/orientdb/databases/Bulk_Transfer_Test1",
"dbType": "graph",
"dbUser": "admin",
"dbPassword": "admin",
"serverUser": "admin",
"wal": false,
"serverPassword":"admin",
"classes": [
{"name": "IpAddress", "extends": "V"},
{"name": "PhyLocation", "extends": "V"},
{"name": "Located", "extends": "E"}
], "indexes": [
{"class":"IpAddress", "fields":["ip:string"], "type":"UNIQUE" },
{"class":"PhyLocation", "fields":["loc:string"], "type":"UNIQUE" }
]
}
}
}
然后我将顶点分成文件并仅针对顶点运行 ETL 作业,这次速度接近 12500 行/秒。这相当快,这对我有用。(当我删除索引时,速度几乎翻了一番)我使用的配置是:
{
"source": { "file": { "path": "/home/ubuntu/labvolume1/orientdb/bin/only_ip_05.csv" } },
"extractor": { "csv": {"columnsOnFirstLine": false, "columns":["ip:string"] } },
"transformers": [
{ "vertex": { "class": "IpAddress" } }],
"loader": {
"orientdb": {
"dbURL": "plocal:/home/ubuntu/labvolume1/orientdb/databases/Bulk_Transfer_Test7",
"dbType": "graph",
"dbUser": "admin",
"dbPassword": "admin",
"serverUser": "admin",
"wal": false,
"serverPassword":"admin",
"classes": [
{"name": "IpAddress", "extends": "V"}
],
"indexes": [
{"class":"IpAddress", "fields":["ip:string"], "type":"UNIQUE" }
]
}
}
}
然而,当我尝试单独插入边缘时,速度变得非常慢,为 2200 行/秒。事实证明,这甚至比一次运行整个操作还要低。配置文件附在下面:
{
"source": { "file": { "path": "/home/ubuntu/labvolume1/orientdb/bin/edge5.csv" } },
"extractor": { "csv": {"columnsOnFirstLine": false, "columns":["ip:string", "loc:string", "prob:string"] } },
"transformers": [
{ "merge": { "joinFieldName":"ip", "lookup":"IpAddress.ip" } },
{ "vertex": { "class" : "IpAddress", "skipDuplicates" : true }},
{ "edge": { "class": "Located",
"joinFieldName": "loc",
"lookup": "PhyLocation.loc",
"direction": "out",
"edgeFields": { "confidence": "${input.prob}" },
"unresolvedLinkAction": "NOTHING"
}
}
],
"loader": {
"orientdb": {
"dbURL": "plocal:/home/ubuntu/labvolume1/orientdb/databases/Bulk_Transfer_Test7",
"dbType": "graph",
"dbUser": "admin",
"dbPassword": "admin",
"serverUser": "admin",
"wal": false,
"tx":false,
"batchCommit":10000,
"serverPassword":"admin",
"classes": [
{"name": "IpAddress", "extends": "V"},
{"name": "PhyLocation", "extends": "V"},
{"name": "Located", "extends": "E"}
]
}
}
}
如果我在这里做错了什么,请告诉我,还请提出更好的性能改进方法