2

我有一个包含 850000 条单独更新语句的文本文件，更新了一个包含 1200 万条记录的表。这种查询的一个例子是：

update bag.pand
set mutatiedatum = to_date('04-03-2013 10:03:48','DD-MM-YYYY HH:MI:SS')
where key='0321100000015282_2013022600000000_N_0'
 ;

其中'key'有一个唯一的索引。

我在 9.1 postgres/postgis 数据库上使用 psql.exe -q -1 -f '/path/to/sql/file' 执行文件。

这执行速度非常慢：每秒大约 14-20 条语句。

当我解释分析单独的语句时，它足够快：

Update on pand  (cost=0.00..6.12 rows=1 width=814) (actual time=0.101..0.101 rows=0 loops=1)
  ->  Index Scan using idx_pand_key on pand  (cost=0.00..6.12 rows=1 width=814) (actual time=0.093..0.093 rows=0 loops=1)
        Index Cond: ((key)::text = '0321100000015282_2013022600000000_N_0'::text)
Total runtime: 0.237 ms

日志文件不包含警告或错误。

我知道通过使用 -f 所有 850000 语句都在一个事务中执行。这可能是缓慢的原因吗？是否有其他解决方案或提示可以加快速度？

4

1 回答 1

1

如果所有更新都具有相同的形状，您可以将它们加载到临时表中并使用它来批量bag.pand更新表，如下所示：

CREATE TABLE bag.mutaties
        ( zkey varchar NOT NULL PRIMARY KEY
        , zdate timestamp NOT NULL
        );
COPY bag.mutaties(zkey,zdate) FROM 'the_big_file';

update bag.pand dst
FROM bag.mutaties src
set mutatiedatum = src.zdate
WHERE dst.key = src.zkey
        ;

以上假设日期采用 ISO 'yyyy-mm-dd hh:mm:ss' 格式。

如果您无法创建具有此日期格式的数据文件，您可以将现有的“dd-mm-yyyy”日期读入字符串并在更新语句中将它们转换（类似于您的一次一行更新：

CREATE TABLE bag.mutaties
        ( zkey varchar NOT NULL PRIMARY KEY
        , dutchdate varchar NOT NULL
        );

COPY bag.mutaties(zkey,dutchdate) FROM 'the_big_file';

update bag.pand dst
FROM bag.mutaties src
set mutatiedatum = to_date(src.dutchdate, 'DD-MM-YYYY HH:MI:SS')
WHERE dst.key = src.key
        ;

额外奖励更新：

        -- This will read in the existing SQL-SCRIPT (!!)
        -- and transform it into a table with {key,datetimestamp}
CREATE TABLE bag.tekstmutaties
        ( id SERIAL NOT NULL PRIMARY KEY
        , typ INTEGER NOT NULL DEFAULT 0
        , num INTEGER NOT NULL DEFAULT 0
        , tekst varchar
        );

        -- Read in the existing script file
        -- (this needs to be in /tmp/ to avoid permission problems)
COPY bag.tekstmutaties(tekst) FROM '/tmp/bagmut.txt';

        -- Remove bagger
delete from bag.tekstmutaties
where LEFT(tekst,1) NOT IN ( 's' , 'w')
        ;

        -- Extract the timestamp
UPDATE bag.tekstmutaties
SET typ = 1
        , tekst = regexp_replace( tekst, E' *set.*to_date..', '', 'ig')
WHERE LEFT(tekst,1) IN ( 's')
        ;
UPDATE bag.tekstmutaties
SET tekst = LEFT( tekst , 19)
WHERE typ = 1
        ;

        -- Extract the key
UPDATE bag.tekstmutaties
SET typ = 2
        , tekst = regexp_replace( tekst, E' *where key..', '', 'ig')
WHERE LEFT(tekst,1) IN ( 'w')
        ;
UPDATE bag.tekstmutaties
SET tekst = regexp_replace( tekst , '[^0-9A-Z_]' , '' , 'g' )
WHERE typ = 2
        ;

        -- number the records
UPDATE bag.tekstmutaties
SET num = id
WHERE TYP=1
        ;
        -- number the records
UPDATE bag.tekstmutaties uu
SET num = src.val
FROM (
        SELECT id, lag(id) OVER (ORDER BY id) AS val
        FROM bag.tekstmutaties
        ) src
WHERE uu.TYP=2
AND src.id = uu.id
        ;

SELECT * FROM bag.tekstmutaties ORDER BY id;

        -- The final table with the {key,timestamp} pairs
CREATE TABLE bag.mutaties
        ( zkey varchar NOT NULL PRIMARY KEY
        , zdate timestamp NOT NULL
        );

        -- Fill it with self-join of teksttable
INSERT INTO bag.mutaties (zkey, zdate)
SELECT k.tekst AS zkey
        , to_date(d.tekst, E'DD-MM-YYYY HH:MI:SS' ) AS zdate
FROM bag.tekstmutaties k
JOIN bag.tekstmutaties d ON k.num = d.num
WHERE k.typ=2
AND d.typ = 1
        ;
SELECT * FROM bag.mutaties;

-- now **after verification** you can use the mutaties-table
-- to batch-update the bag.pand table
EXPLAIN ANALYZE
update bag.pand dst
FROM mutaties src
set mutatiedatum = src.zdate
WHERE dst.key = src.key
        ;
于 2013-05-11T12:07:29.667 回答