php - 如何将大 CSV 文件导入 MySQL

Question

我的 Web 应用程序接收逗号分隔的电话列表，必须将其导入到联系人表中。该列表可以包含 100,000 到 1,000,000 个项目。我实现了存储过程。但它对我来说仍然太慢了。你能帮我改进一下吗？

DELIMITER $$

CREATE DEFINER=`root`@`localhost` PROCEDURE `import_phones`
(IN ContactGroupId int, IN strPhones mediumtext, OUT total int, OUT inserted int)
main: BEGIN
    declare delim varchar(1) default(',');
    declare delimPtr int unsigned default(0); -- pointer to current spliter position
    declare startPtr int unsigned default(1);
    declare phone1 nvarchar(20);

    set total = 0; -- counter of total rows
    set inserted = 0; -- counter of inserted rows

    if strPhones is null or length(strPhones) < 1 then 
        leave main;
    end if;

    drop table if exists insertphones;
    create temporary table insertphones(phone nvarchar(20))
        engine = memory;
/***
-- split strPhones by delimiter
*/
    SET delimPtr = locate(delim, strPhones,startPtr);
    loop_label: while(delimPtr > 0) do
        insert into insertphones (phone) values (substring(strPhones,startPtr,delimPtr-startPtr));
        -- select delimPtr,startPtr, substring(strPhones,startPtr,delimPtr-startPtr); 
        set startPtr = delimPtr+1;
        set delimPtr = locate(delim, strPhones,startPtr);
    end while;
    if delimPtr = 0 then
        insert into insertphones (phone) values (substring(strPhones,startPtr,delimPtr-startPtr));
    end if;

    -- select delimPtr,startPtr; 
    select count(*) from insertphones into total;
/***
--  insert phones 
*/

    insert into contacts (Phone, ContactGroupId)
        select distinct(phone), ContactGroupId from insertphones where 
        phone not in (select Phone from contacts where ContactGroupId = ContactGroupId);
    SELECT ROW_COUNT() into inserted ;
    -- select total, inserted;
END

Answer 1

想一想load data infile，喜欢

load data infile 'filename' into tablename lines terminated by ','

Answer 2

行。该解决方案的工作速度比初始版本快得多（大约 10 倍至 20 倍）。

    $tempFile = tempnam(sys_get_temp_dir(), 'phones');
$tempTable = 'tmp'.$contactGroupId.time();
file_put_contents($tempFile, $phones);
try {
    chmod($tempFile, 0777); //required for MySQL import
    $sql = sprintf(
"set @total = 0;
set @inserted = 0;
drop table if exists %s;
create temporary table %s(phone nvarchar(20)) engine = memory;
LOAD DATA INFILE '%s' INTO TABLE %s FIELDS TERMINATED BY '' LINES TERMINATED BY ',';
SELECT count(*) FROM %s INTO @total;
DELETE FROM %s WHERE length(phone) < 10;
UPDATE %s SET phone = CONCAT('7',phone) WHERE length(phone) = 10;
UPDATE %s SET phone = INSERT(phone,1,1,'7') WHERE phone like '8%%' and length(phone) = 11;
insert into contacts (Phone, ContactGroupId) select distinct(phone), %d from %s where phone not in (select Phone from contacts where ContactGroupId = %d);
SELECT ROW_COUNT() into @inserted;"
    , $tempTable, $tempTable, str_replace('\\','\\\\',$tempFile), $tempTable, $tempTable, $tempTable, $tempTable, $tempTable, $contactGroupId, $tempTable, $contactGroupId );

$command = Yii::app()->db->createCommand($sql);
$command->execute();
$command = false;
$result = Yii::app()->db->createCommand("select @total as total, @inserted as inserted")->queryRow();
$contactCounter['all'] += $result['total'];
$contactCounter['created'] += $result['inserted'];