1

我在此链接中问了一个问题Is-there-a-better-way-to-process-a-300-000-line-text-file-data-and-insert-it-into-a-mysql-database和发布了我的代码。

现在我想将查询更改为 PDO，但在更改 InsertData 函数中的查询时遇到了困难。我是 PDO 的新手，所以有人可以帮助我吗？不知道如何处理执行批量插入的插入语句。

我在想的是一个一个地插入它，但是当我为了提高性能效率而发布这个问题时，它是否有效？

//function for DB writing in table data
function InsertData($rec, &$errorDataCnt, &$sql, $y, $i, $x, &$dCnt)
{

    $dDate = (!isset($rec[0]) ? 0 : (trim($rec[0]) == "" ? 0 : trim($rec[0]))); 
    $dTime = (!isset($rec[1]) ? 0 : (trim($rec[1]) == "" ? 0 : trim($rec[1]))); 
    $address = (!isset($rec[2]) ? 0 : (trim($rec[2]) == "" ? 0 : trim($rec[2]))); 
    $co2SV = (!isset($rec[3]) ? 0 : (trim($rec[3]) == "" ? 0 : trim($rec[3]))); 
    $co2PV = (!isset($rec[4]) ? 0 : (trim($rec[4]) == "" ? 0 : trim($rec[4]))); 
    $tempSV = (!isset($rec[5]) ? 0 : (trim($rec[5]) == "" ? 0 : trim($rec[5]))); 
    $tempPV = (!isset($rec[6]) ? 0 : (trim($rec[6]) == "" ? 0 : trim($rec[6]))); 
    $rhSV = (!isset($rec[7]) ? 0 : (trim($rec[7]) == "" ? 0 : trim($rec[7]))); 
    $rhPV = (!isset($rec[8]) ? 0 : (trim($rec[8]) == "" ? 0 : trim($rec[8]))); 


        /* include('connect.php'); */
        set_time_limit(36000);
        ini_set('max_execution_time','43200');
        $e_results = mysql_query("SELECT ID FROM tbl_reading WHERE (READING_DATE = '".date("Y-m-d",strtotime($dDate))."' AND READING_TIME = '".date("H:i:s",strtotime($dTime))."') AND READING_ADDRESS = $address LIMIT 1"); 
        if(mysql_num_rows($e_results) <= 0 ){
          if (!($dDate == 0 || $dTime == 0 || $address == 0) ) {
            if ($y == 0){
                $sql = "INSERT INTO tbl_reading (READING_DATE, READING_TIME, READING_ADDRESS, CO2_SET_VALUE, CO2_PROCESS_VALUE, TEMP_SET_VALUE, TEMP_PROCESS_VALUE, RH_SET_VALUE, RH_PROCESS_VALUE) VALUES ('".date("Y/m/d",strtotime($dDate))."','".date("H:i:s",strtotime($dTime))."', ". mysql_real_escape_string($address).",". mysql_real_escape_string($co2SV).",". mysql_real_escape_string($co2PV).",". mysql_real_escape_string($tempSV).",". mysql_real_escape_string($tempPV).",". mysql_real_escape_string($rhSV).",". mysql_real_escape_string($rhPV).")";
            }
            else {
                $sql .= ", ('".date("Y/m/d",strtotime($dDate))."','".date("H:i:s",strtotime($dTime))."', ". mysql_real_escape_string($address).",". mysql_real_escape_string($co2SV).",". mysql_real_escape_string($co2PV).",". mysql_real_escape_string($tempSV).",". mysql_real_escape_string($tempPV).",". mysql_real_escape_string($rhSV).",". mysql_real_escape_string($rhPV).")";

            }
           }
          }

            if(($x + 1) == $i){
                //echo ($x + 1)." = ".$i."<br>";
                if (substr($sql, 0, 1) == ",")
                    $sql = "INSERT INTO tbl_reading (READING_DATE, READING_TIME, READING_ADDRESS, CO2_SET_VALUE, CO2_PROCESS_VALUE, TEMP_SET_VALUE, TEMP_PROCESS_VALUE, RH_SET_VALUE, RH_PROCESS_VALUE) VALUES".substr($sql, 1);
                //echo $sql."<br>";
                set_time_limit(36000);
                try {

                    $result = mysql_query($sql) ;
                    $dCnt = mysql_affected_rows();
                    if( $dCnt  == 0)
                    {
                        $errorDataCnt = $errorDataCnt + 1;
                    }
                }
                catch (Exception $e)
                {
                    $conn->query("INSERT INTO tbl_log (LOG_DATE, DETAILS) VALUES ( NOW(), '".$e->getMessage()."' )");
                }
                //mysql_free_result($result);
            }

    unset($dDate); 
    unset($dTime); 
    unset($address); 
    unset($co2SV); 
    unset($co2PV); 
    unset($tempSV); 
    unset($tempPV); 
    unset($rhSV); 
    unset($rhPV);  

}

//function for looping into the records per line
function loop($data)
{
    $errorDataCnt = 0; $sql = ""; $exist = 0;
    $i = count( $data); $x = 0; $y = 0; $tmpAdd = ""; $cnt = 0; $t = 0; $dCnt = 0; 

    ini_set('max_execution_time','43200');
    while($x < $i) 
    {
        $rec = explode(",", $data[$x]); 
        InsertData($rec, $errorDataCnt, $sql, $y, $i, $x, $dCnt);
        $x++; 
        $y++;
        unset($rec);
    }

        $errFetch = ($i - $dCnt);
    if($errorDataCnt > 0)
        $conn->query("INSERT INTO tbl_log (LOG_DATE, DETAILS) VALUES ( NOW(), 'Error inserting $errFetch records. Check if there is a NULL or empty value or if it is the correct data type.' )");
    if($dCnt > 0)
        $conn->query("INSERT INTO tbl_log (LOG_DATE, DETAILS) VALUES ( NOW(), 'Saved $dCnt of $i records into the database. Total $exist records already existing in the database.' )");


}

// functions in looping records and passing into $contents variable
function DataLoop($file)
{
    ini_set("auto_detect_line_endings", true);
    set_time_limit(36000);
    ini_set('max_execution_time','43200');
    $contents = ''; $j = 0;
    if ($handle = fopen($file,"rb")){
        while (!feof($handle)) {
            $rdata = fgets($handle, 3359232);//filesize($file));
            //$rdata = fread($handle, filesize($file));
            if(trim($rdata) != "" || $rdata === FALSE){
                if (feof($handle)) break;
                else {
                $contents .= $rdata;
                $j = $j + 1; }}
        }   
        fclose($handle);
        $data = explode("\n", $contents);
        unset($contents);
        unset($rdata);
    }
    /* echo count($contents)." ".count($data); */
    /* $query = "SELECT MAX(`ID`) AS `max` FROM `tbl_reading`";
    $result = mysql_query($query) or die(mysql_error());
    $row = mysql_fetch_assoc($result);
    $max = $row['max']; */
    /* $res =   mysql_fetch_assoc(mysql_query("SELECT COUNT(*) as total FROM tbl_reading")) or die(mysql_error());
    echo "<script>alert('".$res['total']."')</script>"; */
    $p = 0;
    ini_set('memory_limit','512M');
    if($j != 0)
    {
        foreach(array_chunk($data, ceil(count($data)/200)) as $rec_data){
            loop($rec_data);
            $p++;
        }
    } 

}
4

2 回答 2

1

你可以使用PDO::prepare它。http://php.net/manual/en/pdo.prepare.php

它是为这样的事情而设计的，应该更快一些（实际上并不是从经验中谈起）。只需用一些样本数据来衡量它。

$data = array(
     array(
         ':key1' => 'value1',
         ':key2' => 'value2',
     ),
     array(
         ':key1' => 'value3',
         ':key2' => 'value4',
     ),
);

$sql = 'INSERT INTO data_table (key1, key1) VALUES (:key1, key2);';
$stmt = $dbh->prepare($sql);
foreach($data AS $dataset) {
    $stmt->execute($dataset);
}
于 2013-01-23T07:30:47.490 回答
1

这不是您运行查询（PDO、mysql、批量插入或其他）的方式，而是 InnoDB 引擎使插入速度太慢。
在开始插入之前，您需要做一些准备。
至于特定的插入语法 - 这是无关紧要的。

在插入之前设置此选项

innodb_flush_log_at_trx_commit=0

或将您的所有插入都放入一个事务中。
无论您选择什么语法或驱动程序，它都会非常快

于 2013-01-23T07:54:55.397 回答