python - 当我尝试运行代码时，出现错误：PicklingError: Can't pickle: 属性查找 __builtin__.function 失败

Question

我的代码有什么问题？谢谢

import os
import os.path
import time
global item_count
#-*- coding: cp936 -*-
import MySQLdb
import MySQLdb.cursors
import threading
import multiprocessing
from time import sleep,ctime


def qucun():

    #connect to mysql
    conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong")
    cursor=conn.cursor()
    try:
        cursor.execute("""create database if not exists quad""")
    except:
        print 'Quad is exist'


    conn.select_db('quad')
    conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong",db="quad")
    #get cursor
    cursor=conn.cursor()

    try:
        cursor.execute("""create table if not exists record(fn1 varchar(100),
        fn2 varchar(100),fn3 varchar(100),fn4 varchar(100),
        fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""")
    except:
        print 'Table record is exist'

    loops=['2013071818_1.txt','2013071818_2.txt','2013071818_3.txt','2013071818_4.txt','2013071818_5.txt']




    def loop(nloop,filename):


        print 'This loop%s start at:'%nloop,ctime()
        #connect to quad
        conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong",db="quad")
        conn.select_db('quad')
        #get cursor
        cursor=conn.cursor()

        newitem=open('C:\\Python27\\caihong\\%s'%filename,'r')
        data=[line.strip() for line in newitem.readlines()]
        print data
        ##put data into value
        values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4],
                '%s'%data[5],'%s'%data[6],'%s'%data[7]]

        cursor.execute("""insert into record values(%s,%s,%s,%s,%s,%s,%s,%s)""",values);   

        conn.commit()
        cursor.close()
        sleep(2)
        print 'This loop done at',ctime()



    if __name__=='__main__':

        print 'starting at:',ctime()
        threads=[]
        nloops=range(len(loops))
        pool=multiprocessing.Pool(processes=2)
        for i in nloops:
            t=pool.apply_async(loop,(i,loops[i]))


        pool.close()
        pool.join()

        if t.successful():
            print 'successful'




        print 'all Done at:',ctime()
        os.system("pause")

qucun()

Answer 1

1. 您正在尝试异步调用本地定义的函数。
2. 您正在尝试在进程之间共享开放连接。

第一个在 2.7 中实现起来很棘手，第二个在任何多处理中都是不可能的

您必须为进程池中的每个进程使用单独的连接。

import os
import os.path
import time
global item_count
#-*- coding: cp936 -*-
import MySQLdb
import MySQLdb.cursors
import threading
import multiprocessing
from time import sleep,ctime

CONNECTION = None

def close_connection():
    CONNECTION.close()

def get_connection():
    global CONNECTION

    #If this process pool member launched for a first time - create connection
    if CONNECTION is None:

        conn = MySQLdb.connect( host="localhost",
                                user="root",
                                passwd="caihong")
        cursor = conn.cursor()
        try:
            cursor.execute("""create database if not exists quad""")
        except:
            print 'Quad is exist'

        conn.select_db('quad')

        CONNECTION = MySQLdb.connect(host="localhost",
                                     user="root",
                                     passwd="caihong",
                                     db="quad")

        cursor = CONNECTION.cursor()

        try:
            cursor.execute("""create table if not exists record(fn1 varchar(100),
            fn2 varchar(100),fn3 varchar(100),fn4 varchar(100),
            fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""")
        except:
            print 'Table record is exist'

        # we dont need to close connection after each insert.
        # insted - register a finalizer once
        # so it will be called right before Pool.close()
        multiprocessing.util.Finalize(CONNECTION, close_connection, exitpriority=1)

    #use existing connection
    return CONNECTION

def loop(nloop, filename):
    conn = get_connection()
    cursor = conn.cursor()

    print 'This loop %s start at: %s'%(nloop, ctime())
    with open('C:\\Python27\\caihong\\%s'%filename, 'r') as newitem:
        data = [line.strip() for line in newitem.readlines()]

        # values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4],
        #         '%s'%data[5],'%s'%data[6],'%s'%data[7]]
        # ^^^ Thats a bad way to stringify list

        cursor.execute('insert into record values(%s)', ','.join(data));

    conn.commit()

    # we dont need to close connection after each insert. 
    # cursor.close()
    print 'This loop done at', ctime()


LOOPS = ['2013071818_1.txt', '2013071818_2.txt', '2013071818_3.txt', '2013071818_4.txt', '2013071818_5.txt']

if __name__=='__main__':
    pool = multiprocessing.Pool(processes=2)
    results = []
    for i, loopfile in enumerate(LOOPS):
        results.apply(pool.apply_async(loop, (i, loopfile)))

    pool.close()
    pool.join()

    if all((res.successful() for res in results)):
        print 'successful'

    print 'all Done at:', ctime()
        os.system('pause')