2

我想下载大型科学摘要数据，让我们说大约 2000 个 Pubmed ID。我的 python 代码很草率，而且工作起来似乎很慢。有没有什么快速有效的方法来收获这些摘要？

如果这是最快的方法，我如何测量它以便我能够与其他人或家庭与工作情况进行比较（不同的 ISP 可能会影响速度）？

下面附上我的代码。

import sqlite3
from Bio.Entrez import read,efetch,email,tool
from metapub import PubMedFetcher
import pandas as pd
import requests
from datetime import  date
import xml.etree.ElementTree as ET
import time
import sys
reload(sys)
sys.setdefaultencoding('utf8')
Abstract_data = pd.DataFrame(columns=["name","pmid","abstract"])

def abstract_download(self,dict_pmids):
    """
        This method returns abstract for a given pmid and add to the abstract data
    """
    index=0
    baseUrl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    for names in dict_pmids:
        for pmid in dict_pmids[names]:
            try:
                abstract = []
                url = baseUrl+"efetch.fcgi?db=pubmed&id="+pmid+"&rettype=xml"+
                response=requests.request("GET",url,timeout=500).text
                response=response.encode('utf-8')
                root=ET.fromstring(response)
                root_find=root.findall('./PubmedArticle/MedlineCitation/Article/Abstract/')
                if len(root_find)==0:
                    root_find=root.findall('./PubmedArticle/MedlineCitation/Article/ArticleTitle')
                for i in range(len(root_find)):
                    if root_find[i].text != None:
                        abstract.append(root_find[i].text)
                if abstract is not None:
                    Abstract_data.loc[index]=names,pmid,"".join(abstract)
                index+=1
            except:
                print "Connection Refused"
                time.sleep(5)
                continue
    return Abstract_data

编辑：此脚本发生的一般错误似乎是“连接被拒绝”。请参阅下面的 ZF007 的答案是如何解决的。

4

1 回答 1

1

下面的代码有效。您的脚本挂在格式错误的 URL 构造上。此外，如果脚本内部出现问题，则响应是拒绝连接。事实上并非如此，因为它是处理检索到的数据的代码。我做了一些调整以使代码为我工作，并在您需要调整的地方留下了注释，因为缺少dict_pmids 列表。

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, time, requests, sqlite3

import pandas as pd
import xml.etree.ElementTree as ET

from metapub import PubMedFetcher
from datetime import  date
from Bio.Entrez import read,efetch,email,tool


def abstract_download(pmids):

    """
        This method returns abstract for a given pmid and add to the abstract data
    """

    index   = 0
    baseUrl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    collected_abstract = []

    # code below diabled to get general abstract extraction from pubmed working. I don't have the dict_pmid list.

    """
    for names in dict_pmids:
        for pmid in dict_pmids[names]:

    move below working code to the right to get it in place with above two requirements prior to providing dict_pmid list.

    # from here code works upto the next comment. I don't have the dict_pmid list.
    """

    for pmid in pmids:

        print 'pmid : %s\n' % pmid

        abstract  = []
        root = ''

        try:

            url       = '%sefetch.fcgi?db=pubmed&id=%s&rettype=xml' % (baseUrl, pmid)
             # checks my url... line to parse into a webbrowser like firefox.
            print 'url', url

            response  = requests.request("GET", url, timeout=500).text
             # check if I got a response.
            print 'response', response

#                response  = response.encode('utf-8')
            root      = ET.fromstring(response)

        except Exception as inst:
            # besides a refused connection.... the "why" it was connected comes in handly to resolve issues at hand
            # if and when they happen.
            print "Connection Refused", inst

            time.sleep(5)
            continue

        root_find = root.findall('./PubmedArticle/MedlineCitation/Article/Abstract/')

        if len(root_find)==0:

                root_find = root.findall('./PubmedArticle/MedlineCitation/Article/ArticleTitle')

        # check if I found something
        print 'root_find : %s\n\n' % root_find

        for i in range(len(root_find)):

            if root_find[i].text != None:

                abstract.append(root_find[i].text)

        Abstract_data = pd.DataFrame(columns=["name","pmid","abstract"])

        # check if I found something
        #print 'abstract : %s\n' % abstract

        # code works up to the print statement ''abstract', abstract' teh rest is disabled because I don't have the dict_pmid list.

        if abstract is not None:

#                Abstract_data.loc[index] = names,pmid,"".join(abstract)

            index += 1
            collected_abstract.append(abstract)

    # change back return Abstract_data when dict_pmid list is administered.
#    return Abstract_data
    return collected_abstract

if __name__ == '__main__':

    sys.stdout.flush()

    reload(sys)
    sys.setdefaultencoding('utf8')

    pubmedIDs = range(21491000, 21491001)

    mydata = abstract_download(pubmedIDs)

    print 'mydata : %s' % (mydata)
于 2017-12-01T11:08:07.070 回答