0

嗨，我尝试使用以下代码抓取 digg.com 上的首页图片。问题是 0.jpg 到 6.jpg 是正常的。从 7.jpg 开始到 47.jpg 已损坏。不知道为什么。

这是代码。Github在这里：https ://github.com/kenpeter/py_mm

# os
import os
# http request
import requests
#
import pprint

import time

# import html from lxml
from lxml import html

# global
global_page_num = 0
pp = pprint.PrettyPrinter(indent=4)

# write to file
def download_image(img_urls):
    # total img urls
    amount = len(img_urls)

    # loop
    for index, value in enumerate(img_urls, start=0):
        # file name
        filename = 'img/%s.jpg' % (index)
        # dir
        os.makedirs(os.path.dirname(filename), exist_ok=True)

        print('--- start ---')
        print('filename: %s' % filename)
        print('Downloading: %s out of %s' % (index, amount))

        # open file
        with open(filename, 'wb') as f:
            # f write
            # time.sleep(1)
            f.write(requests.get(value).content)


def get_page_number(num):
    url = 'http://digg.com'
    response = requests.get(url).content
    selector = html.fromstring(response)

    img_urls = []
    img_urls = selector.xpath("//div[@class='digg-story__image--thumb']/a/img/@src")

    news_texts = []
    news_texts = selector.xpath("//div[@itemprop='description']/text()")

    # test
    # print('--- something ---')
    # pp.pprint(img_urls)
    # pp.pprint(news_texts)

    download_image(img_urls)

    return img_urls


if __name__ == '__main__':
    # input, page_number, everything into the var
    # page_number = input('Please enter the page number that you want to scrape:')

    # global_page_num
    # global_page_num = page_number;
    # print('hell world!');

    page_number = 4 # hardcode
    get_page_number(page_number)
4

1 回答 1

0

图像“损坏”的原因是页面内的方案发生了变化，图像开始“隐藏”在属性中，data-src而不是src您使用代码获取的内容。请参阅此处具有两个属性的抓取页面的源代码示例：

<img
class="digg-story__image-img js--digg-story__image-img lazy-image-img need-offset"
data-src="http://static.digg.com/images/f0b92c2d8a2c4b7f829abbc0e58a408c_2oijd0Z_1_www_large_thumb.jpeg"
src="http://static.digg.com/static/fe/944294/images/x_455x248.png"
width="312"
height="170"
alt=""
/>

换句话说，您必须在创建图像 URL 列表时检查这两个属性src并data-src给予data-src优先级。src

此代码执行“技巧”并下载正确的图像：

# os
import os
# http request
import requests
#
import pprint

import time

# import html from lxml
from lxml import html

# global
global_page_num = 0
pp = pprint.PrettyPrinter(indent=4)

# write to file
def download_image(img_urls):
    # total img urls
    amount = len(img_urls)

    # loop
    for index, value in enumerate(img_urls, start=0):
        # file name
        filename = 'img/%s.jpg' % (index)
        # dir
        os.makedirs(os.path.dirname(filename), exist_ok=True)

        print('--- start ---')
        print('filename: %s' % filename)
        print('Downloading: %s out of %s' % (index, amount))

        # open file
        with open(filename, 'wb') as f:
            # f write
            # time.sleep(1)
            f.write(requests.get(value).content)


def get_page_number(num):
    url = 'http://digg.com'
    response = requests.get(url).content
    selector = html.fromstring(response)

    img_urls = []
    img_urls_1a = selector.xpath("//div[@class='digg-story__image--thumb']/a/img/@src")
    img_urls_1b = [item for item in img_urls_1a if 'x_455x248.png' not in item]
    img_urls_2 = selector.xpath("//div[@class='digg-story__image--thumb']/a/img/@data-src")
    img_urls = img_urls_1b + img_urls_2
    # print(img_urls)
    news_texts = []
    news_texts = selector.xpath("//div[@itemprop='description']/text()")

    # test
    # print('--- something ---')
    # pp.pprint(img_urls)
    # pp.pprint(news_texts)

    download_image(img_urls)

    return img_urls


if __name__ == '__main__':
    # input, page_number, everything into the var
    # page_number = input('Please enter the page number that you want to scrape:')

    # global_page_num
    # global_page_num = page_number;
    # print('hell world!');

    page_number = 4 # hardcode
    get_page_number(page_number)
于 2017-04-21T03:58:03.097 回答