1

So I'm very confused here. When I use the scrapy shell and input the xpath the correct data is returned, but when I set that same xpath equal to a variable within the script, it outputs a blank. I'm really not sure what is going on. 

import scrapy

class FestivalSpider(scrapy.Spider): name = 'basketball'

custom_settings = {
    "DOWNLOAD_DELAY": 3,
    "CONCURRENT_REQUESTS_PER_DOMAIN": 3,
    "HTTPCACHE_ENABLED": True
}

start_urls = [
    'http://www.basketball-reference.com/leagues/NBA_2017_per_game.html'
]

def parse(self, response):
    start_urls = [
    'http://www.basketball-reference.com/leagues/NBA_2017_per_game.html']
    for href in start_urls:
        yield scrapy.Request(
            url=href,
            callback=self.parse_festival,
            meta={'url': href}
        )


def parse_festival(self, response):

    url = response.request.meta['url']

    name = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "player"]/a/text()').extract()

    pos = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "pos"]/text()').extract()

    age = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "age"]/text()').extract()

    #team = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "team_id"]/text()').extract()

    games = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "g"]/text()').extract()

    games_s = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "gs"]/text()').extract()

    fg_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_per_mp"]/text()').extract()

    #fga_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fga_per_mp"]/text()').extract()

    #fg_pct = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_pct"]/text()').extract()

    #fg3_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg3_per_mp"]/text()').extract()

    #fg3a_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg3a_per_mp"]/text()').extract()

    #fg3_pct = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg3_pct"]/text()').extract()
    #location = (
    #    response.xpath('//div[@id="festival-basics"]/text()').extract()[3])

    #dates = (
    #    response.xpath('//div[@id="festival-basics"]/text()').extract()[5])

    #tickets = (
    #    response.xpath('//div[@id="festival-basics"]/text()').extract()[7])

    #website = (
    #    response.xpath(
    #        '//div[@id="festival-basics"]/a/@href').extract()[0]
    #)

    #logo = (
    #    response.xpath(
    #        '//div[@id="festival-basics"]/img/@src').extract()[0]
    #)

    #lineup = (
    #    response.xpath(
    #        '//div[@class="lineupguide"]/ul/li/text()').extract() +
    #    response.xpath(
    #        '//div[@class="lineupguide"]/ul/li/a/text()').extract()
    #)
    print(response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_per_mp"]/text()').extract())

    yield {
        'url': url,
        'name': name,
        'pos': pos,
        'age' : age,
        'games' : games,
        'games_s': games_s, 
        'fg_per_mp': fg_per_mp
        #'fga_per_mp': fga_per_mp,
        #'fg_pct': fg_pct,
        #'fg3_per_mp': fg3_per_mp,
        #'fg3a_per_mp': fg3a_per_mp
        #'team' : team
        #'location': location,
        #'dates': dates,
        #'tickets': tickets,
        #'website': website,
        #'logo': logo,
        #'lineup': lineup
        }

The item in question is the fg_per_mp, when I use the response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_per_mp"]/text()').extract() in the shell it works, but the same line in the script returns an empty list.

What am I doing wrong?

4

0 回答 0