python - 如何从代码中的 bs4 标签元素中提取文本？使用内容功能不起作用

Question

调用时出现以下错误text.strip()：

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-278-135ac185ec3f> in <module>
     20         if isinstance(b, Tag):
     21 
---> 22             location = [a.text.strip() for a in b.find('span', attrs = {'class': 'location'})]
     23             job_title = [a.text.strip() for a in b.find('a', attrs = {'data-tn-element':'jobTitle'})]
     24 

TypeError: 'NoneType' object is not iterable

请参阅下面的代码：

import requests
from bs4 import BeautifulSoup, NavigableString, Tag, Comment
import pandas as pd

    df = pd.DataFrame(columns=["location", 'company', 'job_title', 'salary'])

    for start in range(1,100,10):
        url = 'https://www.indeed.com/jobs?q=python+sql&l=San+Francisco&start={}'

        #format url above to request the various search pages
        new_url = url.format(start)

        #conducting a request of the stated URL above:
        page = requests.get(new_url)

        #specifying a desired format of “page” using the html parser - this allows python to read the various components of the page, rather than treating it as one long string.
        soup = BeautifulSoup(page.text, 'html.parser')

        #loop through the tag elements
        for b in soup.find_all(name = 'div', attrs={'class':'jobsearch-SerpJobCard'}):
            print(type(b))
            if isinstance(b,NavigableString):
                continue
            if isinstance(b, Tag):    

                location = [a.text.strip() for a in b.find('span', attrs = {'class': 'location'})]
                job_title = [a.text.strip() for a in b.find('a', attrs = {'data-tn-element':'jobTitle'})]

                try:
                    company = [a.text.strip() for a in b.find('span', attrs = {'class':'company'})]
                except:
                    company = 'NA'
                try:
                    salary = [a.text.strip() for a in b.find('span', attrs = {'class' : 'salaryText'}).find('nobr')]
                except:
                    salary = 'NA'
                df = df.append({"location":location,"company":company, "job_title": job_title, "salary": salary}, ignore_index=True)

Answer 1

您将需要添加对 None 值的检查，find如果未找到任何元素，则返回 None。

location = [a.text.strip() 
            for a in b.find('span', attrs = {'class': 'location'}) 
            if a]

Answer 2

找不到它，因为页面上没有将类属性设置为“位置”。有些 's 的类属性设置为 'location'。这是我的修改版本，仍然不完美，因为某些位置没有被抓取。如果这两个参数是必要的，一个想法是跳过那些没有工作或位置的人。您可以通过将 except 操作从分配 'NA' 替换为continue

import requests
from bs4 import BeautifulSoup, NavigableString, Tag, Comment
import pandas as pd

df = pd.DataFrame(columns=["location", 'company', 'job_title', 'salary'])

for start in range(1,100,10):
    url = 'https://www.indeed.com/jobs?q=python+sql&l=San+Francisco&start={}'

    #format url above to request the various search pages
    new_url = url.format(start)

    #conducting a request of the stated URL above:
    page = requests.get(new_url)

    #specifying a desired format of “page” using the html parser - this allows python to read the various components of the page, rather than treating it as one long string.
    soup = BeautifulSoup(page.text, 'html.parser')

    #loop through the tag elements
    for b in soup.find_all(name = 'div', attrs={'class':'jobsearch-SerpJobCard'}):
        print(type(b))
        if isinstance(b,NavigableString):
            continue
        if isinstance(b, Tag):
            try:
                location = [a.strip() for a in b.find('div', attrs = {'class': 'location'})]
            except TypeError:
                location = 'NA'
            try:
                job_title = [a.strip() for a in b.find('a', attrs = {'data-tn-element':'jobTitle'})]
            except TypeError:
                job_title = 'NA'

            try:
                company = [a.text.strip() for a in b.find('span', attrs = {'class':'company'})]
            except:
                company = 'NA'
            try:
                salary = [a.text.strip() for a in b.find('span', attrs = {'class' : 'salaryText'}).find('nobr')]
            except:
                salary = 'NA'
            df = df.append({"location":location,"company":company, "job_title": job_title, "salary": salary}, ignore_index=True)