2

本网站使用JS设置cookie。

如何运行 JS 来模拟浏览器以避免 429 错误？

from requests_html import HTMLSession

with HTMLSession() as s:
  url = 'https://www.realestate.com.au/auction-results/nsw'
  r = s.get(url)
  print(r.status_code)
  print(r.text)

  r.html.render()
  print(r.text)
4

1 回答 1

1

如果没有某种形式的浏览器模拟，似乎几乎不可能绕过指纹（即使使用 seleniumm，我也必须设置一些选项）。这是我想出的使用 Selenium 获取发出请求所需的唯一关键信息（名为“FGJK”的 cookie）的方法，该信息在后续请求标头中发送，并异步获取所有郊区结果页面。

from requests_html import AsyncHTMLSession
import asyncio
from selenium import webdriver
import nest_asyncio

#I'm using IPython which doesn't like async unless the following is applied:
nest_asyncio.apply()

async def get_token():
    options = webdriver.ChromeOptions()
    options.add_experimental_option('excludeSwitches', ['enable-automation']) 
    driver = webdriver.Chrome(options=options)
    driver.get('https://www.realestate.com.au/auction-results/nsw')
    cookies = driver.get_cookies()
    while True:
        for cookie in cookies:
            if cookie['name'] == 'FGJK':
               token = cookie['value'] 
               return token         
            else:
                cookies = driver.get_cookies()


async def get_results(s, endpoint, headers):
    r = await s.get(f'https://www.realestate.com.au/auction-results/{endpoint}', headers=headers)
    #do something with r.html
    print(r, endpoint)


async def main():
    token = await get_token()
    s = AsyncHTMLSession()
    headers = {'Cookie': f'FGJK={token}',
               'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}    

    r = await s.get(f'https://sales-events-api.realestate.com.au/sales-events/nsw')
    suburbs = r.json()['data']['suburbResults']
    endpoints = [burb['suburb']['urlValue'] for burb in suburbs]    
    asyncio.gather(*(get_results(s, endpoint, headers) for endpoint in endpoints))


asyncio.run(main())
于 2019-08-23T14:24:22.007 回答