通过 request.post 发送数据字段

Question

from bs4 import BeautifulSoup as bs  #importing the required libraries
from urllib.request import  urlopen
import requests
urls1="https://www.makemytrip.com/hotels/" #initial url which contains the form where we could give our preferences.

#passing the data parameters
data={'checkin': '08152020',
'city': 'CTGOI',
'checkout': '08162020',
'roomStayQualifier': '2e0e',
'locusId': 'CTGOI',
'country': 'IN',
'locusType': 'city',
'searchText': 'Goa, India',
'visitorId': '5c68c2fb-0551-4ef2-8dae-1a55bb744e66'
}
req=requests.post(urls1,data, headers={'User-Agent': 'XYZ/3.0'})
page_soup = bs(req.content,"html.parser")
print(page_soup)

实际上我想抓取上述数据字段下的酒店，这就是为什么我将带有requests.post方法的数据参数发送到初始url，这样当我收到响应对象时，我会得到下一页的内容将包含符合上述要求的酒店。

Answer 1

您正在抓取的网站使用 GET 方法执行搜索。

它还使用不同的 URL 进行酒店搜索，https://www.makemytrip.com/hotels/hotel-listing/

稍微修改您的示例以应用 GET 请求而不是 POST 请求，我们能够获得酒店列表结果。

from bs4 import BeautifulSoup as bs

headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"}
# setting a "browser" header seems to be required for this site.

data = {'checkin': '08192020',
        'city': 'CTGOI',
        'checkout': '08202020',
        'roomStayQualifier': '2e0e',
        'locusId': 'CTGOI',
        'country': 'IN',
        'locusType': 'city',
        'searchText': 'Goa, India',
        'visitorId': 'aaab4f61-2069-4033-bb97-0791f0f70'}

url = 'https://www.makemytrip.com/hotels/hotel-listing/'

# adding the params argument and supplying the dictionary of search data formats the resulting URL into something that makemytrip.com can understand. 
# adding a timeout just in case makemytrip.com doesn't respond
req = requests.get(url, params=data, headers=headers, timeout=5)


page_soup = bs(req.content,'html.parser')

# this finds all the divs in the result with a class name of "listingRow".  
listing_results = page_soup.findAll('div', class_='listingRow')

# this results array can then be looped through to find more details about each listing.
for listing in listing_results:
    print(listing.find("p", itemprop="name").getText())