跟踪网站中链接的重复过程（BeautifulSoup）

Question

我正在用 Python 编写代码，以使用 Beautiful soup 获取 URL 中的所有“a”标签，然后我使用位置 3 的链接，然后我应该点击该链接，我将重复这个过程大约 18 次.我包含了下面的代码，该过程重复了两次。我想不出一种方法可以在一个循环中重复相同的过程 18 次。任何帮助将不胜感激。

import re
import urllib

from BeautifulSoup import *
htm1= urllib.urlopen('https://pr4e.dr-chuck.com/tsugi/mod/python-data/data/known_by_Fikret.html ').read()
soup =BeautifulSoup(htm1)
tags = soup('a')
list1=list()
for tag in tags:
    x = tag.get('href', None)
    list1.append(x)

M= list1[2]

htm2= urllib.urlopen(M).read()
soup =BeautifulSoup(htm2)
tags1 = soup('a')
list2=list()
for tag1 in tags1:
    x2 = tag1.get('href', None)
    list2.append(x2)

y= list2[2]
print y

好的，我刚刚编写了这段代码，它正在工作，但我在结果中得到了相同的 4 个链接。循环中似乎有问题（请注意：我正在尝试循环 4 次）

import re
import urllib
from BeautifulSoup import *
list1=list()
url = 'https://pr4e.dr-chuck.com/tsugi/mod/python-data/data/known_by_Fikret.html'

for i in range (4):  # repeat 4 times
    htm2= urllib.urlopen(url).read()
    soup1=BeautifulSoup(htm2)
    tags1= soup1('a')
    for tag1 in tags1:
        x2 = tag1.get('href', None)
        list1.append(x2)
    y= list1[2]
    if len(x2) < 3:  # no 3rd link
        break  # exit the loop
    else:
        url=y             
    print y

Answer 1

我想不出一种方法可以在一个循环中重复相同的过程 18 次。

要在 Python 中重复 18 次，您可以使用 for _ in range(18) 循环：

#!/usr/bin/env python2
from urllib2 import urlopen
from urlparse import urljoin
from bs4 import BeautifulSoup # $ pip install beautifulsoup4

url = 'http://example.com'
for _ in range(18):  # repeat 18 times
    soup = BeautifulSoup(urlopen(url))
    a = soup.find_all('a', href=True)  # all <a href> links
    if len(a) < 3:  # no 3rd link
        break  # exit the loop
    url = urljoin(url, a[2]['href'])  # 3rd link, note: ignore <base href>

Answer 2

我会在 for 循环中使用 for 循环来重复，并使用 count 来中断以使其停止在您想要的链接处。

from bs4 import BeautifulSoup
import urllib.request, urllib.parse, urllib.error
import ssl
import re

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
url = "your first link here"
#to repeat 18 times#
for i in range(18):
    html = urllib.request.urlopen(url, context=ctx).read()
    soup = BeautifulSoup(html, 'html.parser')
    tags = soup('a')
    count = 0
    for tag in tags:
        count = count +1
        #make it stop at position 3#
        if count>3:
            break
        url = tag.get('href', None)
print(url)

Answer 3

url = input('Enter - ')

def functiontofollowlink(url):

    html = urllib.request.urlopen(url, context=ctx).read()
    soup = BeautifulSoup(html, 'html.parser')
    count=0
    # Retrieve all of the anchor tags
    tags = soup('a')
    for tag in tags:
        #print(tag.get('href', None))
        count = count+1
        if count==18:
           url=tag.get('href', None)
return url
numberoftimes = int(input('Enter number of times to repeat:'))
#for fisrt you need to give the link then for numberoftimes decrease number by 1
#since we know previously which position to check so directly used in the function count = 18 else you can give number externally
#positionoflink = input('Enter position of the link:')

while numberoftimes>=0:
    numberoftimes=numberoftimes-1
    url=functiontofollowlink(url)

print(url)

这将为您提供所需的准确查询输出

Answer 4

import urllib
from BeautifulSoup import *

url='http://python-data.dr-chuck.net/known_by_Eesa.html'
counts=raw_input('Enter number of pages to jump: ')
counts=int(counts)
pos=raw_input('Enter position: ')
pos=int(pos)
y1= list()
y2=list()
count=0
while True:
   data=urllib.urlopen(url).read()
   soup= BeautifulSoup(data)
   tags=soup('a')
   for tag in tags:
       value=tag.get('href',None)
       value=str(value)
       y1.append(value)
   t=y1[pos-1]
   y2.append(t)
   y1=[]
   count=count+1 
   if count==counts:break
   else:
    url=t
    continue
print y2

Answer 5

你应该使用递归http://www.python-course.eu/recursive_functions.php

def GetLinks(initialPage):
    htm1= urllib.urlopen(initialPage).read()
    soup =BeautifulSoup(htm1)
    tags = soup('a')
    list1=list()
    for tag in tags:
        x = tag.get('href', None)
        list1.append(x)
        list1.append(GetLinks(x))
    return list1

如果你想设置你想去的级别的限制，你可以通过你所在的级别作为参数，例如：

def GetLinks(initialPage, level):
    level += 1
    htm1= urllib.urlopen(initialPage).read()
    soup =BeautifulSoup(htm1)
    tags = soup('a')
    list1=list()
    for tag in tags:
        x = tag.get('href', None)
        list1.append(x)
        if level < 18:
            list1.append(GetLinks(x))
    return list1

Answer 6

import urllib
from BeautifulSoup import *

URL = raw_input("Enter the URL:") #Put insurance
link_line = int(raw_input("Enter the line of the desired link:")) - 1 #Put insurance
count = int(raw_input("Enter the loop repeat times:")) #Put insurance

while count >= 0:
    html = urllib.urlopen(URL).read()
    soup = BeautifulSoup(html)
    tags = soup('a')
    print URL
    URL = tags[link_line].get("href", None)
    count = count - 1

Answer 7

import urllib
from BeautifulSoup import *
url = raw_input('http://example')
for i in range(18):
    html = urllib.urlopen(url).read()
    soup = BeautifulSoup(html)

    tags = soup('a')
    s = []

    for tag in tags:
        x = tag.get('href', None)
        s.append(x)
    print s[3]
    url = s[3]
print "ansver:", s[3]

Answer 8

from bs4 import BeautifulSoup
import urllib.request, urllib.parse, urllib.error
import ssl

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
url = input('Enter :')
for i in range(7): #to repeat 7 times
    html = urllib.request.urlopen(url, context=ctx).read()
    soup = BeautifulSoup(html, 'html.parser')
    tags = soup('a')
    count = 0
    for tag in tags:
        count = count +1
        if count>18: #to stop after printing 18th name
            break
        url = tag.get('href', None)
        if count == 18: #to print 18th name
            print(url)

Answer 9

我发现使用 while 循环可以使代码更简洁，并且可以选择更改该代码的输入。

    from urllib.request import urlopen
    from bs4 import BeautifulSoup
    import ssl
    
    # Ignore SSL certificate errors
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    
    url = input('Enter url: ')
    
    count = input('Enter count: ')
    count = int(count)
    
    pos = input('Enter position:')
    pos = int(pos)
    
    while count > 0:
        # Re-opens the link
        html = urlopen(url, context=ctx).read()
        soup = BeautifulSoup(html, "html.parser")
        
        #Extract 'href=' values
        lst= list()
        tags = soup("a")
        for tag in tags:
            href = tag.get("href", None)
            lst.append(href)
        #prints only the url 'http:/...'
        url = lst[pos]
        #prints out the url on that position
        print('Retrieving:', url)
        
        #makes sure the loop isn't infinite
        count = count - 1