使用 CSV 文件时列出超出范围的错误

Question

我正在运行一个 python 抓取工具，它从网页中抓取引号并将结果输出到 CSV 文件中。 我自己没有写这个，因为我是一个初学者，但是当我运行这段代码来测试它并自己使用它的一部分时，我得到了这个错误。我知道这个错误是什么意思，但我很不知道如何解决这个问题。我想将更新推送到作者的 github 以提供帮助。

Traceback (most recent call last):
  File "quotes.py", line 100, in <module>
    get_authors()
  File "quotes.py", line 58, in get_authors
    quote_details = fetch_quote(url)
  File "quotes.py", line 77, in fetch_quote
    tempString += ("\"%s\","%next(q.find_class('b-qt')[0].iter('a')).text)
IndexError: list index out of range

问题发生在它开始获取引号时。创建作者列表和 URL 作品列表，没有任何问题。 IndexError 在它创建 CSV 文件之后发生，那是错误被抛出的那一刻，所以我认为问题出在这部分代码：

tempString += ("\"%s\","%next(q.find_class('b-qt')[0].iter('a')).text)

这听起来对吗？除了 typeErrors 和一些更简单的 IndexErrors 之外，我完全不知道如何解决 Python 中的错误。我很想学习，但我在 stackoverflow 上的所有搜索都显示很多人在 CSV 文件方面遇到了同样的问题。但所有的答案都非常具体。

#!/usr/bin/python

import requests
from lxml import html
import time
import string

def get_authors():
    baseUrl = 'http://www.brainyquote.com'
    urlString = 'http://www.brainyquote.com/authors/'
    authorsUrl = [urlString + x for x in list(string.lowercase[:26])]

    urlsList = [] # authors list page urls
    print ""
    print "Scanning Started for page links"
    print ""
    for url in authorsUrl:
        print "Scanning URL: %s"%url
        urlsList.append(url)
        urlsList.extend(pagination(url, False))

    authorsList = []
    print ""
    print "Scanning Started for Author Pages"
    print ""
    for url in urlsList:
        print "Scanning URL: %s"%url
        authorsList.extend(get_authors_links(url))
    # Write all authors links
    authorsFile = open("authors.txt","a+")
    for urls in authorsList:
        authorsFile.write(baseUrl + urls.encode('utf-8') + "\n")
    authorsFile.close()

    quoteLinks = []
    # Write all authors links
    print ""
    print "Scanning Started for Quote Page Links"
    print ""
    for url in authorsList:
        newUrl = (baseUrl + url)
        print "Scanning URL: %s"%newUrl
        quoteLinks.append(newUrl)
        arr = pagination(newUrl, True)
        quoteLinks.extend(arr)
    # Write all quotes link
    linksFile = open("quotes_links.txt","a+")
    for url in quoteLinks:
        linksFile.write(url.encode('utf-8') + "\n")
    linksFile.close()

    print ""
    print "Scanning Started for fetching quotes"
    print ""
    # Write all quotes
    quotesFile = open("quotes.csv","a+")
    for url in quoteLinks:
        quote_details = fetch_quote(url)
        quotesFile.write(quote_details.encode('utf-8') + "\n")

    print ""
    print "All Done \nThanks for using it...!!!"
    print ""

def get_authors_links(url):
    page = requests.get(url)
    tree = html.fromstring(page.text)
    arr = tree.xpath('//table[@class="table table-hover table-bordered"]//td/a/@href')
    return arr

def fetch_quote(url):
    page = requests.get(url)
    tree = html.fromstring(page.text)
    quotes = tree.find_class('bqQt')
    tempString = ""
    for q in quotes:
        tempString += ("\"%s\","%next(q.find_class('b-qt')[0].iter('a')).text)
        tempString += ("%s,"%next(q.find_class('bq-aut')[0].iter('a')).text)
        for element in q.find_class('oncl_k'):
            tempString += "%s "%element.text
        tempString += "\n"
    return tempString

def pagination(url, htmlPage): # .html or not - htmlPage True or False
    arr = []
    page = requests.get(url)
    tree = html.fromstring(page.text)
    end = tree.xpath('//div[@class="row paginationContainer"]//nav//ul/li[last()-1]/a/text()')
    if len(end):
        if(htmlPage):
            url = url.split('.html')[0]
            for count in range(2, int(end[0])+1):
                arr.append(url+"%s.html"%(count))
        else:
            for count in range(2, int(end[0])+1):
                arr.append(url+"%s"%(count))
    return  arr

if __name__ == '__main__':
    get_authors()#!/usr/bin/python

import requests
from lxml import html
import time
import string

def get_authors():
    baseUrl = 'http://www.brainyquote.com'
    urlString = 'http://www.brainyquote.com/authors/'
    authorsUrl = [urlString + x for x in list(string.lowercase[:26])]

    urlsList = [] # authors list page urls
    print ""
    print "Scanning Started for page links"
    print ""
    for url in authorsUrl:
        print "Scanning URL: %s"%url
        urlsList.append(url)
        urlsList.extend(pagination(url, False))

    authorsList = []
    print ""
    print "Scanning Started for Author Pages"
    print ""
    for url in urlsList:
        print "Scanning URL: %s"%url
        authorsList.extend(get_authors_links(url))
    # Write all authors links
    authorsFile = open("authors.txt","a+")
    for urls in authorsList:
        authorsFile.write(baseUrl + urls.encode('utf-8') + "\n")
    authorsFile.close()

    quoteLinks = []
    # Write all authors links
    print ""
    print "Scanning Started for Quote Page Links"
    print ""
    for url in authorsList:
        newUrl = (baseUrl + url)
        print "Scanning URL: %s"%newUrl
        quoteLinks.append(newUrl)
        arr = pagination(newUrl, True)
        quoteLinks.extend(arr)
    # Write all quotes link
    linksFile = open("quotes_links.txt","a+")
    for url in quoteLinks:
        linksFile.write(url.encode('utf-8') + "\n")
    linksFile.close()

    print ""
    print "Scanning Started for fetching quotes"
    print ""
    # Write all quotes
    quotesFile = open("quotes.csv","a+")
    for url in quoteLinks:
        quote_details = fetch_quote(url)
        quotesFile.write(quote_details.encode('utf-8') + "\n")

    print ""
    print "All Done \nThanks for using it...!!!"
    print ""

def get_authors_links(url):
    page = requests.get(url)
    tree = html.fromstring(page.text)
    arr = tree.xpath('//table[@class="table table-hover table-bordered"]//td/a/@href')
    return arr

def fetch_quote(url):
    page = requests.get(url)
    tree = html.fromstring(page.text)
    quotes = tree.find_class('bqQt')
    tempString = ""
    for q in quotes:
        tempString += ("\"%s\","%next(q.find_class('b-qt')[0].iter('a')).text)
        tempString += ("%s,"%next(q.find_class('bq-aut')[0].iter('a')).text)
        for element in q.find_class('oncl_k'):
            tempString += "%s "%element.text
        tempString += "\n"
    return tempString

def pagination(url, htmlPage): # .html or not - htmlPage True or False
    arr = []
    page = requests.get(url)
    tree = html.fromstring(page.text)
    end = tree.xpath('//div[@class="row paginationContainer"]//nav//ul/li[last()-1]/a/text()')
    if len(end):
        if(htmlPage):
            url = url.split('.html')[0]
            for count in range(2, int(end[0])+1):
                arr.append(url+"%s.html"%(count))
        else:
            for count in range(2, int(end[0])+1):
                arr.append(url+"%s"%(count))
    return  arr

if __name__ == '__main__':
    get_authors()

任何想法或指针将不胜感激。据我所知，这应该不难解决，但作为初学者，在比我习惯的更长的代码中更改 3 行的想法是非常令人生畏的。

感谢作者，希望能在你的帮助下解决问题：

https://github.com/ravingupta/brainyquote/

Answer 1

这可行（代码转换为 python 3）

import requests
from lxml import html
import string


def get_authors():
    baseUrl = 'http://www.brainyquote.com'
    urlString = 'http://www.brainyquote.com/authors/'
    authorsUrl = [urlString + x for x in list(string.ascii_lowercase[:26])]

    urlsList = []  # authors list page urls
    print("")
    print("Scanning Started for page links")
    print("")
    for url in authorsUrl:
        print("Scanning URL: %s" % url)
        urlsList.append(url)
        urlsList.extend(pagination(url, False))

    authorsList = []
    print("")
    print("Scanning Started for Author Pages")
    print("")
    for url in urlsList:
        print("Scanning URL: %s" % url)
        authorsList.extend(get_authors_links(url))
    # Write all authors links
    authorsFile = open("authors.txt", "a+")
    for urls in authorsList:
        authorsFile.write(baseUrl + str(urls.encode('utf-8')) + "\n")
    authorsFile.close()

    quoteLinks = []
    # Write all authors links
    print("")
    print("Scanning Started for Quote Page Links")
    print("")
    for url in authorsList:
        newUrl = (baseUrl + url)
        print("Scanning URL: %s" % newUrl)
        quoteLinks.append(newUrl)
        arr = pagination(newUrl, True)
        quoteLinks.extend(arr)
    # Write all quotes link
    linksFile = open("quotes_links.txt", "a+")
    for url in quoteLinks:
        linksFile.write(str(url.encode('utf-8')) + "\n")
    linksFile.close()

    print("")
    print("Scanning Started for fetching quotes")
    print("")
    # Write all quotes
    quotesFile = open("quotes.csv", "a+")
    for url in quoteLinks:
        quote_details = fetch_quote(url)
        quotesFile.write(str(quote_details.encode('utf-8')) + "\n")

    print("")
    print("All Done \nThanks for using it...!!!")
    print("")


def get_authors_links(url):
    page = requests.get(url)
    tree = html.fromstring(page.text)
    arr = tree.xpath('//table[@class="table table-hover table-bordered"]//td/a/@href')
    return arr


def fetch_quote(url):
    page = requests.get(url)
    tree = html.fromstring(page.text)
    quotes = tree.find_class('bqQt')
    tempString = ""
    for q in quotes:
        tempString += ("\"%s\"," % next(q.find_class('b-qt')[0].iter('a')).text)
        tempString += ("%s," % next(q.find_class('bq-aut')[0].iter('a')).text)
        for element in q.find_class('oncl_k'):
            tempString += "%s " % element.text
        tempString += "\n"
    return tempString


def pagination(url, htmlPage):  # .html or not - htmlPage True or False
    arr = []
    page = requests.get(url)
    tree = html.fromstring(page.text)
    end = tree.xpath('//div[@class="row paginationContainer"]//nav//ul/li[last()-1]/a/text()')
    if len(end):
        if (htmlPage):
            url = url.split('.html')[0]
            for count in range(2, int(end[0]) + 1):
                arr.append(url + "%s.html" % (count))
        else:
            for count in range(2, int(end[0]) + 1):
                arr.append(url + "%s" % (count))
    return arr


if __name__ == '__main__':
    get_authors()