Django JsonResponse 有时会发送重复的对象片段而不是其他对象

Question

这是我遇到的一个非常奇怪的错误。我有一个 python 脚本，它生成一个我想使用JsonReponse 响应的 JSON 对象。此响应成功返回，但有时它包含对象的多个部分，有时它包含一个，并且就它发送回的副本数量而言，它只是上下波动。不知道这怎么可能。

这是一个网络爬虫，它为图形的节点和边发送回 JSON，重复的节点/边似乎是随机返回的。进行网络爬取并生成 JSON 的脚本每次都能正常工作，但只有在向 Django 发出请求的上下文中使用它时才会出现问题。

我的 views.py 有一个类似这样的函数来处理网络爬虫脚本。

def webcrawler(request):
    source = request.GET.get('source')
    method = request.GET.get('method')
    nodes = request.GET.get('nodes')
    depth = request.GET.get('depth')
    keyword = request.GET.get('keyword')
    webcrawler = WebCrawler(source, method, nodes, depth, keyword)
    data = webcrawler.jsonSerialize()
    return JsonResponse(data, safe=False)

我的jsonSerialize() 函数如下所示：

def jsonSerialize(self):
    for n in self.graph.nodes:
        n.sourceNodes = []
    self.graph.edges = list(self.graph.edges)
    return json.dumps(self.graph, default=lambda o: o.__dict__)

为什么我会从这个响应中得到随机数量的副本？我每次都发出一个新请求并创建一个新对象，但似乎发回的对象数量只是增加了，有时会上升或下降。如果我打开一个新选项卡并提出新请求，也会发生同样的事情。这可能是什么原因？

例如，如果我继续发出相同的请求webcrawler?source=http://www.google.com&method=BFS&nodes=5&depth=0&keyword=google

有时看起来像这样：

{
  "nodes": [
    {
      "keyword": false,
      "url": "http://www.google.com",
      "sourceNodes": [],
      "title": "Google",
      "index": 0
    },
    {
      "keyword": false,
      "url": "http://www.google.com/imghp?hl=en&tab=wi",
      "sourceNodes": [],
      "title": "Google Images",
      "index": 4
    },
    {
      "keyword": false,
      "url": "http://www.youtube.com/?tab=w1",
      "sourceNodes": [],
      "title": "YouTube",
      "index": 3
    },
    {
      "keyword": false,
      "url": "http://news.google.com/nwshp?hl=en&tab=wn",
      "sourceNodes": [],
      "title": "Google News",
      "index": 2
    },
    {
      "keyword": false,
      "url": "http://maps.google.com/maps?hl=en&tab=wl",
      "sourceNodes": [],
      "title": "Google Maps",
      "index": 1
    }
  ],
  "edges": [
    {
      "source": 0,
      "target": 1
    },
    {
      "source": 0,
      "target": 3
    },
    {
      "source": 0,
      "target": 2
    },
    {
      "source": 0,
      "target": 4
    }
  ]
}

有时它看起来像这样……有重复的节点和边。

{
  "nodes": [
    {
      "keyword": false,
      "url": "https://mail.google.com/mail/?tab=wm",
      "sourceNodes": [],
      "title": "Gmail",
      "index": 1
    },
    {
      "keyword": false,
      "url": "https://www.google.com/intl/en/options/",
      "sourceNodes": [],
      "title": "\n      Our Products | Google\n    ",
      "index": 7
    },
    {
      "keyword": false,
      "url": "http://www.google.com/imghp?hl=en&tab=wi",
      "sourceNodes": [],
      "title": "Google Images",
      "index": 6
    },
    {
      "keyword": false,
      "url": "https://drive.google.com/?tab=wo",
      "sourceNodes": [],
      "title": "Meet Google Drive \u2013 One place for all your files",
      "index": 2
    },
    {
      "keyword": false,
      "url": "http://news.google.com/nwshp?hl=en&tab=wn",
      "sourceNodes": [],
      "title": "Google News",
      "index": 8
    },
    {
      "keyword": false,
      "url": "http://maps.google.com/maps?hl=en&tab=wl",
      "sourceNodes": [],
      "title": "Google Maps",
      "index": 3
    },
    {
      "keyword": true,
      "url": "https://play.google.com/?hl=en&tab=w8",
      "sourceNodes": [],
      "title": "Google Play",
      "index": 9
    },
    {
      "keyword": false,
      "url": "https://accounts.google.com/ServiceLogin?hl=en&passive=true&continue=http://www.google.com/",
      "sourceNodes": [],
      "title": "Sign in - Google Accounts",
      "index": 4
    },
    {
      "keyword": false,
      "url": "http://www.google.com",
      "sourceNodes": [],
      "title": "Google",
      "index": 0
    },
    {
      "keyword": false,
      "url": "http://www.google.com/history/optout?hl=en",
      "sourceNodes": [],
      "title": " Google - Search Customization ",
      "index": 5
    },
    {
      "keyword": false,
      "url": "http://www.youtube.com/?tab=w1",
      "sourceNodes": [],
      "title": "YouTube",
      "index": 3
    }
  ],
  "edges": [
    {
      "source": 0,
      "target": 1
    },
    {
      "source": 0,
      "target": 7
    },
    {
      "source": 0,
      "target": 6
    },
    {
      "source": 0,
      "target": 5
    },
    {
      "source": 0,
      "target": 4
    },
    {
      "source": 0,
      "target": 9
    },
    {
      "source": 0,
      "target": 3
    },
    {
      "source": 0,
      "target": 8
    },
    {
      "source": 0,
      "target": 2
    }
  ]
}

如果我多次使用相同的参数在本地运行 python 脚本，我不会看到这种不良行为。这让我觉得它与request/Django/JsonResponse有关，因为脚本似乎运行正常。

这是我的网络爬虫的代码：

import urllib
# from urllib.request import urlopen
from urlparse import urlparse
from bs4 import BeautifulSoup
import requests
import collections
from Graph import Graph
from Node import Node
import sys
from time import gmtime, strftime
from timeout import timeout
from multiprocessing import Pool
from multiprocessing import Process
import json
import re
import pdb

class WebCrawler:
    def __init__(self, originUrl, method, totalNodes, depthLimit=None, keyword=None):
        self.originUrl = originUrl
        self.method = method
        self.totalNodes = int(totalNodes)
        self.nodeCount = 0
        self.depthLimit = int(depthLimit)
        self.currentDepth = 0
        self.keyword = keyword
        self.keywordUrls = []
        self.nodeUrlMap = {}
        self.nodesToVisit = []
        self.visitedUrls = set()
        self.graph = Graph()
        self.nodeIndex = 0
        self.storeCookie()
        originTitle = self.getTitle(originUrl)
        startNode = Node(originUrl, None, originTitle)
        self.crawl(startNode)

    def crawl(self, node):
        print("crawl(): " + strftime("%H:%M:%S", gmtime()))
        visited = node.url in self.visitedUrls
        if not visited:
            self.graph.addNode(node, self.nodeIndex)
            self.nodeIndex += 1
            self.nodeCount += 1
            self.visitedUrls.add(node.url)
        if node.sourceNodes: # If this is not the starting node
            sourceNode = node.sourceNodes.pop()
            if sourceNode.index is not None and node.index is not None:
                self.graph.addEdge(sourceNode.index, node.index) # Add an edge between sourceNode and node
        if not visited:
            soup = self.generateSoup(node.url)
            hasKeyword = self.checkForKeyword(soup, node.url)
            if hasKeyword:
                node.keyword = True
            links = self.findLinks(soup)
            links = self.validLinks(links)
            links = {l for l in links} # Remove duplicate links
            if links:
                if self.method == "BFS":
                    self.bfs(node, links)
                else: # DFS
                    self.currentDepth += 1
                    if self.currentDepth >= self.depthLimit: # If depth limit reached, getNextNode (up a level)
                        self.currentDepth = 0 # Reset currentDepth
                        self.getNextNode()
                    else: # Otherwise, keep going deeper
                        self.dfs(node, links)
            else: # No links present
                self.getNextNode()
        else: # Avoid infinite loop
            self.getNextNode()

    def validLinks(self, links):
        print("validLinks(): " + strftime("%H:%M:%S", gmtime()))
        validLinks = []
        for link in links:
            # Only add links while there is still room
            if self.nodeCount + len(validLinks) <= self.totalNodes:
                if self.isValidUrl(link):
                    validLinks.append(link)
        return validLinks

    def isValidUrl(self, url):
        print("isValidUrl(): " + strftime("%H:%M:%S", gmtime()))
        extensionBlacklist = ["zip", "dmg", "msi", "tar", "exe", "sisx"]
        for x in extensionBlacklist:
            if x in url:
                return False
        if "http" not in url: return False
        parsed_url = urlparse(url)
        if not bool(parsed_url.scheme): return False
        try:
            self.testRequest(url)
        except:
            return False
        return True

    @timeout(1)
    def testRequest(self, url):
        requests.head(url)

    def getNextNode(self):
        print("getNextNode(): " + strftime("%H:%M:%S", gmtime()))
        if len(self.nodesToVisit) is not 0 and not self.nodeLimitReached():
            # We use the same data structure to store urlsToVisit for BFS and DFS,
            # and pop elements off the same way.  How the elements are added is
            # what's important.
            nextNode = self.nodesToVisit.pop()
            self.crawl(nextNode)
        else: # Crawl is over
            return

    def printGraph(self):
        for node in self.graph.nodes:
            print("\nNode:")
            if node.title:
                print("Index: " + str(node.index))
                print("Title: " + node.title)
                print("URL: " + node.url)
                print("Keyword: " + str(node.keyword))
        if self.graph.edges:
            print("\nEdges:")
            edgeCount = 0
            for e in self.graph.edges:
                print("Source: " + str(e.source) + " Target: " + str(e.target))
        if self.keywordUrls:
            print("\nKeyword URLs:")
            for k in self.keywordUrls:
                print("URL: " + k)
        print("\nJSON:")
        print(self.jsonSerialize())

    def jsonSerialize(self):
        for n in self.graph.nodes:
            n.sourceNodes = []
        self.graph.edges = list(self.graph.edges)
        self.graph.nodes = list(self.graph.nodes)
        return json.dumps(self.graph, default=lambda o: o.__dict__)

    def storeCookie(self):
        # Store graph as cookie (do this one)
        pass

    def nodeLimitReached(self):
        return self.nodeCount >= self.totalNodes

    # Convert URL into soup
    def generateSoup(self, url):
        print("generateSoup(): " + strftime("%H:%M:%S", gmtime()))
        sourceCode = requests.get(url)
        plainText = sourceCode.text
        soup = BeautifulSoup(plainText, "html.parser")
        return soup

    # Parse soup to find links
    def findLinks(self, soup):
        print("findLinks(): " + strftime("%H:%M:%S", gmtime()))
        links = soup.findAll('a')
        hrefs = []
        for link in links:
            href = link.get('href', '')
            hrefs.append(href)
        return hrefs

    def getTitle(self, url):
        print("getTitle(): " + strftime("%H:%M:%S", gmtime()))
        soup = self.generateSoup(url)
        title = soup.title  # same as soup.find("title")
        if title is not None:
            return title.get_text()

    def bfs(self, currentNode, links):
        print("bfs(): " + strftime("%H:%M:%S", gmtime()))
        for link in links:
            # If url is not already visited, and nodesToVisit+nodeCount hasn't exceeded totalNodes
            if link not in self.visitedUrls and self.nodeCount + len(self.nodesToVisit) <= self.totalNodes:
                title = self.getTitle(link)
                newNode = Node(link, [currentNode], title)
                newNode.sourceNodes.insert(0, currentNode)
                self.nodesToVisit.insert(0, newNode)
                self.nodeUrlMap[link] = newNode
            elif link in self.nodeUrlMap: # Repeat URL, get existing node
                existingNode = self.nodeUrlMap[link]
                existingNode.sourceNodes.insert(0, currentNode)
                self.nodesToVisit.insert(0, existingNode)
        self.getNextNode()

    def dfs(self, currentNode, links):
        print("dfs(): " + strftime("%H:%M:%S", gmtime()))
        for link in links:
            if link not in self.visitedUrls:
                title = self.getTitle(link)
                newNode = Node(link, [currentNode], title)
                newNode.sourceNodes.append(currentNode)
                self.nodesToVisit.append(newNode)
            elif link in self.nodeUrlMap: # Repeat URL, get existing node
                existingNode = self.nodeUrlMap[link]
                existingNode.sourceNodes.append(currentNode)
                self.nodesToVisit.append(existingNode)
        self.getNextNode()

    def checkForKeyword(self, soup, url):
        if self.keyword != "":
            # If keyword found in soup, append url to keywordUrls
            if soup.body and soup.body.find_all(string=re.compile('.*{0}.*'.format(self.keyword)), recursive=True):
                self.keywordUrls.append(url)
                return True

Answer 1

我猜WebCrawler 错误地初始化了它的graph 属性。而不是：

class WebCrawler:
    graph = Something()  # graph is a static class variable!!

试试：

class WebCrawler:
    def __init__(self, ...)
        self.graph = Something()  # graph is an instance variable

（如果不是 - 检查graph.edges/graph.nodes 的初始化）。