带有代理和线程的请求

Question

我正在尝试使用请求来检查庞大的代理列表。为了做到这一点，我正在使用线程。我真的需要这些线程，因为我使用的是相同的代码结构，以便在我正在执行 1 个线程 = 1 个 IP 的网站上发出许多请求。

所以我的排序代码是：

def proxyList(proxies, nbThread):
    with open('proxyList.txt', 'w') as f:
        f.write('')
        f.close()
    proxies = list(set(proxies))
    prox = []
    lenS = len(proxies)
    pas = int(lenS/nbThread)
    subSeq = [proxies[i*pas:(i+1)*pas] for i in range(nbThread)]
    subSeq[nbThread-1]+=proxies[nbThread*pas:]
    threads = [0 for i in range(nbThread)]

    for i in range(nbThread):
        threads[i] = proxy(subSeq[i], )
    for i in range(nbThread):
        threads[i].start();
    for i in range(nbThread):
        threads[i].join();

    return list(set(prox))

class proxy(Thread):
    def __init__(self, proxies):
        Thread.__init__(self)
        self.proxies = proxies

    def run(self):
        k=0
        prox = []
        for proxy in self.proxies:
            k+=1
            try:
                requests.get("https://api.ipify.org/?format=json", timeout=15, proxies={"https":str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0])})
                try:
                    requests.get("https://api.ipify.org/?format=json", timeout=15, proxies={"https":str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0])})
                    prox+=[str(proxy)]
                    print("Bon proxy : " + str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0]))
                    with open('proxyList.txt', 'a') as f:
                        f.writelines(str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0])+'\n');
                        f.close()
                except:
                    t = "a"
            except:
                print("Mauvais proxy : "+ str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0]))
                print(sys.exc_info()[0])
        print("Terminé: "+str(k), prox)

有效，但我的输出结果并不总是相同，这与我设置的线程数高度相关。

你们有什么想法吗，我发现 requests 在这里可能不是最好的选择，但我真的需要我的线程和我的代理。

谢谢， 抖音

Answer 1

我很确定请求是最好的方法。这是下面的讨论；

https://gist.github.com/kennethreitz/973705

但是我已经尝试对您的代码进行一些改进，减少了一直在做同样工作的循环，并阻止了两次调用“get”方法。

希望对你有帮助

def proxyList(proxies, nbThread):
    with open('proxyList.txt', 'w') as f:
        f.write('')
        f.close()
    proxies = list(set(proxies))
    prox = []
    lenS = len(proxies)
    pas = int(lenS/nbThread)
    subSeq = [proxies[i*pas:(i+1)*pas] for i in range(nbThread)]
    subSeq[nbThread-1]+=proxies[nbThread*pas:]
    threads = [0 for i in range(nbThread)]

    for i in range(nbThread):
        threads[i] = proxy(subSeq[i], )
        threads[i].start()
        threads[i].join()

    return list(set(prox))

class proxy(Thread):
    def __init__(self, proxies):
        Thread.__init__(self)
        self.proxies = proxies

    def run(self):
        k=0
        prox = []
        for proxy in self.proxies:
            k+=1
            try:
                s = requests.Session()
                try:
                    s.get("https://api.ipify.org/?format=json", timeout=15, proxies={"https":str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0])})
                    prox+=[str(proxy)]
                    print("Bon proxy : " + str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0]))
                    with open('proxyList.txt', 'a') as f:
                        f.writelines(str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0])+'\n');
                        f.close()
                except:
                    t = "a"
            except:
                print("Mauvais proxy : "+ str(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', proxy)[0]))
                print(sys.exc_info()[0])
        print("Terminé: "+str(k), prox)