通过 python 处理数据库会给出错误的结果

Question

这种将数据放入数据库的方法有什么问题吗？

程序的部分是这样的。它将在一个循环中执行多次。

    db= MySQLdb.connect("localhost","root","ahmed","practice")
    cursor=db.cursor()

    #checking phase to stop scrapping
    sql = """SELECT Short_link FROM Properties WHERE Short_link=%s"""
    print rows
    rows = cursor.execute(sql,(link_result))
    print rows
    if rows>=1:
        print "Already present - The program is terminating"
        sys.exit()
    else:
        query="""INSERT INTO Properties (Sale_Rent, Type, Title,Price, PricePerSqrFt, Bedroom,Agency_Fee, Bathroom, Size,ZonedFor, Freehold, Prop_ref,Furnished_status,Rent_payment,Building_info,Amenities,Trade_name,Licence, RERA_ID,Phone_info,Short_link) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
        cursor.execute(query,(Sale_Rent_result,Type_result, title_result, price_result, Pricepersq_result, bedroom_result, agencyfee_result, bathroom_result, size_result,Zoned_for_result, Freehold_result, propertyref_result, furnished_result, rent_is_paid_result, building_result, Amenities_result, tradename_result, licencenum_result, reraid_result, phone_result, link_result))


    db.commit()
    cursor.close()
    db.close()

当我运行这个程序时，它运行良好。但是当我使用下面的脚本以并行方式运行该程序的 5 个副本时，其中一个程序从一开始就具有 row=1（而数据库为空且 row 不应为 1）。

all.sh
python python1.py &
python python2.py &
python python3.py &
python python4.py &
python python5.py &

每个link_result's结果在提取后都是唯一的，一旦输入数据库，它就会创建一列唯一链接。

假设数据库是空的并且我一起运行这些文件，row 永远不会等于 1。当我再次运行程序时它应该得到 1。当我再次运行它们时，它将输入新数据并在link_result与已经存在的链接结果冲突时停止（通过sql的select方法检查）

我假设在数据库为空时row 变量变为 1 的数据库打开和关闭存在一些问题。我无法理解这种行为。

这是整个程序供参考

#!/usr/bin/python
import urllib
from bs4 import BeautifulSoup

import MySQLdb
import re
import pdb
import sys





def getting_urls_of_all_pages(): 

    i=1
    while i<=40: #40 is the total number of main pages
        url_rent_flat='http://dubai.dubizzle.com/property-for-rent/residential/apartmentflat/?page='+str(i) #url of the main page (iterating to 40)


        link=[]
        htmlfile=urllib.urlopen(url_rent_flat).read()
        soup=BeautifulSoup(htmlfile)

        link=soup.find_all('a',xtclib=re.compile("listing_list_\d+_title_link"),href=True) #stores all the links (25) links of the page

        """
        Part 2: passing each property url to process for data extraction
        """

        for a in link:
            every_property_in_a_page_data_extraction(a['href']) 

        i+=1


def every_property_in_a_page_data_extraction(url):






    title_result=""
    price_result=""
    bedroom_result="" 
    agencyfee_result="" 
    bathroom_result="" 
    size_result=""
    propertyref_result=""
    furnished_result=""
    rent_is_paid_result=""
    building_result=""
    Amenities_result=""
    tradename_result=""
    licencenum_result=""
    reraid_result=""
    phone_result=""
    link_result=""
    Zoned_for_result=""
    Freehold_result=""
    Pricepersq_result=""
    Type_result="Apartment"
    Sale_Rent_result="Rent"
    rows=0




    """
    Part1: Extracting data
    """

    htmlfile=urllib.urlopen(url).read()
    soup=BeautifulSoup(htmlfile)

    """
    Part2: Extracting the components
    """


    # Sale/Rent
    print "Sale/Rent: ", Sale_Rent_result

    # Type of property
    print "Type of property: ", Type_result

    #title
    try:
        title= soup.find('span',{'id':'listing-title-wrap'})
        title_result= str(title.get_text().strip().encode("utf-8"))
        print "Title: ",title_result
    except StandardError as e:
        title_result="Error was {0}".format(e)
        print title_result

    #price
    try:
        price = soup.find('span',{'id':'actualprice'})
        price_result= str(price.get_text())
        print "Price: ",price_result
    except StandardError as e:
        price_result="Error was {0}".format(e)
        print price_result

    #Agency Fee, Bedroom, Bathroom, Size
    spans_ABBS= []
    for a in soup.select(".important-fields li span"):
        spans_ABBS.append(a.text.strip())

    strongs_ABBS=[]
    for a in soup.select(".important-fields li strong"):
        strongs_ABBS.append(a.text.strip())


    for name, value in zip(spans_ABBS, strongs_ABBS):
        if name=="Agency Fees:":
            try:
                agencyfee_result= str(value)
                print "Agency Fees: ", agencyfee_result
            except StandardError as e:
                agencyfee_result="Error was {0}".format(e)
                print agencyfee_result

        elif name=="Bedrooms:":
            try:
                bedroom_result= str(value)
                print "Number of Bedrooms: ",bedroom_result
            except StandardError as e:
                bedroom_result="Error was {0}".format(e)
                print bedroom_result

        elif name=="Bathrooms:":
            try:
                bathroom_result= str(value)
                print "Number of Bathrooms: ", bathroom_result
            except StandardError as e:
                bathroom_result="Error was {0}".format(e)
                print bathroom_result

        elif name=="Size:":
            try:
                size_result= str(value)
                print "Size of the property: ",size_result
            except StandardError as e:
                size_result="Error was {0}".format(e)
                print size_result

        elif name=="Zoned For:":
            try:
                Zoned_for_result= str(value)
                print "Zoned For:",Zoned_for_result
            except StandardError as e:
                Zoned_for_result="Error was {0}".format(e)
                print Zoned_for_result

        elif name=="Freehold:":
            try:
                Freehold_result= str(value)
                print "Freehold: ",Freehold_result
            except StandardError as e:
                Freehold_result="Error was {0}".format(e)
                print Freehold_result

        elif name=="Price / SqFt:":
            try:
                Pricepersq_result= str(value)
                print "Price Per Sqft: ",Pricepersq_result
            except StandardError as e:
                Pricepersq_result="Error was {0}".format(e)
                print Pricepersq_result

    #Property Reference, Furnished, Listed By, Rent Is Paid, Building, Amenities: 
    spans_others=[]
    for a in soup.select("#listing-details-list li span"):
            spans_others.append(a.text.strip())

    strongs_others=[]
    for a in soup.select("#listing-details-list li strong"):
        strongs_others.append(a.text.strip())



    for name, value in zip(spans_others, strongs_others):
        if name=="Listed by:":
            break

        elif name=="Property Reference:":
            try:
                propertyref_result=str(value.strip())
                print "Property reference in Dubizel: ",propertyref_result
            except StandardError as e:
                propertyref_result="Error was {0}".format(e)
                print propertyref_result

        elif name=="Furnished:":
            try:
                furnished_result=str(value.strip())
                print "Furnished status: ",furnished_result
            except StandardError as e:
                furnished_result="Error was {0}".format(e)
                print furnished_result


        elif name=="Rent Is Paid:":
            try:
                rent_is_paid_result=str(value.strip())
                print "Rent payment: ",rent_is_paid_result
            except StandardError as e:
                rent_is_paid_result="Error was {0}".format(e)
                print rent_is_paid_result

        elif name=="Building:":
            try:
                building_result=str(value.strip())
                print "Building info: ",building_result
            except StandardError as e:
                building_result="Error was {0}".format(e)
                print building_result
        elif name=="Amenities:":
            try:
                for a in value.split(","):
                    Amenities_result+=a.strip()+","
                print Amenities_result
            except StandardError as e:
                Amenities_result="Error was {0}".format(e)
                print Amenities_result



    #Agents info --> TTrade Name, DED Licence Number, RERA Registration Number
    spans_broker=[]
    for a in soup.select("#broker-details li span"):
            spans_broker.append(a.text.strip())

    strongs_broker=[]
    for a in soup.select("#broker-details  li strong"):
        strongs_broker.append(a.text.strip())


    for name, value in zip(spans_broker, strongs_broker):
        if name=="Trade Name:":
            try:
                tradename_result=str(value.strip())
                print "Trade name: ",tradename_result
            except StandardError as e:
                tradename_result="Error was {0}".format(e)
                print tradename_result

        elif name=="DED Licence Number:":
            try:
                licencenum_result=str(value.strip())
                print "Licence #: ",licencenum_result
            except StandardError as e:
                licencenum_result="Error was {0}".format(e)
                print licencenum_result

        elif name=="RERA Registration Number:":
            try:
                reraid_result=str(value.strip())
                print "RERA ID #: ",reraid_result
            except StandardError as e:
                reraid_result="Error was {0}".format(e)
                print reraid_result


    # phone num
    try:
        phone=soup.find_all("div", "phone-content")
        for a in phone:
            phone_result= str(a.get_text().strip().encode("utf-8"))
        print "Phone information:", phone_result
    except StandardError as e:
        phone_result="Error was {0}".format(e)
        print phone_result



    #link
    try:
        link = soup.find('input',{'id':'short-link-input'})
        link_result= str(link.get('value'))
        print "Short Reference link: ", link_result
    except StandardError as e:
        link_result="Error was {0}".format(e)
        print link_result





    """
    Connecting to Database and putting data into in
    """

    db= MySQLdb.connect("localhost","root","ahmed","practice")
    cursor=db.cursor()

    #checking phase to stop scrapping
    sql = """SELECT Short_link FROM Properties WHERE Short_link=%s"""
    print rows
    rows = cursor.execute(sql,(link_result))
    print rows
    if rows>=1:
        print "Already present - The program is terminating"
        sys.exit()
    else:
        query="""INSERT INTO Properties (Sale_Rent, Type, Title,Price, PricePerSqrFt, Bedroom,Agency_Fee, Bathroom, Size,ZonedFor, Freehold, Prop_ref,Furnished_status,Rent_payment,Building_info,Amenities,Trade_name,Licence, RERA_ID,Phone_info,Short_link) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
        cursor.execute(query,(Sale_Rent_result,Type_result, title_result, price_result, Pricepersq_result, bedroom_result, agencyfee_result, bathroom_result, size_result,Zoned_for_result, Freehold_result, propertyref_result, furnished_result, rent_is_paid_result, building_result, Amenities_result, tradename_result, licencenum_result, reraid_result, phone_result, link_result))


    db.commit()
    cursor.close()
    db.close()



    #-----------------------------------------------------------





        getting_urls_of_all_pages()

Answer 1

您没有将link_result 正确传递给execute() 方法：

rows = cursor.execute(sql,(link_result))

括号是可选的，Python 将其视为：

rows = cursor.execute(sql, link_result)

因此在查询中只使用了link_result 的第一个字符（其他数据库会告诉你传入的参数过多）。

您需要使用逗号使其成为正确的元组：

rows = cursor.execute(sql, (link_result,))