如何打开sqlite数据库并将其转换为pandas数据框

Question

我已经下载了一些数据作为 sqlite 数据库（data.db），我想在 python 中打开这个数据库，然后将其转换为 pandas 数据框。

到目前为止，我已经完成了

import sqlite3
import pandas    
dat = sqlite3.connect('data.db') #connected to database with out error
pandas.DataFrame.from_records(dat, index=None, exclude=None, columns=None, coerce_float=False, nrows=None)

但是它抛出了这个错误

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/pandas/core/frame.py", line 980, in from_records
    coerce_float=coerce_float)
  File "/usr/local/lib/python2.7/dist-packages/pandas/core/frame.py", line 5353, in _to_arrays
    if not len(data):
TypeError: object of type 'sqlite3.Connection' has no len()

如何将 sqlite 数据库转换为 pandas 数据框

Answer 1

尽管 sqlite 是 Python 标准库的一部分，并且是 SQLite 数据库的一个简单易用的接口，但 Pandas 教程states：

注意为了使用 read_sql_table()，你必须有 SQLAlchemy 安装了可选的依赖项。

但如果你想避免安装 SQLAlchemy，Pandas 仍然支持 sqlite3 访问：

import sqlite3
import pandas as pd
# Create your connection.
cnx = sqlite3.connect('file.db')

df = pd.read_sql_query("SELECT * FROM table_name", cnx)

如here所说，但需要提前知道使用表的名称。

Answer 2

线

data = sqlite3.connect('data.db')

打开到数据库的连接。没有查询到此的记录。因此，您必须在之后执行查询并将其提供给 pandas DataFrame 构造函数。

应该和这个类似

import sqlite3
import pandas as pd

dat = sqlite3.connect('data.db')
query = dat.execute("SELECT * From <TABLENAME>")
cols = [column[0] for column in query.description]
results= pd.DataFrame.from_records(data = query.fetchall(), columns = cols)

我对SQL命令不是很坚定，所以你应该检查查询的正确性。应该是数据库中表的名称。

Answer 3

在 google 中搜索 sqlalchemy、engine 和数据库名称（本例中为 sqlite）：

import pandas as pd
import sqlalchemy

db_name = "data.db"
table_name = "LITTLE_BOBBY_TABLES"

engine = sqlalchemy.create_engine("sqlite:///%s" % db_name, execution_options={"sqlite_raw_colnames": True})
df = pd.read_sql_table(table_name, engine)

Answer 4

在不知道表名的情况下将 sqlite .db 解析为数据帧字典：

def read_sqlite(dbfile):
    import sqlite3
    from pandas import read_sql_query, read_sql_table

    with sqlite3.connect(dbfile) as dbcon:
        tables = list(read_sql_query("SELECT name FROM sqlite_master WHERE type='table';", dbcon)['name'])
        out = {tbl : read_sql_query(f"SELECT * from {tbl}", dbcon) for tbl in tables}

   return out

Answer 5

我编写了一段代码，将表格保存在 .sqlite 或 .db 等数据库文件中，并从中创建一个 excel 文件，其中每个表格作为表格或将单个表格制作为 csvs。

注意：不需要提前知道表名！

import os, fnmatch
import sqlite3
import pandas as pd

#creates a directory without throwing an error
def create_dir(dir):
  if not os.path.exists(dir):
    os.makedirs(dir)
    print("Created Directory : ", dir)
  else:
    print("Directory already existed : ", dir)
  return dir

#finds files in a directory corresponding to a regex query
def find(pattern, path):
    result = []
    for root, dirs, files in os.walk(path):
        for name in files:
            if fnmatch.fnmatch(name, pattern):
                result.append(os.path.join(root, name))
    return result



#convert sqlite databases(.db,.sqlite) to pandas dataframe(excel with each table as a different sheet or individual csv sheets)
def save_db(dbpath=None,excel_path=None,csv_path=None,extension="*.sqlite",csvs=True,excels=True):
    if (excels==False and csvs==False):
      print("Atleast one of the parameters need to be true: csvs or excels")
      return -1

    #little code to find files by extension
    if dbpath==None:
      files=find(extension,os.getcwd())
      if len(files)>1:
        print("Multiple files found! Selecting the first one found!")
        print("To locate your file, set dbpath=<yourpath>")
      dbpath = find(extension,os.getcwd())[0] if dbpath==None else dbpath
      print("Reading database file from location :",dbpath)

    #path handling

    external_folder,base_name=os.path.split(os.path.abspath(dbpath))
    file_name=os.path.splitext(base_name)[0] #firstname without .
    exten=os.path.splitext(base_name)[-1]   #.file_extension

    internal_folder="Saved_Dataframes_"+file_name
    main_path=os.path.join(external_folder,internal_folder)
    create_dir(main_path)


    excel_path=os.path.join(main_path,"Excel_Multiple_Sheets.xlsx") if excel_path==None else excel_path
    csv_path=main_path if csv_path==None else csv_path

    db = sqlite3.connect(dbpath)
    cursor = db.cursor()
    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
    tables = cursor.fetchall()
    print(len(tables),"Tables found :")

    if excels==True:
      #for writing to excel(xlsx) we will be needing this!
      try:
        import XlsxWriter
      except ModuleNotFoundError:
        !pip install XlsxWriter

    if (excels==True and csvs==True):
      writer = pd.ExcelWriter(excel_path, engine='xlsxwriter')
      i=0
      for table_name in tables:
          table_name = table_name[0]
          table = pd.read_sql_query("SELECT * from %s" % table_name, db)
          i+=1
          print("Parsing Excel Sheet ",i," : ",table_name)
          table.to_excel(writer, sheet_name=table_name, index=False)
          print("Parsing CSV File ",i," : ",table_name)
          table.to_csv(os.path.join(csv_path,table_name + '.csv'), index_label='index')

      writer.save()


    elif excels==True:
      writer = pd.ExcelWriter(excel_path, engine='xlsxwriter')
      i=0
      for table_name in tables:
          table_name = table_name[0]
          table = pd.read_sql_query("SELECT * from %s" % table_name, db)
          i+=1
          print("Parsing Excel Sheet ",i," : ",table_name)
          table.to_excel(writer, sheet_name=table_name, index=False)

      writer.save()

    elif csvs==True:
      i=0
      for table_name in tables:
          table_name = table_name[0]
          table = pd.read_sql_query("SELECT * from %s" % table_name, db)
          i+=1
          print("Parsing CSV File ",i," : ",table_name)
          table.to_csv(os.path.join(csv_path,table_name + '.csv'), index_label='index')
    cursor.close()
    db.close()
    return 0
save_db(); 

Answer 6

如果data.db 是您的SQLite 数据库并且table_name 是其中一张表，那么您可以这样做：

import pandas as pd
df = pd.read_sql_table('table_name', 'sqlite:///data.db')

无需其他导入。

Answer 7

我已将我的数据存储在 database.sqlite 表名是评论

import sqlite3
con=sqlite3.connect("database.sqlite")

data=pd.read_sql_query("SELECT * FROM Reviews",con)
print(data)