提高存储数百种不同格式的文件系统的速度

Question

我正在为实验室质量控制系统编写一个文件系统，其中数百台不同的机器正在发送需要在一个中心位置收集的信息。每条消息的格式可能不同，新的格式随时可能推出。

规格：

1. 用 Python 编写

2. 一个记录文件： 可变字段数 可变长度字段 变量类型字段（字符串、整数、浮点数）

3. 返回一个列表： 其中第一个元素是记录类型（或 ID） 后跟每个字段

我想提高读取（获取）过程的速度

def Get():
    L =[]
    try:
        nos = ord(f.read(1))           # read no of fields
    except:
        return []                      # end of record
    for i in range(nos):
        try: 
            x = f.read(1)              # read field lengt
            y = f.read(ord(x))         # read field
            try:
                L.append(int(y))       # apend ifinteger
            except:
                try:
                    L.append(float(y)) # append if float    
                except:
                    L.append(y)        # append if string        
        except:
            pass
    return L
#----------------------------------------------------
def Put(*x):
    f.write(chr(len(x)))         # write no of fields
    for i in range(len(x)):      # for each field
        y = str(x[i])                 # get field length
        f.write(chr(len(y)))          # write field length
        f.write(y)                    # write field
#----------------------------------------------------
f = open('x', 'w')
Put(1,'aaaaaaaa', 1, -1, 1.1, -1.1)
Put(2,'bbbbbbbbbbbbbb', 2, -2, 2.2, -2.2, 'abc', 12)   
f.close()
#----------------------------------------------------
f = open('x', 'r')
while True:
    LL = Get()
    if LL != []:
        print(LL)
        for i in range(len(LL)):
            print(type(LL[i]))
    else:
        break  
f.close()

Answer 1

假设

• 您主要关心功能
• 您不限于发布的文件格式

然后，使用 pickle 进行序列化/反序列化可以使读写性能提高约 7 倍，如下所示。

原始代码（有更正）

# Corrections:
#   1. Fixed Bare try/except issue
#   2. Filehandle should be passed into the functions rather than using a global
def Get(f):
    L =[]
    try:
        nos = ord(f.read(1))           # read no of fields
    except ValueError:
        return []                      # end of record
    for i in range(nos):
        try: 
            x = f.read(1)              # read field lengt
            y = f.read(ord(x))         # read field
            try:
                L.append(int(y))       # apend ifinteger
            except ValueError:
                try:
                    L.append(float(y)) # append if float    
                except ValueError:
                    L.append(y)        # append if string        
        except ValueError:
            pass     # error reading file or with ord conversion
    return L
#----------------------------------------------------
def Put(f, *x):
    f.write(chr(len(x)))         # write no of fields
    for i in range(len(x)):      # for each field
        y = str(x[i])                 # get field length
        f.write(chr(len(y)))          # write field length
        f.write(y)

Pickle 版本（更快）

import pickle

def Get_pickle(f):
    try:
        return pickle.load(f)
    except EOFError:
        return None
#----------------------------------------------------
def Put_pickle(f, *x):
    pickle.dump(x, f)

使用泡菜版

with open('x.pickle', 'wb') as f:
    Put_pickle(1,'aaaaaaaa', 1, -1, 1.1, -1.1)
    Put_pickle(2,'bbbbbbbbbbbbbb', 2, -2, 2.2, -2.2, 'abc', 12)   
#----------------------------------------------------
with open('x.pickle', 'rb') as f:
    while True:
        LL = Get_pickle()
        if LL:
            print(LL)
            for i in range(len(LL)):
                print(type(LL[i]))
        else:
            break  

输出（与原始输出相同）

(1, 'aaaaaaaa', 1, -1, 1.1, -1.1)
<class 'int'>
<class 'str'>
<class 'int'>
<class 'int'>
<class 'float'>
<class 'float'>
(2, 'bbbbbbbbbbbbbb', 2, -2, 2.2, -2.2, 'abc', 12)
<class 'int'>
<class 'str'>
<class 'int'>
<class 'int'>
<class 'float'>
<class 'float'>
<class 'str'>
<class 'int'>

时序测试

n = 100000 # number of repeats of posted data

时间原创

def test_write():
    with open('x.txt', 'w') as f:
        for _ in range(n):
            Put(f, 1,'aaaaaaaa', 1, -1, 1.1, -1.1)
            Put(f, 2,'bbbbbbbbbbbbbb', 2, -2, 2.2, -2.2, 'abc', 12)  
            
def test_get():
    with open('x.txt', 'r') as f:
        while Get(f):
            pass
            
%timeit test_write() # Output: 3.45 s ± 244 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
%timeit test_get()   # Output: 4.3 s ± 97.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

时间泡菜版

import timeit

def test_write_pickle():
    with open('x.pickle', 'wb') as f:
        for _ in range(n):
            Put_pickle(f, 1,'aaaaaaaa', 1, -1, 1.1, -1.1)
            Put_pickle(f, 2,'bbbbbbbbbbbbbb', 2, -2, 2.2, -2.2, 'abc', 12)   
            
def test_get_pickle():
    with open('x.pickle', 'rb') as f:
        while Get_pickle(f):
            pass

%timeit test_write_pickle() # Output: 474 ms ± 57.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
%timeit test_get_pickle()    # Output: 562 ms ± 82.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)