加快学校项目数据文件读取程序

Question

我正在学习较低级别的编码课程 (Python)，并且有一个重大项目要在三天后完成。我们的评分标准之一是程序速度。我的程序运行大约 30 秒，理想情况下它会在 15 秒或更短的时间内执行。这是我的代码：

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
import time

start_time = time.time()#for printing execution time

#function for appending any number of files to a dataframe

def read_data_files(pre, start, end): #reading in the data
    data = pd.DataFrame()#dataframe with all the data from files
    x = start

    while x <= end:
        filename = pre + str(x) + ".csv" #string manipulation
        dpath = pd.read_csv("C:\\Users\\jacks\\Downloads\\BMEN 207 Project 1 
        Data\\" + filename )

        for y in dpath:
            dpath = dpath.rename(columns = {y: y})
    
        data = data.append(dpath)
        x += 1
       
    return data

data = read_data_files("Data_", 5, 163) #start, end, prefix...

#converting to human time and adding to new column in dataframe
human_time = []

for i in data[' Time']:
    i = int(i)
    j = datetime.utcfromtimestamp(i).strftime('%Y-%m-%d %H:%M:%S')
    human_time.append(j)

human_timen = np.array(human_time) #had issues here for some reason, so i 
created another array to replace the time column in the dataframe
data[' Time'] = human_timen

hours = [] #for use as x-axis in plot
stdlist = [] #for use as y-axis in plot
histlist = [] #for storing magnitudes of most active hour

def magfind(row): #separate function to calculate the magnitude of each row in 
each dataframe
    return (row[' Acc X'] ** 2 + row[' Acc Y'] ** 2 + row[' Acc Z'] ** 2) ** .5

def filterfunction(intro1, intro2, first, last): #two different intros to deal 
with the issue of '00:' versus '10:' timestamps
    k = first
    meanmax = 0 
    active = 0

    while k <= last:
        if 0 <= k < 7: #data from hours 0 to 6, none after
            hr = intro1 + str(k) + ':'
            tfilter = data[' Time'].str.contains(hr)
            acc = data.loc[tfilter, [' Acc X', ' Acc Y', ' Acc Z']]
            acc['magnitude'] = acc.apply(magfind, axis = 1) #creates magnitude 
column using prior function, column has magnitudes for every row of every file
            p = acc.loc[:, 'magnitude'].std()#finds std dev for the column and 
appends to a list for graphing
            m = acc.loc[:, 'magnitude'].mean()
            stdlist.append(p)            
      
        elif k == 12 or 20 < k <= last: #data at 12 and beyond hour 20
            hr = intro2 + str(k) + ":"
            tfilter = data[' Time'].str.contains(hr)
            acc = data.loc[tfilter, [' Acc X', ' Acc Y', ' Acc Z']]
            acc['magnitude'] = acc.apply(magfind, axis = 1)
            p = acc.loc[:, 'magnitude'].std()
            m = acc.loc[:, 'magnitude'].mean()
            stdlist.append(p)
        
        else: #in the case that we are dealing with an hour that has no data
            p = 0
            m = 0
            stdlist.append(p) #need this so that the hours with no data still 
get graphed 
        if m > meanmax: # for determining which hour was the most active, and 
appending those magnitudes to a list for histogramming
            meanmax = m
            active = k #most active hour
            for i in acc['magnitude']:
                histlist.append(i) #adding all the magnitudes for histogramming
        
        
        hours.append(k)
        k += 1
    print("mean magnitude", meanmax)
    print("most active hour", active)

    return hours, stdlist, histlist

filterfunction(' 0', ' ', 0, 23)

速度慢源于“filterfunction”功能。该程序所做的是从 100 多个文件中读取数据，该函数专门将数据分类到数据框中，并按时间（每个小时）进行分析，以计算该小时所有行中的数据。我相信可以通过改变数据过滤方式以按小时搜索来加快速度，但我不确定。我有声明不包含某些 k 值的原因是有几个小时没有数据可操作，这会弄乱标准偏差计算列表以及该数据将生成的图。任何加快速度的提示或想法将不胜感激！

Answer 1

一个加快速度的建议是删除这一行，因为它没有在程序中的任何地方使用：

import matplotlib.pyplot as plt

matplotlib 是一个大库，因此删除它应该会提高性能。

另外我认为你可以摆脱 numpy，因为它只使用一次......考虑使用元组

Answer 2

我无法测试，因为我现在在移动设备上。然而，我的主要想法不是让代码变得更好或更简洁。我更改了流程的功能部分。

将“多处理”库（方法）集成到您的代码中，并计算系统 cpu 内核并在它们之间划分所有进程。

多处理库详细文档：https://docs.python.org/2/library/multiprocessing.html

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import psutil
from datetime import datetime
from multiprocessing import Pool

cores = psutil.cpu_count()

start_time = time.time()#for printing execution time

#function for appending any number of files to a dataframe

def read_data_files(pre, start, end): #reading in the data
    data = pd.DataFrame()#dataframe with all the data from files
    x = start

    while x <= end:
        filename = pre + str(x) + ".csv" #string manipulation
        dpath = pd.read_csv("C:\\Users\\jacks\\Downloads\\BMEN 207 Project 1 
        Data\\" + filename )

        for y in dpath:
            dpath = dpath.rename(columns = {y: y})
    
        data = data.append(dpath)
        x += 1
       
    return data

data = read_data_files("Data_", 5, 163) #start, end, prefix...

#converting to human time and adding to new column in dataframe
human_time = []

for i in data[' Time']:
    i = int(i)
    j = datetime.utcfromtimestamp(i).strftime('%Y-%m-%d %H:%M:%S')
    human_time.append(j)

human_timen = np.array(human_time) #had issues here for some reason, so i 
created another array to replace the time column in the dataframe
data[' Time'] = human_timen

hours = [] #for use as x-axis in plot
stdlist = [] #for use as y-axis in plot
histlist = [] #for storing magnitudes of most active hour

def magfind(row): #separate function to calculate the magnitude of each row in 
each dataframe
    return (row[' Acc X'] ** 2 + row[' Acc Y'] ** 2 + row[' Acc Z'] ** 2) ** .5


def filterfunction(intro1, intro2, first, last): #two different intros to deal 
with the issue of '00:' versus '10:' timestamps
    k = first
    meanmax = 0 
    active = 0

    while k <= last:
        if 0 <= k < 7: #data from hours 0 to 6, none after
            hr = intro1 + str(k) + ':'
            tfilter = data[' Time'].str.contains(hr)
            acc = data.loc[tfilter, [' Acc X', ' Acc Y', ' Acc Z']]
            acc['magnitude'] = acc.apply(magfind, axis = 1) #creates magnitude 
column using prior function, column has magnitudes for every row of every file
            p = acc.loc[:, 'magnitude'].std()#finds std dev for the column and 
appends to a list for graphing
            m = acc.loc[:, 'magnitude'].mean()
            stdlist.append(p)            
      
        elif k == 12 or 20 < k <= last: #data at 12 and beyond hour 20
            hr = intro2 + str(k) + ":"
            tfilter = data[' Time'].str.contains(hr)
            acc = data.loc[tfilter, [' Acc X', ' Acc Y', ' Acc Z']]
            acc['magnitude'] = acc.apply(magfind, axis = 1)
            p = acc.loc[:, 'magnitude'].std()
            m = acc.loc[:, 'magnitude'].mean()
            stdlist.append(p)
        
        else: #in the case that we are dealing with an hour that has no data
            p = 0
            m = 0
            stdlist.append(p) #need this so that the hours with no data still 
get graphed 
        if m > meanmax: # for determining which hour was the most active, and 
appending those magnitudes to a list for histogramming
            meanmax = m
            active = k #most active hour
            for i in acc['magnitude']:
                histlist.append(i) #adding all the magnitudes for histogramming
        
        
        hours.append(k)
        k += 1
    print("mean magnitude", meanmax)
    print("most active hour", active)

    return hours, stdlist, histlist

                            
# Run this with a pool of 5 agents having a chunksize of 3 until finished
agents = cores
chunksize = (len(data) / cores)

with Pool(processes=agents) as pool:
    pool.map(filterfunction, (' 0', ' ', 0, 23))

Answer 3

不要使用apply，它不是矢量化的。相反，尽可能使用矢量化操作。在这种情况下，不要执行df.apply(magfind, 1)，而是执行：

def add_magnitude(df):
    df['magnitude'] = (df[' Acc X'] ** 2 + df[' Acc Y'] ** 2 + df[' Acc Z'] ** 2) ** .5