使用文件计算函数

Question

我已经为使用以下两个数据集的某些计算编写了一个 python 函数。我想使用 data_1 的 row1、row2、ror3、row4、row5 计算 data_2 中的每个数据的 z。但是，因为我是 python 新手我试着写，但中间失败了。请帮助。谢谢。

       data_1                                      data_2        
file    a    b    c    d                             x
file1  0.5  0.6  0.8  0.3                           0.5
file1  0.2  0.2  0.4  0.1                           0.8
file1  0.1  0.4  0.5  0.2                           0.9

我尝试过的代码在这里：

import numpy as np
file1=np.loadtxt('data_1',skiprows=1,usecols=(1,2,3))
file2=np.loadtxt('data_2',skiprows=1,usecols=(0))

def calculation(a,b,c,x):
    z=(a+b+c)*x
    return z

for value in file2:
    print(value)
    calculation

我的预期输出应该是这样的

   data_3                                            
file    a    b    c    d       z                          
file1  0.5  0.6  0.8  0.3      -
file1  0.5  0.6  0.8  0.3      -                     
file1  0.5  0.6  0.8  0.3      -                     
file1  0.2  0.2  0.4  0.1      -
file1  0.2  0.2  0.4  0.1      -                     
file1  0.2  0.2  0.4  0.1      -                        
file1  0.1  0.4  0.5  0.2      -
file1  0.1  0.4  0.5  0.2      -                       
file1  0.1  0.4  0.5  0.2      -                     

Answer 1

Python 是一种动态语言，numpy 倾向于覆盖普通运算符以将操作应用于整个数据集合。通常，如果您有一个 for 循环，您就没有利用它。

numpy 数组只能保存一种数据类型，但您在第 0 列中有一个字符串。pandas 包装了numpy，使多种数据类型更易于处理。所以我转而阅读pandas.DataFrame 对象而不是数组。

看起来您想要file2["x"] 的笛卡尔积与file1 中的行。一种方法是在具有匹配值的两个数据框中创建一个虚拟列，然后合并。对a + b + c 使用sum 方法，然后与x 相乘，即可得到结果。

import pandas as pd

# read space separated tables
file1=pd.read_table('data_1', sep=r"\s+")
file2=pd.read_table('data_2', sep=r"\s+")

# we want (a+b+c)*x, for each value in file2["x"]. Do the sum, then
# use `merge` with a temporary key to create the cartesian product 
# with x. For each `x`, merge will create a row for each matching
# key and since all keys match, we've got a cartesian product.
# Finally, multiply.
file1["_tmpsums"] = file1[["a", "b", "c"]].sum(axis=1)
file1["_tmpmergekey"] = file2["_tmpmergekey"] = 1
file1 = pd.merge(file1, file2, on="_tmpmergekey")
file1["z"] = file1["_tmpsums"] * file1["x"]
file1 = file1.drop(["_tmpsums", "_tmpmergekey", "x"], axis=1)

print("   data_3")
print(file1.to_string(col_space=6, index=False, justify="center"))

结果

   data_3
 file     a      b      c      d      z  
 file1   0.5    0.6    0.8    0.3   0.95 
 file1   0.5    0.6    0.8    0.3   1.52 
 file1   0.5    0.6    0.8    0.3   1.71 
 file1   0.2    0.2    0.4    0.1   0.40 
 file1   0.2    0.2    0.4    0.1   0.64 
 file1   0.2    0.2    0.4    0.1   0.72 
 file1   0.1    0.4    0.5    0.2   0.50 
 file1   0.1    0.4    0.5    0.2   0.80 
 file1   0.1    0.4    0.5    0.2   0.90 

Answer 2

如下使用pandas

import pandas as pd

# Load Data
data_1 = pd.read_csv('data_1.txt', delimiter = r"\s+")
data_2 = pd.read_csv('data_2.txt', delimiter = r"\s+")

# Compute the cartesian product of data_1 with data_2
# since for each row in data_1, we need sequence of rows in data_2
# We do this using DataFrame merge by injecting a key that is repeated for each row
# i.e. 'merge_key'
data_1['merge_key'] = pd.Series([1]*len(data_1))
data_2['merge_key'] = pd.Series([1]*len(data_2))
df = pd.merge(data_1, data_2, on = 'merge_key')
# Drop merge key from result
df.drop('merge_key', axis = 'columns', inplace = True)

# DataFrame df now has columns File, a, b, c, d, x
# We can apply function calulation to each row using apply
# and specifying the columns to send to calculation
df['z'] = df.apply(lambda row: calculation(row['a'], row['b'], row['c'], row['x']), axis = 'columns')

# Drop x column
df.drop('x', axis = 'columns', inplace = True)

# Write to CSV file
df.to_csv('data_3.txt', index=False, sep = " ")

输出

Pandas DataFrame df

    file    a   b   c   d   z
0   file1   0.5 0.6 0.8 0.3 0.95
1   file1   0.5 0.6 0.8 0.3 1.52
2   file1   0.5 0.6 0.8 0.3 1.71
3   file1   0.2 0.2 0.4 0.1 0.40
4   file1   0.2 0.2 0.4 0.1 0.64
5   file1   0.2 0.2 0.4 0.1 0.72
6   file1   0.1 0.4 0.5 0.2 0.50
7   file1   0.1 0.4 0.5 0.2 0.80
8   file1   0.1 0.4 0.5 0.2 0.90

CSV 文件 data_3.txt

file a b c d z
file1 0.5 0.6 0.8 0.3 0.9500000000000001
file1 0.5 0.6 0.8 0.3 1.5200000000000002
file1 0.5 0.6 0.8 0.3 1.7100000000000002
file1 0.2 0.2 0.4 0.1 0.4
file1 0.2 0.2 0.4 0.1 0.6400000000000001
file1 0.2 0.2 0.4 0.1 0.7200000000000001
file1 0.1 0.4 0.5 0.2 0.5
file1 0.1 0.4 0.5 0.2 0.8
file1 0.1 0.4 0.5 0.2 0.9

基础 Python

同样的输出

# Get data from first file
with open('data_1.txt', 'r') as f:
    # first file header
    header1 = f.readline()
    
    # Let's get the lines of data
    data_1 = []
    for line in f:
        new_data = line.rstrip().split()  # strip '\n' and split on parens
        for i in range(1, len(new_data)):
            new_data[i] = float(new_data[i])  # convert columns after file to float
        data_1.append(new_data)
  
# Get data from second file
with open('data_2.txt', 'r') as f:
    # second file header
    header2 = f.readline()
    
    # Let's get the lines of data
    data_2 = []
    for line in f:
        new_data = float(line.rstrip())  # only one value per line
        data_2.append(new_data)


with open('data_3.txt', 'w') as f:
    # Output file
    # Write Header
    f.write("file a b c d z\n")
    
    # Use double loop to loop through all rows of data_2 for each row in data_1
    for v1 in data_1:
        # For each row in data_1
        file, a, b, c, d = v1  # unpacking the values in v1 to individual variables
        for v2 in data_2:
            # for each row in data_2
            x = v2  # data2 just has a single value per row
           
            # Calculation using posted formula
            z = calculation(a, b, c, x)
            
            # Write result
            f.write(f"{file} {a} {b} {c} {d} {z}\n")
       

Numpy 版本

import numpy as np
file1=np.loadtxt('data_1.txt',skiprows=1,usecols=(1,2,3, 4))
file2=np.loadtxt('data_2.txt',skiprows=1,usecols=(0))

with open('data_3.txt', 'w') as f:
    # Write header
    f.write("file a b c d z\n")
    
    # Double loop to through the values of file1 and file2
    for val1 in file1:
        for val2 in file2:
            # Only use first 3 values (val1[:3] to only use first 3 value so ignore d)
            z = calculation(*val1[:3], val2)  # *val[:3] is unpacking values to go into calculation
            # Write result
            # map(str, val1) - converts values to string
            # str(z) converts z to string
            #' '.join([*map(str, val1), str(z)] - creates a space separated string
            f.write(' '.join([*map(str, val1), str(z)]) + "\n")