将 5 级字典（以 pd.Series 作为值转换为 pandas DataFrame

Question

原始问题：我使用的是 python 3。我有一些 4 级字典和 5 级字典。我想将这个多级字典转换成带有递归函数的 pandas DataFrame

为了简化我的问题并测试我的函数，我生成了一个如下所示的 3 级字典并尝试我的递归函数。我知道有了这 3 层嵌套字典，还有很多其他方法可以解决这个问题。但是，我觉得只有递归函数才能轻松解决 4 级、5 级或更多级字典的问题

创建一个简化的 3 级字典：

from collections import defaultdict
def ddict():
    return defaultdict(ddict)

tree = ddict()
tree['level1_1']['level2_1']['level3_1'] = <pd.Series1>
tree['level1_1']['level2_1']['level3_2'] = <pd.Series2>
tree['level1_1']['level2_2']['level3_1'] = <pd.Series3>
tree['level1_1']['level2_2']['level3_2'] = <pd.Series4>
tree['level1_2']['level2_1']['level3_1'] = <pd.Series5>
tree['level1_2']['level2_1']['level3_2'] = <pd.Series6>
tree['level1_2']['level2_2']['level3_1'] = <pd.Series7>
tree['level1_2']['level2_2']['level3_2'] = <pd.Series8>

受下面Bart Cubrich的启发，我修改了xx的代码并将我的解决方案放在这里

import collections
def tree2df (d, colname):
    """
    Inputs:
        1. d               (a nested dict, or a tree, all values are pd.Series)
        2. colname         (a list)

    Return:
        1. a pd.DataFrame  
    """
    def flatten(d, parent_key='', sep='-'):
        items = []
        for k, v in d.items():
            new_key = str(parent_key) + str(sep) + str(k) if parent_key else k
            if isinstance(v, collections.MutableMapping):
                items.extend(flatten(v, new_key, sep=sep).items())
            else:
                items.append((new_key, v))
        return dict(items)
    flat_dict = flatten (d)  
    levels, vals = zip(*[(tuple(level.split('-')),val) for level, val in flat_dict.items()])
    max_level = np.max(np.array([len(l) for l in levels]))
    if len(colname) != max_level:
        print ("The numbers of column name is invalid because of moer than maximum level: %s.\nNothing will be returned. Please revise the colname!"%max_level)
    else:
        colname += ['Old index']
        s = pd.concat(list(vals), keys = list(levels), names = colname)
        s = pd.DataFrame(s)
        s.reset_index(inplace=True)
        s.rename(columns={0:'Value'},inplace=True)
        return s

#Example
BlockEvent_TS_df = tree2df (BlockEvent_TS_tree, ['ID','Session','Trial type','Block', 'Event name'])

5 级嵌套字典与 3 级嵌套字典的思想相同：

tree['level1_1']['level2_1']['level3_1']['level4_1']['level5_1'] = <pd.Series1>
                ...
tree['level1_2']['level2_2']['level3_2']['level4_2']['level5_2'] = <pd.Series32>

因为我有一个很大的数据集，所以在这里显示整个嵌套字典非常复杂。但是，想法是这样的。稍后，我希望有 6 列，5 列来存储每个级别，并且一列用于值。

我已经尝试了上面的代码，它对我来说效果很好。速度也很不错。

感谢您的帮助！

Answer 1

这个版本可以在有不同的关卡深度时工作，虽然看起来很乱。

import pandas as pd

from collections import defaultdict
def ddict():
    return defaultdict(ddict)

tree = ddict()
tree['level1_1']['level2_1']['level3_1'] = 1
tree['level1_1']['level2_1']['level3_2'] = 2
tree['level1_1']['level2_2']['level3_1'] = 3
tree['level1_1']['level2_2']['level3_2'] = 4
tree['level1_2']['level2_1']['level3_1'] = 5
tree['level1_2']['level2_1']['level3_2'] = 6
tree['level1_2']['level2_2']['level3_1'] = 7
tree['level1_2']['level2_2']['level3_2']['Level4_1'] = 8

import collections

def flatten(d, parent_key='', sep='-'):
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, sep=sep).items())
        else:
            items.append((new_key, v))
    return dict(items)

flat_dict=flatten(tree)

#df=pd.DataFrame()
levels=[]
vals=[]
for key in flat_dict.keys():
    levels.append(key.split('-'))
    vals.append(flat_dict.get(key))



max_level=0
for level in levels:
    if len(level)>max_level: max_level=len(level)

df=pd.DataFrame(columns=range(max_level+1))

index=0

for level,val in zip(levels,vals):
    for i in range(max_level):
        try: 
            level[i]
            df.loc[index,i]=level[i]
        except IndexError:
            print('means this level has less than max')

        df.loc[index,max_level]=val

    index+=1

df

Out:

          0         1         2         3  4
0  level1_1  level2_1  level3_1       NaN  1
1  level1_1  level2_1  level3_2       NaN  2
2  level1_1  level2_2  level3_1       NaN  3
3  level1_1  level2_2  level3_2       NaN  4
4  level1_2  level2_1  level3_1       NaN  5
5  level1_2  level2_1  level3_2       NaN  6
6  level1_2  level2_2  level3_1       NaN  7
7  level1_2  level2_2  level3_2  Level4_1  8

我从Here得到了扁平化的想法

Answer 2

所以我的解决方案是遍历树，查看所有键并将每个元素路径构建为数组，然后从记录创建 DataFrame。我将这些步骤中的每一个都拆分为自己的方法。

可能有更有效的方法，但这应该可以完成工作。希望这会有所帮助。

def traverse_tree(d, prefix='', results=[]):
    if type(d) is int:
        record = str(prefix).split(',')
        record.append(d)
        results.append(record)
        return results
    keys = d.keys()
    for key in keys:
        temp = prefix + ',' if prefix != '' else ''
        results = traverse_tree(d[key], temp + str(key), results)
    return results


def dict_to_df(d):
    res = traverse_tree(tree)
    labels = []
    for i in range(len(res[0]) - 1):
        labels.append('L' + str(i+1))
    labels.append('Value')
    print(res)
    print(labels)
    return pd.DataFrame.from_records(res, columns=labels)


if __name__ == '__main__':
    tree = ddict()
    tree['level1_1']['level2_1']['level3_1'] = 1
    tree['level1_1']['level2_1']['level3_2'] = 2
    tree['level1_1']['level2_2']['level3_1'] = 3
    tree['level1_1']['level2_2']['level3_2'] = 4
    tree['level1_2']['level2_1']['level3_1'] = 5
    tree['level1_2']['level2_1']['level3_2'] = 6
    tree['level1_2']['level2_2']['level3_1'] = 7
    tree['level1_2']['level2_2']['level3_2'] = 8
    df = dict_to_df(tree)
    print(df)

Answer 3

你需要：

format_ = {(level1_key, level2_key, level3_key): values
 for level1_key, level2_dict in tree.items()
 for level2_key, level3_dict in level2_dict.items()
 for level3_key, values      in level3_dict.items()}
df = pd.DataFrame(format_, index=['Value']).T.reset_index()

输出：

     level_0    level_1      level_2    Value
0   level1_1    level2_1    level3_1    1
1   level1_1    level2_1    level3_2    2
2   level1_1    level2_2    level3_1    3
3   level1_1    level2_2    level3_2    4
4   level1_2    level2_1    level3_1    5
5   level1_2    level2_1    level3_2    6
6   level1_2    level2_2    level3_1    7
7   level1_2    level2_2    level3_2    8