如何使用 pandas Series/DataFrame 从类 dict 类的对象中提取数据

Question

这是我正在做的学校作业......

所以基本上我被要求扫描给定目录并找到其中的所有 .py 文件，并计算给定属性，即文件中定义的类和函数（包括类中的方法），以及总行数和字符数每个文件。并在终端上打印表格中的所有数据。

为了打印表格，我的讲师建议使用一个名为 prettytable 的包，尽管对我来说它一点也不漂亮。

我想使用熊猫。
原因很简单：为每个文件计算它的 4 个属性 --> 这里很自然地想起了一个嵌套字典。 pandas.DataFrame 100% 完美地记录嵌套字典。

扫描和总结是最容易的部分，真正让我陷入困境的是如何使数据容器灵活和可扩展。

内置 dict 无法使用其中的 4 个键值对进行初始化，因此我构建了一个 class CountAttr(MutableMapping) 并使用另一个 class FileCounter 来为每个文件创建并计算每个属性。

但是，pandas.DataFrame 只识别这个类字典对象的第一层。而且我已经阅读了 DataFrame 和 Series 的源文件，仍然无法弄清楚如何解决这个问题。

所以我的问题是，
如何让 pandas.DataFrame/Series 从值是类字典对象的字典中提取数据？

附：我对以下代码、编码风格、实现方式等所有方面的建议持开放态度。非常感谢！

from collections.abc import MutableMapping
from collections import defaultdict
import pandas as pd
import os

class CounterAttr(MutableMapping):
""" Initialize a dictionary with 4 keys whose values are all 0,

    keys:value
    - 'class': 0
    - 'function': 0
    - 'line': 0
    - 'char': 0

    interfaces to get and set these attributes """

    def __init__(self):
        """ Initially there are 4 attributes in the storage"""
        # key: counted attributes | value: counting number
        self.__dict__ = {'class': 0, 'function': 0, 'line': 0, 'char': 0}

    def __getitem__(self, key):
        if key in self.__dict__:
            return self.__dict__[key]
        else:
            raise KeyError

    def get(self, key, defaut = None):
        if key in self.__dict__:
            return self.__dict__[key]
        else:
            return defaut

    def __setitem__(self, key, value):
        self.__dict__[key] = value

    def __delitem__(self, key):
        del self.__dict__[key]

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        return iter(self.__dict__)

    def get_all(self):
        """ return a copy of the self._storagem, in case the internal data got polluted"""
        copy = self.__dict__.copy()
        return copy

    def to_dict(self):
        return self.__dict__

    def __repr__(self):
        return '{0.__class__.__name__}()'.format(self)

class FileCounter(MutableMapping):
""" Discribe the object the store all the counters for all .py files

    Attributes:
    - 

"""
    def __init__(self):
        self._storage = dict()

    def __setitem__(self, key, value = CounterAttr()):
        if key not in self._storage.keys():
            self._storage[key] = value
        else:
            print("Attribute exist!")

    def __getitem__(self, key):
        if key in self._storage.keys():
            return self._storage[key]
        else:
            self._storage[key] = CounterAttr()

    def __delitem__(self, key):
        del self._storage[key]

    def __len__(self):
        return len(self._storage)

    def __iter__(self):
        return iter(self._storage)






def scan_summerize_pyfile(directory, give_me_dict = False):
""" Scan the passing directory, find all .py file, count the classes, funcs, lines, chars in each file
    and print out with a table
"""
    file_counter = FileCounter()


    if os.path.isdir(directory):                                            # if the given directory is a valid one

        os.chdir(directory)                                                 # change the CWD
        print("\nThe current working directory is {}\n".format(os.getcwd()))

        file_lst = os.listdir(directory)                                    # get all files in the CWD

        for a_file in file_lst:                                             # traverse the list and find all pyfiles
            if a_file.endswith(".py"):

                file_counter[a_file] 

                try:
                    open_file = open(a_file, 'r')
                except FileNotFoundError:
                    print("File {0} can't be opened!".format(a_file))

                else:

                    with open_file:
                        for line in open_file:

                            if line.lstrip().startswith("class"):           # count the classes
                                file_counter[a_file]['class'] += 1

                            if line.lstrip().startswith("def"):             # count the functions
                                file_counter[a_file]['function'] += 1

                            file_counter[a_file]['line'] += 1               # count the lines

                            file_counter[a_file]['char'] += len(line)       # count the chars, no whitespace

    else:
        print("The directory", directory, "is not existed.\nI'm sorry, program ends.")


    return file_counter

# Haven't had the pandas codes part yet

Answer 1

所以这是我对这个问题的解决方案。 我没有为 pandas 所做的事情而苦苦挣扎，而是试图弄清楚如何调整我的解决方案并让 pandas 轻松读取我的数据。感谢@RockyLi 的建议

class FileCounter(object):
""" A class that contains the .py files counted 
    - .py files that are found in the given directory
    - attributes counted for each .py file
    - methods that scan and sumerized .py file
"""
def __init__(self, directory):
    self._directory = directory
    self._data = dict()        # key: file name | value: dict of counted attributes
    self._update_data()

def _read_file(self, filename):
    """ return a dictionary of attributes statistical data

        return type: dictionary
            - key: attributes' name
            - value: counting number of attributes

        it's not available to add a counting attributes interactively
    """

    class_, function_, line_, char_ = 0, 0, 0, 0
    try:
        open_file = open(filename, 'r')
    except FileNotFoundError:
        print("File {0} can't be opened!".format(filename))
    else:

        with open_file:
            for line in open_file:

                if line.lstrip().startswith("class "):           # count the classes
                    class_ += 1

                if line.lstrip().startswith("def "):             # count the functions
                    function_ += 1

                line_ += 1                                       # count the lines

                char_ += len(line)                               # count the chars, no whitespace
    return {'class': class_, 'function': function_, 'line': line_, 'char': char_}

def _scan_dir(self):
    """ return all of the file in the directory
        if the directory is not valid, raise and OSError
    """
    if os.path.isdir(self._directory):
        os.chdir(self._directory)
        return os.listdir(self._directory)

    else:
        raise OSError("The directory doesn't exist!")

def _find_py(self, lst_of_file):
    """ find all of the .py files in the directory"""
    lst_of_pyfile = list()

    for filename in lst_of_file:
        if filename.endswith('.py'):
            lst_of_pyfile.append(filename)

    return lst_of_pyfile

def _update_data(self):
    """ manipulate the _data\n
        this is the ONLY method that manipulate _data
    """
    lst_of_pyfile = self._find_py(self._scan_dir())

    for filename in lst_of_pyfile:
        self._data[filename] = self._read_file(filename)        # only place manipulate _data

def pretty_print(self):
    """ Print the data!"""

    df_prettyprint = pd.DataFrame.from_dict(self._data, orient = 'index')

    if not df_prettyprint.empty:
        print(df_prettyprint)
    else:
        print("Oops, seems like you don't get any .py file.\n You must be Java people :p")

def get_data(self):
    return self._data.copy()                                    # never give them the original data!

该类构建了两个接口A.打印表B.获取数据以备使用，同时保护数据直接被访问和修改。

Answer 2

我不知道你为什么需要像你写的那样的东西。在我看来，这一切都被过度设计了。

假设read_file()返回你想要的4个属性class, function, line, chars并且你在list_of_files中有一个python文件列表，你可以这样做：

result = []
for file in list_of_files:
    c, f, l, num_c = read_file(file)
    curr_dict = {'class':c, 'function':f, 'line':l, 'chars':num_c}
    result.append(curr_dict)
your_table = pd.DataFrame(result)

这就是你所需要的。

你应该生成文件列表和函数来分别读取它们，每个不同的东西都应该存在于它自己的函数中——这绝对有助于分离逻辑。