python脚本从巨大的（60000）JSON文件目录中提取特征到csv

Question

我正在尝试解析大量 json 文件。大约 60000 个 json 文件（大小范围 100 KB-700 MB）总共 1.8 TB，所以我制作了这个脚本来解析 json 文件并提取一些特征并将它们导出到 csv 文件中，它工作正常，但它的一些 json 文件非常慢解析需要 30 多分钟，我试图让它更快，但由于我的 python 经验短，我做不到。无论如何我可以让它更快，因为我需要更快地解析这些庞大的集合。我正在发布我的代码的 sn-p 我知道它有点转储，但我对 python 很陌生，并尽我所能。提前考虑

这是我的 json 文件的示例，请随时查看 https://gofile.io/d/vddzHY

count1=0
my_file_list = [f for f in glob.glob(r"E:\JsonOrgnized\Pach\*.json")]
final_result = []
for filename in my_file_list:
    try:
        with open(filename, 'r', encoding='utf8', errors='ignore') as f:
            row = {}
            info = ijson.items(f, 'info')
            f.seek(0)
            for o in info:
                 row['AA-Added']= float(o.get('added'))
                 row['AB-Started']= float(o.get('started'))
                 row['AC-Duration']= o.get('duration')
                 row['AD-Ended']= float(o.get('ended'))

            f.seek(0)
            domains = ijson.items(f, 'network.domains.item')
            domain_count = 0
            for domain in domains:
                domain_count+=1
            row['AE-DomainCount'] = domain_count

            f.seek(0)
            signatures = ijson.items(f, 'signatures.item')
            signature_count = 0
            for signature in signatures:
                signature_count+=1
            row['AF-SignatureCount'] = signature_count

            f.seek(0)
            domains = ijson.items(f, 'behavior.generic.item')
            domain_count = 0
            for domain in domains:
                domain_count+=1
            row['AG-GenericCount'] = domain_count

            f.seek(0)
            apistats = ijson.items(f, 'behavior.apistats')
            apistat_count = 0
            for apistat in apistats:
                for inner_apistat in apistat:
                    apistat_count+=1
            row['AH-ApistatCount'] = apistat_count

            f.seek(0)
            processes = ijson.items(f, 'behavior.processes.item')
            process_count = 0
            for process in processes:
                process_count+=1
            row['AI-ProcessCount'] = process_count

            f.seek(0)
            summaries = ijson.items(f, 'behavior.summary')
            summary_count = 0
            for summary in summaries:
                for inner_summary in summary:
                    summary_count+=1
            row['AJ-SummaryCount'] = summary_count

            f.seek(0)
            apistats_element = ijson.items(f, 'behavior.apistats')
            for inner_apistats in apistats_element:
                for index, inner_fields in inner_apistats.items():
                    row = dict(Counter(row)+Counter(inner_fields))
            
            row['AK-Filename'] = os.path.basename(filename)

            
    except Exception as e:
        #pass
        #print(f"Filename {filename} has issue with {e}")
        row = {}
    
    if row:        
        final_result.append(row) 
    count1+=1
    print("File Number" , count1 , "Is Finished!")
Print("<<<<<<<<<<<<<<<<<<<DONE>>>>>>>>>>>>>>>>>>")

Answer 1

这似乎有点快，我觉得更干净。

我们将使用来自ijson 的更多“低级”调用之一。并根据我们得到的路径采取某种行动。

我们会将感兴趣的路径和遇到的操作存储在一个小工作字典中。

import ijson
import os

def fn_set_value(row, key, value):
    row[key] = value

def fn_increment_count(row, key):
    row[key] = row.get(key, 0) + 1

# ---------------------
# When these keys (tuples) are encountered, we will take the corresponding action.
# ---------------------
work = {
    ("info.added", "number"): lambda row, value: fn_set_value(row, "AA-Added", value),
    ("info.started", "number"): lambda row, value: fn_set_value(row, "AB-Started", value),
    ("info.duration", "number"): lambda row, value: fn_set_value(row, "AC-Duration", value),
    ("info.ended", "number"): lambda row, value: fn_set_value(row, "AD-Ended", value),
    ("network.domains.item", "start_map"): lambda row, value: fn_increment_count(row, "AE-DomainCount"),
    ("signatures.item", "start_map"): lambda row, value: fn_increment_count(row, "AF-SignatureCount"),
    ("behavior.generic.item", "start_map"): lambda row, value: fn_increment_count(row, "AG-GenericCount"),
    ("behavior.apistats", "map_key"): lambda row, value: fn_increment_count(row, "AH-ApistatCount"),
    ("behavior.processes.item", "start_map"): lambda row, value: fn_increment_count(row, "AI-ProcessCount"),
    ("behavior.summary", "map_key"): lambda row, value: fn_increment_count(row, "AJ-SummaryCount"),
}
# ---------------------

# ---------------------
# Your initial set of files
# ---------------------
my_file_list = [
    "d:/temp/foo/report1.json",
    "d:/temp/foo/report2.json",
    "d:/temp/foo/report3.json",
    "d:/temp/foo/report4.json",
    "d:/temp/foo/report5.json"
]
# ---------------------

final_result = []
for index, filename in enumerate(my_file_list):
    print(f"Processing file {index+1} from {filename}")
    try:
        row = {}
        with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
            for i in ijson.parse(f):
                key = (i[0], i[1])
                if key in work.keys():      # if the tuple is an interesting one
                    work[key](row, i[2])    # use it to take an action on row
        row["AK-Filename"] = os.path.basename(filename)
        final_result.append(row)
    except Exception as e:
        print(f"\tUnable to process \"{filename}\": {e}")
        # retry with ascii or having stripped out the bad character?
        pass

print("<<<<<<<<<<<<<<<<<<<DONE>>>>>>>>>>>>>>>>>>")
print(final_result)

这会在几秒钟内产生这个结果。

[
    {
        'AA-Added': Decimal('1631536343.897729'),
        'AB-Started': Decimal('1631536440.728626'),
        'AC-Duration': 21,
        'AD-Ended': Decimal('1631536461.778441'),
        'AE-DomainCount': 3,
        'AF-SignatureCount': 5,
        'AG-GenericCount': 3,
        'AH-ApistatCount': 2,
        'AI-ProcessCount': 3,
        'AJ-SummaryCount': 14,
        'AK-Filename': 'report1.json'
    },
    {
        'AA-Added': Decimal('1631536343.90739'),
        'AB-Started': Decimal('1631536461.849837'),
        'AC-Duration': 12,
        'AD-Ended': Decimal('1631536474.755813'),
        'AE-DomainCount': 3,
        'AF-SignatureCount': 2,
        'AG-GenericCount': 2,
        'AH-ApistatCount': 1,
        'AI-ProcessCount': 2,
        'AJ-SummaryCount': 2,
        'AK-Filename': 'report2.json'
    },
    {
        'AA-Added': Decimal('1631536343.962804'),
        'AB-Started': Decimal('1631536692.972615'),
        'AC-Duration': 312,
        'AD-Ended': Decimal('1631537005.710977'),
        'AE-DomainCount': 4,
        'AF-SignatureCount': 36,
        'AG-GenericCount': 13,
        'AH-ApistatCount': 12,
        'AI-ProcessCount': 13,
        'AJ-SummaryCount': 22,
        'AK-Filename': 'report3.json'
    },
    {
        'AA-Added': Decimal('1631536344.049105'),
        'AB-Started': Decimal('1631537026.918725'),
        'AC-Duration': 316,
        'AD-Ended': Decimal('1631537342.92093'),
        'AE-DomainCount': 3,
        'AF-SignatureCount': 16,
        'AG-GenericCount': 4,
        'AH-ApistatCount': 3,
        'AI-ProcessCount': 4,
        'AJ-SummaryCount': 16,
        'AK-Filename': 'report4.json'
    },
    {
        'AA-Added': Decimal('1631536344.112968'),
        'AB-Started': Decimal('1631537322.81162'),
        'AC-Duration': 14,
        'AD-Ended': Decimal('1631537337.342377'),
        'AE-DomainCount': 3,
        'AF-SignatureCount': 1,
        'AG-GenericCount': 2,
        'AH-ApistatCount': 1,
        'AI-ProcessCount': 2,
        'AJ-SummaryCount': 7,
        'AK-Filename': 'report5.json'
    }
]