在python中通过变量释放内存使用量

Question

我目前正在尝试将一些数据存储到 .h5 文件中，我很快意识到可能必须将我的数据存储到部分中，因为无法在我的 ram 中处理它。我开始使用numpy.array 来压缩内存使用量，但这导致在格式化数据上花费了几天的时间。

所以我回去使用list，但让程序监控内存使用情况， 当它高于指定值时，是否将部分存储为numpy 格式 - 以便另一个进程可以加载并使用它。这样做的问题是，我认为会释放我的内存的东西并没有释放内存。出于某种原因，即使我重置了变量和del 变量，内存也是一样的。为什么这里没有释放内存？

import numpy as np
import os
import resource
import sys
import gc
import math
import h5py
import SecureString
import objgraph
from numpy.lib.stride_tricks import as_strided as ast

total_frames = 15
total_frames_with_deltas = total_frames*3
dim = 40
window_height = 5


def store_file(file_name,data):
    with h5py.File(file_name,'w') as f:
        f["train_input"] = np.concatenate(data,axis=1)

def load_data_overlap(saved):
    #os.chdir(numpy_train)
    print "Inside function!..."
    if saved == False:
        train_files = np.random.randint(255,size=(1,40,690,4))
        train_input_data_interweawed_normalized = []
        print "Storing train pic to numpy"
        part = 0
        for i in xrange(100000):
            print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
            if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 2298842112/10:
                print "Max ram storing part: " + str(part) + " At entry: " + str(i)
                print "Storing Train input"
                file_name = 'train_input_'+'part_'+str(part)+'_'+str(dim)+'_'+str(total_frames_with_deltas)+'_window_height_'+str(window_height)+'.h5'
                store_file(file_name,train_input_data_interweawed_normalized)
                part = part + 1             
                del train_input_data_interweawed_normalized
                gc.collect()
                train_input_data_interweawed_normalized = []
                raw_input("something")
            for plot in train_files:
                overlaps_reshaped = np.random.randint(10,size=(45,200,5,3))
                for ind_plot in overlaps_reshaped.reshape(overlaps_reshaped.shape[1],overlaps_reshaped.shape[0],overlaps_reshaped.shape[2],overlaps_reshaped.shape[3]): 
                    ind_plot_reshaped = ind_plot.reshape(ind_plot.shape[0],1,ind_plot.shape[1],ind_plot.shape[2])
                    train_input_data_interweawed_normalized.append(ind_plot_reshaped)
    print len(train_input_data_interweawed_normalized)

    return train_input_data_interweawed_normalized_print
#------------------------------------------------------------------------------------------------------------------------------------------------------------

saved = False
train_input = load_data_overlap(saved)

输出：

.....
223662080
224772096
225882112
226996224
228106240
229216256
230326272
Max ram storing part: 0 At entry: 135
Storing Train input
something
377118720
Max ram storing part: 1 At entry: 136
Storing Train input
something
377118720
Max ram storing part: 2 At entry: 137
Storing Train input
something

Answer 1

你需要显式强制垃圾回收，见here：

根据Python官方文档，可以通过gc.collect()强制垃圾回收器释放未引用的内存