从 N 个 numpy 数组生成相同大小的批次

Question

我有 N 个形状为 data[n,m,3] 的 NumPy 数组。我想将它们拟合/挤压/拆分/切片/重塑成 N' 个形状为 new_data_#[1000,m,3] 的数组，其中 # 是新数组的索引。问题是 n 可以小于或大于 1000。当它以某种方式较小时，我应该用下一个数组填充 new_array 的其余 1000 个容量，当它大于 1000 时，我应该创建一个 new_data_# 并添加休息到那个。我不知道如何管理这个。这是一个伪代码，但不能以这种方式完成，例如，while 可能不是必需的。输出可以写入磁盘或以新的数据格式返回。

def array2blocks(array_files)
 for each N in array_files:
    N = data = np.random.rand(n, m, 3)
    new_data = np.zeros((1000, m, 3), dtype=np.float32)
    j=0
    index = 0
    while j <= new_data.shape[0]:
        for i in range(data.shape[0]):
            print("--->", data[i,:,:])
            print (i)
            if i <= new_data.shape[0]:
                # here first we should check the left capacity of new_data and then insert data into it
                # new_data[i, :, :] = data[i, :, :] #this overrides previous items so not correct
                print(new_data)
            else:
                print('n>1000')
                new_data_name = 'new_data' + '_' + str(index)
                # here fill rest of the data in the new_data
                ...
                index += 1
            #when capacity is full write it to the disk
    print(new_data)

UPDATE 与 Aaron 的旧答案： 我将 1000 替换为 batch_size = 5 以使其变得简单。

def numpyarrays2blocks(array_files):
    N1 = np.random.rand(7, 4, 3)
    N2 = np.random.rand(7, 4, 3)
    N3 = np.random.rand(4, 4, 3)
    # array_files = []
    array_files.append(N1)
    array_files.append(N2)
    array_files.append(N3)
    for N in array_files:
        n = N.shape[0]
        m = N.shape[1]
        batch_size = 5
        # N = data = np.random.rand(n, m, 3)
        data = N
        # print(data)
        new_arrays = []
        i = 0  # the current row index to insert
        while i < n:
            new_data = np.zeros((batch_size, m, 3), dtype=np.float32)
            j = min(i + batch_size, n)  # the last row (exclusive) to copy to new_data
            # j - i is the number of rows to copy
            new_data[:j - i, :, :] = data[i:j, :, :]
            print('NEW DATA: ', new_data)
            i = j  # update the index
            new_arrays.append(new_data)
    print(new_arrays)

Answer 1

1. data 用于存储临时结果，data_start 是向data 插入行的索引。
2. 如果是None，则分配data
3. yield data 如果已满。

merge_and_split 是一个生成器，因此内存需求应该很低。

import random
from typing import Iterator

import numpy as np


def merge_and_split(arrays, batch_size) -> Iterator:
    arrays = tuple(arrays)

    dtype = arrays[0].dtype

    data_shape = (batch_size,) + arrays[0].shape[1:]

    assert all(a.shape[1:] == data_shape[1:] for a in arrays), "Shape mismatch"

    data = None
    data_start = 0

    for src in arrays:
        src_index = 0
        src_avail = src.shape[0]

        while src_avail >= 1:
            if data is None:
                # allocate if None
                data = np.zeros(data_shape, dtype=dtype)
                data_start = 0

            num_moved = min(batch_size - data_start, src_avail)
            data[data_start:data_start + num_moved, ...] = src[src_index:src_index + num_moved, ...]

            data_start += num_moved
            src_index += num_moved
            src_avail -= num_moved

            if data_start >= batch_size:
                yield data
                data = None

    if data is not None:
        yield data


def input_arrays():
    number = 10

    r = random.Random(13)

    return [np.random.randint(0, 10, size=(r.randint(1, 5), 4, 3)) for _ in range(number)]


def main():
    # Testing input and output
    arrays = input_arrays()

    # for i, item in enumerate(arrays):
    #     print('input', i, item.shape)
    #     print(item)

    result = list(merge_and_split(arrays, 5))

    # for i, item in enumerate(result):
    #     print('result', i, item.shape)
    #     print(item)

    src_concat = np.vstack(arrays)
    row_number = sum(s.shape[0] for s in arrays)
    print('concatenated', src_concat.shape, row_number)

    out_concat = np.vstack(result)
    print(out_concat.shape)
    print((out_concat[0:row_number, ...] == src_concat).all())  # They are indeed the same


if __name__ == '__main__':
    main()

Answer 2

你可以concatenate你所有的原始数组split他们：

ars = ... # list of N arrays
ars = np.concatenate(ars, axis=0)
ars = np.split(ars, np.arange(1000, ars.shape[0], 1000))

最后一行可以写成ars = np.split(ars, 1000)，但前提是你确定元素的总数是1000的倍数，否则np.split会出错。与np.arange 一样，指定显式分割点可以让您拥有更短的最终段。