我参与了您提出的其他问题之一，老实说，我认为您在这里打的是一场失败的战斗。除了将所有内容读入内存之外，您可能还需要探索处理这些数据的其他途径。
如果我理解正确，您有多个线程同时处理数据，这就是为什么您不想直接处理文件，因为我假设存在 I/O 争用。
您是否考虑过或是否有可能将数据块读入内存，让线程处理该块，然后读取下一个块或由线程处理？这样，在任何时候，您的内存中永远不会超过一个块，但所有线程都可以访问该块。这不是最佳的，但我把它作为一个起点。如果这是可行的，那么可以探索优化它的选项。
更新： 使用平台调用分配非托管内存并从 .NET 使用它的示例。
既然您非常确定需要将这么多数据加载到内存中，我想我会编写一个小型测试应用程序来验证这是否可行。为此，您将需要以下内容
- 使用
/unsafe 编译选项进行编译
- 如果您想分配超过 2 GB 的空间，您还需要将目标平台切换到 x64
*上面的第 2 点稍微复杂一些，在 64 位操作系统上，您仍然可以针对 x86 平台并访问完整的 4 GB 内存。这需要您使用EDITBIN.EXE 之类的工具在PE 标头中设置LargeAddressAware 标志。
此代码使用VirtualAllocEx 分配非托管内存并使用UnmanagedMemoryStream 使用.NET 流隐喻访问非托管内存。请注意，此代码仅在具有 4 GB RAM 的目标 64 位环境中完成了一些非常基本的快速测试。最重要的是，该进程的内存利用率仅达到了大约 2.6 GB。
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Diagnostics;
using System.ComponentModel;
namespace MemoryMappedFileTests
{
class Program
{
static void Main(string[] args)
{
IntPtr ptr = IntPtr.Zero;
try
{
// Allocate and Commit the memory directly.
ptr = VirtualAllocEx(
Process.GetCurrentProcess().Handle,
IntPtr.Zero,
new IntPtr(0xD0000000L),
AllocationType.Commit | AllocationType.Reserve,
MemoryProtection.ReadWrite);
if (ptr == IntPtr.Zero)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
// Query some information about the allocation, used for testing.
MEMORY_BASIC_INFORMATION mbi = new MEMORY_BASIC_INFORMATION();
IntPtr result = VirtualQueryEx(
Process.GetCurrentProcess().Handle,
ptr,
out mbi,
new IntPtr(Marshal.SizeOf(mbi)));
if (result == IntPtr.Zero)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
// Use unsafe code to get a pointer to the unmanaged memory.
// This requires compiling with /unsafe option.
unsafe
{
// Pointer to the allocated memory
byte* pBytes = (byte*)ptr.ToPointer();
// Create Read/Write stream to access the memory.
UnmanagedMemoryStream stm = new UnmanagedMemoryStream(
pBytes,
mbi.RegionSize.ToInt64(),
mbi.RegionSize.ToInt64(),
FileAccess.ReadWrite);
// Create a StreamWriter to write to the unmanaged memory.
StreamWriter sw = new StreamWriter(stm);
sw.Write("Everything seems to be working!\r\n");
sw.Flush();
// Reset the stream position and create a reader to check that the
// data was written correctly.
stm.Position = 0;
StreamReader rd = new StreamReader(stm);
Console.WriteLine(rd.ReadLine());
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
if (ptr != IntPtr.Zero)
{
VirtualFreeEx(
Process.GetCurrentProcess().Handle,
ptr,
IntPtr.Zero,
FreeType.Release);
}
}
Console.ReadKey();
}
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
static extern IntPtr VirtualAllocEx(
IntPtr hProcess,
IntPtr lpAddress,
IntPtr dwSize,
AllocationType dwAllocationType,
MemoryProtection flProtect);
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
static extern bool VirtualFreeEx(
IntPtr hProcess,
IntPtr lpAddress,
IntPtr dwSize,
FreeType dwFreeType);
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
static extern IntPtr VirtualQueryEx(
IntPtr hProcess,
IntPtr lpAddress,
out MEMORY_BASIC_INFORMATION lpBuffer,
IntPtr dwLength);
[StructLayout(LayoutKind.Sequential)]
public struct MEMORY_BASIC_INFORMATION
{
public IntPtr BaseAddress;
public IntPtr AllocationBase;
public int AllocationProtect;
public IntPtr RegionSize;
public int State;
public int Protect;
public int Type;
}
[Flags]
public enum AllocationType
{
Commit = 0x1000,
Reserve = 0x2000,
Decommit = 0x4000,
Release = 0x8000,
Reset = 0x80000,
Physical = 0x400000,
TopDown = 0x100000,
WriteWatch = 0x200000,
LargePages = 0x20000000
}
[Flags]
public enum MemoryProtection
{
Execute = 0x10,
ExecuteRead = 0x20,
ExecuteReadWrite = 0x40,
ExecuteWriteCopy = 0x80,
NoAccess = 0x01,
ReadOnly = 0x02,
ReadWrite = 0x04,
WriteCopy = 0x08,
GuardModifierflag = 0x100,
NoCacheModifierflag = 0x200,
WriteCombineModifierflag = 0x400
}
[Flags]
public enum FreeType
{
Decommit = 0x4000,
Release = 0x8000
}
}
}