使用本文和 Json.Net 从文件中存储和检索大型 Json ( 100 mb +) 的有效方法

Question

我一直在尝试找出使用 JSon.net 从包含 Json 的文件中存储和检索数据的最佳方法。在我的研究中，我发现了几种方法。

1. http://www.drdobbs.com/windows/parsing-big-records-with-jsonnet/240165316
2. http://www.ngdata.com/parsing-a-large-json-file-efficiently-and-easily/Jackson Api（仅适用于 Java）

这种方法中的任何一种都可以与 Json.Net 一起使用吗？ DrDoobs 上的文章是否有可用的实际实现？我想不通 HandleToken(reader.TokenType, reader.Value) 方法。

当前统计：

• 编写大小约为 60Mb 的 Json 大约需要 5 分钟

• 读取 Json 大约需要 3 分钟。

当前代码：

public static T DeserializeJsonFromStream<T>(Stream s)
    {
        using (StreamReader reader = new StreamReader(s))
        {
            using (JsonTextReader jsonReader = new JsonTextReader(reader))
            {
                JsonSerializer ser = new JsonSerializer();
                ser.Formatting = Newtonsoft.Json.Formatting.None;
                ser.PreserveReferencesHandling = PreserveReferencesHandling.Objects;
                ser.TypeNameHandling = TypeNameHandling.All;
                ser.NullValueHandling = NullValueHandling.Ignore;
                ser.Error += ReportJsonErrors;
                ser.DateFormatHandling = DateFormatHandling.IsoDateFormat;
                return ser.Deserialize<T>(jsonReader);
            }
        }
    }
    public static void SerializeJsonIntoStream(object value, Stream s)
    {

        using (StreamWriter writer = new StreamWriter(s))
        {
            using (JsonTextWriter jsonWriter = new JsonTextWriter(writer))
            {
                JsonSerializer ser = new JsonSerializer();

                ser.Formatting = Newtonsoft.Json.Formatting.None;
                ser.PreserveReferencesHandling = PreserveReferencesHandling.Objects;
                ser.TypeNameHandling = TypeNameHandling.All;
                ser.NullValueHandling = NullValueHandling.Ignore;
                ser.Error += ReportJsonErrors;

                ser.Serialize(jsonWriter, value);
                jsonWriter.Flush();
            }
        }
    }

Answer 1

由于我不需要手动读取序列化文件，使用协议缓冲区解决了这个问题，序列化/反序列化相同数量的数据需要 6 秒。

它可以在 Nuget 上通过 http://www.nuget.org/packages/protobuf-net 获得

    public static async Task<T> DeserializeFromBinary<T>(string filename, StorageFolder storageFolder)
    {
        try
        {

            var file = await storageFolder.GetFileAsync(filename);
            if (file != null)
            {
                var stream = await file.OpenStreamForReadAsync();
                T content = Serializer.Deserialize<T>(stream);
                return content;
            }
        }

        catch (NullReferenceException nullException)
        {
            logger.LogError("Exception happened while de-serializing input object, Error: " + nullException.Message);
        }
        catch (FileNotFoundException fileNotFound)
        {
            logger.LogError("Exception happened while de-serializing input object, Error: " + fileNotFound.Message);
        }
        catch (Exception e)
        {
            logger.LogError("Exception happened while de-serializing input object, Error: " + e.Message, e.ToString());
        }
        return default(T);
    }


    public static async Task<bool> SerializeIntoBinary<T>(string fileName, StorageFolder destinationFolder, Content content)
    {
        bool shouldRetry = true;
        while (shouldRetry)
        {
            try
            {

                StorageFile file = await destinationFolder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting);

                using (var stream = await file.OpenStreamForWriteAsync())
                {
                    Serializer.Serialize(stream, content);
                    shouldRetry = false;
                    return true;
                }
            }
            catch (UnauthorizedAccessException unAuthorizedAccess)
            {
                logger.LogError("UnauthorizedAccessException happened while serializing input object, will wait for 5 seconds, Error: " + unAuthorizedAccess.Message);
                shouldRetry = true;
            }
            catch (NullReferenceException nullException)
            {
                shouldRetry = false;
                logger.LogError("Exception happened while serializing input object, Error: " + nullException.Message);
            }
            catch (Exception e)
            {
                shouldRetry = false;
                logger.LogError("Exception happened while serializing input object, Error: " + e.Message, e.ToString());
            }

            if (shouldRetry)
                await Task.Delay(5000);
        }
        return false;
    }