【问题标题】:Benchmark inconsistent results基准不一致的结果
【发布时间】:2012-12-02 05:35:16
【问题描述】:

我正在尝试分析一个函数。该函数旨在将结构转换为数组。我有两种不同的方法,使用编组或 BitConverter。当然,marshaling 方法创建了一个函数,可以在特定条件下与几乎所有结构一起使用。 BitConverter 需要每个结构的自定义函数。我最初的想法是 BitConverter 会更快,但我的测试结果并不一致。

这是基准的剪切和粘贴。

我以多种不同的形式尝试了以下基准。 当我首先对 BitConverter 函数进行基准测试时,它往往会更快。 当我首先对 Marshaling 函数进行基准测试时,它往往会更快。 我错过了什么?

显示流程的摘要。这不是基准流程的实际代码。

main()
{
    Stopwatch watch = new Stopwatch;
    // To take care of JIT
    bitConverterFunction();
    marshalingFunction();

    //Thread.Sleep(0);   // I've tried this thinking it had to do with context switching issues but the results were basically the same.
    watch.Start();
    for(i=0; i<iterations; i++)
    {
         bitConverterFunction();
    }
    watch.Stop();

    Timespan bitConverterTime = watch.Elapsed;
    //Thread.Sleep(0);   // I've tried this thinking it had to do with context switching issues
    watch.Restart();
    for(i=0; i<iterations; i++)
    {
         marshalingFunction();
    }
    watch.Stop();
    Timespan marshalingTime = watch.Elapsed;


    // it seems that whichever function is run first, tends to be the quickest.

}

如果你想测试真实代码

using System;
using BenchmarkTool;

namespace BenchmarkConsole
{
    class Program
    {
        static void Main(string[] args)
        {
            Benchmarks.StructToArrayConversion(100);
            Benchmarks.StructToArrayConversion(1000);
            Benchmarks.StructToArrayConversion(10000);
            Benchmarks.StructToArrayConversion(100000);

            Console.WriteLine("Press any key to continue.");
            Console.ReadKey();
        }
    }
}


using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
using NUnit.Framework;

namespace BenchmarkTool
{
    [TestFixture]
    public static class Benchmarks
    {
        [TestCase(100)]
        [TestCase(1000)]
        [TestCase(10000)]
        [TestCase(100000)]
        [TestCase(1000000)]
        public static void StructToArrayConversion(int iteration = 100)
        {
            Stopwatch watch = new Stopwatch();
            EntityStatePDU6 state = new EntityStatePDU6()
                {
                    Version = 0,
                    ExerciseID = 0x01,
                    PDUType = 0x02,
                    Family = 0x03,
                    Timestamp = 0x07060504,
                    Length = 0x0908,
                    Site = 0x0D0C,
                    Application = 0X0F0E,
                    Entity = 0X1110,
                    NumArticulationParams = 0X13,
                    VelocityX = BitConverter.ToSingle(new byte[] {0x14, 0x15, 0x16, 0x17}, 0),
                    VelocityY = BitConverter.ToSingle(new byte[] {0x18, 0x19, 0x1A, 0x1B}, 0),
                    VelocityZ = BitConverter.ToSingle(new byte[] {0x1C, 0x1D, 0x1E, 0x1F}, 0),
                    LocationX = BitConverter.ToSingle(new byte[] {0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27}, 0),
                    LocationY = BitConverter.ToSingle(new byte[] {0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F}, 0),
                    LocationZ = BitConverter.ToSingle(new byte[] {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37}, 0),
                    Roll = BitConverter.ToSingle(new byte[] {0x38, 0x39, 0x3A, 0x3B}, 0),
                    Pitch = BitConverter.ToSingle(new byte[] {0x3C, 0x3D, 0x3E, 0x3F}, 0),
                    Heading = BitConverter.ToSingle(new byte[] {0x40, 0x41, 0x42, 0x43}, 0),
                    Appearance = 0X47464544
                };

            // To take care of JIT
            ToArrayBitConverter(state);
            state.ToByteArray();

            Console.WriteLine("*** Benchmark Start ***");
            Console.WriteLine("BitConverter Benchmark");
            byte[] bitconverterArray = ToArrayBitConverter(state);
            //Thread.Sleep(0);
            watch.Start();
            for(int i = 0; i < iteration; i++)
            {
                bitconverterArray = ToArrayBitConverter(state);
            }
            watch.Stop();
            TimeSpan bitConverterTime = watch.Elapsed;

            Console.WriteLine("{0} Iterations: {1}", iteration, watch.Elapsed.TotalSeconds.ToString("0.0000000"));
            Console.WriteLine();

            Console.WriteLine("Marshal StructToPtr Benchmark");
            byte[] marshalArray = null;
            //Thread.Sleep(0);
            watch.Restart();
            for (int i = 0; i < iteration; i++)
            {
                marshalArray = state.ToByteArray();
            }
            watch.Stop();
            TimeSpan marshalTime = watch.Elapsed;

            Console.WriteLine("{0} Iterations: {1}", iteration, watch.Elapsed.TotalSeconds.ToString("0.0000000"));

            Console.WriteLine();
            Console.WriteLine("Results");
            Console.WriteLine("{0} Faster", marshalTime < bitConverterTime ? "Marshaling" : "BitConverter");
            Console.WriteLine("Speed Ratio: {0}", marshalTime < bitConverterTime ? bitConverterTime.TotalSeconds / marshalTime.TotalSeconds : marshalTime.TotalSeconds / bitConverterTime.TotalSeconds);

            Console.WriteLine("**********************************");
            Console.WriteLine();
            Assert.AreEqual(bitconverterArray.Length, marshalArray.Length);
            for(int i = 0; i < bitconverterArray.Length; i++)
            {
                Assert.AreEqual(marshalArray[i],bitconverterArray[i], "@index " + i);
            }
        }

        public static byte[] ToArrayBitConverter(EntityStatePDU6 entity)
        {
            int size = Marshal.SizeOf(typeof (EntityStatePDU6));
            byte[] array = new byte[size];
            array[0] = entity.Version;
            array[1] = entity.ExerciseID;
            array[2] = entity.PDUType;
            array[3] = entity.Family;
            array[4] = (byte)((0xFF & entity.Timestamp));
            array[5] = (byte)((0xFF00 & entity.Timestamp) >> 8);
            array[6] = (byte)((0xFF0000 & entity.Timestamp) >> 16);
            array[7] = (byte)((0xFF000000 & entity.Timestamp) >> 24);
            array[8] = (byte)((0xFF & entity.Length));
            array[9] = (byte)((0xFF00 & entity.Length) >> 8);
            // Padding1: array[10], array[11]
            array[12] = (byte)((0xFF & entity.Site));
            array[13] = (byte)((0xFF00 & entity.Site) >> 8);
            array[14] = (byte)((0xFF & entity.Application));
            array[15] = (byte)((0xFF00 & entity.Application) >> 8);
            array[16] = (byte)((0xFF & entity.Entity));
            array[17] = (byte)((0xFF00 & entity.Entity) >> 8);
            //padding2 array[18]
            array[19] = entity.NumArticulationParams;

            byte[] bytes = BitConverter.GetBytes(entity.VelocityX);
            array[20] = bytes[0];
            array[21] = bytes[1];
            array[22] = bytes[2];
            array[23] = bytes[3];
            bytes = BitConverter.GetBytes(entity.VelocityY);
            array[24] = bytes[0];
            array[25] = bytes[1];
            array[26] = bytes[2];
            array[27] = bytes[3];
            bytes = BitConverter.GetBytes(entity.VelocityZ);
            array[28] = bytes[0];
            array[29] = bytes[1];
            array[30] = bytes[2];
            array[31] = bytes[3];

            bytes = BitConverter.GetBytes(entity.LocationX);
            array[32] = bytes[0];
            array[33] = bytes[1];
            array[34] = bytes[2];
            array[35] = bytes[3];
            array[36] = bytes[4];
            array[37] = bytes[5];
            array[38] = bytes[6];
            array[39] = bytes[7];

            bytes = BitConverter.GetBytes(entity.LocationY);
            array[40] = bytes[0];
            array[41] = bytes[1];
            array[42] = bytes[2];
            array[43] = bytes[3];
            array[44] = bytes[4];
            array[45] = bytes[5];
            array[46] = bytes[6];
            array[47] = bytes[7];

            bytes = BitConverter.GetBytes(entity.LocationZ);
            array[48] = bytes[0];
            array[49] = bytes[1];
            array[50] = bytes[2];
            array[51] = bytes[3];
            array[52] = bytes[4];
            array[53] = bytes[5];
            array[54] = bytes[6];
            array[55] = bytes[7];

            bytes = BitConverter.GetBytes(entity.Roll);
            array[56] = bytes[0];
            array[57] = bytes[1];
            array[58] = bytes[2];
            array[59] = bytes[3];
            bytes = BitConverter.GetBytes(entity.Pitch);
            array[60] = bytes[0];
            array[61] = bytes[1];
            array[62] = bytes[2];
            array[63] = bytes[3];
            bytes = BitConverter.GetBytes(entity.Heading);
            array[64] = bytes[0];
            array[65] = bytes[1];
            array[66] = bytes[2];
            array[67] = bytes[3];

            array[68] = (byte)((0xFF & entity.Appearance));
            array[69] = (byte)((0xFF00 & entity.Appearance) >> 8);
            array[70] = (byte)((0xFF0000 & entity.Appearance) >> 16);
            array[71] = (byte)((0xFF000000 & entity.Appearance) >> 24);

            return array;
        }

        public static Byte[] ToByteArray<T>(this T obj) where T : struct
        {
            int size = Marshal.SizeOf(obj);
            var arr = new byte[size];
            IntPtr ptr = Marshal.AllocHGlobal(size);

            Marshal.StructureToPtr(obj, ptr, false);
            Marshal.Copy(ptr, arr, 0, size);
            Marshal.FreeHGlobal(ptr);

            return arr;
        }
    }

    public struct EntityStatePDU6
    {
        // PDU Header 12 Bytes
        public byte Version;
        public byte ExerciseID;
        public byte PDUType;
        public byte Family;
        public uint Timestamp;
        public ushort Length;
        public ushort Padding1;

        // Entity ID 6 bytes
        public ushort Site;
        public ushort Application;
        public ushort Entity;

        public byte Padding2;

        public byte NumArticulationParams;

        public float VelocityX;
        public float VelocityY;
        public float VelocityZ;

        public double LocationX;
        public double LocationY;
        public double LocationZ;

        public float Roll;
        public float Pitch;
        public float Heading;

        public uint Appearance;


    }


}

【问题讨论】:

  • 您的代码太长,我无法阅读。但这可能是一个 JIT 问题。在开始测量持续时间之前,我会运行每个方法一次。
  • 我做到了。我在开始第一个基准测试的秒表类之前运行它们。
  • 我将添加一个快速摘要以显示我的流程,也许它会显示任何缺陷。
  • 在我非常慢的 Pentium B950 上,基准测试几乎很快就完成了?最长的运行(100k 次迭代)为 0.16 秒。您不应该运行固定数量的迭代,而是运行固定的 TIME 并计数迭代,以获得“好”的结果。 EDIT 运行 10.000.000 次迭代让 marshal 版本以 10%+ 获胜,即使 BitConverter 版本首先运行。
  • 我同意,我通常以 1000000+ 次迭代运行基准测试。尽管除非先执行,否则我还没有编组获胜。但是,我确实喜欢在指定时间内处理的想法。这是个好主意。

标签: c# .net benchmarking


【解决方案1】:

任何低于 100000 的情况都太小而无法获得一致的结果。

即使在相同代码的运行之间,结果也非常不一致(> 2x 时间差异)。这让我觉得产生了大量的垃圾,结果主要取决于垃圾收集何时开始以及垃圾收集器的性能。

我在停止秒表后添加了一些 GC.Collect 调用,这使得结果更加一致(运行之间的差异为 +/- 10%)。 100000 和 1000000 次迭代的编组速度通常快 1.5 - 2 倍。这是在为 Release|x86 编译的 Mono 2.10.8.1 上,所以你的里程可能会有所不同。

【讨论】:

  • 非常感谢您的分析。 GC其实有点意思。我将尝试单独运行基准测试,而不是在同一个过程中。我认为这应该会给出更一致的结果。虽然当我运行基准测试时,我很少得到 > +-10% 的结果。在 1000000 和 10000000 次运行时,我也得到了 ~ 1.5-2.0 的比率,但是,最快的基准是我首先运行的基准(这就是我喜欢 GC 建议的原因)。
  • 有趣的是,我在不同的进程中运行基准测试,并且执行基准测试是一个单独的线程。我使用 BitConverter 获得了更好的结果。
猜你喜欢
  • 1970-01-01
  • 1970-01-01
  • 1970-01-01
  • 2019-08-02
  • 1970-01-01
  • 2020-12-24
  • 1970-01-01
  • 1970-01-01
  • 2014-09-21
相关资源
最近更新 更多