动态 LINQ 性能

Question

我有一个例程（在其他人的慷慨帮助下编写），它允许我获取一个 List 对象，并以任何顺序使用任意数量的属性，它动态地构建一个 TreeView 结构，每个节点都有一个 Count。这种动态能力是严格的用户要求。

所以一个源列表：

{Prop1 = "A", Prop2 = "I", Prop3 = "X"},  
{Prop1 = "A", Prop2 = "J", Prop3 = "X"},  
{Prop1 = "B", Prop2 = "I", Prop3 = "X"},  
{Prop1 = "B", Prop2 = "I", Prop3 = "Y"},   
{Prop1 = "C", Prop2 = "K", Prop3 = "Z"}

Gives the following when the Selection is by Prop1 by Prop3:

Total (5)  
- A(2)  
- - X(2)  
- B(2)  
- - X(1)  
- - Y(1)  
- C(1)  
- - Z(1) 

从功能上讲，这很好用。但是，当不同值的数量增加时，性能还有很多不足之处。例如 - 在具有 5K 对象和 Prop1 中的 1K 不同值的数据集中进行一次特定运行将需要几秒钟。

下面是例程：

    public static class TreeBuilder
    {
        public static Dictionary<string, TreeItem> BuildTree<TSource>(List<TSource> source, List<string> columns)
        {
            return new Dictionary<string, TreeItem>()
                {
                    { "Total",
                      new TreeItem()
                        {
                            Key = "Total",
                            RawKey = "Total",
                            Count = source.Count,
                            Items = GroupBySelector<TSource>(source, columns, 0, "Total")
                        }
                    }
                };
        }

        public static MethodInfo GetGenericMethod(this Type type, string name, Type[] genericTypeArgs, Type[] paramTypes)
        {
            foreach (MethodInfo method in type.GetMethods())
                if (method.Name == name)
                {
                    var pa = method.GetParameters();
                    if (pa.Length == paramTypes.Length)
                    {
                        var genericMethod = method.MakeGenericMethod(genericTypeArgs);
                        if (genericMethod.GetParameters().Select(p => p.ParameterType).SequenceEqual(paramTypes))
                            return genericMethod;
                    }
                }
            return null;
        }

        private static MethodInfo GetGroupByMethodStatically<TElement, TKey>()
            => typeof(Enumerable).GetGenericMethod("GroupBy", new[] { typeof(TElement), typeof(TKey) }, new[] { typeof(IEnumerable<TElement>), typeof(Func<TElement, TKey>) });

        private static MethodInfo GetEnumerableMethod(string methodName, Type tElement, Type tTKey)
        {
            var tIElement = typeof(IEnumerable<>).MakeGenericType(tElement);
            var tFunction = typeof(Func<,>).MakeGenericType(tElement, tTKey);
            return typeof(Enumerable).GetGenericMethod(methodName, new[] { tElement, tTKey }, new[] { tIElement, tFunction });
        }

        private static MethodInfo GetEnumerableMethod(string methodName, Type tElement)
        {
            var tIELEMENT = typeof(IEnumerable<>).MakeGenericType(tElement);
            return typeof(Enumerable).GetGenericMethod(methodName, new[] { tElement }, new[] { tIELEMENT });
        }

        public static Dictionary<string, TreeItem> GroupBySelector<TElement>(IEnumerable<TElement> source, IList<string> columnNames, int entry = 0, string key = "")
        {
            if (source == null) throw new ArgumentNullException(nameof(source));

            List<string> columnParameters = columnNames[entry].Split('|').ToList();
            string columnName = columnParameters[0];

            if (columnName == null) throw new ArgumentNullException(nameof(columnName));
            if (columnName.Length == 0) throw new ArgumentException(nameof(columnName));

            int nextEntry = entry + 1;

            var tElement = typeof(TElement);
            var tIElement = typeof(IEnumerable<TElement>);

            var keyParm = Expression.Parameter(tElement);
            var prop = Expression.Property(keyParm, columnName);

            var param = Expression.Parameter(tIElement, "p");
            var groupByMethod = GetEnumerableMethod("GroupBy", tElement, prop.Type);
            var groupByExpr = Expression.Lambda(prop, keyParm);
            var bodyExprCall = Expression.Call(groupByMethod, param, groupByExpr);

            var tSelectInput = typeof(IGrouping<,>).MakeGenericType(prop.Type, tElement);
            var selectParam = Expression.Parameter(tSelectInput, "p");

            var tKey = typeof(TreeItem).GetMember("Key").Single();
            var tRawKey = typeof(TreeItem).GetMember("RawKey").Single();
            var tCount = typeof(TreeItem).GetMember("Count").Single();
            var tParentKey = typeof(TreeItem).GetMember("ParentKey").Single();
            var tItems = typeof(TreeItem).GetMember("Items").Single();

            Expression selectParamKey = Expression.Property(selectParam, "Key");
            Expression selectParamRawKey = selectParamKey;

            if (selectParamKey.Type != typeof(string))
            {
                var toStringMethod = selectParamKey.Type.GetMethod("ToString", Type.EmptyTypes);
                selectParamKey = Expression.Call(selectParamKey, toStringMethod);
            }

            if (selectParamRawKey.Type != typeof(string))
            {
                var toStringMethod = selectParamRawKey.Type.GetMethod("ToString", Type.EmptyTypes);
                selectParamRawKey = Expression.Call(selectParamRawKey, toStringMethod);
            }

            var countMethod = GetEnumerableMethod("Count", tElement);
            var countMethodExpr = Expression.Call(countMethod, selectParam);

            var concatFullKeyExpr = Expression.Call(typeof(string).GetMethod("Concat", new[] { typeof(string), typeof(string), typeof(string) }),
                                                    Expression.Constant(key),
                                                    Expression.Constant("|"),
                                                    selectParamRawKey);

            var groupBySelectorMethod = GetGenericMethod(MethodBase.GetCurrentMethod().DeclaringType, "GroupBySelector", new[] { tElement }, new[] { tIElement, typeof(IList<string>), typeof(int), typeof(string) });
            var groupBySelectorMethodExpr = Expression.Call(groupBySelectorMethod, selectParam, Expression.Constant(columnNames), Expression.Constant(nextEntry), concatFullKeyExpr);

            var newMenuItemExpr = Expression.New(typeof(TreeItem));

            var selectBodyExpr = Expression.MemberInit(newMenuItemExpr, new[] {
                                                                                    Expression.Bind(tKey, selectParamKey),
                                                                                    Expression.Bind(tRawKey, selectParamRawKey),
                                                                                    Expression.Bind(tParentKey, Expression.Constant(key) ),
                                                                                    Expression.Bind(tCount, countMethodExpr),
                                                                                    Expression.Bind(tItems, groupBySelectorMethodExpr)
                                                                                 });
            var selectBodyExprLamba = Expression.Lambda(selectBodyExpr, selectParam);

            var selectBodyLastExpr = Expression.MemberInit(newMenuItemExpr, new[] {
                                                                                    Expression.Bind(tKey, selectParamKey),
                                                                                    Expression.Bind(tRawKey, selectParamRawKey),
                                                                                    Expression.Bind(tParentKey, Expression.Constant(key) ),
                                                                                    Expression.Bind(tCount, countMethodExpr)
                                                                                    });
            var selectBodyLastExprLambda = Expression.Lambda(selectBodyLastExpr, selectParam);

            var selectMethod = GetEnumerableMethod("Select", tSelectInput, typeof(TreeItem));
            bodyExprCall = Expression.Call(selectMethod, bodyExprCall, (nextEntry < columnNames.Count) ? selectBodyExprLamba : selectBodyLastExprLambda);

            var selectParamout = Expression.Parameter(typeof(TreeItem), "o");
            Expression selectParamKeyout = Expression.Property(selectParamout, "FullKey");
            var selectParamKeyLambda = Expression.Lambda(selectParamKeyout, selectParamout);

            var lmi = GetEnumerableMethod("ToDictionary", typeof(TreeItem), typeof(string));
            bodyExprCall = Expression.Call(lmi, bodyExprCall, selectParamKeyLambda);

            var returnFunc = Expression.Lambda<Func<IEnumerable<TElement>, Dictionary<string, TreeItem>>>(bodyExprCall, param).Compile();

            return returnFunc(source);
        }
    }    

该例程用于从 DB 表中获取数据并将其转换为层次结构以用于 WPF TreeView。

 Dictionary<string, TreeItem> treeItems = new Dictionary<string, TreeItem>();
 treeItems = TreeBuilder.BuildTree<IDBRecord>(DBService.GetDBRecordList(), PropertySortList);

任何人都可以就如何提高此例程的性能提供任何建议吗？或者建议以更高效的方式实现相同结果的任何替代方法？

谢谢

Answer 1

可以进行一些优化。调用Compile 花费了很多时间，而您正在为树中每个级别的每个键调用Compile，这增加了很多开销，在我对 5k 个项目的测试中大约需要 7 秒。我首先更改了代码以提取所有具有固定类型的静态反射，因此每次程序运行只执行一次。这只产生了很小的差异，因为构建 Expression 树不是主要问题。

然后我更改了方法，将构建Expression 与编译Expression 并调用生成的lambda 分开。这允许我修改对 Expression 构建器的递归调用，改为内联 Invoke 的新 lambda 用于新级别。然后我在生成的Expression 上调用了一次编译并执行它。新版本不再带entry参数，但可以放回去。

这将总时间从大约 7.6 秒减少到 0.14 秒，速度提高了大约 50 倍。对所有三个属性进行测试后，速度提高了 280 倍。

如果仍然可以重复调用该方法，那么添加缓存会带来更多好处，尽管快速测试仅显示节省了大约 14% 的时间，而且实时时间仅为百分之一秒。

static MemberInfo tKey = typeof(TreeItem).GetMember("Key").Single();
static MemberInfo tRawKey = typeof(TreeItem).GetMember("RawKey").Single();
static MemberInfo tCount = typeof(TreeItem).GetMember("Count").Single();
static MemberInfo tParentKey = typeof(TreeItem).GetMember("ParentKey").Single();
static MemberInfo tItems = typeof(TreeItem).GetMember("Items").Single();
// Concat(string, string, string)
static MethodInfo Concat3MI = ((Func<string, string, string, string>)String.Concat).Method;
// new TreeItem() { ... }
static NewExpression newMenuItemExpr = Expression.New(typeof(TreeItem));

// Enumerable.ToDictionary<TreeItem>(IEnumerable<TreeItem>, Func<TreeItem,string>)
static MethodInfo ToDictionaryMI = GetEnumerableMethod("ToDictionary", typeof(TreeItem), typeof(string));

static Expression<Func<IEnumerable<TElement>, Dictionary<string, TreeItem>>> BuildGroupBySelector<TElement>(IList<string> columnNames, int entry, Expression key) {
    List<string> columnParameters = columnNames[entry].Split('|').ToList();
    string columnName = columnParameters[0];

    if (columnName == null) throw new ArgumentNullException(nameof(columnName));
    if (columnName.Length == 0) throw new ArgumentException(nameof(columnName));

    int nextEntry = entry + 1;

    var tElement = typeof(TElement);
    var tIElement = typeof(IEnumerable<TElement>);

    // (TElement kp)
    var keyParm = Expression.Parameter(tElement, "kp");
    // kp.columnName
    var prop = Expression.Property(keyParm, columnName);

    // (IEnumerable<TElement> p)
    var IEParam = Expression.Parameter(tIElement, "p");
    // GroupBy<TElement>(IEnumerable<TElement>, Func<TElement, typeof(kp.columnName)>)
    var groupByMethod = GetEnumerableMethod("GroupBy", tElement, prop.Type);
    // kp => kp.columnName
    var groupByExpr = Expression.Lambda(prop, keyParm);
    // GroupBy(p, kp => kp.columnName)
    var bodyExprCall = Expression.Call(groupByMethod, IEParam, groupByExpr);

    // typeof(IGrouping<typeof(kp.columnName), TElement>)
    var tSelectInput = typeof(IGrouping<,>).MakeGenericType(prop.Type, tElement);
    // (IGrouping<typeof(kp.columnName), TElement> sp)
    var selectParam = Expression.Parameter(tSelectInput, "sp");

    // sp.Key
    Expression selectParamKey = Expression.Property(selectParam, "Key");
    Expression selectParamRawKey = selectParamKey;

    if (selectParamKey.Type != typeof(string)) {
        var toStringMethod = selectParamKey.Type.GetMethod("ToString", Type.EmptyTypes);
        // sp.Key.ToString()
        selectParamKey = Expression.Call(selectParamKey, toStringMethod);
        selectParamRawKey = selectParamKey;
    }

    // Count<TElement>()
    var countMethod = GetEnumerableMethod("Count", tElement);
    // sp.Count()
    var countMethodExpr = Expression.Call(countMethod, selectParam);

    LambdaExpression selectBodyExprLamba;
    if (nextEntry < columnNames.Count) {
        // Concat(key, "|", sp.Key.ToString())
        var concatFullKeyExpr = Expression.Call(Concat3MI, key, Expression.Constant("|"), selectParamRawKey);

        // p# => p#.GroupBy().Select().ToDictionary()
        var groupBySelectorLambdaExpr = BuildGroupBySelector<TElement>(columnNames, nextEntry, (Expression)concatFullKeyExpr);
        // Invoke(p# => p#..., sp#)
        var groupBySelectorInvokeExpr = Expression.Invoke(groupBySelectorLambdaExpr, selectParam);

        var selectBodyExpr = Expression.MemberInit(newMenuItemExpr, new[] {
                                                                                Expression.Bind(tKey, selectParamKey),
                                                                                Expression.Bind(tRawKey, selectParamRawKey),
                                                                                Expression.Bind(tParentKey, key ),
                                                                                Expression.Bind(tCount, countMethodExpr),
                                                                                Expression.Bind(tItems, groupBySelectorInvokeExpr)
                                                                             });
        // sp => new TreeItem { Key = sp.Key.ToString(), RawKey = sp.Key.ToString(), ParentKey = key, Count = sp.Count(), Items = Invoke(p# => p#..., sp)) }
        selectBodyExprLamba = Expression.Lambda(selectBodyExpr, selectParam);
    }
    else { // Last Level
        var selectBodyExpr = Expression.MemberInit(newMenuItemExpr, new[] {
                                                                                Expression.Bind(tKey, selectParamKey),
                                                                                Expression.Bind(tRawKey, selectParamRawKey),
                                                                                Expression.Bind(tParentKey, key ),
                                                                                Expression.Bind(tCount, countMethodExpr)
                                                                                });
        // sp => new TreeItem { Key = sp.Key.ToString(), RawKey = sp.Key.ToString(), ParentKey = key, Count = sp.Count() }
        selectBodyExprLamba = Expression.Lambda(selectBodyExpr, selectParam);
    }

    // Enumerable.Select<IGrouping<typeof<kp.columnName>, TElement>>(IEnumerable<IGrouping<>>, Func<IGrouping<>, TreeItem>)
    var selectMethod = GetEnumerableMethod("Select", tSelectInput, typeof(TreeItem));
    // p.GroupBy(kp => kp => kp.columnName).Select(sp => ...)
    bodyExprCall = Expression.Call(selectMethod, bodyExprCall, selectBodyExprLamba);

    // (TreeItem o)
    var selectParamout = Expression.Parameter(typeof(TreeItem), "o");
    // o.FullKey
    Expression selectParamKeyout = Expression.Property(selectParamout, "FullKey");
    // o => o.FullKey
    var selectParamKeyLambda = Expression.Lambda(selectParamKeyout, selectParamout);

    // p.GroupBy(...).Select(...).ToDictionary(o => o.FullKey)
    bodyExprCall = Expression.Call(ToDictionaryMI, bodyExprCall, selectParamKeyLambda);

    // p => p.GroupBy(kp => kp => kp.columnName).Select(sp => ...).ToDictionary(o => o.FullKey)
    return Expression.Lambda<Func<IEnumerable<TElement>, Dictionary<string, TreeItem>>>(bodyExprCall, IEParam);
}

public static Dictionary<string, TreeItem> GroupBySelector<TElement>(IEnumerable<TElement> source, IList<string> columnNames, string key = "") {
    if (source == null) throw new ArgumentNullException(nameof(source));

    // p => p.GroupBy(kp => kp => kp.columnName).Select(sp => ...).ToDictionary(o => o.FullKey)
    var returnFunc = BuildGroupBySelector<TElement>(columnNames, 0, Expression.Constant(key)).Compile();

    return returnFunc(source);
}