Clojure - 在大地图上使用归约的替代方法或提高性能的方法

Question

我有一个中等大小的地图（~1M 键值相对较小，例如(first mymap) => ["7:21658846-21658846" {["C" "T"] {"central_nervous_system" 1}}]），它使用下面的函数简化为另一个地图（省略了一些辅助函数，因为它们与问题并不真正相关 - 它们是不影响性能）

新地图是使用归约 (reduce-kv) 创建的，因为在解析原始地图中的每个元素时，我需要累积增加地图中的各种嵌套值。

对于大约需要 40 秒的大约 100K 条目来说，这并不是一个真正的问题。对于我的 1M 大小的地图（未来的运行只会变得更大）来说，这是一个大问题，它在 20 分钟后还没有完成。

是否有任何显而易见的问题很容易解决？可以更改以大幅提高性能的非惯用方法？它似乎不是特别可并行化-但其中的某些东西确实效率低下-我希望是累积图。

任何建议表示赞赏。

*编辑 - 添加 assoc-in-sum defn

(defn add-mut-freq-firstTS
  "Builds a map of Transcript -> {aapos -> {:aaposn_count :codon :aa {genomic_SNP_posn -> {:SNP_posn_count :frame {:genomic_ref :genomic_mut :SNP_count :aa_mut :codon_mut}}}}.
  but only using one transcript per SNP. ***ONLY USED FOR STATS CALCS***"
  [CDS-ref snp-freq]
  (reduce-kv (fn [m k v](let [aa-ref (first (cosu/map-ts-aa-pos CDS-ref [k v]))] (add-mut-freq** m aa-ref aa-ref [k v]))) {} snp-freq) )

(defn add-mut-freq**
  "Adds data for frequency of different mutations at a given position to a     cumulative map m for a given transcript. Updates running totals
  for frequency at aa position and genomic position as well."
  [m ts first-ts snp-freq]
  (let [[ts_ID SNP_aa_posn SNP_aa_frame _ gene strand] ts
    [posn nt-mut-freq] snp-freq
    m-pre (if (= ts first-ts) (assoc-in m [:ts ts_ID :snp-aa-pos SNP_aa_posn :snp-nt-posn posn :first] true) m)
    m-init (assoc-in m-pre [:ts ts_ID :gene] gene)]
(reduce-kv (fn [m1 k v](
          let [mut k
               tiss-freq v
               snp-count (apply + (vals tiss-freq))]
                     (-> m1 (u/assoc-in-sum [:ts ts_ID :ts-cnt] snp-count)
                         (assoc-in [:ts ts_ID :ts-strand] strand)
                         (u/assoc-in-sum [:ts ts_ID :snp-aa-pos SNP_aa_posn :aa-cnt] snp-count)
                         (u/assoc-in-sum [:ts ts_ID :snp-aa-pos SNP_aa_posn :snp-nt-posn posn :pos-cnt] snp-count)
                         (assoc-in [:ts ts_ID :snp-aa-pos SNP_aa_posn :snp-nt-posn posn :ts-frame] SNP_aa_frame)
                         (u/assoc-in-sum [:ts ts_ID :snp-aa-pos SNP_aa_posn :snp-nt-posn posn :mut-nt mut :posnt-cnt] snp-count)
                         (add-tissue-counts ts_ID SNP_aa_posn tiss-freq)
                         (assoc-in [:ts ts_ID :snp-aa-pos SNP_aa_posn :snp-nt-posn posn :mut-nt mut :mut-tiss-freq] tiss-freq))
          )) m-init nt-mut-freq)))

(defn add-tissue-counts
  [m ts_ID SNP_aa_posn tiss-map]
  (reduce-kv (fn [m1 k v] (-> m1 (u/assoc-in-sum [:ts ts_ID :snp-aa-pos SNP_aa_posn :aa-tiss-cnt k] v)
           (u/assoc-in-sum [:ts ts_ID :ts-tiss-cnt k] v)
           (u/assoc-in-sum [:tiss-cnt k] v)
           )) m tiss-map))

(defn assoc-in-sum
  "Same as assoc-in except that if the key already exists, the value is added to instead of replaced"
  [m key-vec v]
  (let [ex-val (get-in m key-vec)
        new-val (if ex-val (+ ex-val v) v)]
  (assoc-in m key-vec new-val))
  )

Answer 1

与基于持久数据的缩减相比，您正在做大量缩减，并且可以通过使用有状态转换器来提高速度。

这是一个关于如何使用 volatile 创建转换器函数的小示例，也许它可以为创建更快的代码版本提供一些想法。

(defn test-xf
  [rf]
  (let [sum (volatile! 0)]
    (fn
      ([] (rf))

      ([result] (rf (assoc! result :total-sum @sum)))

      ([result [k m]]

        ;; calculate sums etc.
       (vswap! sum + (get-in m [["C" "T"] "x"]))

        ;; Result is transient map while in reduction!
       (-> result
           (assoc! :mydata "hello")
           (assoc! k m))
        ))))

(defn data [n] 
  (for [i (range n)]
    [(str "key-" i) {["C" "T"] {"x" 1}}]))

(time
  (:total-sum
    (into {} test-xf (data 1000000))))

"Elapsed time: 1750.867127 msecs"
=> 1000000