JSON 转储：TypeError：int64 类型的对象不是 JSON 可序列化的

Question

我有以下课程来训练主题模型：

class LDA_Model:
    def __init__(self, data_words_model):
        self.data_words = data_words_model
        self.id2word = corpora.Dictionary(self.data_words)
        self.corpus = [self.id2word.doc2bow(text) for text in self.data_words]
    
    def get_coherence(self, model, coherence):
        coh_model = CoherenceModel(model=model, texts= self.data_words, dictionary=self.id2word, coherence=coherence)
        return coh_model.get_coherence()
        
    def build_models(self, start, limit, step, runs):           
        crossval_dict = dict()
        for run in range(runs):
            coherence_dict = dict()
            topic_dict = dict()
        
            for num_topics in range(start, limit, step):
                start = time.time()

                model = LdaModel(corpus=self.corpus,
                                 id2word=self.id2word,
                                 num_topics=num_topics, 
                                 random_state=100,
                                 update_every=1,
                                 chunksize=100,
                                 passes=50,
                                 alpha='auto',
                                 per_word_topics=True)

                coherence_dict[num_topics] = {'c_v':self.get_coherence(model, 'c_v'), 
                                              'u_mass':self.get_coherence(model, 'u_mass'), 
                                              'c_uci':self.get_coherence(model, 'c_uci'), 
                                              'c_npmi':self.get_coherence(model, 'c_npmi')}

                topic_dict[num_topics] = model.show_topics()

                dump = open(PATH+"LDA_coherences.json", "w")
                json.dump(coherence_dict, dump)
                dump.close()

                dump = open(PATH+"LDA_topics.json", "w")
                json.dump(topic_dict, dump)
                dump.close()
                end = time.time()
            
                print('Calculating topics for ', str(num_topics), 'number of topics, took: ', str(int(end-start)), ' seconds.')
    
            crossval_dict[run] = {'coherence_dict': coherence_dict,
                              'topic_dict':topic_dict}
            dump = open(PATH+"10runs_LDA.json", "w")
            json.dump(crossval_dict, dump)
            dump.close()
        return coherence_dict

我跑完之后：

ldamodel = LDA_Model(data_words)
coherence = ldamodel.build_models(start = 5, limit= 71, step= 5, runs=10)

该模型对于前两个 num_topcis（5 和 10）运行良好。到目前为止的输出：

>>> Calculating topics for  5 number of topics, took:  1109  seconds.
>>> Calculating topics for  10 number of topics, took:  1485  seconds.

但是，如果有 15 个主题，它会终止并引发以下错误：

TypeError                                 Traceback (most recent call last)
<ipython-input-32-eb870528e17d> in <module>
----> 1 coherence = ldamodel.build_models(start = 5, limit= 71, step= 5, runs=10)

<ipython-input-30-09de1b2cf22a> in build_models(self, start, limit, step, runs)
     55 
     56                 dump = open(PATH+"LDA_topics.json", "w")
---> 57                 json.dump(topic_dict, dump)
     58                 dump.close()
     59                 end = time.time()

~\Anaconda3\lib\json\__init__.py in dump(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
    177     # could accelerate with writelines in some versions of Python, at
    178     # a debuggability cost
--> 179     for chunk in iterable:
    180         fp.write(chunk)
    181 

~\Anaconda3\lib\json\encoder.py in _iterencode(o, _current_indent_level)
    429             yield from _iterencode_list(o, _current_indent_level)
    430         elif isinstance(o, dict):
--> 431             yield from _iterencode_dict(o, _current_indent_level)
    432         else:
    433             if markers is not None:

~\Anaconda3\lib\json\encoder.py in _iterencode_dict(dct, _current_indent_level)
    403                 else:
    404                     chunks = _iterencode(value, _current_indent_level)
--> 405                 yield from chunks
    406         if newline_indent is not None:
    407             _current_indent_level -= 1

~\Anaconda3\lib\json\encoder.py in _iterencode_list(lst, _current_indent_level)
    323                 else:
    324                     chunks = _iterencode(value, _current_indent_level)
--> 325                 yield from chunks
    326         if newline_indent is not None:
    327             _current_indent_level -= 1

~\Anaconda3\lib\json\encoder.py in _iterencode_list(lst, _current_indent_level)
    323                 else:
    324                     chunks = _iterencode(value, _current_indent_level)
--> 325                 yield from chunks
    326         if newline_indent is not None:
    327             _current_indent_level -= 1

~\Anaconda3\lib\json\encoder.py in _iterencode(o, _current_indent_level)
    436                     raise ValueError("Circular reference detected")
    437                 markers[markerid] = o
--> 438             o = _default(o)
    439             yield from _iterencode(o, _current_indent_level)
    440             if markers is not None:

~\Anaconda3\lib\json\encoder.py in default(self, o)
    177 
    178         """
--> 179         raise TypeError(f'Object of type {o.__class__.__name__} '
    180                         f'is not JSON serializable')
    181 

TypeError: Object of type int64 is not JSON serializable

所以，我似乎无法序列化 topic_dict，因为它包含一个“int64”类型的对象。然而，这让我感到惊讶，因为它在前两次运行中确实有效。

是什么导致了错误？我该怎么做才能克服这个错误？

Answer 1

Gensim 的 show_topics 方法默认返回最多 10 个主题。但是，当产生更多主题时，它只选择 10 个主题。我没有检查源代码，但预计由于此选择是 Numpy 操作，因此每个索引的类型也会发生变化：

for top in ldamodel.topic_dict[5]:
    print(type(top[0]),type(top[1]))
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>


for top in ldamodel.topic_dict[10]:
    print(type(top[0]),type(top[1]))
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>
<class 'int'> <class 'str'>

for top in ldamodel.topic_dict[15]:
    print(type(top[0]),type(top[1]))
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>
<class 'numpy.int64'> <class 'str'>

为了克服这个问题，我在课堂上使用了以下代码：

topic_dict[num_topics] = model.show_topics(num_topics = num_topics)