TensorBoard 标量摘要是单个数据点。怎么修？

Question

我使用tf.summary.scalar 方法和tf.train.LoggingTensorHook 记录一些张量。这是tf.estimator.Estimator 框架。

tf.train.LoggingTensorHook 的东西甚至没有出现在 AFAIK。其他内容正在显示，但显然没有时间步骤。

图表和其他所有内容（权重）在张量板上看起来都不错。

更新：看起来多次调用 train 会产生一个图表。 steps 和 every_n_iter 是否存在未按预期交互的内容？

import numpy as np
import tensorflow as tf

m = 10000
n = 5
X = np.random.randn(m, n)
A = np.random.randn(n)
y = X.dot(A) + np.random.randn(m) * 0.1

batch_size = 1024

def input_fn(batch_size):
    ds = tf.data.Dataset.from_tensor_slices(dict(X=X, y=y))
    ds = ds.repeat(-1)
    ds = ds.batch(batch_size)
    return ds

def model_fn(features, labels, mode, params):
    X = features['X']
    y = features['y']
    l = X
    for i, k in enumerate([32, 16, 16]):
        l = tf.layers.dense(inputs=l, units=k, name=f'l_{i}', activation=tf.nn.tanh)
    some_thing = tf.reduce_sum(l, axis=1, name='some_thing')
    l = tf.layers.dense(inputs=l, units=1, name='l_final')
    predictions = tf.squeeze(l, axis=-1)
    loss = tf.losses.mean_squared_error(y, predictions, weights=1.0)
    metric_ops = {"mse": tf.metrics.mean_squared_error(labels=y, predictions=predictions)}
    if mode == tf.estimator.ModeKeys.TRAIN:
        optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
        train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op, eval_metric_ops=metric_ops)
    if mode == tf.estimator.ModeKeys.PREDICT:
        predictions = {}
        return tf.estimator.EstimatorSpec(mode, predictions=predictions)
    if mode == tf.estimator.ModeKeys.EVAL:
        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=metric_ops)
    raise Exception('should not hit this')

model = tf.estimator.Estimator(
        model_fn=model_fn,
        model_dir='/tmp/junk',
        config=None,
        params=dict(),
        warm_start_from=None
        )

tensors_to_log = dict(some_thing='some_thing')
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=10)

train_input_fn = lambda: input_fn(batch_size)
test_input_fn = lambda: input_fn(batch_size)

train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, hooks=[logging_hook], max_steps=100)
eval_spec = tf.estimator.EvalSpec(input_fn=test_input_fn, hooks=[logging_hook])
out = tf.estimator.train_and_evaluate(model, train_spec, eval_spec)

更新：这个直到运行结束才显示在张量板上，然后它也只显示一个点。

import numpy as np
import tensorflow as tf
# tf.enable_eager_execution()
tf.logging.set_verbosity(tf.logging.INFO)

m = 10000
n = 5
X = np.random.randn(m, n)
A = np.random.randn(n)
y = X.dot(A) + np.random.randn(m) * 0.1

steps = 1000
batch_size = 1024

def input_fn(repeat, batch_size):
    ds = tf.data.Dataset.from_tensor_slices(dict(X=X, y=y))
    ds = ds.repeat(repeat)
    ds = ds.batch(batch_size)
    return ds

def model_fn(features, labels, mode, params):
    X = features['X']
    y = features['y']
    l = X
    for i, k in enumerate([32, 16, 16]):
        l = tf.layers.dense(inputs=l, units=k, name=f'l_{i}', activation=tf.nn.tanh)
    some_thing = tf.reduce_sum(l, axis=1, name='some_thing')
    l = tf.layers.dense(inputs=l, units=1, name='l_final')
    predictions = tf.squeeze(l, axis=-1)
    loss = tf.losses.mean_squared_error(y, predictions, weights=1.0)
    metric_ops = {"mse": tf.metrics.mean_squared_error(labels=y, predictions=predictions)}

    tf.summary.scalar('summary_loss', loss) # plot a dist across the batch
    if mode == tf.estimator.ModeKeys.TRAIN:
        optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
        train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op, eval_metric_ops=metric_ops)
    if mode == tf.estimator.ModeKeys.PREDICT:
        predictions = {}
        return tf.estimator.EstimatorSpec(mode, predictions=predictions)
    if mode == tf.estimator.ModeKeys.EVAL:
        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=metric_ops)
    raise Exception('should not hit this')

model = tf.estimator.Estimator(
        model_fn=model_fn,
        model_dir='/tmp/junk',
        config=None,
        params=dict(),
        warm_start_from=None
        )

tensors_to_log = dict(some_thing='some_thing')
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=10)

train_input_fn = lambda: input_fn(steps, batch_size)
test_input_fn = lambda: input_fn(steps, batch_size)

train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, hooks=[logging_hook], max_steps=None)
eval_spec = tf.estimator.EvalSpec(input_fn=test_input_fn, hooks=[logging_hook])
out = tf.estimator.train_and_evaluate(model, train_spec, eval_spec)

Answer 1

我遇到了一个非常相似的问题。我很惊讶解决方案非常简单。关闭 TensorBoard 并再次启动，并等待几分钟。赶上需要时间。出于某种原因，如果您在训练期间启动 TensorBoard，它会卡住。我希望这将有所帮助。 我在谷歌云上运行代码

from google.datalab.ml import TensorBoard
TensorBoard().start('gs://{}/directoy_where_my_models_are'.format(BUCKET))