Has anyone figured out a way to use pr_curve_streaming_op for multiclass classification ?
I am trying to follow this demo and this demo. But so far I am getting bumped at the data collection stage itself. Its seems like I need to be able to collect data at runtime and I am facing several issues like inability to modify a bool tensor at runtime etc. Any help will be very usefull. Thanks
Here is my code snippet and the errors I am getting:
self.input_y = tf.placeholder(tf.float32, [None, num_classes], name='input_y')
with tf.device('/cpu:0'), tf.name_scope('embedding'):
self.emb_var = tf.Variable(embedding_mat, name='emb_var')
# if not non_static:
# self.emb_var = tf.constant(embedding_mat, name='emb_var')
# else:
# self.emb_var = tf.Variable(embedding_mat, name='emb_var', trainable=True)
self.embedded_chars = tf.nn.embedding_lookup(self.emb_var, self.input_x)
self.emb = tf.expand_dims(self.embedded_chars, -1)
pooled_concat = []
reduced = np.int32(np.ceil((sequence_length) * 1.0 / max_pool_size))
###
.
.
Removed some code for simplicity...
.
.
###
with tf.name_scope('probabilities'):
self.probabilities = tf.nn.softmax(self.scores)
with tf.name_scope('confidence'):
self.conf = tf.reduce_max(self.probabilities, reduction_indices=[1])
with tf.name_scope('Low_confidence'):
self.conf_low = tf.reduce_min(self.conf, name='low_conf')
with tf.name_scope('Avg_confidence'):
self.Avg_conf = tf.reduce_mean(self.conf, name='confidence')
with tf.name_scope('high_confidence'):
self.conf_high = tf.reduce_max(self.conf, name='high_conf')
with tf.name_scope('loss'):
losses = tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y, logits=self.scores) # only named arguments accepted
self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss
with tf.name_scope('accuracy'):
correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name='accuracy')
with tf.name_scope('num_correct'):
correct = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
self.num_correct = tf.reduce_sum(tf.cast(correct, 'float'))
''' start '''
with tf.name_scope('pr_curve'):
self.pr_curve, update_op = summary_lib.pr_curve_streaming_op('pr_curve', predictions=self.probabilities, labels=tf.cast(self.input_y,tf.bool), num_thresholds=11)
''' end '''
Here are the errors I am getting:
2017-12-25 18:37:30.433250: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433565: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433613: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433630: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433652: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433699: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433729: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433767: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
2017-12-25 18:37:30.433805: W tensorflow/core/framework/op_kernel.cc:1192] Failed precondition: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
Traceback (most recent call last):
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1323, in _do_call
return fn(*args)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1302, in _run_fn
status, run_metadata)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 473, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
[[Node: rnn/LessEqual_4/_37 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_105_rnn/LessEqual_4", tensor_type=DT_BOOL, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./code/train.py", line 357, in <module>
train_cnn_rnn()
File "./code/train.py", line 276, in train_cnn_rnn
train_step(x_train_batch, y_train_batch)
File "./code/train.py", line 223, in train_step
_, step, summaries = sess.run([train_op, global_step, train_summary_op], feed_dict)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/debug/wrappers/framework.py", line 534, in run
run_metadata=run_metadata)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 889, in run
run_metadata_ptr)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1120, in _run
feed_dict_tensor, options, run_metadata)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1317, in _do_run
options, run_metadata)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1336, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
[[Node: rnn/LessEqual_4/_37 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_105_rnn/LessEqual_4", tensor_type=DT_BOOL, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
Caused by op 'pr_curve/pr_curve/true_negatives/true_negatives/read', defined at:
File "./code/train.py", line 357, in <module>
train_cnn_rnn()
File "./code/train.py", line 170, in train_cnn_rnn
l2_reg_lambda=params['l2_reg_lambda'])
File "/media/hemant/MVV/MyValueVest-local/main/code/text_cnn_rnn.py", line 127, in __init__
self.pr_curve, update_op = summary_lib.pr_curve_streaming_op('pr_curve', predictions=self.probabilities, labels=tf.cast(self.input_y,tf.bool), num_thresholds=11)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorboard/plugins/pr_curve/summary.py", line 234, in streaming_op
weights=weights)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/metrics_impl.py", line 1482, in true_negatives_at_thresholds
labels, predictions, thresholds, weights=weights, includes=('tn',))
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/metrics_impl.py", line 543, in _confusion_matrix_at_thresholds
true_n = _create_local('true_negatives', shape=[num_thresholds])
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/metrics_impl.py", line 196, in _create_local
validate_shape=validate_shape)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 1927, in variable
caching_device=caching_device, name=name, dtype=dtype)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 213, in __init__
constraint=constraint)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 356, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 125, in identity
return gen_array_ops.identity(input, name=name)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 2071, in identity
"Identity", input=input, name=name)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2956, in create_op
op_def=op_def)
File "/home/hemant/anaconda3/envs/tf14/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1470, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value pr_curve/pr_curve/true_negatives/true_negatives
[[Node: pr_curve/pr_curve/true_negatives/true_negatives/read = Identity[T=DT_FLOAT, _class=["loc:@pr_curve/pr_curve/true_negatives/true_negatives"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](pr_curve/pr_curve/true_negatives/true_negatives)]]
[[Node: rnn/LessEqual_4/_37 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_105_rnn/LessEqual_4", tensor_type=DT_BOOL, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
Figured it out... - Created a separate session since I needed the values of all the predictions. - Looped through all my classes.
pr_graph = tf.Graph()
with pr_graph.as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
pr_sess = tf.Session(config=session_conf)
with pr_sess.as_default():
for cat in range(y_train.shape[1]):
with tf.name_scope('%s' % labels[cat]):
_, update_op = summary_lib.pr_curve_streaming_op('pr_curve', predictions=probs[:, cat], labels=tf.cast(y_test[:, cat], tf.bool), num_thresholds=500, metrics_collections='pr')
pr_summary_op = tf.summary.merge_all()
pr_summary_dir = os.path.join(checkpoint_dir, "s", "pr")
pr_summary_writer = tf.summary.FileWriter(pr_summary_dir, pr_sess.graph)
pr_sess.run(tf.local_variables_initializer())
pr_sess.run([update_op])
pr_summary_writer.add_summary(pr_sess.run(pr_summary_op))
pr_summary_writer.close()