Python tensorflow.python.framework.errors.FailedPreconditionError() Examples

def _ready(op, sess, msg):
  """Checks if the model is ready or not, as determined by op.

  Args:
    op: An op, either _ready_op or _ready_for_local_init_op, which defines the
      readiness of the model.
    sess: A `Session`.
    msg: A message to log to warning if not ready

  Returns:
    A tuple (is_ready, msg), where is_ready is True if ready and False
    otherwise, and msg is `None` if the model is ready, a `String` with the
    reason why it is not ready otherwise.
  """
  if op is None:
    return True, None
  else:
    try:
      ready_value = sess.run(op)
      # The model is considered ready if ready_op returns an empty 1-D tensor.
      # Also compare to `None` and dtype being int32 for backward
      # compatibility.
      if (ready_value is None or ready_value.dtype == np.int32 or
          ready_value.size == 0):
        return True, None
      else:
        # TODO(sherrym): If a custom ready_op returns other types of tensor,
        # or strings other than variable names, this message could be
        # confusing.
        non_initialized_varnames = ", ".join(
            [i.decode("utf-8") for i in ready_value])
        return False, "Variables not initialized: " + non_initialized_varnames
    except errors.FailedPreconditionError as e:
      if "uninitialized" not in str(e):
        logging.warning("%s : error [%s]", msg, str(e))
        raise e
      return False, str(e) 

def _ready(op, sess, msg):
  """Checks if the model is ready or not, as determined by op.

  Args:
    op: An op, either _ready_op or _ready_for_local_init_op, which defines the
      readiness of the model.
    sess: A `Session`.
    msg: A message to log to warning if not ready

  Returns:
    A tuple (is_ready, msg), where is_ready is True if ready and False
    otherwise, and msg is `None` if the model is ready, a `String` with the
    reason why it is not ready otherwise.
  """
  if op is None:
    return True, None
  else:
    try:
      ready_value = sess.run(op)
      # The model is considered ready if ready_op returns an empty 1-D tensor.
      # Also compare to `None` and dtype being int32 for backward
      # compatibility.
      if (ready_value is None or ready_value.dtype == np.int32 or
          ready_value.size == 0):
        return True, None
      else:
        # TODO(sherrym): If a custom ready_op returns other types of tensor,
        # or strings other than variable names, this message could be
        # confusing.
        non_initialized_varnames = ", ".join(
            [i.decode("utf-8") for i in ready_value])
        return False, "Variables not initialized: " + non_initialized_varnames
    except errors.FailedPreconditionError as e:
      if "uninitialized" not in str(e):
        logging.warning("%s : error [%s]", msg, str(e))
        raise e
      return False, str(e) 

def testUninitialized(self):
    with self.assertRaisesRegexp(
        errors.FailedPreconditionError,
        "Attempting to use uninitialized value Variable"):
      with self.test_session() as sess:
        v = tf.Variable([1, 2])
        sess.run(v[:].assign([1, 2])) 

def _ready(self, op, sess, msg):
    """Checks if the model is ready or not, as determined by op.

    Args:
      op: An op, either _ready_op or _ready_for_local_init_op, which defines the
        readiness of the model.
      sess: A `Session`.
      msg: A message to log to warning if not ready

    Returns:
      A tuple (is_ready, msg), where is_ready is True if ready and False
      otherwise, and msg is `None` if the model is ready, a `String` with the
      reason why it is not ready otherwise.
    """
    if op is None:
      return True, None
    else:
      try:
        ready_value = sess.run(op)
        # The model is considered ready if ready_op returns an empty 1-D tensor.
        # Also compare to `None` and dtype being int32 for backward
        # compatibility.
        if (ready_value is None or ready_value.dtype == np.int32 or
            ready_value.size == 0):
          return True, None
        else:
          # TODO(sherrym): If a custom ready_op returns other types of tensor,
          # or strings other than variable names, this message could be
          # confusing.
          non_initialized_varnames = ", ".join(
              [i.decode("utf-8") for i in ready_value])
          return False, "Variables not initialized: " + non_initialized_varnames
      except errors.FailedPreconditionError as e:
        if "uninitialized" not in str(e):
          logging.warning("%s : error [%s]", msg, str(e))
          raise  e
        return False, str(e) 

def get_weights(self):
#         if len(self.weights) != self.nHLayers + 1:
        self.weights = []
        for n in xrange(self.nHLayers + 1):
            if self.get_layers[n].get_w:
                try:
                    self.weights.append(self.session.run(self.get_layers[n].get_w))
                except FailedPreconditionError:
                    break
            else:
                break

        return self.weights 

def get_biases(self):
#         if len(self.biases) != self.nHLayers + 1:
        self.biases = []
        for n in xrange(self.nHLayers + 1):
            if self.get_layers[n].get_b:
                try:
                    self.biases.append(self.session.run(self.get_layers[n].get_b))
                except FailedPreconditionError:
                    break
            else:
                break

        return self.biases 

def testDebugNumericSummaryFailureIsToleratedWhenOrdered(self):
    with session.Session() as sess:
      a = variables.Variable("1", name="a")
      b = variables.Variable("3", name="b")
      c = variables.Variable("2", name="c")

      d = math_ops.add(a, b, name="d")
      e = math_ops.add(d, c, name="e")
      n = parsing_ops.string_to_number(e, name="n")
      m = math_ops.add(n, n, name="m")

      sess.run(variables.global_variables_initializer())

      # Using DebugNumericSummary on sess.run(m) with the default
      # tolerate_debug_op_creation_failures=False should error out due to the
      # presence of string-dtype Tensors in the graph.
      run_metadata = config_pb2.RunMetadata()
      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary"],
          debug_urls=self._debug_urls())
      with self.assertRaises(errors.FailedPreconditionError):
        sess.run(m, options=run_options, run_metadata=run_metadata)

      # Using tolerate_debug_op_creation_failures=True should get rid of the
      # error.
      m_result, dump = self._debug_run_and_get_dump(
          sess, m, debug_ops=["DebugNumericSummary"],
          tolerate_debug_op_creation_failures=True)
      self.assertEqual(264, m_result)

      # The integer-dtype Tensors in the graph should have been dumped
      # properly.
      self.assertIn("n:0:DebugNumericSummary", dump.debug_watch_keys("n"))
      self.assertIn("m:0:DebugNumericSummary", dump.debug_watch_keys("m")) 

def _ready(op, sess, msg):
  """Checks if the model is ready or not, as determined by op.

  Args:
    op: An op, either _ready_op or _ready_for_local_init_op, which defines the
      readiness of the model.
    sess: A `Session`.
    msg: A message to log to warning if not ready

  Returns:
    A tuple (is_ready, msg), where is_ready is True if ready and False
    otherwise, and msg is `None` if the model is ready, a `String` with the
    reason why it is not ready otherwise.
  """
  if op is None:
    return True, None
  else:
    try:
      ready_value = sess.run(op)
      # The model is considered ready if ready_op returns an empty 1-D tensor.
      # Also compare to `None` and dtype being int32 for backward
      # compatibility.
      if (ready_value is None or ready_value.dtype == np.int32 or
          ready_value.size == 0):
        return True, None
      else:
        # TODO(sherrym): If a custom ready_op returns other types of tensor,
        # or strings other than variable names, this message could be
        # confusing.
        non_initialized_varnames = ", ".join(
            [i.decode("utf-8") for i in ready_value])
        return False, "Variables not initialized: " + non_initialized_varnames
    except errors.FailedPreconditionError as e:
      if "uninitialized" not in str(e):
        logging.warning("%s : error [%s]", msg, str(e))
        raise e
      return False, str(e) 

def _ready(op, sess, msg):
  """Checks if the model is ready or not, as determined by op.

  Args:
    op: An op, either _ready_op or _ready_for_local_init_op, which defines the
      readiness of the model.
    sess: A `Session`.
    msg: A message to log to warning if not ready

  Returns:
    A tuple (is_ready, msg), where is_ready is True if ready and False
    otherwise, and msg is `None` if the model is ready, a `String` with the
    reason why it is not ready otherwise.
  """
  if op is None:
    return True, None
  else:
    try:
      ready_value = sess.run(op)
      # The model is considered ready if ready_op returns an empty 1-D tensor.
      # Also compare to `None` and dtype being int32 for backward
      # compatibility.
      if (ready_value is None or ready_value.dtype == np.int32 or
          ready_value.size == 0):
        return True, None
      else:
        # TODO(sherrym): If a custom ready_op returns other types of tensor,
        # or strings other than variable names, this message could be
        # confusing.
        non_initialized_varnames = ", ".join(
            [i.decode("utf-8") for i in ready_value])
        return False, "Variables not initialized: " + non_initialized_varnames
    except errors.FailedPreconditionError as e:
      if "uninitialized" not in str(e):
        logging.warning("%s : error [%s]", msg, str(e))
        raise e
      return False, str(e) 

def testDebugNumericSummaryFailureIsToleratedWhenOrdered(self):
    with session.Session() as sess:
      a = variables.Variable("1", name="a")
      b = variables.Variable("3", name="b")
      c = variables.Variable("2", name="c")

      d = math_ops.add(a, b, name="d")
      e = math_ops.add(d, c, name="e")
      n = parsing_ops.string_to_number(e, name="n")
      m = math_ops.add(n, n, name="m")

      sess.run(variables.global_variables_initializer())

      # Using DebugNumericSummary on sess.run(m) with the default
      # tolerate_debug_op_creation_failures=False should error out due to the
      # presence of string-dtype Tensors in the graph.
      run_metadata = config_pb2.RunMetadata()
      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary"],
          debug_urls=self._debug_urls())
      with self.assertRaises(errors.FailedPreconditionError):
        sess.run(m, options=run_options, run_metadata=run_metadata)

      # Using tolerate_debug_op_creation_failures=True should get rid of the
      # error.
      new_run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          new_run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary"],
          debug_urls=self._debug_urls(),
          tolerate_debug_op_creation_failures=True)

      self.assertEqual(264,
                       sess.run(
                           m,
                           options=new_run_options,
                           run_metadata=run_metadata))

      # The integer-dtype Tensors in the graph should have been dumped
      # properly.
      dump = debug_data.DebugDumpDir(
          self._dump_root, partition_graphs=run_metadata.partition_graphs)
      self.assertIn("n:0:DebugNumericSummary", dump.debug_watch_keys("n"))
      self.assertIn("m:0:DebugNumericSummary", dump.debug_watch_keys("m")) 

def testDebugNumericSummaryInvalidAttributesStringAreCaught(self):
    with session.Session() as sess:
      a = variables.Variable(10.0, name="a")
      b = variables.Variable(0.0, name="b")
      c = variables.Variable(0.0, name="c")

      x = math_ops.divide(a, b, name="x")
      y = math_ops.multiply(x, c, name="y")

      sess.run(variables.global_variables_initializer())

      run_metadata = config_pb2.RunMetadata()
      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary(foo=1.0)"],
          debug_urls=self._debug_urls())
      with self.assertRaisesRegexp(
          errors.FailedPreconditionError,
          r"1 attribute key\(s\) were not valid for debug node "
          r"__dbg_a:0_0_DebugNumericSummary: foo"):
        sess.run(y, options=run_options, run_metadata=run_metadata)

      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary(foo=1.0; bar=false)"],
          debug_urls=self._debug_urls())
      with self.assertRaisesRegexp(
          errors.FailedPreconditionError,
          r"2 attribute key\(s\) were not valid for debug node "
          r"__dbg_a:0_0_DebugNumericSummary:"):
        sess.run(y, options=run_options, run_metadata=run_metadata)

      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary(foo=1.0; mute_if_healthy=true)"],
          debug_urls=self._debug_urls())
      with self.assertRaisesRegexp(
          errors.FailedPreconditionError,
          r"1 attribute key\(s\) were not valid for debug node "
          r"__dbg_a:0_0_DebugNumericSummary: foo"):
        sess.run(y, options=run_options, run_metadata=run_metadata) 

def analyze(sdae, datafile_norm,\
            labels, mapped_labels=None,\
            bias_node=False, prefix=None):

    """
        Speeks to R, and submits it analysis jobs.
    """

    # Get some R functions on the Python environment
    def_colors = robjects.globalenv['def_colors']
    do_analysis = robjects.globalenv['do_analysis']

    # labels.reset_index(level=0, inplace=True)
    def_colors(labels)
    act = np.float32(datafile_norm)

    try:
        do_analysis(act, sdae.get_weights, sdae.get_biases,\
                    pjoin(FLAGS.output_dir, "{}_R_Layer_".format(prefix)),\
                    bias_node=bias_node)
    except RRuntimeError as e:
        pass

#     for layer in sdae.get_layers:
#         fixed = False if layer.which > sdae.nHLayers - 1 else True
#  
#         try:
#             act = sdae.get_activation(act, layer.which, use_fixed=fixed)
#             print("Analysis for layer {}:".format(layer.which + 1))
#             temp = pd.DataFrame(data=act)
#             do_analysis(temp, pjoin(FLAGS.output_dir,\
#                                     "{}_Layer_{}"\
#                                     .format(prefix, layer.which)))
#              
# #             if not fixed:
# #                 weights = sdae.get_weights[layer.which]
# #                 for node in weights.transpose():
# #                     sns.distplot(node, kde=False,\
#                                     fit=stats.gamma, rug=True);
# #                     sns.plt.show()
#             try:
#                 plot_tSNE(act, mapped_labels,\
#                             plot_name="Pyhton_{}_tSNE_layer_{}"\
#                             .format(prefix, layer.which))
#             except IndexError as e:
#                 pass
#         except FailedPreconditionError as e:
#             break 

def testDebugNumericSummaryInvalidAttributesStringAreCaught(self):
    with session.Session(config=no_rewrite_session_config()) as sess:
      a = variables.Variable(10.0, name="a")
      b = variables.Variable(0.0, name="b")
      c = variables.Variable(0.0, name="c")

      x = math_ops.divide(a, b, name="x")
      y = math_ops.multiply(x, c, name="y")

      sess.run(variables.global_variables_initializer())

      run_metadata = config_pb2.RunMetadata()
      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary(foo=1.0)"],
          debug_urls=self._debug_urls())
      with self.assertRaisesRegexp(
          errors.FailedPreconditionError,
          r"1 attribute key\(s\) were not valid for debug node "
          r"__dbg_.:0_0_DebugNumericSummary: foo"):
        sess.run(y, options=run_options, run_metadata=run_metadata)

      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary(foo=1.0; bar=false)"],
          debug_urls=self._debug_urls())
      with self.assertRaisesRegexp(
          errors.FailedPreconditionError,
          r"2 attribute key\(s\) were not valid for debug node "
          r"__dbg_.:0_0_DebugNumericSummary:"):
        sess.run(y, options=run_options, run_metadata=run_metadata)

      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugNumericSummary(foo=1.0; mute_if_healthy=true)"],
          debug_urls=self._debug_urls())
      with self.assertRaisesRegexp(
          errors.FailedPreconditionError,
          r"1 attribute key\(s\) were not valid for debug node "
          r"__dbg_.:0_0_DebugNumericSummary: foo"):
        sess.run(y, options=run_options, run_metadata=run_metadata) 

def _poll_server_till_success(max_attempts,
                              sleep_per_poll_sec,
                              debug_server_url,
                              dump_dir,
                              server,
                              gpu_memory_fraction=1.0):
  """Poll server until success or exceeding max polling count.

  Args:
    max_attempts: (int) How many times to poll at maximum
    sleep_per_poll_sec: (float) How many seconds to sleep for after each
      unsuccessful poll.
    debug_server_url: (str) gRPC URL to the debug server.
    dump_dir: (str) Dump directory to look for files in. If None, will directly
      check data from the server object.
    server: The server object.
    gpu_memory_fraction: (float) Fraction of GPU memory to be
      allocated for the Session used in server polling.

  Returns:
    (bool) Whether the polling succeeded within max_polls attempts.
  """
  poll_count = 0

  config = config_pb2.ConfigProto(gpu_options=config_pb2.GPUOptions(
      per_process_gpu_memory_fraction=gpu_memory_fraction))
  with session.Session(config=config) as sess:
    for poll_count in range(max_attempts):
      server.clear_data()
      print("Polling: poll_count = %d" % poll_count)

      x_init_name = "x_init_%d" % poll_count
      x_init = constant_op.constant([42.0], shape=[1], name=x_init_name)
      x = variables.Variable(x_init, name=x_init_name)

      run_options = config_pb2.RunOptions()
      debug_utils.add_debug_tensor_watch(
          run_options, x_init_name, 0, debug_urls=[debug_server_url])
      try:
        sess.run(x.initializer, options=run_options)
      except errors.FailedPreconditionError:
        pass

      if dump_dir:
        if os.path.isdir(
            dump_dir) and debug_data.DebugDumpDir(dump_dir).size > 0:
          shutil.rmtree(dump_dir)
          print("Poll succeeded.")
          return True
        else:
          print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec)
          time.sleep(sleep_per_poll_sec)
      else:
        if server.debug_tensor_values:
          print("Poll succeeded.")
          return True
        else:
          print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec)
          time.sleep(sleep_per_poll_sec)

    return False