Python tensorflow.python.data.ops.dataset_ops.DatasetV2() Examples

def from_input_fn(return_values):
    """Returns an `_Inputs` instance according to `input_fn` return value."""
    if isinstance(return_values, dataset_ops.DatasetV2):
      dataset = return_values
      return _Inputs(dataset=dataset)

    features, labels = _Inputs._parse_inputs(return_values)
    return _Inputs(features, labels) 

def from_input_fn(return_values):
    """Returns an `_Inputs` instance according to `input_fn` return value."""
    if isinstance(return_values, dataset_ops.DatasetV2):
      dataset = return_values
      return _Inputs(dataset=dataset)

    features, labels = _Inputs._parse_inputs(return_values)
    return _Inputs(features, labels) 

def from_input_fn(return_values):
    """Returns an `_Inputs` instance according to `input_fn` return value."""
    if isinstance(return_values, dataset_ops.DatasetV2):
      dataset = return_values
      return _Inputs(dataset=dataset)

    features, labels = _Inputs._parse_inputs(return_values)
    return _Inputs(features, labels) 

def parse_input_fn_result(result):
  """Gets features, labels, and hooks from the result of an Estimator input_fn.

  Args:
    result: output of an input_fn to an estimator, which should be one of:
      * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a tuple
        (features, labels) with same constraints as below.
      * A tuple (features, labels): Where `features` is a `Tensor` or a
        dictionary of string feature name to `Tensor` and `labels` is a `Tensor`
        or a dictionary of string label name to `Tensor`. Both `features` and
        `labels` are consumed by `model_fn`. They should satisfy the expectation
        of `model_fn` from inputs.

  Returns:
    Tuple of features, labels, and input_hooks, where features are as described
    above, labels are as described above or None, and input_hooks are a list
    of SessionRunHooks to be included when running.

  Raises:
    ValueError: if the result is a list or tuple of length != 2.
  """
  input_hooks = []
  if isinstance(result, dataset_ops.DatasetV2):
    iterator = dataset_ops.make_initializable_iterator(result)
    input_hooks.append(_DatasetInitializerHook(iterator))
    result = iterator.get_next()
  return parse_iterator_result(result) + (input_hooks,) 

def from_input_fn(return_values):
    """Returns an `_Inputs` instance according to `input_fn` return value."""
    if isinstance(return_values, dataset_ops.DatasetV2):
      dataset = return_values
      return _Inputs(dataset=dataset)

    features, labels = _Inputs._parse_inputs(return_values)
    return _Inputs(features, labels) 

def on_epoch_end(self, epoch, logs={}):
    '''computing token error'''

    cur_session = tf.keras.backend.get_session()
    target_seq_list, predict_seq_list = [], []

    is_py_sequence = True
    if isinstance(self.eval_ds, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
      eval_gen = self.eval_ds.make_one_shot_iterator()
      self.next_batch_gen = eval_gen.get_next()[0]
      is_py_sequence = False
    elif isinstance(self.eval_ds,
                    (iterator_ops.IteratorV2, iterator_ops.Iterator)):
      self.next_batch_gen = self.ds.get_next()[0]
      is_py_sequence = False

    for index in range(len(self.eval_task)):
      batch_data = None
      if is_py_sequence:
        batch_data = self.eval_ds[index][0]
      else:
        batch_data = cur_session.run(self.next_batch_gen)
      batch_input = batch_data['inputs']
      batch_target = batch_data['targets'].tolist()
      batch_predict = self.func(batch_input)[0]

      if self.decoder_type == 'argmax':
        predict_seq_list += py_ctc.ctc_greedy_decode(
            batch_predict, 0, unique=True)
      else:
        sequence_lens = [len(pre_sequence) for pre_sequence in batch_predict]
        batch_decoder, _ = tf_ctc.ctc_beam_search_decode(
            tf.constant(batch_predict),
            tf.constant(sequence_lens),
            beam_width=3,
            top_paths=3)
        predict_seq_list += cur_session.run(batch_decoder)[0].tolist()
      target_seq_list += batch_target

    val_token_errors = metrics_lib.token_error(
        predict_seq_list=predict_seq_list,
        target_seq_list=target_seq_list,
        eos_id=0)
    logs['val_token_err'] = val_token_errors

    if 'val_loss' in logs:
      logging.info("Epoch {}: on eval, val_loss is {}.".format(
          epoch + 1, logs['val_loss']))
    logging.info("Epoch {}: on eval, token_err is {}.".format(
        epoch + 1, val_token_errors))
    logging.info("Epoch {}: loss on train is {}".format(epoch + 1,
                                                        logs['loss'])) 

def on_epoch_end(self, epoch, logs={}):
    '''computing every class prec/rec'''

    cur_session = tf.keras.backend.get_session()
    truth, predict = [], []

    is_py_sequence = True
    if isinstance(self.eval_task, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
      eval_gen = self.eval_task.make_one_shot_iterator()
      self.next_batch_gen = eval_gen.get_next()
      is_py_sequence = False
    elif isinstance(self.eval_task,
                    (iterator_ops.IteratorV2, iterator_ops.Iterator)):
      self.next_batch_gen = self.ds.get_next()
      is_py_sequence = False

    for index in range(len(self.eval_task)):
      batch_data = None
      if is_py_sequence:
        batch_data, batch_truth = self.eval_task[index]
      else:
        batch_data = cur_session.run(self.next_batch_gen)
      #print("batch_data", batch_data)
      batch_input = batch_data
      batch_truth = batch_truth.tolist()

      text = self.model.get_layer('text').input
      speech = self.model.get_layer('speech').input
      y_pred = self.model(batch_input)
      f = K.function([text, speech], y_pred)
      batch_predict = f([batch_input['inputs'], batch_input['texts']])
      truth.extend(batch_truth)
      predict.extend(batch_predict)
    y_true = np.argmax(np.asarray(truth), axis=1)
    y_pred = np.argmax(np.asarray(predict), axis=1)
    accuracy = metrics.accuracy_score(y_true, y_pred)
    unw_accuracy = metrics.precision_score(y_true, y_pred, average='macro')
    logs['ClassReport'] = accuracy
    logging.info("Epoch {}: on eval.".format(
        epoch + 1))
    logging.info("Weighted accuracy: {}".format(accuracy))
    logging.info("Unweighted accuracy: {}".format(unw_accuracy))
    logging.info("Specific results: {}".format('\n' + metrics.classification_report(
      y_true, y_pred, digits=4)))