Python tensorflow.reduce_join() Examples

def get_words_from_chars(characters_list: List[str], sequence_lengths: List[int], name='chars_conversion'):
    with tf.name_scope(name=name):
        def join_charcaters_fn(coords):
            return tf.reduce_join(characters_list[coords[0]:coords[1]])

        def coords_several_sequences():
            end_coords = tf.cumsum(sequence_lengths)
            start_coords = tf.concat([[0], end_coords[:-1]], axis=0)
            coords = tf.stack([start_coords, end_coords], axis=1)
            coords = tf.cast(coords, dtype=tf.int32)
            return tf.map_fn(join_charcaters_fn, coords, dtype=tf.string)

        def coords_single_sequence():
            return tf.reduce_join(characters_list, keep_dims=True)

        words = tf.cond(tf.shape(sequence_lengths)[0] > 1,
                        true_fn=lambda: coords_several_sequences(),
                        false_fn=lambda: coords_single_sequence())

        return words 

def _mapper(example_proto):
  features = {
      'samples': tf.FixedLenSequenceFeature([1], tf.float32, allow_missing=True),
      'label': tf.FixedLenSequenceFeature([], tf.string, allow_missing=True)
  }
  example = tf.parse_single_example(example_proto, features)

  wav = example['samples'][:, 0]

  wav = wav[:16384]
  wav_len = tf.shape(wav)[0]
  wav = tf.pad(wav, [[0, 16384 - wav_len]])

  label = tf.reduce_join(example['label'], 0)

  return wav, label 

def get_words_from_chars(characters_list: List[str], sequence_lengths: List[int], name='chars_conversion'):
    with tf.name_scope(name=name):
        def join_charcaters_fn(coords):
            return tf.reduce_join(characters_list[coords[0]:coords[1]])

        def coords_several_sequences():
            end_coords = tf.cumsum(sequence_lengths)
            start_coords = tf.concat([[0], end_coords[:-1]], axis=0)
            coords = tf.stack([start_coords, end_coords], axis=1)
            coords = tf.cast(coords, dtype=tf.int32)
            return tf.map_fn(join_charcaters_fn, coords, dtype=tf.string)

        def coords_single_sequence():
            return tf.reduce_join(characters_list, keep_dims=True)

        words = tf.cond(tf.shape(sequence_lengths)[0] > 1,
                        true_fn=lambda: coords_several_sequences(),
                        false_fn=lambda: coords_single_sequence())

        return words 

def _joined_array(num_dims, reduce_dim):
  """Creates an ndarray with the result from reduce_join on input_array.

  Args:
    num_dims: The number of dimensions of the original input array.
    reduce_dim: The dimension to reduce.

  Returns:
    An ndarray of shape [2] * (num_dims - 1).
  """
  formatter = "{:0%db}" % (num_dims - 1)
  result = np.zeros(shape=[2] * (num_dims - 1), dtype="S%d" % (2 * num_dims))
  flat = result.ravel()
  for i in xrange(2 ** (num_dims - 1)):
    dims = formatter.format(i)
    flat[i] = "".join([(dims[:reduce_dim] + "%d" + dims[reduce_dim:]) % j
                       for j in xrange(2)])
  return result 

def _testReduceJoin(self, input_array, truth, truth_shape,
                      reduction_indices, keep_dims=False, separator=""):
    """Compares the output of reduce_join to an expected result.

    Args:
      input_array: The string input to be joined.
      truth: An array or np.array of the expected result.
      truth_shape: An array or np.array of the expected shape.
      reduction_indices: The indices to reduce over.
      keep_dims: Whether or not to retain reduced dimensions.
      separator: The separator to use for joining.
    """
    with self.test_session():
      output = tf.reduce_join(inputs=input_array,
                              reduction_indices=reduction_indices,
                              keep_dims=keep_dims,
                              separator=separator)
      output_array = output.eval()

    self.assertAllEqualUnicode(truth, output_array)
    self.assertAllEqual(truth_shape, output.get_shape()) 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
        self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def create_metric_ops(self, _inputs, labels, predictions):
    """Creates (value, update_op) tensors
    """
    with tf.variable_scope(self._name):

      # Join tokens into single strings
      predictions_flat = tf.reduce_join(
          predictions["predicted_tokens"], 1, separator=self._separator)
      labels_flat = tf.reduce_join(
          labels["target_tokens"], 1, separator=self._separator)

      sources_value, sources_update = accumulate_strings(
          values=predictions_flat, name="sources")
      targets_value, targets_update = accumulate_strings(
          values=labels_flat, name="targets")

      metric_value = tf.py_func(
          func=self._py_func,
          inp=[sources_value, targets_value],
          Tout=tf.float32,
          name="value")

    with tf.control_dependencies([sources_update, targets_update]):
      update_op = tf.identity(metric_value, name="update_op")

    return metric_value, update_op 

def labels_to_text(self, labels):
    """Convert labels to text strings.
    Args:
      labels: int32 tensor with shape [batch_size, max_label_length]
    Returns:
      text: string tensor with shape [batch_size]
    """
    if labels.dtype == tf.int32 or labels.dtype == tf.int64:
      labels = tf.cast(labels, tf.int64)
    else:
      raise ValueError('Wrong dtype of labels: {}'.format(labels.dtype))
    chars = self._label_to_char_table.lookup(labels)
    text = tf.reduce_join(chars, axis=1)
    return text 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
        self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def create_metric_ops(self, _inputs, labels, predictions):
    """Creates (value, update_op) tensors
    """
    with tf.variable_scope(self._name):

      # Join tokens into single strings
      predictions_flat = tf.reduce_join(
          predictions["predicted_tokens"], 1, separator=self._separator)
      labels_flat = tf.reduce_join(
          labels["target_tokens"], 1, separator=self._separator)

      sources_value, sources_update = accumulate_strings(
          values=predictions_flat, name="sources")
      targets_value, targets_update = accumulate_strings(
          values=labels_flat, name="targets")

      metric_value = tf.py_func(
          func=self._py_func,
          inp=[sources_value, targets_value],
          Tout=tf.float32,
          name="value")

    with tf.control_dependencies([sources_update, targets_update]):
      update_op = tf.identity(metric_value, name="update_op")

    return metric_value, update_op 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
        self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def lowercase(self, raw_post):
        split_chars = tf.string_split(tf.reshape(raw_post, [-1]), delimiter="").values
        upchar_inds = self.upchars_lut.lookup(split_chars)
        return tf.reduce_join(tf.map_fn(lambda x: tf.cond(x[0] > 25,
                                                          lambda: x[1],
                                                          lambda: self.lchars[x[0]]),
                                        (upchar_inds, split_chars), dtype=tf.string)) 

def labels_to_text(self, labels):
    """Convert labels to text strings.
    Args:
      labels: int32 tensor with shape [batch_size, max_label_length]
    Returns:
      text: string tensor with shape [batch_size]
    """
    if labels.dtype == tf.int32 or labels.dtype == tf.int64:
      labels = tf.cast(labels, tf.int64)
    else:
      raise ValueError('Wrong dtype of labels: {}'.format(labels.dtype))
    chars = self._label_to_char_table.lookup(labels)
    text = tf.reduce_join(chars, axis=1)
    return text 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
      self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
      self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def create_metric_ops(self, _inputs, labels, predictions):
    """Creates (value, update_op) tensors
    """
    with tf.variable_scope(self._name):

      # Join tokens into single strings
      predictions_flat = tf.reduce_join(
          predictions["predicted_tokens"], 1, separator=self._separator)
      labels_flat = tf.reduce_join(
          labels["target_tokens"], 1, separator=self._separator)

      sources_value, sources_update = accumulate_strings(
          values=predictions_flat, name="sources")
      targets_value, targets_update = accumulate_strings(
          values=labels_flat, name="targets")

      metric_value = tf.py_func(
          func=self._py_func,
          inp=[sources_value, targets_value],
          Tout=tf.float32,
          name="value")

    with tf.control_dependencies([sources_update, targets_update]):
      update_op = tf.identity(metric_value, name="update_op")

    return metric_value, update_op 

def create_metric_ops(self, _inputs, labels, predictions):
    """Creates (value, update_op) tensors
    """
    with tf.variable_scope(self._name):

      # Join tokens into single strings
      predictions_flat = tf.reduce_join(
          predictions["predicted_tokens"], 1, separator=self._separator)
      labels_flat = tf.reduce_join(
          labels["target_tokens"], 1, separator=self._separator)

      sources_value, sources_update = accumulate_strings(
          values=predictions_flat, name="sources")
      targets_value, targets_update = accumulate_strings(
          values=labels_flat, name="targets")

      metric_value = tf.py_func(
          func=self._py_func,
          inp=[sources_value, targets_value],
          Tout=tf.float32,
          name="value")

    with tf.control_dependencies([sources_update, targets_update]):
      update_op = tf.identity(metric_value, name="update_op")

    return metric_value, update_op 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
      self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def resize_sen(self, raw, mxlen):
        """
        Splits and rejoins a string to ensure that tokens meet
        the required max len.
        """
        raw_tokens = tf.string_split(tf.reshape(raw, [-1])).values
        # sentence length > mxlen
        raw_post = tf.reduce_join(raw_tokens[:mxlen], separator=" ")
        return raw_post 

def _convert_to_tokens(self, buffer_size):
        # The following 3 steps act as a python String lower() function
        # Split to characters
        self.text_set = self.text_set.map(lambda src, tgt:
                                          (tf.string_split([src], delimiter='').values,
                                           tf.string_split([tgt], delimiter='').values)
                                          ).prefetch(buffer_size)
        # Convert all upper case characters to lower case characters
        self.text_set = self.text_set.map(lambda src, tgt:
                                          (self.case_table.lookup(src), self.case_table.lookup(tgt))
                                          ).prefetch(buffer_size)
        # Join characters back to strings
        self.text_set = self.text_set.map(lambda src, tgt:
                                          (tf.reduce_join([src]), tf.reduce_join([tgt]))
                                          ).prefetch(buffer_size)

        # Split to word tokens
        self.text_set = self.text_set.map(lambda src, tgt:
                                          (tf.string_split([src]).values, tf.string_split([tgt]).values)
                                          ).prefetch(buffer_size)
        # Remove sentences longer than the model allows
        self.text_set = self.text_set.map(lambda src, tgt:
                                          (src[:self.src_max_len], tgt[:self.tgt_max_len])
                                          ).prefetch(buffer_size)

        # Reverse the source sentence if applicable
        if self.hparams.source_reverse:
            self.text_set = self.text_set.map(lambda src, tgt:
                                              (tf.reverse(src, axis=[0]), tgt)
                                              ).prefetch(buffer_size)

        # Convert the word strings to ids.  Word strings that are not in the vocab get
        # the lookup table's default_value integer.
        self.id_set = self.text_set.map(lambda src, tgt:
                                        (tf.cast(self.vocab_table.lookup(src), tf.int32),
                                         tf.cast(self.vocab_table.lookup(tgt), tf.int32))
                                        ).prefetch(buffer_size) 

def get_text(self, ids):
    """ Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
    """
    return tf.reduce_join(
        self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def testInvalidArgsUnknownIndices(self):
    with self.test_session():
      placeholder = tf.placeholder(tf.int32, name="placeholder")
      reduced = tf.reduce_join(["test", "test2"],
                               reduction_indices=placeholder)

      with self.assertRaisesOpError("reduction dimension -2"):
        reduced.eval(feed_dict={placeholder.name: -2})
      with self.assertRaisesOpError("reduction dimension 2"):
        reduced.eval(feed_dict={placeholder.name: 2}) 

def testInvalidArgsUnknownShape(self):
    with self.test_session():
      placeholder = tf.placeholder(tf.string, name="placeholder")
      index_too_high = tf.reduce_join(placeholder, reduction_indices=1)
      duplicate_index = tf.reduce_join(placeholder, reduction_indices=[-1, 1])
      with self.assertRaisesOpError("Invalid reduction dimension 1"):
        index_too_high.eval(feed_dict={placeholder.name: [""]})
      with self.assertRaisesOpError("Duplicate reduction dimension 1"):
        duplicate_index.eval(feed_dict={placeholder.name: [[""]]}) 

def testZeroDims(self):
    valid_truth_shape = [0]
    with self.test_session():
      inputs = np.zeros([0, 1], dtype=str)
      with self.assertRaisesRegexp(ValueError, "dimension 0 with size 0"):
        tf.reduce_join(inputs=inputs, reduction_indices=0)
      valid = tf.reduce_join(inputs=inputs, reduction_indices=1)
      valid_array_shape = valid.eval().shape
      self.assertAllEqualUnicode(valid_truth_shape, valid_array_shape) 

def testUnknownIndices(self):
    input_array = [["this", "is", "a", "test"],
                   ["please", "do", "not", "panic"]]
    truth_dim_zero = ["thisplease", "isdo", "anot", "testpanic"]
    truth_dim_one = ["thisisatest", "pleasedonotpanic"]
    truth_shape = None
    with self.test_session():
      placeholder = tf.placeholder(tf.int32, name="placeholder")
      reduced = tf.reduce_join(input_array, reduction_indices=placeholder)
      output_array_dim_zero = reduced.eval(feed_dict={placeholder.name: [0]})
      output_array_dim_one = reduced.eval(feed_dict={placeholder.name: [1]})
      self.assertAllEqualUnicode(truth_dim_zero, output_array_dim_zero)
      self.assertAllEqualUnicode(truth_dim_one, output_array_dim_one)
      self.assertAllEqual(truth_shape, reduced.get_shape()) 

def testUnknownShape(self):
    input_array = [["a"], ["b"]]
    truth = ["ab"]
    truth_shape = None
    with self.test_session():
      placeholder = tf.placeholder(tf.string, name="placeholder")
      reduced = tf.reduce_join(placeholder, reduction_indices=0)
      output_array = reduced.eval(feed_dict={placeholder.name: input_array})
      self.assertAllEqualUnicode(truth, output_array)
      self.assertAllEqual(truth_shape, reduced.get_shape()) 

def _testMultipleReduceJoin(self, input_array, reduction_indices,
                              separator=" "):
    """Tests reduce_join for one input and multiple reduction_indices.

    Does so by comparing the output to that from nested reduce_string_joins.
    The correctness of single-dimension reduce_join is verified by other
    tests below using _testReduceJoin.

    Args:
      input_array: The input to test.
      reduction_indices: The indices to reduce.
      separator: The separator to use when joining.
    """
    num_dims = len(input_array.shape)
    truth_red_indices = reduction_indices or list(reversed(xrange(num_dims)))
    with self.test_session():
      output = tf.reduce_join(
          inputs=input_array, reduction_indices=reduction_indices,
          keep_dims=False, separator=separator)
      output_keep_dims = tf.reduce_join(
          inputs=input_array, reduction_indices=reduction_indices,
          keep_dims=True, separator=separator)

      truth = input_array
      for index in truth_red_indices:
        truth = tf.reduce_join(
            inputs=truth, reduction_indices=index, keep_dims=True,
            separator=separator)
      truth_squeezed = tf.squeeze(truth, squeeze_dims=truth_red_indices)
      output_array = output.eval()
      output_keep_dims_array = output_keep_dims.eval()
      truth_array = truth.eval()
      truth_squeezed_array = truth_squeezed.eval()
    self.assertAllEqualUnicode(truth_array, output_keep_dims_array)
    self.assertAllEqualUnicode(truth_squeezed_array, output_array)
    self.assertAllEqual(truth.get_shape(), output_keep_dims.get_shape())
    self.assertAllEqual(truth_squeezed.get_shape(), output.get_shape()) 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
      self.table.lookup(tf.to_int64(ids)), reduction_indices=1) 

def record_to_xy(example_proto, labels):
  features = {
      'samples': tf.FixedLenSequenceFeature([1], tf.float32, allow_missing=True),
      'label': tf.FixedLenSequenceFeature([], tf.string, allow_missing=True)
  }
  example = tf.parse_single_example(example_proto, features)

  wav = example['samples'][:, 0]
  wav = wav[:16384]
  wav = tf.pad(wav, [[0, 16384 - tf.shape(wav)[0]]])
  wav.set_shape([16384])

  label_chars = example['label']
  # Truncate labels for TIMIT
  label_lens = [len(l) for l in labels]
  if len(set(label_lens)) == 1:
    label_chars = label_chars[:label_lens[0]]

  label = tf.reduce_join(label_chars, 0)
  label_id = tf.constant(0, dtype=tf.int32)
  nmatches = tf.constant(0)

  for i, label_candidate in enumerate(labels):
    match = tf.cast(tf.equal(label, label_candidate), tf.int32)
    label_id += i * match
    nmatches += match

  with tf.control_dependencies([tf.assert_equal(nmatches, 1)]):
    return wav, label_id 

def get_text(self, ids):
    """Returns a string corresponding to a sequence of character ids.

        Args:
          ids: a tensor with shape [batch_size, max_sequence_length]
        """
    return tf.reduce_join(
      self.table.lookup(tf.to_int64(ids)), reduction_indices=1)