Python tensorflow.compat.v1.string() Examples

def main(_):
  # Define schema.
  raw_metadata = dataset_metadata.DatasetMetadata(
      dataset_schema.from_feature_spec({
          'text': tf.FixedLenFeature([], tf.string),
          'language_code': tf.FixedLenFeature([], tf.string),
      }))

  # Add in padding tokens.
  reserved_tokens = FLAGS.reserved_tokens
  if FLAGS.num_pad_tokens:
    padded_tokens = ['<pad>']
    padded_tokens += ['<pad%d>' % i for i in range(1, FLAGS.num_pad_tokens)]
    reserved_tokens = padded_tokens + reserved_tokens

  params = learner.Params(FLAGS.upper_thresh, FLAGS.lower_thresh,
                          FLAGS.num_iterations, FLAGS.max_input_tokens,
                          FLAGS.max_token_length, FLAGS.max_unique_chars,
                          FLAGS.vocab_size, FLAGS.slack_ratio,
                          FLAGS.include_joiner_token, FLAGS.joiner,
                          reserved_tokens)

  generate_vocab(FLAGS.data_file, FLAGS.vocab_file, FLAGS.metrics_file,
                 raw_metadata, params) 

def _extract_dict_from_config(config, prefix, keys):
  """Return a subset of key/value pairs from `config` as a dict.

  Args:
    config: A Config object.
    prefix: A string to which `keys` are added to form keys in `config`.
    keys: The potential keys in the resulting dict.

  Returns:
    A dict with `key`/`value` pairs where `prefix + key` has value
    `value` in `config`.
  """
  subset = {}
  for key in keys:
    config_key = prefix + key
    subset[key] = config[config_key]
  return subset 

def create_border(video, color="blue", border_percent=2):
  """Creates a border around each frame to differentiate input and target.

  Args:
    video: 5-D NumPy array.
    color: string, "blue", "red" or "green".
    border_percent: Percentarge of the frame covered by the border.
  Returns:
    video: 5-D NumPy array.
  """
  # Do not create border if the video is not in RGB format
  if video.shape[-1] != 3:
    return video
  color_to_axis = {"blue": 2, "red": 0, "green": 1}
  axis = color_to_axis[color]
  _, _, height, width, _ = video.shape
  border_height = np.ceil(border_percent * height / 100.0).astype(np.int)
  border_width = np.ceil(border_percent * width / 100.0).astype(np.int)
  video[:, :, :border_height, :, axis] = 255
  video[:, :, -border_height:, :, axis] = 255
  video[:, :, :, :border_width, axis] = 255
  video[:, :, :, -border_width:, axis] = 255
  return video 

def eval_autoregressive(self, features=None, decode_length=50):
    """Autoregressive eval.

    Quadratic time in decode_length.

    Args:
      features: an map of string to `Tensor`
      decode_length: an integer.  How many additional timesteps to decode.

    Returns:
      logits: `Tensor`
      losses: a dictionary: {loss-name (string): floating point `Scalar`}.
          Contains a single key "training".
    """
    results = self._slow_greedy_infer(features, decode_length=decode_length)
    return results["logits"], results["losses"] 

def _encode_gif(images, fps):
  """Encodes numpy images into gif string.

  Args:
    images: A 4-D `uint8` `np.array` (or a list of 3-D images) of shape
      `[time, height, width, channels]` where `channels` is 1 or 3.
    fps: frames per second of the animation

  Returns:
    The encoded gif string.

  Raises:
    IOError: If the ffmpeg command returns an error.
  """
  writer = WholeVideoWriter(fps)
  writer.write_multi(images)
  return writer.finish() 

def _beam_decode(self,
                   features,
                   decode_length,
                   beam_size,
                   top_beams,
                   alpha,
                   use_tpu=False):
    """Beam search decoding.

    Models should ideally implement a more efficient version of this function.

    Args:
      features: an map of string to `Tensor`
      decode_length: an integer.  How many additional timesteps to decode.
      beam_size: number of beams.
      top_beams: an integer. How many of the beams to return.
      alpha: Float that controls the length penalty. larger the alpha, stronger
        the preference for longer translations.
      use_tpu: A bool, whether to do beam decode on TPU.

    Returns:
       samples: an integer `Tensor`. Top samples from the beam search
    """
    return self._beam_decode_slow(features, decode_length, beam_size, top_beams,
                                  alpha, use_tpu) 

def _greedy_infer(self, features, decode_length, use_tpu=False):
    """A greedy inference method.

    Models should ideally implement a more efficient version of this function.

    Args:
      features: an map of string to `Tensor`
      decode_length: an integer.  How many additional timesteps to decode.
      use_tpu: A bool, whether to build the inference graph for TPU.

    Returns:
      A dict of decoding results {
          "outputs": integer `Tensor` of decoded ids of shape
              [batch_size, <= decode_length] if beam_size == 1 or
              [batch_size, top_beams, <= decode_length]
          "scores": None
          "logits": `Tensor` of shape [batch_size, time, 1, 1, vocab_size].
          "losses": a dictionary: {loss-name (string): floating point `Scalar`}
      }
    """
    if use_tpu:
      return self._slow_greedy_infer_tpu(features, decode_length)
    return self._slow_greedy_infer(features, decode_length) 

def summarize_features(features, num_shards=1):
  """Generate summaries for features."""
  if not common_layers.should_generate_summaries():
    return

  with tf.name_scope("input_stats"):
    for (k, v) in sorted(six.iteritems(features)):
      if (isinstance(v, tf.Tensor) and (v.get_shape().ndims > 1) and
          (v.dtype != tf.string)):
        tf.summary.scalar("%s_batch" % k, tf.shape(v)[0] // num_shards)
        tf.summary.scalar("%s_length" % k, tf.shape(v)[1])
        nonpadding = tf.to_float(tf.not_equal(v, 0))
        nonpadding_tokens = tf.reduce_sum(nonpadding)
        tf.summary.scalar("%s_nonpadding_tokens" % k, nonpadding_tokens)
        tf.summary.scalar("%s_nonpadding_fraction" % k,
                          tf.reduce_mean(nonpadding)) 

def decode_jpeg(image_buffer, scope=None):  # , dtype=tf.float32):
  """Decode a JPEG string into one 3-D float image Tensor.

  Args:
    image_buffer: scalar string Tensor.
    scope: Optional scope for op_scope.
  Returns:
    3-D float Tensor with values ranging from [0, 1).
  """
  # with tf.op_scope([image_buffer], scope, 'decode_jpeg'):
  # with tf.name_scope(scope, 'decode_jpeg', [image_buffer]):
  with tf.name_scope(scope or 'decode_jpeg'):
    # Decode the string as an RGB JPEG.
    # Note that the resulting image contains an unknown height and width
    # that is set dynamically by decode_jpeg. In other words, the height
    # and width of image is unknown at compile-time.
    image = tf.image.decode_jpeg(image_buffer, channels=3,
                                 fancy_upscaling=False,
                                 dct_method='INTEGER_FAST')

    # image = tf.Print(image, [tf.shape(image)], 'Image shape: ')

    return image 

def __init__(self):
    # Create a single Session to run all image coding calls.
    self._sess = tf.Session()

    # Initializes function that converts PNG to JPEG data.
    self._png_data = tf.placeholder(dtype=tf.string)
    image = tf.image.decode_png(self._png_data, channels=3)
    self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100)

    # Initializes function that converts CMYK JPEG data to RGB JPEG data.
    self._cmyk_data = tf.placeholder(dtype=tf.string)
    image = tf.image.decode_jpeg(self._cmyk_data, channels=0)
    self._cmyk_to_rgb = tf.image.encode_jpeg(image, format='rgb', quality=100)

    # Initializes function that decodes RGB JPEG data.
    self._decode_jpeg_data = tf.placeholder(dtype=tf.string)
    self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3) 

def _is_cmyk(filename):
  """Determine if file contains a CMYK JPEG format image.

  Args:
    filename: string, path of the image file.

  Returns:
    boolean indicating if the image is a JPEG encoded with CMYK color space.
  """
  # File list from:
  # https://github.com/cytsai/ilsvrc-cmyk-image-list
  blacklist = ['n01739381_1309.JPEG', 'n02077923_14822.JPEG',
               'n02447366_23489.JPEG', 'n02492035_15739.JPEG',
               'n02747177_10752.JPEG', 'n03018349_4028.JPEG',
               'n03062245_4620.JPEG', 'n03347037_9675.JPEG',
               'n03467068_12171.JPEG', 'n03529860_11437.JPEG',
               'n03544143_17228.JPEG', 'n03633091_5218.JPEG',
               'n03710637_5125.JPEG', 'n03961711_5286.JPEG',
               'n04033995_2932.JPEG', 'n04258138_17003.JPEG',
               'n04264628_27969.JPEG', 'n04336792_7448.JPEG',
               'n04371774_5854.JPEG', 'n04596742_4225.JPEG',
               'n07583066_647.JPEG', 'n13037406_4650.JPEG']
  return filename.split('/')[-1] in blacklist 

def _find_human_readable_labels(synsets, synset_to_human):
  """Build a list of human-readable labels.

  Args:
    synsets: list of strings; each string is a unique WordNet ID.
    synset_to_human: dict of synset to human labels, e.g.,
      'n02119022' --> 'red fox, Vulpes vulpes'

  Returns:
    List of human-readable strings corresponding to each synset.
  """
  humans = []
  for s in synsets:
    assert s in synset_to_human, ('Failed to find: %s' % s)
    humans.append(synset_to_human[s])
  return humans 

def serving_input_receiver_fn():
  """Creates an input function for serving."""
  seq_len = FLAGS.max_seq_length
  serialized_example = tf.placeholder(
      dtype=tf.string, shape=[None], name="serialized_example")
  features = {
      "input_ids": tf.FixedLenFeature([seq_len], dtype=tf.int64),
      "input_mask": tf.FixedLenFeature([seq_len], dtype=tf.int64),
      "segment_ids": tf.FixedLenFeature([seq_len], dtype=tf.int64),
  }
  feature_map = tf.parse_example(serialized_example, features=features)
  feature_map["is_real_example"] = tf.constant(1, dtype=tf.int32)
  feature_map["label_ids"] = tf.constant(0, dtype=tf.int32)

  # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
  # So cast all int64 to int32.
  for name in feature_map.keys():
    t = feature_map[name]
    if t.dtype == tf.int64:
      t = tf.to_int32(t)
    feature_map[name] = t

  return tf.estimator.export.ServingInputReceiver(
      features=feature_map, receiver_tensors=serialized_example) 

def DecodeExample(self, serialized_example, item_handler, image_format):
    """Decodes the given serialized example with the specified item handler.

    Args:
      serialized_example: a serialized TF example string.
      item_handler: the item handler used to decode the image.
      image_format: the image format being decoded.

    Returns:
      the decoded image found in the serialized Example.
    """
    serialized_example = array_ops.reshape(serialized_example, shape=[])
    decoder = tfexample_decoder.TFExampleDecoder(
        keys_to_features={
            'image/encoded':
                parsing_ops.FixedLenFeature((), tf.string, default_value=''),
            'image/format':
                parsing_ops.FixedLenFeature((),
                                            tf.string,
                                            default_value=image_format),
        },
        items_to_handlers={'image': item_handler})
    [tf_image] = decoder.decode(serialized_example, ['image'])
    return tf_image 

def _get_feature(tensor_spec,
                 decode_images = True):
  """Get FixedLenfeature or FixedLenSequenceFeature for a tensor spec."""
  varlen_default_value = getattr(tensor_spec, 'varlen_default_value', None)
  if getattr(tensor_spec, 'is_sequence', False):
    cls = tf.FixedLenSequenceFeature
  elif varlen_default_value is not None:
    cls = tf.VarLenFeature
  else:
    cls = tf.FixedLenFeature
  if decode_images and is_encoded_image_spec(tensor_spec):
    if varlen_default_value is not None:
      # Contains a variable length list of images.
      return cls(tf.string)
    elif len(tensor_spec.shape) > 3:
      # Contains a fixed length list of images.
      return cls((tensor_spec.shape[0]), tf.string)
    else:
      return cls((), tf.string)
  elif varlen_default_value is not None:
    return cls(tensor_spec.dtype)
  else:
    return cls(tensor_spec.shape, tensor_spec.dtype) 

def filter_required_flat_tensor_spec(flat_tensor_spec):
  """Process a flat tensor spec structure and return only the required subset.

  Args:
    flat_tensor_spec: A flattened sequence (result of flatten_spec_structure)
      with the joined string paths as OrderedDict. Since we use OrderedDicts we
      can safely call flatten_spec_structure multiple times.

  Raises:
    ValueError: If the passed flat_tensor_spec is not a valid flat tensor_spec
      structure.

  Returns:
    filtered_flat_required_tensor_spec: The same flattened sequence but only
      the {key: tensor_spec} pairs for the non optional tensor_spec.
  """
  if not is_flat_spec_or_tensors_structure(flat_tensor_spec):
    raise ValueError('Only flat tensor_spec structures are allowed.')
  filtered_flat_required_tensor_spec = TensorSpecStruct()
  for key, value in flat_tensor_spec.items():
    if hasattr(value, 'is_optional') and value.is_optional:
      continue
    filtered_flat_required_tensor_spec[key] = value
  return filtered_flat_required_tensor_spec 

def test_tensorspec_to_feature_dict(self):
    features, tensor_spec_dict = utils.tensorspec_to_feature_dict(
        mock_nested_subset_spec, decode_images=True)
    self.assertDictEqual(tensor_spec_dict, {
        'images': T1,
        'actions': T2,
    })
    self.assertDictEqual(
        features, {
            'images': tf.FixedLenFeature((), tf.string),
            'actions': tf.FixedLenFeature(T2.shape, T2.dtype),
        })
    features, tensor_spec_dict = utils.tensorspec_to_feature_dict(
        mock_nested_subset_spec, decode_images=False)
    self.assertDictEqual(tensor_spec_dict, {
        'images': T1,
        'actions': T2,
    })
    self.assertDictEqual(
        features, {
            'images': tf.FixedLenFeature(T1.shape, T1.dtype),
            'actions': tf.FixedLenFeature(T2.shape, T2.dtype),
        }) 

def parse(data_dict):
  """Parse dataset from _data_gen into the same format as sst_binary."""
  sentiment = data_dict['label']
  sentence = data_dict['sentence']
  dense_chars = tf.decode_raw(sentence, tf.uint8)
  dense_chars.set_shape((None,))
  chars = tfp.math.dense_to_sparse(dense_chars)
  if six.PY3:
    safe_chr = lambda c: '?' if c >= 128 else chr(c)
  else:
    safe_chr = chr
  to_char = np.vectorize(safe_chr)
  chars = tf.SparseTensor(indices=chars.indices,
                          values=tf.py_func(to_char, [chars.values], tf.string),
                          dense_shape=chars.dense_shape)
  return {'sentiment': sentiment,
          'sentence': chars} 

def __init__(self, include_mask=False, regenerate_source_id=False):
    self._include_mask = include_mask
    self._regenerate_source_id = regenerate_source_id
    self._keys_to_features = {
        'image/encoded': tf.FixedLenFeature((), tf.string),
        'image/source_id': tf.FixedLenFeature((), tf.string, ''),
        'image/height': tf.FixedLenFeature((), tf.int64, -1),
        'image/width': tf.FixedLenFeature((), tf.int64, -1),
        'image/object/bbox/xmin': tf.VarLenFeature(tf.float32),
        'image/object/bbox/xmax': tf.VarLenFeature(tf.float32),
        'image/object/bbox/ymin': tf.VarLenFeature(tf.float32),
        'image/object/bbox/ymax': tf.VarLenFeature(tf.float32),
        'image/object/class/label': tf.VarLenFeature(tf.int64),
        'image/object/area': tf.VarLenFeature(tf.float32),
        'image/object/is_crowd': tf.VarLenFeature(tf.int64),
    }
    if include_mask:
      self._keys_to_features.update({
          'image/object/mask':
              tf.VarLenFeature(tf.string),
      }) 

def transform_wav_data_op(wav_data_tensor, hparams, jitter_amount_sec):
  """Transforms with audio for data augmentation. Only for training."""

  def transform_wav_data(wav_data):
    """Transforms with sox."""
    if jitter_amount_sec:
      wav_data = audio_io.jitter_wav_data(wav_data, hparams.sample_rate,
                                          jitter_amount_sec)
    wav_data = audio_transform.transform_wav_audio(wav_data, hparams)

    return [wav_data]

  return tf.py_func(
      transform_wav_data, [wav_data_tensor],
      tf.string,
      name='transform_wav_data_op') 

def vector_to_svg(vectors, stop_at_eos=False, categorical=False):
  """Tranforms a given vector to an svg string."""
  new_path = []
  for vector in vectors:
    if stop_at_eos:
      if categorical:
        try:
          is_eos = np.argmax(vector[:len(CMDS_LIST) + 1]) == 0
        except:
          raise Exception(vector)
      else:
        is_eos = vector[0] < -0.5

      if is_eos:
        break
    new_path.append(' '.join(vector_to_cmd(vector, categorical=categorical)))
  new_path = ' '.join(new_path)
  return SVG_PREFIX_BIG + PATH_PREFIX_1 + new_path + PATH_POSFIX_1 + SVG_POSFIX 

def add_missing_cmd(command_list):
  """Adds missing cmd tags to the given command list."""
  # E.g.: given:
  #   ['a', '0', '0', '0', '0', '0', '0', '0',
  #         '0', '0', '0', '0', '0', '0', '0']
  # Converts to:
  #   [['a', '0', '0', '0', '0', '0', '0', '0'],
  #    ['a', '0', '0', '0', '0', '0', '0', '0']]
  # And returns a string that joins these elements with spaces.
  cmd_tag = command_list[0]
  args = command_list[1:]

  final_cmds = []
  for arg_batch in grouper(args, NUM_ARGS[cmd_tag]):
    final_cmds.append([cmd_tag] + list(arg_batch))

  if not final_cmds:
    # command has no args (e.g.: 'z')
    final_cmds = [[cmd_tag]]

  return final_cmds 

def _find_image_bounding_boxes(filenames, image_to_bboxes):
  """Find the bounding boxes for a given image file.

  Args:
    filenames: list of strings; each string is a path to an image file.
    image_to_bboxes: dictionary mapping image file names to a list of
      bounding boxes. This list contains 0+ bounding boxes.
  Returns:
    List of bounding boxes for each image. Note that each entry in this
    list might contain from 0+ entries corresponding to the number of bounding
    box annotations for the image.
  """
  num_image_bbox = 0
  bboxes = []
  for f in filenames:
    basename = os.path.basename(f)
    if basename in image_to_bboxes:
      bboxes.append(image_to_bboxes[basename])
      num_image_bbox += 1
    else:
      bboxes.append([])
  print('Found %d images with bboxes out of %d images' % (
      num_image_bbox, len(filenames)))
  return bboxes 

def make_text_summary_value(svg, tag):
  """Converts the given str to a text tf.summary.Summary.Value."""
  svg_proto = tf.make_tensor_proto(svg, tf.string)
  value = tf.summary.Summary.Value(tag=tag, tensor=svg_proto)
  value.metadata.plugin_data.plugin_name = 'text'
  return value 

def parse_preprocessed_example(example_proto):
  """Process an already preprocessed Example proto into input tensors."""
  features = {
      'spec': tf.VarLenFeature(dtype=tf.float32),
      'spectrogram_hash': tf.FixedLenFeature(shape=(), dtype=tf.int64),
      'labels': tf.VarLenFeature(dtype=tf.float32),
      'label_weights': tf.VarLenFeature(dtype=tf.float32),
      'length': tf.FixedLenFeature(shape=(), dtype=tf.int64),
      'onsets': tf.VarLenFeature(dtype=tf.float32),
      'offsets': tf.VarLenFeature(dtype=tf.float32),
      'velocities': tf.VarLenFeature(dtype=tf.float32),
      'sequence_id': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'note_sequence': tf.FixedLenFeature(shape=(), dtype=tf.string),
  }
  record = tf.parse_single_example(example_proto, features)
  input_tensors = InputTensors(
      spec=tf.sparse.to_dense(record['spec']),
      spectrogram_hash=record['spectrogram_hash'],
      labels=tf.sparse.to_dense(record['labels']),
      label_weights=tf.sparse.to_dense(record['label_weights']),
      length=record['length'],
      onsets=tf.sparse.to_dense(record['onsets']),
      offsets=tf.sparse.to_dense(record['offsets']),
      velocities=tf.sparse.to_dense(record['velocities']),
      sequence_id=record['sequence_id'],
      note_sequence=record['note_sequence'])
  return input_tensors 

def parse_example(example_proto):
  features = {
      'id': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'sequence': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'audio': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'velocity_range': tf.FixedLenFeature(shape=(), dtype=tf.string),
  }
  record = tf.parse_single_example(example_proto, features)
  return record 

def truncate_note_sequence_op(sequence_tensor, truncated_length_frames,
                              hparams):
  """Truncates a NoteSequence to the given length."""
  def truncate(sequence_tensor, num_frames):
    sequence = music_pb2.NoteSequence.FromString(sequence_tensor)
    num_secs = num_frames / hparams_frames_per_second(hparams)
    return truncate_note_sequence(sequence, num_secs).SerializeToString()
  res = tf.py_func(
      truncate,
      [sequence_tensor, truncated_length_frames],
      tf.string)
  res.set_shape(())
  return res 

def jitter_label_op(sequence_tensor, jitter_amount_sec):

  def jitter_label(sequence_tensor):
    sequence = music_pb2.NoteSequence.FromString(sequence_tensor)
    sequence = sequences_lib.shift_sequence_times(sequence, jitter_amount_sec)
    return sequence.SerializeToString()

  return tf.py_func(jitter_label, [sequence_tensor], tf.string) 

def wav_to_num_frames_op(wav_audio, frames_per_second):
  """Transforms a wav-encoded audio string into number of frames."""
  res = tf.py_func(
      functools.partial(wav_to_num_frames, frames_per_second=frames_per_second),
      [wav_audio],
      tf.int32,
      name='wav_to_num_frames_op')
  res.set_shape(())
  return res 

def main(_):
  # Generate schema of input data.
  raw_metadata = dataset_metadata.DatasetMetadata(
      dataset_schema.from_feature_spec({
          'text': tf.FixedLenFeature([], tf.string),
          'language_code': tf.FixedLenFeature([], tf.string),
      }))

  pipeline = word_count(FLAGS.input_path, FLAGS.output_path, raw_metadata)
  pipeline.run().wait_until_finish()