Python tensorflow.compat.v1.FixedLenFeature() Examples

def __call__(self, example_string):
    """Processes a single example string.

    Extracts and processes the feature, and ignores the label.

    Args:
      example_string: str, an Example protocol buffer.

    Returns:
      feat: The feature tensor.
    """
    feat = tf.parse_single_example(
        example_string,
        features={
            'image/embedding':
                tf.FixedLenFeature([self.feat_len], dtype=tf.float32),
            'image/class/label':
                tf.FixedLenFeature([], tf.int64)
        })['image/embedding']

    return feat 

def example_reading_spec(self):
    extra_data_fields, extra_data_items_to_decoders = self.extra_reading_spec

    data_fields = {
        "image/encoded": tf.FixedLenFeature((), tf.string),
        "image/format": tf.FixedLenFeature((), tf.string),
    }
    data_fields.update(extra_data_fields)

    data_items_to_decoders = {
        "frame":
            contrib.slim().tfexample_decoder.Image(
                image_key="image/encoded",
                format_key="image/format",
                shape=[self.frame_height, self.frame_width, self.num_channels],
                channels=self.num_channels),
    }
    data_items_to_decoders.update(extra_data_items_to_decoders)

    return data_fields, data_items_to_decoders 

def _get_feature(tensor_spec,
                 decode_images = True):
  """Get FixedLenfeature or FixedLenSequenceFeature for a tensor spec."""
  varlen_default_value = getattr(tensor_spec, 'varlen_default_value', None)
  if getattr(tensor_spec, 'is_sequence', False):
    cls = tf.FixedLenSequenceFeature
  elif varlen_default_value is not None:
    cls = tf.VarLenFeature
  else:
    cls = tf.FixedLenFeature
  if decode_images and is_encoded_image_spec(tensor_spec):
    if varlen_default_value is not None:
      # Contains a variable length list of images.
      return cls(tf.string)
    elif len(tensor_spec.shape) > 3:
      # Contains a fixed length list of images.
      return cls((tensor_spec.shape[0]), tf.string)
    else:
      return cls((), tf.string)
  elif varlen_default_value is not None:
    return cls(tensor_spec.dtype)
  else:
    return cls(tensor_spec.shape, tensor_spec.dtype) 

def example_reading_spec(self):
    data_fields, data_items_to_decoders = (
        super(ImageVqav2Tokens10kLabels3k, self).example_reading_spec())
    data_fields["image/image_id"] = tf.FixedLenFeature((), tf.int64)
    data_fields["image/question_id"] = tf.FixedLenFeature((), tf.int64)
    data_fields["image/question"] = tf.FixedLenSequenceFeature(
        (), tf.int64, allow_missing=True)
    data_fields["image/answer"] = tf.FixedLenSequenceFeature(
        (), tf.int64, allow_missing=True)

    slim = contrib.slim()
    data_items_to_decoders["question"] = slim.tfexample_decoder.Tensor(
        "image/question")
    data_items_to_decoders["targets"] = slim.tfexample_decoder.Tensor(
        "image/answer")
    return data_fields, data_items_to_decoders 

def example_reading_spec(self):
    """Return a mix of env and video data fields and decoders."""
    slim = contrib.slim()
    video_fields, video_decoders = (
        video_utils.VideoProblem.example_reading_spec(self))
    env_fields, env_decoders = (
        gym_env_problem.GymEnvProblem.example_reading_spec(self))

    # Remove raw observations field since we want to capture them as videos.
    env_fields.pop(env_problem.OBSERVATION_FIELD)
    env_decoders.pop(env_problem.OBSERVATION_FIELD)

    # Add frame number spec and decoder.
    env_fields[_FRAME_NUMBER_FIELD] = tf.FixedLenFeature((1,), tf.int64)
    env_decoders[_FRAME_NUMBER_FIELD] = slim.tfexample_decoder.Tensor(
        _FRAME_NUMBER_FIELD)

    # Add video fields and decoders
    env_fields.update(video_fields)
    env_decoders.update(video_decoders)
    return env_fields, env_decoders 

def serving_input_receiver_fn():
  """Creates an input function for serving."""
  seq_len = FLAGS.max_seq_length
  serialized_example = tf.placeholder(
      dtype=tf.string, shape=[None], name="serialized_example")
  features = {
      "input_ids": tf.FixedLenFeature([seq_len], dtype=tf.int64),
      "input_mask": tf.FixedLenFeature([seq_len], dtype=tf.int64),
      "segment_ids": tf.FixedLenFeature([seq_len], dtype=tf.int64),
  }
  feature_map = tf.parse_example(serialized_example, features=features)
  feature_map["is_real_example"] = tf.constant(1, dtype=tf.int32)
  feature_map["label_ids"] = tf.constant(0, dtype=tf.int32)

  # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
  # So cast all int64 to int32.
  for name in feature_map.keys():
    t = feature_map[name]
    if t.dtype == tf.int64:
      t = tf.to_int32(t)
    feature_map[name] = t

  return tf.estimator.export.ServingInputReceiver(
      features=feature_map, receiver_tensors=serialized_example) 

def __init__(self, include_mask=False, regenerate_source_id=False):
    self._include_mask = include_mask
    self._regenerate_source_id = regenerate_source_id
    self._keys_to_features = {
        'image/encoded': tf.FixedLenFeature((), tf.string),
        'image/source_id': tf.FixedLenFeature((), tf.string, ''),
        'image/height': tf.FixedLenFeature((), tf.int64, -1),
        'image/width': tf.FixedLenFeature((), tf.int64, -1),
        'image/object/bbox/xmin': tf.VarLenFeature(tf.float32),
        'image/object/bbox/xmax': tf.VarLenFeature(tf.float32),
        'image/object/bbox/ymin': tf.VarLenFeature(tf.float32),
        'image/object/bbox/ymax': tf.VarLenFeature(tf.float32),
        'image/object/class/label': tf.VarLenFeature(tf.int64),
        'image/object/area': tf.VarLenFeature(tf.float32),
        'image/object/is_crowd': tf.VarLenFeature(tf.int64),
    }
    if include_mask:
      self._keys_to_features.update({
          'image/object/mask':
              tf.VarLenFeature(tf.string),
      }) 

def test_tensorspec_to_feature_dict(self):
    features, tensor_spec_dict = utils.tensorspec_to_feature_dict(
        mock_nested_subset_spec, decode_images=True)
    self.assertDictEqual(tensor_spec_dict, {
        'images': T1,
        'actions': T2,
    })
    self.assertDictEqual(
        features, {
            'images': tf.FixedLenFeature((), tf.string),
            'actions': tf.FixedLenFeature(T2.shape, T2.dtype),
        })
    features, tensor_spec_dict = utils.tensorspec_to_feature_dict(
        mock_nested_subset_spec, decode_images=False)
    self.assertDictEqual(tensor_spec_dict, {
        'images': T1,
        'actions': T2,
    })
    self.assertDictEqual(
        features, {
            'images': tf.FixedLenFeature(T1.shape, T1.dtype),
            'actions': tf.FixedLenFeature(T2.shape, T2.dtype),
        }) 

def main(_):
  # Define schema.
  raw_metadata = dataset_metadata.DatasetMetadata(
      dataset_schema.from_feature_spec({
          'text': tf.FixedLenFeature([], tf.string),
          'language_code': tf.FixedLenFeature([], tf.string),
      }))

  # Add in padding tokens.
  reserved_tokens = FLAGS.reserved_tokens
  if FLAGS.num_pad_tokens:
    padded_tokens = ['<pad>']
    padded_tokens += ['<pad%d>' % i for i in range(1, FLAGS.num_pad_tokens)]
    reserved_tokens = padded_tokens + reserved_tokens

  params = learner.Params(FLAGS.upper_thresh, FLAGS.lower_thresh,
                          FLAGS.num_iterations, FLAGS.max_input_tokens,
                          FLAGS.max_token_length, FLAGS.max_unique_chars,
                          FLAGS.vocab_size, FLAGS.slack_ratio,
                          FLAGS.include_joiner_token, FLAGS.joiner,
                          reserved_tokens)

  generate_vocab(FLAGS.data_file, FLAGS.vocab_file, FLAGS.metrics_file,
                 raw_metadata, params) 

def parse_example(serialized_example):
  """Parse example."""
  features = tf.parse_single_example(
      serialized_example,
      features={
          "question":
              tf.FixedLenFeature([], tf.string),
          "context":
              tf.FixedLenSequenceFeature(
                  dtype=tf.string, shape=[], allow_missing=True),
          "long_answer_indices":
              tf.FixedLenSequenceFeature(
                  dtype=tf.int64, shape=[], allow_missing=True)
      })
  features["question"] = features["question"]
  features["context"] = features["context"]
  features["long_answer_indices"] = tf.to_int32(features["long_answer_indices"])
  return features 

def _parse_fn(self, value):
    """Parses an image and its label from a serialized TFExample.

    Args:
      value: serialized string containing an TFExample.

    Returns:
      Returns a tuple of (image, label) from the TFExample.
    """
    if FLAGS.get_flag_value('pseudo_label_key', None):
      self.ORIGINAL_LABEL_KEY = FLAGS.get_flag_value(
          'original_label_key', None)
      assert self.ORIGINAL_LABEL_KEY is not None, (
          'You must set original_label_key for pseudo labeling.')

      #Replace original label_key with pseudo_label_key.
      self.LABEL_KEY = FLAGS.get_flag_value('pseudo_label_key', None)
      self.FEATURE_MAP.update({
          self.LABEL_KEY: tf.FixedLenFeature(shape=[], dtype=tf.int64),
          self.ORIGINAL_LABEL_KEY: tf.FixedLenFeature(
              shape=[], dtype=tf.int64),
          self.FLAG_KEY: tf.FixedLenFeature(shape=[], dtype=tf.int64),
      })
    return tf.parse_single_example(value, self.FEATURE_MAP) 

def parse_example(serialized_example):
  """Parse example."""
  features = tf.parse_single_example(
      serialized_example,
      features={
          "question": tf.FixedLenFeature([], tf.string),
          "context": tf.FixedLenFeature([], tf.string),
          "answer_start": tf.FixedLenFeature([], tf.int64),
          "answer_end": tf.FixedLenFeature([], tf.int64),
      })
  features["question"] = split_on_whitespace(features["question"])
  features["context"] = split_on_whitespace(features["context"])
  features["answer_start"] = tf.to_int32(features["answer_start"])
  features["answer_end"] = tf.to_int32(features["answer_end"])
  return features 

def main(_):
  # Generate schema of input data.
  raw_metadata = dataset_metadata.DatasetMetadata(
      dataset_schema.from_feature_spec({
          'text': tf.FixedLenFeature([], tf.string),
          'language_code': tf.FixedLenFeature([], tf.string),
      }))

  pipeline = word_count(FLAGS.input_path, FLAGS.output_path, raw_metadata)
  pipeline.run().wait_until_finish() 

def example_reading_spec(self):
    data_fields, data_items_to_decoders = (
        super(BabiQa, self).example_reading_spec())
    data_fields["targets"] = tf.FixedLenFeature([1], tf.int64)
    return (data_fields, data_items_to_decoders) 

def decode(self, serialized_example, items=None):
    """Decodes the given serialized TF-example."""
    context, sequence = tf.parse_single_sequence_example(
        serialized_example, self._context_keys_to_features,
        self._sequence_keys_to_features)

    # Merge context and sequence features
    example = {}
    example.update(context)
    example.update(sequence)

    all_features = {}
    all_features.update(self._context_keys_to_features)
    all_features.update(self._sequence_keys_to_features)

    # Reshape non-sparse elements just once:
    for k, value in all_features.items():
      if isinstance(value, tf.FixedLenFeature):
        example[k] = tf.reshape(example[k], value.shape)

    if not items:
      items = list(self._items_to_handlers.keys())

    outputs = []
    for item in items:
      handler = self._items_to_handlers[item]
      keys_to_tensors = {key: example[key] for key in handler.keys}
      outputs.append(handler.tensors_to_item(keys_to_tensors))

    return outputs 

def example_reading_spec(self):
    data_fields = {
        "image/encoded": tf.FixedLenFeature((), tf.string),
        "image/format": tf.FixedLenFeature((), tf.string),
    }

    data_items_to_decoders = {
        "inputs":
            contrib.slim().tfexample_decoder.Image(
                image_key="image/encoded",
                format_key="image/format",
                channels=self.num_channels),
    }

    return data_fields, data_items_to_decoders 

def build_input(tfrecord_paths):
  """Builds the graph's input.

  Args:
    tfrecord_paths: List of paths to the input TFRecords

  Returns:
    serialized_example_tensor: The next serialized example. String scalar Tensor
    image_tensor: The decoded image of the example. Uint8 tensor,
        shape=[1, None, None,3]
  """
  filename_queue = tf.train.string_input_producer(
      tfrecord_paths, shuffle=False, num_epochs=1)

  tf_record_reader = tf.TFRecordReader()
  _, serialized_example_tensor = tf_record_reader.read(filename_queue)
  features = tf.parse_single_example(
      serialized_example_tensor,
      features={
          standard_fields.TfExampleFields.image_encoded:
              tf.FixedLenFeature([], tf.string),
      })
  encoded_image = features[standard_fields.TfExampleFields.image_encoded]
  image_tensor = tf.image.decode_image(encoded_image, channels=3)
  image_tensor.set_shape([None, None, 3])
  image_tensor = tf.expand_dims(image_tensor, 0)

  return serialized_example_tensor, image_tensor 

def decode(self, serialized_example, items=None):
    """Decodes the given serialized TF-SequenceExample.

    Args:
      serialized_example: A serialized TF-SequenceExample tensor.
      items: The list of items to decode. These must be a subset of the item
        keys in self._items_to_handlers. If `items` is left as None, then all
        of the items in self._items_to_handlers are decoded.
    Returns:
      The decoded items, a list of tensor.
    """
    context, feature_list = tf.parse_single_sequence_example(
        serialized_example, self._keys_to_context_features,
        self._keys_to_sequence_features)
    # Reshape non-sparse elements just once:
    for k in self._keys_to_context_features:
      v = self._keys_to_context_features[k]
      if isinstance(v, tf.FixedLenFeature):
        context[k] = tf.reshape(context[k], v.shape)
    if not items:
      items = self._items_to_handlers.keys()
    outputs = []
    for item in items:
      handler = self._items_to_handlers[item]
      keys_to_tensors = {
          key: context[key] if key in context else feature_list[key]
          for key in handler.keys
      }
      outputs.append(handler.tensors_to_item(keys_to_tensors))
    return outputs 

def read_single_example(example_string):
  """Parses the record string."""
  return tf.parse_single_example(
      example_string,
      features={
          'image': tf.FixedLenFeature([], dtype=tf.string),
          'label': tf.FixedLenFeature([], tf.int64)
      }) 

def example_reading_spec(self):
    label_key = "image/class/label"
    data_fields, data_items_to_decoders = (
        super(Video2ClassProblem, self).example_reading_spec())
    data_fields[label_key] = tf.FixedLenFeature((1,), tf.int64)
    data_items_to_decoders["targets"] = contrib.slim().tfexample_decoder.Tensor(
        label_key)
    return data_fields, data_items_to_decoders 

def tensorspec_to_feature_dict(tensor_spec_struct, decode_images = True):
  """Converts collection of tensorspecs to a dict of FixedLenFeatures specs.

  Args:
    tensor_spec_struct: A (possibly nested) collection of TensorSpec.
    decode_images: If True, TensorSpec with data_format 'JPEG' or 'PNG' are
      interpreted as encoded image strings.

  Returns:
    features: A dict mapping feature keys to FixedLenFeature and
      FixedLenSequenceFeature values.

  Raises:
    ValueError: If duplicate keys are found in the TensorSpecs.
  """
  assert_valid_spec_structure(tensor_spec_struct)
  features = {}
  tensor_spec_dict = {}

  # Note it is valid to iterate over all tensors since
  # assert_valid_spec_structure will ensure that non unique tensor_spec names
  # have the identical properties.
  flat_tensor_spec_struct = flatten_spec_structure(tensor_spec_struct)
  for key, tensor_spec in flat_tensor_spec_struct.items():
    if tensor_spec.name is None:
      # Do not attempt to parse TensorSpecs whose name attribute is not set.
      logging.info(
          'TensorSpec name attribute for %s is not set; will not parse this '
          'Tensor from TFExamples.', key)
      continue
    features[tensor_spec.name] = _get_feature(tensor_spec, decode_images)
    tensor_spec_dict[tensor_spec.name] = tensor_spec
  return features, tensor_spec_dict 

def parse_and_preprocess(self, value, batch_position):
    """Parse an TFRecord."""
    del batch_position
    assert self.supports_datasets()
    context_features = {
        'labels': tf.VarLenFeature(dtype=tf.int64),
        'input_length': tf.FixedLenFeature([], dtype=tf.int64),
        'label_length': tf.FixedLenFeature([], dtype=tf.int64),
    }
    sequence_features = {
        'features': tf.FixedLenSequenceFeature([161], dtype=tf.float32)
    }
    context_parsed, sequence_parsed = tf.parse_single_sequence_example(
        serialized=value,
        context_features=context_features,
        sequence_features=sequence_features,
    )

    return [
        # Input
        tf.expand_dims(sequence_parsed['features'], axis=2),
        # Label
        tf.cast(
            tf.reshape(
                tf.sparse_tensor_to_dense(context_parsed['labels']), [-1]),
            dtype=tf.int32),
        # Input length
        tf.cast(
            tf.reshape(context_parsed['input_length'], [1]),
            dtype=tf.int32),
        # Label length
        tf.cast(
            tf.reshape(context_parsed['label_length'], [1]),
            dtype=tf.int32),
    ] 

def _references_content(ref_files):
  """Returns dict<str ref_url, str ref_content>."""
  example_spec = {
      "url": tf.FixedLenFeature([], tf.string),
      "content": tf.FixedLenFeature([], tf.string),
  }
  data = {}
  for ex in generator_utils.tfrecord_iterator(
      ref_files, gzipped=True, example_spec=example_spec):
    data[ex["url"]] = text_encoder.to_unicode(ex["content"])
  return data 

def extra_reading_spec(self):
    """Additional data fields to store on disk and their decoders."""
    data_fields = {
        "frame_number": tf.FixedLenFeature([1], tf.int64),
        "action": tf.FixedLenFeature([4], tf.float32),
    }
    decoders = {
        "frame_number":
            contrib.slim().tfexample_decoder.Tensor(tensor_key="frame_number"),
        "action":
            contrib.slim().tfexample_decoder.Tensor(tensor_key="action"),
    }
    return data_fields, decoders 

def parse_preprocessed_example(example_proto):
  """Process an already preprocessed Example proto into input tensors."""
  features = {
      'spec': tf.VarLenFeature(dtype=tf.float32),
      'spectrogram_hash': tf.FixedLenFeature(shape=(), dtype=tf.int64),
      'labels': tf.VarLenFeature(dtype=tf.float32),
      'label_weights': tf.VarLenFeature(dtype=tf.float32),
      'length': tf.FixedLenFeature(shape=(), dtype=tf.int64),
      'onsets': tf.VarLenFeature(dtype=tf.float32),
      'offsets': tf.VarLenFeature(dtype=tf.float32),
      'velocities': tf.VarLenFeature(dtype=tf.float32),
      'sequence_id': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'note_sequence': tf.FixedLenFeature(shape=(), dtype=tf.string),
  }
  record = tf.parse_single_example(example_proto, features)
  input_tensors = InputTensors(
      spec=tf.sparse.to_dense(record['spec']),
      spectrogram_hash=record['spectrogram_hash'],
      labels=tf.sparse.to_dense(record['labels']),
      label_weights=tf.sparse.to_dense(record['label_weights']),
      length=record['length'],
      onsets=tf.sparse.to_dense(record['onsets']),
      offsets=tf.sparse.to_dense(record['offsets']),
      velocities=tf.sparse.to_dense(record['velocities']),
      sequence_id=record['sequence_id'],
      note_sequence=record['note_sequence'])
  return input_tensors 

def parse_example(example_proto):
  features = {
      'id': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'sequence': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'audio': tf.FixedLenFeature(shape=(), dtype=tf.string),
      'velocity_range': tf.FixedLenFeature(shape=(), dtype=tf.string),
  }
  record = tf.parse_single_example(example_proto, features)
  return record 

def example_reading_spec(self):
    data_fields = {'targets_rel': tf.FixedLenFeature([51*10], tf.float32),
                   'targets_rnd': tf.FixedLenFeature([64*64], tf.float32),
                   'targets_sln': tf.FixedLenFeature([1], tf.int64),
                   'targets_cls': tf.FixedLenFeature([1], tf.int64)}

    data_items_to_decoders = None
    return (data_fields, data_items_to_decoders) 

def parse_function(image_size, raw_image_key_name):
  """Generate parse function for parsing the TFRecord training dataset.

  Read the image example and resize it to desired size.

  Args:
    image_size: int, target size to resize the image to
    raw_image_key_name: str, name of the JPEG image in each TFRecord entry

  Returns:
    A map function to use with tf.data.Dataset.map() .
  """

  def func(example_proto):
    """A generator to be used as representative_dataset for TFLiteConverter."""
    image_raw = tf.io.parse_single_example(
        example_proto,
        features={raw_image_key_name: tf.FixedLenFeature([], tf.string)},
    )
    image = tf.image.decode_jpeg(image_raw[raw_image_key_name])
    image = tf.expand_dims(image, axis=0)
    image = tf.image.resize_bilinear(image, (image_size, image_size))
    image = tf.squeeze(image, axis=0)
    image = image / 255.0
    return image

  return func 

def extra_reading_spec(self):
    """Additional data fields to store on disk and their decoders."""
    data_fields = {
        "frame_number": tf.FixedLenFeature([1], tf.int64),
    }
    decoders = {
        "frame_number":
            contrib.slim().tfexample_decoder.Tensor(tensor_key="frame_number"),
    }
    return data_fields, decoders 

def example_reading_spec(self):
    data_fields = {
        "inputs": tf.VarLenFeature(tf.int64),
        "audio/sample_count": tf.FixedLenFeature((), tf.int64),
        "audio/sample_width": tf.FixedLenFeature((), tf.int64),
        "targets": tf.VarLenFeature(tf.int64),
    }
    return data_fields, None