Python tensorflow.uint16() Examples

def read_and_decode_distillation(self, filename_queue):
        img1_name = tf.string_join([self.img_dir, '/', filename_queue[0]])
        img2_name = tf.string_join([self.img_dir, '/', filename_queue[1]])     
        img1 = tf.image.decode_png(tf.read_file(img1_name), channels=3)
        img1 = tf.cast(img1, tf.float32)
        img2 = tf.image.decode_png(tf.read_file(img2_name), channels=3)
        img2 = tf.cast(img2, tf.float32)    
        
        flow_occ_fw_name = tf.string_join([self.fake_flow_occ_dir, '/flow_occ_fw_', filename_queue[2], '.png'])
        flow_occ_bw_name = tf.string_join([self.fake_flow_occ_dir, '/flow_occ_bw_', filename_queue[2], '.png'])
        flow_occ_fw = tf.image.decode_png(tf.read_file(flow_occ_fw_name), dtype=tf.uint16, channels=3)
        flow_occ_fw = tf.cast(flow_occ_fw, tf.float32)   
        flow_occ_bw = tf.image.decode_png(tf.read_file(flow_occ_bw_name), dtype=tf.uint16, channels=3)
        flow_occ_bw = tf.cast(flow_occ_bw, tf.float32)             
        flow_fw, occ_fw = self.extract_flow_and_mask(flow_occ_fw)
        flow_bw, occ_bw = self.extract_flow_and_mask(flow_occ_bw)
        return img1, img2, flow_fw, flow_bw, occ_fw, occ_bw 

def setUp(self):
    super(TPUEncodeTest, self).setUp()
    self.data = (
        # Supported on TPU
        tf.random.uniform([128], maxval=100000, dtype=tf.int32),
        # Not supported on TPU
        tf.cast(
            tf.random.uniform([128], maxval=65535, dtype=tf.int32), tf.uint16),
        # Not supported on TPU
        tf.cast(
            tf.random.uniform([64, 84, 84, 4], maxval=256, dtype=tf.int32),
            tf.uint8),
        # Not supported on TPU
        tf.cast(tf.random.uniform([1], maxval=256, dtype=tf.int32), tf.uint8),
        # Not supported on TPU
        tf.cast(
            tf.random.uniform([100, 128, 1, 1, 1], maxval=256, dtype=tf.int32),
            tf.uint8),
        # Not supported on TPU
        tf.cast(
            tf.random.uniform([128, 100, 1, 1, 1], maxval=256, dtype=tf.int32),
            tf.uint8),
    ) 

def testIntTypes(self):
    for dtype, nptype in [
        (tf.int32, np.int32),
        (tf.uint8, np.uint8),
        (tf.uint16, np.uint16),
        (tf.int16, np.int16),
        (tf.int8, np.int8)]:
      # Test with array.
      t = tensor_util.make_tensor_proto([10, 20, 30], dtype=dtype)
      self.assertEquals(dtype, t.dtype)
      self.assertProtoEquals("dim { size: 3 }", t.tensor_shape)
      a = tensor_util.MakeNdarray(t)
      self.assertEquals(nptype, a.dtype)
      self.assertAllClose(np.array([10, 20, 30], dtype=nptype), a)
      # Test with ndarray.
      t = tensor_util.make_tensor_proto(np.array([10, 20, 30], dtype=nptype))
      self.assertEquals(dtype, t.dtype)
      self.assertProtoEquals("dim { size: 3 }", t.tensor_shape)
      a = tensor_util.MakeNdarray(t)
      self.assertEquals(nptype, a.dtype)
      self.assertAllClose(np.array([10, 20, 30], dtype=nptype), a) 

def reduce_mean_support_empty(input, keepdims=False):
    return tf.cond(tf.size(input) > 0, lambda: tf.reduce_mean(input, keepdims=keepdims), lambda: tf.zeros_like(input))


# def bit_tensor_list(input):
#     assert input.dtype in [tf.uint8, tf.uint16, tf.uint32, tf.uint64], 'unsupported data type, must be uint*'
#     num_bits = 0
#     if input.dtype == tf.int8:
#         num_bits = 8
#     elif input.dtype == tf.int16:
#         num_bits = 16
#     elif input.dtype == tf.uint32:
#         num_bits = 32
#     elif input.dtype == tf.uint64:
#         num_bits = 64
#     bit_tensors = []
#     for i in range(num_bits):
#         current_bit = 1 << i
#         current_bit_tensor = tf.bitwise.bitwise_and(input, current_bit) == 1
#         bit_tensors.append(current_bit_tensor)
#     print(bit_tensors)
#     return bit_tensors 

def test16bit(self):
    img_bytes = [[0, 255], [1024, 1024 + 255]]
    # Encoded PNG bytes resulting from encoding the above img_bytes
    # using go's image/png encoder.
    encoded_bytes = [137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68,
                     82, 0, 0, 0, 2, 0, 0, 0, 2, 16, 0, 0, 0, 0, 7, 77, 142,
                     187, 0, 0, 0, 21, 73, 68, 65, 84, 120, 156, 98, 98, 96, 96,
                     248, 207, 194, 2, 36, 1, 1, 0, 0, 255, 255, 6, 60, 1, 10,
                     68, 160, 26, 131, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96,
                     130]

    byte_string = bytes(bytearray(encoded_bytes))
    img_in = tf.constant(byte_string, dtype=tf.string)
    decode = tf.squeeze(tf.image.decode_png(img_in, dtype=tf.uint16))

    with self.test_session():
      decoded = decode.eval()
      self.assertAllEqual(decoded, img_bytes) 

def __init__(self, gray, isInstance=False):
    self._sess = tf.Session()
    self.isGrayscale = gray
    self._png_data = tf.placeholder(dtype=tf.string)
    if isInstance:
      self._isInstance = True
      self._image = tf.placeholder(dtype=tf.uint8)
      self._decode_png = tf.image.decode_png(self._png_data, channels=0, dtype=tf.uint16)
      self._decode_png = tf.image.resize_images(tf.cast(self._decode_png, tf.float32), size=[256, 512])
      self._encode_png = tf.image.encode_png(self._image)
    elif self.isGrayscale:
      self._image = tf.placeholder(dtype=tf.uint8)
      self._decode_png = tf.image.decode_png(self._png_data, channels=0)
      self._decode_png = tf.image.resize_images(self._decode_png, size=[256, 512])
      self._encode_png = tf.image.encode_png(self._image)
    else:
      self._image = tf.placeholder(dtype=tf.uint8)
      self._decode_png = tf.image.decode_png(self._png_data, channels=3)
      self._decode_png = tf.image.resize_images(self._decode_png, size=[256, 512])
      self._encode_png = tf.image.encode_png(self._image) 

def _convert_string_dtype(dtype):
    if dtype == 'float16':
        return tf.float16
    if dtype == 'float32':
        return tf.float32
    elif dtype == 'float64':
        return tf.float64
    elif dtype == 'int16':
        return tf.int16
    elif dtype == 'int32':
        return tf.int32
    elif dtype == 'int64':
        return tf.int64
    elif dtype == 'uint8':
        return tf.int8
    elif dtype == 'uint16':
        return tf.uint16
    else:
        raise ValueError('Unsupported dtype:', dtype) 

def test__dtype_to_bytes():
    np_tf_dt = [
        (np.uint8, tf.uint8, b"uint8"),
        (np.uint16, tf.uint16, b"uint16"),
        (np.uint32, tf.uint32, b"uint32"),
        (np.uint64, tf.uint64, b"uint64"),
        (np.int8, tf.int8, b"int8"),
        (np.int16, tf.int16, b"int16"),
        (np.int32, tf.int32, b"int32"),
        (np.int64, tf.int64, b"int64"),
        (np.float16, tf.float16, b"float16"),
        (np.float32, tf.float32, b"float32"),
        (np.float64, tf.float64, b"float64"),
    ]

    for npd, tfd, dt in np_tf_dt:
        npd = np.dtype(npd)
        assert tfrecord._dtype_to_bytes(npd) == dt
        assert tfrecord._dtype_to_bytes(tfd) == dt

    assert tfrecord._dtype_to_bytes("float32") == b"float32"
    assert tfrecord._dtype_to_bytes("foobar") == b"foobar" 

def load_tensorflow_image(self, channel_label: str,
                            image_name: str) -> lt.LabeledTensor:
    # All images will be cropped to this size.
    crop_size = 1024

    filename_op = tf.train.string_input_producer([self.data_path(image_name)])
    wfr = tf.WholeFileReader()
    _, encoded_png_op = wfr.read(filename_op)
    image_op = tf.image.decode_png(
        tf.reshape(encoded_png_op, shape=[]), channels=1, dtype=tf.uint16)
    image_op = image_op[:crop_size, :crop_size, :]
    image_op = tf.to_float(image_op) / np.iinfo(np.uint16).max
    image_op = tf.reshape(image_op, [1, 1024, 1024, 1])

    return lt.LabeledTensor(
        image_op, ['batch', 'row', 'column', ('channel', [channel_label])]) 

def _load_sample(self, files):
        left_file_name = files[0]
        right_file_name = files[1]
        gt_file_name = files[2]

        #read rgb images
        left_image = read_image_from_disc(left_file_name)
        right_image = read_image_from_disc(right_file_name)

        #read gt 
        if self._usePfm:
            gt_image = tf.py_func(lambda x: readPFM(x)[0], [gt_file_name], tf.float32)
            gt_image.set_shape([None,None,1])
        else:
            read_type = tf.uint16 if self._double_prec_gt else tf.uint8
            gt_image = read_image_from_disc(gt_file_name,shape=[None,None,1], dtype=read_type)
            gt_image = tf.cast(gt_image,tf.float32)
            if self._double_prec_gt:
                gt_image = gt_image/256.0
        
        #crop gt to fit with image (SGM adds some paddings who know why...)
        gt_image = gt_image[:,:tf.shape(left_image)[1],:]

        if self._resize_shape[0] is not None:
            scale_factor = tf.cast(tf.shape(gt_image_left)[1],tf.float32)/float(self._resize_shape[1])
            left_image = preprocessing.rescale_image(left_image,self._resize_shape)
            right_image = preprocessing.rescale_image(right_image, self._resize_shape)
            gt_image = tf.image.resize_nearest_neighbor(tf.expand_dims(gt_image,axis=0), self._resize_shape)[0]/scale_factor
        
        if self._crop_shape[0] is not None:
            if self._is_training:
                left_image,right_image,gt_image = preprocessing.random_crop(self._crop_shape, [left_image,right_image,gt_image])
            else:
                (left_image,right_image,gt_image) = [tf.image.resize_image_with_crop_or_pad(x,self._crop_shape[0],self._crop_shape[1]) for x in [left_image,right_image,gt_image]]
        
        if self._augment:
            left_image,right_image=preprocessing.augment(left_image,right_image)

        return [left_image,right_image,gt_image] 

def _build_input_pipeline(self):
        left_files, right_files, gt_files, _ = read_list_file(self._path_file)
        self._couples = [[l, r, gt] for l, r, gt in zip(left_files, right_files, gt_files)]
        #flags 
        self._usePfm = gt_files[0].endswith('pfm') or gt_files[0].endswith('PFM')
        if not self._usePfm:
            gg = cv2.imread(gt_files[0],-1)
            self._double_prec_gt = (gg.dtype == np.uint16)

        #create dataset
        dataset = tf.data.Dataset.from_tensor_slices(self._couples).repeat(self._num_epochs)
        if self._shuffle:
            dataset = dataset.shuffle(self._batch_size*50)
        
        #load images
        dataset = dataset.map(self._load_image)

        #transform data
        dataset = dataset.batch(self._batch_size, drop_remainder=True)
        dataset = dataset.prefetch(buffer_size=30)

        #get iterator and batches
        iterator = dataset.make_one_shot_iterator()
        images = iterator.get_next()
        self._left_batch = images[0]
        self._right_batch = images[1]
        self._gt_batch = images[2]

    ################# PUBLIC METHOD ####################### 

def _read_flow(filenames, num_epochs=None):
    """Given a list of filenames, constructs a reader op for ground truth."""
    filename_queue = tf.train.string_input_producer(filenames,
        shuffle=False, capacity=len(filenames), num_epochs=num_epochs)
    reader = tf.WholeFileReader()
    _, value = reader.read(filename_queue)
    gt_uint16 = tf.image.decode_png(value, dtype=tf.uint16)
    gt = tf.cast(gt_uint16, tf.float32)
    flow = (gt[:, :, 0:2] - 2 ** 15) / 64.0
    mask = gt[:, :, 2:3]
    return flow, mask 

def flow_to_int16(flow):
    _, h, w, _ = tf.unstack(tf.shape(flow))
    u, v = tf.unstack(flow, num=2, axis=3)
    r = tf.cast(tf.maximum(0.0, tf.minimum(u * 64.0 + 32768.0, 65535.0)), tf.uint16)
    g = tf.cast(tf.maximum(0.0, tf.minimum(v * 64.0 + 32768.0, 65535.0)), tf.uint16)
    b = tf.ones([1, h, w], tf.uint16)
    return tf.stack([r, g, b], axis=3) 

def reduce_batch_minus_min_and_max(x, reduce_instance_dims):
  """Computes the -min and max of a tensor x.

  Args:
    x: A `tf.Tensor`.
    reduce_instance_dims: A bool indicating whether this should collapse the
      batch and instance dimensions to arrive at a single scalar output, or only
      collapse the batch dimension and outputs a vector of the same shape as the
      input.

  Returns:
    The computed `tf.Tensor`s (batch -min, batch max) pair.
  """
  output_dtype = x.dtype

  if x.dtype == tf.uint8 or x.dtype == tf.uint16:
    x = tf.cast(x, tf.int32)

  elif x.dtype == tf.uint32 or x.dtype == tf.uint64:
    raise TypeError('Tensor type %r is not supported' % x.dtype)

  if reduce_instance_dims:
    if isinstance(x, tf.SparseTensor):
      x = x.values

    x_batch_max = tf.reduce_max(input_tensor=x)
    x_batch_minus_min = tf.reduce_max(input_tensor=tf.zeros_like(x) - x)
    x_batch_minus_min, x_batch_max = assert_same_shape(x_batch_minus_min,
                                                       x_batch_max)
  elif isinstance(x, tf.SparseTensor):
    x_batch_minus_min, x_batch_max = (
        _sparse_minus_reduce_min_and_reduce_max(x))
  else:
    x_batch_max = tf.reduce_max(input_tensor=x, axis=0)
    x_batch_minus_min = tf.reduce_max(input_tensor=0 - x, axis=0)

  # TODO(b/112309021): Remove workaround once tf.reduce_max of a tensor of all
  # NaNs produces -inf.
  return (_inf_to_nan(x_batch_minus_min, output_dtype),
          _inf_to_nan(x_batch_max, output_dtype)) 

def sum(x, reduce_instance_dims=True, name=None):  # pylint: disable=redefined-builtin
  """Computes the sum of the values of a `Tensor` over the whole dataset.

  Args:
    x: A `Tensor` or `SparseTensor`. Its type must be floating point
        (float{16|32|64}),integral (int{8|16|32|64}), or
        unsigned integral (uint{8|16})
    reduce_instance_dims: By default collapses the batch and instance dimensions
        to arrive at a single scalar output. If False, only collapses the batch
        dimension and outputs a vector of the same shape as the input.
    name: (Optional) A name for this operation.

  Returns:
    A `Tensor` containing the sum. If `x` is float32 or float64, the sum will
    have the same type as `x`. If `x` is float16, the output is cast to float32.
    If `x` is integral, the output is cast to [u]int64. If `x` is sparse and
    reduce_inst_dims is False will return 0 in place where column has no values
    across batches.

  Raises:
    TypeError: If the type of `x` is not supported.
  """
  with tf.compat.v1.name_scope(name, 'sum'):
    if reduce_instance_dims:
      if isinstance(x, tf.SparseTensor):
        x = x.values
      x = tf.reduce_sum(input_tensor=x)
    elif isinstance(x, tf.SparseTensor):
      if x.dtype == tf.uint8 or x.dtype == tf.uint16:
        x = tf.cast(x, tf.int64)
      elif x.dtype == tf.uint32 or x.dtype == tf.uint64:
        TypeError('Data type %r is not supported' % x.dtype)
      x = tf.sparse.reduce_sum(x, axis=0)
    else:
      x = tf.reduce_sum(input_tensor=x, axis=0)
    output_dtype, sum_fn = _sum_combine_fn_and_dtype(x.dtype)
    return _numeric_combine([x], sum_fn, reduce_instance_dims,
                            [output_dtype])[0] 

def test_uint16(self):
    x = tf.constant([[1, 2], [3, 4]], tf.uint16)
    self.assertAllEqual(utils.make_time_major(x), tf.constant([[1, 3], [2, 4]])) 

def load_annotation(self, img, img_filename, annotation_filename):
    annotation_filename_without_id = tf.string_split([annotation_filename], ':').values[0]
    #ann_data = tf.read_file(annotation_filename_without_id)
    #ann = tf.image.decode_png(ann_data, dtype=tf.uint16, channels=1)
    ann, = tf.py_func(load_ann_with_colormap, [annotation_filename_without_id], [tf.uint8])
    ann.set_shape(img.get_shape().as_list()[:-1] + [1])
    ann = self.postproc_annotation(annotation_filename, ann)
    return ann 

def load_annotation(self, img, img_filename, annotation_filename):
    annotation_filename_without_id = tf.string_split([annotation_filename], ':').values[0]
    ann_data = tf.read_file(annotation_filename_without_id)
    ann = tf.image.decode_png(ann_data, dtype=tf.uint16, channels=1)
    ann.set_shape(img.get_shape().as_list()[:-1] + [1])
    ann = self.postproc_annotation(annotation_filename, ann)
    return ann 

def masked_bit(input, bit_index):
    """
    Returns a boolean tensor, where values are true, on which the bit on bit_index is True.
    :param input: The input tensor to check.
    :param bit_index: The bit index which will be compared with bitwise and. (LSB 0 order)
    :return: The tensor.
    """
    assert input.dtype in [tf.int8, tf.int16, tf.int32, tf.int64, tf.uint8, tf.uint16, tf.uint32, tf.uint64], 'unsupported data type, must be *int*'
    current_bit = tf.bitwise.left_shift(tf.constant(1, dtype=input.dtype), tf.cast(bit_index, dtype=input.dtype))
    return tf.greater(tf.bitwise.bitwise_and(input, current_bit), 0) 

def log_likelihood_constraint(self, ll_cutoff):
        """
        Computes a mask for entries that, if the edge corresponding to the entry is added/removed, would lead to the
        log likelihood constraint to be violated.

        Parameters
        ----------
        ll_cutoff: float
            Cutoff value for the unnoticeability constraint. Smaller means stricter constraint. 0.004 corresponds to a
            p-value of 0.95 in the Chi-square distribution with one degree of freedom.

        Returns
        -------
        allowed_mask: tf.Tensor shape [N, N], dtype float
            ones everywhere except the entries that, if an edge is added/removed, would violate the log likelihood
            constraint. There, the returned tensor has value 0.

        current_ratio: tf.Tensor, scalar, dtype float
            current value of the Chi-square test.
        """

        t_d_min = tf.constant(2, dtype=self.dtype)
        t_possible_edges = tf.constant(np.array(np.triu(np.ones((self.N, self.N)), k=1).nonzero()).T,
                                       dtype=tf.uint16)
        allowed_mask, current_ratio = utils.likelihood_ratio_filter(t_possible_edges,
                                                                    self.modified_adjacency,
                                                                    self.adjacency_orig, t_d_min,
                                                                    ll_cutoff)

        return allowed_mask, current_ratio 

def load_annotation(self, img, img_filename, annotation_filename):
    ann_data = tf.read_file(annotation_filename)
    ann = tf.image.decode_image(ann_data, dtype=tf.uint16, channels=1)
    ann.set_shape(img.get_shape().as_list()[:-1] + [1])
    ann = self.postproc_annotation(annotation_filename, ann)
    return ann 

def load_annotation(self, img, img_filename, annotation_filename):
    annotation_filename_without_id = tf.string_split([annotation_filename], ':').values[0]
    ann_data = tf.read_file(annotation_filename_without_id)
    ann = tf.image.decode_png(ann_data, dtype=tf.uint16, channels=1)
    ann.set_shape(img.get_shape().as_list()[:-1] + [1])
    ann = self.postproc_annotation(annotation_filename, ann)
    return ann 

def generate_fake_flow_occlusion(self, restore_model, save_dir):
        dataset = BasicDataset(data_list_file=self.dataset_config['data_list_file'], img_dir=self.dataset_config['img_dir'])
        save_name_list = dataset.data_list[:, 2]
        iterator = dataset.create_one_shot_iterator(dataset.data_list, num_parallel_calls=self.num_input_threads)
        batch_img1, batch_img2 = iterator.get_next()        
        flow_fw, flow_bw = pyramid_processing_bidirection(batch_img1, batch_img2, 
            train=False, trainable=False, reuse=None, regularizer=None, is_scale=True)  
        occ_fw, occ_bw = occlusion(flow_fw['full_res'], flow_bw['full_res'])        
        
        flow_fw_full_res = flow_fw['full_res'] * 64. + 32768
        flow_occ_fw = tf.concat([flow_fw_full_res, occ_fw], -1)
        flow_occ_fw = tf.cast(flow_occ_fw, tf.uint16)
        flow_bw_full_res = flow_bw['full_res'] * 64. + 32768
        flow_occ_bw = tf.concat([flow_bw_full_res, occ_bw], -1)
        flow_occ_bw = tf.cast(flow_occ_bw, tf.uint16)
        
        restore_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) 
        saver = tf.train.Saver(var_list=restore_vars)
        sess = tf.Session()
        sess.run(tf.global_variables_initializer()) 
        sess.run(iterator.initializer) 
        saver.restore(sess, restore_model)
        #save_dir = '/'.join([self.save_dir, 'sample', self.model_name])
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)           
        for i in range(dataset.data_num):
            np_flow_occ_fw, np_flow_occ_bw, np_occ_fw = sess.run([flow_occ_fw, flow_occ_bw, occ_fw])
            
            # opencv read and save image as bgr format, here we change rgb to bgr
            np_flow_occ_fw = rgb_bgr(np_flow_occ_fw[0])
            np_flow_occ_bw = rgb_bgr(np_flow_occ_bw[0])
            np_flow_occ_fw = np_flow_occ_fw.astype(np.uint16)
            np_flow_occ_bw = np_flow_occ_bw.astype(np.uint16)
            
            cv2.imwrite('%s/flow_occ_fw_%s.png' % (save_dir, save_name_list[i]), np_flow_occ_fw)
            cv2.imwrite('%s/flow_occ_bw_%s.png' % (save_dir, save_name_list[i]), np_flow_occ_bw)
            print('Finish %d/%d' % (i, dataset.data_num)) 

def test_forward_zeros_like():
    if tf.__version__ < LooseVersion('1.2'):
        _test_forward_zeros_like((2, 3), "int32")
        _test_forward_zeros_like((2, 3, 5), "int8")
        _test_forward_zeros_like((2, 3, 5, 7), "uint16")
        _test_forward_zeros_like((2, 3, 11), "float32")
        _test_forward_zeros_like((2, 3, 11), "float64") 

def make_deserialize(params, batch_size, training=False):
  """Construct deserialize function for training and eval fns."""
  feature_map = {
      movielens.USER_COLUMN: tf.FixedLenFeature([], dtype=tf.string),
      movielens.ITEM_COLUMN: tf.FixedLenFeature([], dtype=tf.string),
  }
  if training:
    feature_map["labels"] = tf.FixedLenFeature([], dtype=tf.string)
  else:
    feature_map[rconst.DUPLICATE_MASK] = tf.FixedLenFeature([], dtype=tf.string)

  def deserialize(examples_serialized):
    """Called by Dataset.map() to convert batches of records to tensors."""
    features = tf.parse_single_example(examples_serialized, feature_map)
    users = tf.reshape(tf.decode_raw(
        features[movielens.USER_COLUMN], tf.int32), (batch_size,))
    items = tf.reshape(tf.decode_raw(
        features[movielens.ITEM_COLUMN], tf.uint16), (batch_size,))

    if params["use_tpu"] or params["use_xla_for_gpu"]:
      items = tf.cast(items, tf.int32)  # TPU and XLA disallows uint16 infeed.

    if not training:
      dupe_mask = tf.reshape(tf.cast(tf.decode_raw(
          features[rconst.DUPLICATE_MASK], tf.int8), tf.bool), (batch_size,))
      return {
          movielens.USER_COLUMN: users,
          movielens.ITEM_COLUMN: items,
          rconst.DUPLICATE_MASK: dupe_mask,
      }

    labels = tf.reshape(tf.cast(tf.decode_raw(
        features["labels"], tf.int8), tf.bool), (batch_size,))

    return {
        movielens.USER_COLUMN: users,
        movielens.ITEM_COLUMN: items,
    }, labels
  return deserialize 

def _convert_string_dtype(dtype):
    """Get the type from a string.

    # Arguments
        dtype: A string representation of a type.

    # Returns
        The type requested.

    # Raises
        ValueError: if `dtype` is not supported.
    """
    if dtype == 'float16':
        return tf.float16
    if dtype == 'float32':
        return tf.float32
    elif dtype == 'float64':
        return tf.float64
    elif dtype == 'int16':
        return tf.int16
    elif dtype == 'int32':
        return tf.int32
    elif dtype == 'int64':
        return tf.int64
    elif dtype == 'uint8':
        return tf.int8
    elif dtype == 'uint16':
        return tf.uint16
    else:
        raise ValueError('Unsupported dtype:', dtype) 

def test_compression_fp16(self):
        valid_dtypes = [tf.float16, tf.float32, tf.float64]
        invalid_dtypes = [tf.uint8, tf.int8, tf.uint16, tf.int16,
                          tf.int32, tf.int64, tf.bool]

        tensor_size = [17] * 3
        compression = hvd.Compression.fp16

        with self.test_session(config=self.config) as session:
            for dtype in valid_dtypes:
                tensor = tf.ones(tensor_size, dtype=dtype)

                compressor = compression.get_compressor(dtype)
                tensor_compressed = compressor.compress(tensor)
                self.assertEqual(tensor_compressed.dtype, tf.float16)

                tensor_decompressed = compressor.decompress(tensor_compressed)
                self.assertEqual(tensor_decompressed.dtype, dtype)

                actual = session.run(tensor_decompressed)
                expected = np.ones(tensor_size)
                err = np.linalg.norm(expected - actual)
                self.assertLess(err, 0.00000001)

            for dtype in invalid_dtypes:
                tensor = tf.ones(tensor_size, dtype=dtype)

                compressor = compression.get_compressor(dtype)
                tensor_compressed = compressor.compress(tensor)
                self.assertEqual(tensor_compressed.dtype, dtype)

                tensor_decompressed = compressor.decompress(tensor_compressed)
                self.assertEqual(tensor_decompressed.dtype, dtype)

                actual = session.run(tensor_decompressed)
                expected = np.ones(tensor_size)
                err = np.linalg.norm(expected - actual)
                self.assertLess(err, 0.00000001) 

def _build_input_pipeline(self):
        left_files, right_files, gt_files, _ = read_list_file(self._path_file)
        self._couples = [[l, r, gt] for l, r, gt in zip(left_files, right_files, gt_files)]
        #flags 
        self._usePfm = gt_files[0].endswith('pfm') or gt_files[0].endswith('PFM')
        if not self._usePfm:
            gg = cv2.imread(gt_files[0],-1)
            self._double_prec_gt = (gg.dtype == np.uint16)
        
        print('Input file loaded, starting to build input pipelines')
        print('FLAGS:')
        print('_usePfmGt',self._usePfm)
        print('_double_prec_gt', self._double_prec_gt)

        #create dataset
        dataset = tf.data.Dataset.from_tensor_slices(self._couples).repeat(self._num_epochs)
        if self._shuffle:
            dataset = dataset.shuffle(self._batch_size*50)
        
        #load images
        dataset = dataset.map(self._load_sample)

        #transform data
        dataset = dataset.batch(self._batch_size, drop_remainder=True)
        dataset = dataset.prefetch(buffer_size=30)

        #get iterator and batches
        iterator = dataset.make_one_shot_iterator()
        images = iterator.get_next()
        self._left_batch = images[0]
        self._right_batch = images[1]
        self._gt_batch = images[2]

    ################# PUBLIC METHOD ####################### 

def _decode_gt(self, gt):
        if self._usePfm:
            gt_image_op = tf.py_func(lambda x: read_PFM(x)[0], [gt], tf.float32)
            gt_image_op.set_shape([None,None,1])
        else:
            read_type = tf.uint16 if self._double_prec_gt else tf.uint8
            gt_image_op = read_image_from_disc(gt,shape=[None,None,1], dtype=read_type)
            gt_image_op = tf.cast(gt_image_op,tf.float32)
            if self._double_prec_gt:
                gt_image_op = gt_image_op/256.0
        return gt_image_op 

def _build_input_pipeline(self):
        #fetch one sample to setup flags
        task_sample = self._task_library.get_task()
        gt_sample = task_sample[2,0]
        #flags 
        self._usePfm = gt_sample.endswith('pfm') or gt_sample.endswith('PFM')
        if not self._usePfm:
            gg = cv2.imread(gt_sample,-1)
            self._double_prec_gt = (gg.dtype == np.uint16)
        
        print('Input file loaded, starting to build input pipelines')
        print('FLAGS:')
        print('_usePfmGt',self._usePfm)
        print('_double_prec_gt', self._double_prec_gt)

        #create dataset
        dataset = tf.data.Dataset.from_generator(self._task_library,(tf.string)).repeat(self._num_epochs)
        
        #load images
        dataset = dataset.map(self._load_task)

        #transform data
        dataset = dataset.batch(self._batch_size, drop_remainder=True)
        dataset = dataset.prefetch(buffer_size=10)

        #get iterator and batches
        iterator = dataset.make_one_shot_iterator()
        samples = iterator.get_next()
        self._left_batch = samples[0]
        self._right_batch = samples[1]
        self._gt_batch = samples[2]

    ################# PUBLIC METHOD #######################