Python tensorflow.RandomShuffleQueue() Examples

def testBlockingDequeueManyFromClosedEmptyQueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(10, 5, tf.float32, ((),))
      close_op = q.close()
      dequeued_t = q.dequeue_many(4)

      def dequeue():
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testParallelEnqueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_ops = [q.enqueue((x,)) for x in elems]
      dequeued_t = q.dequeue()

      # Run one producer thread for each element in elems.
      def enqueue(enqueue_op):
        sess.run(enqueue_op)
      threads = [self.checkedThread(target=enqueue, args=(e,))
                 for e in enqueue_ops]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()

      # Dequeue every element using a single thread.
      results = []
      for _ in xrange(len(elems)):
        results.append(dequeued_t.eval())
      self.assertItemsEqual(elems, results) 

def _assign_queue(self, proto_text):
        """
        Args:
            proto_text: object to be enqueued and managed by parallel threads.
        """

        with tf.variable_scope('shuffle_queue'):
            queue = tf.RandomShuffleQueue(
                capacity=self.capacity,
                min_after_dequeue=10*self.batch_size,
                dtypes=tf.string, shapes=[()])

            enqueue_op = queue.enqueue(proto_text)
            example_dq = queue.dequeue()

            qr = tf.train.QueueRunner(queue, [enqueue_op] * 4)
            tf.train.add_queue_runner(qr)

            _sequence_lengths, _sequences = tf.parse_single_sequence_example(
                serialized=example_dq,
                context_features=LENGTHS,
                sequence_features=SEQUENCES)
        return _sequence_lengths, _sequences 

def build_pipeline(self, name):
        """Creates a new input subgraph composed of the following components:
            - Reader queue that feeds protobuf data files.
            - RandomShuffleQueue assigned parallel-thread queuerunners.
            - Dynamic padded-bucketed-batching queue for organizing batches in a time and
              space-efficient manner.

        Args:
            name: filename prefix for data. See Dataset class for naming conventions.

        Returns:
            2-tuple (lengths, sequences):
                lengths: (dict) parsed context feature from protobuf file.
                Supports keys in LENGTHS.
                sequences: (dict) parsed feature_list from protobuf file.
                Supports keys in SEQUENCES.
        """
        with tf.variable_scope(name + '_pipeline'):
            proto_text = self._read_line(self.paths[name + '_tfrecords'])
            context_pair, sequence_pair = self._assign_queue(proto_text)
            input_length = tf.add(context_pair['encoder_sequence_length'],
                                  context_pair['decoder_sequence_length'],
                                  name=name + 'length_add')
            return self._padded_bucket_batches(input_length, sequence_pair) 

def get_multitrack_input(shape, batch_size, name="", input_shape=None):
    '''
    Creates multitrack placeholders and a random shuffle queue based on it
    :param input_shape: Shape of accompaniment and voice magnitudes
    :param batch_size: Number of samples in each batch
    :param name: How to name the placeholders
    :return: [List of mixture,acc,voice placeholders, random shuffle queue, symbolic batch sample from queue]
    '''
    m,a,v = get_multitrack_placeholders(shape, input_shape=input_shape)

    min_after_dequeue = 0
    buffer = 1000
    capacity = min_after_dequeue + buffer

    if input_shape is None:
        input_shape = shape
    queue = tf.RandomShuffleQueue(capacity, min_after_dequeue, [tf.float32, tf.float32, tf.float32], [input_shape, shape, shape])
    input_batch = queue.dequeue_many(batch_size, name="input_batch" + name)

    return [m,a,v], queue, input_batch 

def _setup_base_graph(self):
        """
        Set up queue, variables and session
        """
        self.graph = tf.Graph()
        with self.graph.as_default() as g:
            input_dim = self.input_dim
            batch_size = self.batch_size
            hidden_units = self.hidden_units
            layer_units = [input_dim] + hidden_units + [1]
            layer_num = len(layer_units)

            #make Queue for getting batch
            self.queue = q = tf.RandomShuffleQueue(capacity=self.q_capacity,
                                        min_after_dequeue=self.min_after_dequeue,
                                        dtypes=["float", "float"],
                                        shapes=[[input_dim], [input_dim]])
            #input data
            self.data1, self.data2 = q.dequeue_many(batch_size, name="inputs")

            self._setup_variables()
            self._setup_training()
            self._setup_prediction()
            self._setup_pretraining() 

def testScalarShapes(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(
          10, 0, [tf.int32, tf.int32],
          shapes=[(), (1,)])
      q.enqueue_many([[1, 2, 3, 4], [[5], [6], [7], [8]]]).run()
      q.enqueue([9, [10]]).run()
      dequeue_t = q.dequeue()
      results = []
      for _ in range(2):
        a, b = sess.run(dequeue_t)
        results.append((a, b))
      a, b = sess.run(q.dequeue_many(3))
      for i in range(3):
        results.append((a[i], b[i]))
      self.assertItemsEqual([(1, [5]), (2, [6]), (3, [7]), (4, [8]), (9, [10])],
                            results) 

def get_multitrack_input(shape, batch_size, name="", input_shape=None):
    '''
    Creates multitrack placeholders and a random shuffle queue based on it
    :param input_shape: Shape of accompaniment and voice magnitudes
    :param batch_size: Number of samples in each batch
    :param name: How to name the placeholders
    :return: [List of mixture,acc,voice placeholders, random shuffle queue, symbolic batch sample from queue]
    '''
    m,a,v = get_multitrack_placeholders(shape, input_shape=input_shape)

    min_after_dequeue = 0
    buffer = 1000
    capacity = min_after_dequeue + buffer

    if input_shape is None:
        input_shape = shape
    queue = tf.RandomShuffleQueue(capacity, min_after_dequeue, [tf.float32, tf.float32, tf.float32], [input_shape, shape, shape])
    input_batch = queue.dequeue_many(batch_size, name="input_batch" + name)

    return [m,a,v], queue, input_batch 

def testMultiEnqueueAndDequeue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(
          10, 0, (tf.int32, tf.float32))
      elems = [(5, 10.0), (10, 20.0), (15, 30.0)]
      enqueue_ops = [q.enqueue((x, y)) for x, y in elems]
      dequeued_t = q.dequeue()

      for enqueue_op in enqueue_ops:
        enqueue_op.run()

      results = []
      for _ in xrange(len(elems)):
        x, y = sess.run(dequeued_t)
        results.append((x, y))
      self.assertItemsEqual(elems, results) 

def testEmptyDequeueManyWithNoShape(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      enqueue_op = q.enqueue(
          (tf.constant([10.0, 20.0], shape=(1, 2)),))
      dequeued_t = q.dequeue_many(0)

      # Expect the operation to fail due to the shape not being constrained.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval()

      enqueue_op.run()

      # RandomShuffleQueue does not make any attempt to support DequeueMany
      # with unspecified shapes, even if a shape could be inferred from the
      # elements enqueued.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval() 

def testParallelDequeueMany(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(1000, 0, tf.float32, shapes=())
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(100)

      enqueue_op.run()

      # Dequeue 100 items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

def testParallelDequeueUpTo(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(1000, 0, tf.float32, shapes=())
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_up_to(100)

      enqueue_op.run()

      # Dequeue 100 items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

def testParallelDequeueUpToRandomPartition(self):
    with self.test_session() as sess:
      dequeue_sizes = [random.randint(50, 150) for _ in xrange(10)]
      total_elements = sum(dequeue_sizes)
      q = tf.RandomShuffleQueue(total_elements, 0, tf.float32, shapes=())

      elems = [10.0 * x for x in xrange(total_elements)]
      enqueue_op = q.enqueue_many((elems,))
      dequeue_ops = [q.dequeue_up_to(size) for size in dequeue_sizes]

      enqueue_op.run()

      # Dequeue random number of items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue(dequeue_op):
        dequeued_elems.extend(sess.run(dequeue_op))
      threads = []
      for dequeue_op in dequeue_ops:
        threads.append(self.checkedThread(target=dequeue, args=(dequeue_op,)))
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

def testDequeueUpToWithTensorParameter(self):
    with self.test_session():
      # Define a first queue that contains integer counts.
      dequeue_counts = [random.randint(1, 10) for _ in range(100)]
      count_q = tf.RandomShuffleQueue(100, 0, tf.int32)
      enqueue_counts_op = count_q.enqueue_many((dequeue_counts,))
      total_count = sum(dequeue_counts)

      # Define a second queue that contains total_count elements.
      elems = [random.randint(0, 100) for _ in range(total_count)]
      q = tf.RandomShuffleQueue(
          total_count, 0, tf.int32, ((),))
      enqueue_elems_op = q.enqueue_many((elems,))

      # Define a subgraph that first dequeues a count, then DequeuesUpTo
      # that number of elements.
      dequeued_t = q.dequeue_up_to(count_q.dequeue())

      enqueue_counts_op.run()
      enqueue_elems_op.run()

      dequeued_elems = []
      for _ in dequeue_counts:
        dequeued_elems.extend(dequeued_t.eval())
      self.assertItemsEqual(elems, dequeued_elems) 

def testDequeueFromClosedQueue(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 2, tf.float32)
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue()

      enqueue_op.run()
      close_op.run()
      results = [dequeued_t.eval() for _ in elems]
      expected = [[elem] for elem in elems]
      self.assertItemsEqual(expected, results)

      # Expect the operation to fail due to the queue being closed.
      with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                   "is closed and has insufficient"):
        dequeued_t.eval() 

def testBlockingDequeueFromClosedEmptyQueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      close_op = q.close()
      dequeued_t = q.dequeue()

      finished = []  # Needs to be a mutable type
      def dequeue():
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)
        finished.append(True)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      self.assertEqual(len(finished), 0)
      close_op.run()
      dequeue_thread.join()
      self.assertEqual(len(finished), 1) 

def testBlockingDequeueUpToFromClosedEmptyQueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(10, 5, tf.float32, ((),))
      close_op = q.close()
      dequeued_t = q.dequeue_up_to(4)

      def dequeue():
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testEmptyDequeueUpToWithNoShape(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      enqueue_op = q.enqueue(
          (tf.constant([10.0, 20.0], shape=(1, 2)),))
      dequeued_t = q.dequeue_up_to(0)

      # Expect the operation to fail due to the shape not being constrained.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval()

      enqueue_op.run()

      # RandomShuffleQueue does not make any attempt to support DequeueUpTo
      # with unspecified shapes, even if a shape could be inferred from the
      # elements enqueued.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval() 

def testBigDequeueMany(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(2, 0, tf.int32, ((),))
      elem = np.arange(4, dtype=np.int32)
      enq_list = [q.enqueue((e,)) for e in elem]
      deq = q.dequeue_many(4)

      results = []
      def blocking_dequeue():
        # Will only complete after 4 enqueues complete.
        results.extend(sess.run(deq))
      thread = self.checkedThread(target=blocking_dequeue)
      thread.start()
      # The dequeue should start and then block.
      for enq in enq_list:
        # TODO(mrry): Figure out how to do this without sleeping.
        time.sleep(0.1)
        self.assertEqual(len(results), 0)
        sess.run(enq)

      # Enough enqueued to unblock the dequeue
      thread.join()
      self.assertItemsEqual(elem, results) 

def testDequeueUpToInDifferentOrders(self):
    with self.test_session():
      # Specify seeds to make the test deterministic
      # (https://en.wikipedia.org/wiki/Taxicab_number).
      q1 = tf.RandomShuffleQueue(10, 5, tf.int32, ((),), seed=1729)
      q2 = tf.RandomShuffleQueue(10, 5, tf.int32, ((),), seed=87539319)
      enq1 = q1.enqueue_many(([1, 2, 3, 4, 5],))
      enq2 = q2.enqueue_many(([1, 2, 3, 4, 5],))
      deq1 = q1.dequeue_up_to(5)
      deq2 = q2.dequeue_up_to(5)

      enq1.run()
      enq1.run()
      enq2.run()
      enq2.run()

      results = [[], [], [], []]

      results[0].extend(deq1.eval())
      results[1].extend(deq2.eval())

      q1.close().run()
      q2.close().run()

      results[2].extend(deq1.eval())
      results[3].extend(deq2.eval())

      # No two should match
      for i in range(1, 4):
        for j in range(i):
          self.assertNotEqual(results[i], results[j]) 

def __init__(self, dataset, num_epochs, batch_size=16, queue_capacity=512, shuffle=True,
                 allow_smaller_final_batch=False):
        assert(isinstance(dataset, BaseDataset))

        self.dataset = dataset
        self.num_epochs = num_epochs
        self.batch_size = batch_size
        self.queue_capacity = queue_capacity
        self.epoch = 0
        self.allow_smaller_final_batch = allow_smaller_final_batch

        self.queue_placeholders_dict = {}
        self.queue_placeholders = [] # One-to-one correspondence with dataset.sample_* members.
        self.num_data_variables = len(self.dataset.sample_shape)

        for i in range(self.num_data_variables):
            self.queue_placeholders.append(tf.placeholder(self.dataset.sample_tf_type[i], shape=self.dataset.sample_shape[i]))
            self.queue_placeholders_dict[self.dataset.sample_key[i]] = self.queue_placeholders[-1]

        # Tensorflow Queues complain if we don't have fully defined tensors. In other words, we need to have static
        # shapes. However, batch generators such as tf.train.batch need to know tensor rank. Otherwise, it makes random
        # placeholder assignments (i.e., input is mapped to seq_len placeholder). This seems to be a Tensorflow bug.
        if shuffle:
            self.input_queue = tf.RandomShuffleQueue(queue_capacity,
                                                     min_after_dequeue=int(queue_capacity/2),
                                                     dtypes=self.dataset.sample_tf_type,
                                                     names=self.dataset.sample_key)
        else:
            self.input_queue = tf.FIFOQueue(queue_capacity,
                                            dtypes=self.dataset.sample_tf_type,
                                            names=self.dataset.sample_key)

        self.enqueue_op = self.input_queue.enqueue(self.queue_placeholders_dict)
        self.dequeue_op = self.input_queue.dequeue()

        # Set tensor shapes here.
        for i in range(self.num_data_variables):
            self.dequeue_op[self.dataset.sample_key[i]].set_shape(self.dataset.sample_shape[i]) 

def __init__(self, dataset, num_epochs, batch_size=16, queue_capacity=512, shuffle=True, allow_smaller_final_batch=False):
        """

        Args:
            dataset (Dataset):
            batch_size:
            queue_capacity:
        """
        assert(isinstance(dataset, BaseDataset))

        self.dataset = dataset
        self.num_epochs = num_epochs
        self.batch_size = batch_size
        self.queue_capacity = queue_capacity
        self.epoch = 0
        self.allow_smaller_final_batch = allow_smaller_final_batch

        self.queue_placeholders = []
        self.num_data_variables = len(self.dataset.sample_shape)

        for i in range(self.num_data_variables):
            self.queue_placeholders.append(tf.placeholder(self.dataset.sample_tf_type[i], shape=self.dataset.sample_shape[i]))

        if shuffle:
            self.input_queue = tf.RandomShuffleQueue(queue_capacity, min_after_dequeue=int(queue_capacity/2),
                                                     dtypes=self.dataset.sample_tf_type)
        else:
            self.input_queue = tf.FIFOQueue(queue_capacity, dtypes=self.dataset.sample_tf_type)

        self.enqueue_op = self.input_queue.enqueue(self.queue_placeholders)
        self.dequeue_op = self.input_queue.dequeue() 

def testBlockingEnqueueManyToFullQueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(4, 0, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      blocking_enqueue_op = q.enqueue_many(([50.0, 60.0],))
      dequeued_t = q.dequeue()

      enqueue_op.run()

      def blocking_enqueue():
        sess.run(blocking_enqueue_op)
      thread = self.checkedThread(target=blocking_enqueue)
      thread.start()
      # The dequeue ops should run after the blocking_enqueue_op has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)

      results = []
      for _ in elems:
        time.sleep(0.01)
        results.append(dequeued_t.eval())
      results.append(dequeued_t.eval())
      results.append(dequeued_t.eval())
      self.assertItemsEqual(elems + [50.0, 60.0], results)
      # There wasn't room for 50.0 or 60.0 in the queue when the first
      # element was dequeued.
      self.assertNotEqual(50.0, results[0])
      self.assertNotEqual(60.0, results[0])
      # Similarly for 60.0 and the second element.
      self.assertNotEqual(60.0, results[1]) 

def testBlockingEnqueueToFullQueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(4, 0, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      blocking_enqueue_op = q.enqueue((50.0,))
      dequeued_t = q.dequeue()

      enqueue_op.run()

      def blocking_enqueue():
        sess.run(blocking_enqueue_op)
      thread = self.checkedThread(target=blocking_enqueue)
      thread.start()
      # The dequeue ops should run after the blocking_enqueue_op has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      results = []
      for _ in elems:
        results.append(dequeued_t.eval())
      results.append(dequeued_t.eval())
      self.assertItemsEqual(elems + [50.0], results)
      # There wasn't room for 50.0 in the queue when the first element was
      # dequeued.
      self.assertNotEqual(50.0, results[0])
      thread.join() 

def make_input(model_options):
    '''
    Prepare the input placeholders and queues
    '''
    model_vars = {}
    if model_options['mode'] == 'train':
        images = tf.placeholder("float",[None,224,224,model_options['num_channels']])
        model_vars['images'] = images

        labels = tf.placeholder("uint8",[1])
        model_vars['labels'] = labels

        q = tf.RandomShuffleQueue(200, model_options['min_to_keep'], [tf.float32, tf.uint8],
                                  shapes=[[model_options['example_size'],224,224,\
                                  model_options['num_channels']],1])
        model_vars['queue'] = q
        enqueue_op = q.enqueue([images, labels])
        model_vars['enqueue_op'] = enqueue_op

    else:
        num_crops = 10 if model_options['flip'] else 5;
        images = tf.placeholder("float",[num_crops,model_options['example_size']\
                                         ,224,224,model_options['num_channels']])
        labels = tf.placeholder("uint8",[num_crops,1])
        names = tf.placeholder("string",[num_crops,1])
        model_vars['images'] = images
        model_vars['labels'] = labels
        model_vars['names'] = names

        q = tf.FIFOQueue(100, [tf.float32, tf.uint8, "string"],
                              shapes=[[model_options['example_size'],224,224,\
                              model_options['num_channels']],[1],[1]])

        model_vars['queue'] = q
        enqueue_op = q.enqueue_many([images, labels, names])
        model_vars['enqueue_op'] = enqueue_op

    return model_vars 

def _batch_input(is_train, tfrecords_path, batch_size, time_step):

    if is_train:
        tf_files = glob.glob(os.path.join(tfrecords_path, 'train-*.tfrecords'))
        filename_queue = tf.train.string_input_producer(tf_files, shuffle=True, capacity=16)

        min_queue_examples = config.min_queue_examples
        examples_queue = tf.RandomShuffleQueue(
            capacity=min_queue_examples + 3 * batch_size,
            min_after_dequeue=min_queue_examples,
            dtypes=[tf.string])
        enqueue_ops = []
        for _ in range(config.num_readers):
            _, value = tf.TFRecordReader().read(filename_queue)
            enqueue_ops.append(examples_queue.enqueue([value]))

        tf.train.add_queue_runner(
            tf.train.QueueRunner(examples_queue, enqueue_ops))
        example_serialized = examples_queue.dequeue()
    else:
        tf_files = sorted(glob.glob(os.path.join(tfrecords_path, 'val-*.tfrecords')))
        filename_queue = tf.train.string_input_producer(tf_files, shuffle=False, capacity=8)
        _, example_serialized = tf.TFRecordReader().read(filename_queue)
        # example_serialized = next(tf.python_io.tf_record_iterator(self._tf_files[0]))
    images_and_labels = []
    for thread_id in range(config.num_preprocess_threads):
        sequence, context = _parse_example_proto(example_serialized)
        image_buffers = sequence['images']
        bboxes = sequence['bboxes']
        seq_len = tf.cast(context['seq_len'][0], tf.int32)
        z_exemplars, x_crops, y_crops = _process_images(image_buffers, bboxes, seq_len, thread_id, time_step, is_train)
        images_and_labels.append([z_exemplars, x_crops, y_crops])

    batch_z, batch_x, batch_y = tf.train.batch_join(images_and_labels,
                                                    batch_size=batch_size,
                                                    capacity=2 * config.num_preprocess_threads * batch_size)
    if is_train:
        tf.summary.image('exemplars', batch_z[0], 5)
        tf.summary.image('crops', batch_x[0], 5)

    return batch_z, batch_x, batch_y 

def testEnqueueToClosedQueue(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 4, tf.float32)
      enqueue_op = q.enqueue((10.0,))
      close_op = q.close()

      enqueue_op.run()
      close_op.run()

      # Expect the operation to fail due to the queue being closed.
      with self.assertRaisesRegexp(tf.errors.CancelledError, "is closed"):
        enqueue_op.run() 

def __init__(self, hight, width, batch_size, folder_image, folder_label, format_image = '.jpg' , random = True):
    """
    Args:
      hight             :         hight of samples
      width             :         width of samples
      batch_size        :         batch size
      folder_image      :         the folder where the images are
      folder_label      :         the folder where the ground truth are
      format_image      :         format of images (usually jpg)
      random            :         is the queue shuffled (for training) or not (FIFO for test related tasks)
    """  

    self.hight           =       hight
    self.width           =       width
    self.batch_size      =       batch_size
    self.image           =       np.array([f for f in os.listdir(folder_image) if format_image in f])
    self.f1              =       folder_image
    self.f2              =       folder_label
    self.size_epoch      =       len(self.image)
    if random:
      self.queue           =       tf.RandomShuffleQueue(shapes=[(self.hight,self.width,3), (self.hight,self.width), []],dtypes=[tf.float32, tf.float32, tf.string],capacity=16*self.batch_size, min_after_dequeue=8*self.batch_size)
    else:
      self.queue           =       tf.FIFOQueue(shapes=[(self.hight,self.width,3), (self.hight,self.width), []],dtypes=[tf.float32, tf.float32, tf.string],capacity=16*self.batch_size)
    self.image_pl        =       tf.placeholder(tf.float32, shape=(batch_size,hight,width,3))
    self.label_pl        =       tf.placeholder(tf.float32, shape=(batch_size,hight,width))
    self.name_pl         =       tf.placeholder(tf.string, shape=(batch_size))
    self.enqueue_op      =       self.queue.enqueue_many([self.image_pl, self.label_pl, self.name_pl]) 

def batch_input(self):

        if self._is_train:
            tf_files = glob.glob(os.path.join(config.tfrecords_path, 'train-*.tfrecords'))
            filename_queue = tf.train.string_input_producer(tf_files, shuffle=True, capacity=16)

            min_queue_examples = config.min_queue_examples
            examples_queue = tf.RandomShuffleQueue(
                capacity=min_queue_examples + 3 * self._batch_size,
                min_after_dequeue=min_queue_examples,
                dtypes=[tf.string])
            enqueue_ops = []
            for _ in range(config.num_readers):
                _, value = tf.TFRecordReader().read(filename_queue)
                enqueue_ops.append(examples_queue.enqueue([value]))

            tf.train.add_queue_runner(
                tf.train.QueueRunner(examples_queue, enqueue_ops))
            example_serialized = examples_queue.dequeue()
        else:
            tf_files = sorted(glob.glob(os.path.join(config.tfrecords_path, 'val-*.tfrecords')))
            filename_queue = tf.train.string_input_producer(tf_files, shuffle=False, capacity=8)
            _, example_serialized = tf.TFRecordReader().read(filename_queue)
            # example_serialized = next(tf.python_io.tf_record_iterator(self._tf_files[0]))
        images_and_labels = []
        for thread_id in range(config.num_preprocess_threads):
            sequence, context = self.parse_example_proto(example_serialized)
            image_buffers = sequence['images']
            bboxes = sequence['bboxes']
            seq_len = tf.cast(context['seq_len'][0], tf.int32)
            z_exemplars, x_crops, y_crops = self.process_images(image_buffers, bboxes, seq_len, thread_id)
            images_and_labels.append([z_exemplars, x_crops, y_crops])

        batch_z, batch_x, batch_y = tf.train.batch_join(images_and_labels,
                                                             batch_size=self._batch_size,
                                                             capacity=2 * config.num_preprocess_threads * self._batch_size)
        if self._is_train:
            tf.summary.image('exemplars', batch_z[0], 5)
            tf.summary.image('crops', batch_x[0], 5)

        return batch_z, batch_x, batch_y 

def _shuffle_inputs(input_tensors, capacity, min_after_dequeue, num_threads):
  """Shuffles tensors in `input_tensors`, maintaining grouping."""
  shuffle_queue = tf.RandomShuffleQueue(
      capacity, min_after_dequeue, dtypes=[t.dtype for t in input_tensors])
  enqueue_op = shuffle_queue.enqueue(input_tensors)
  runner = tf.train.QueueRunner(shuffle_queue, [enqueue_op] * num_threads)
  tf.train.add_queue_runner(runner)

  output_tensors = shuffle_queue.dequeue()

  for i in range(len(input_tensors)):
    output_tensors[i].set_shape(input_tensors[i].shape)

  return output_tensors