Python tensorflow.contrib.layers.linear() Examples

def discriminator_res(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    # last layer must be linear
    # H = tf.squeeze(H, [1,2])
    # pdb.set_trace()
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    H_dis_0 = layers.fully_connected(tf.nn.dropout(H, keep_prob=dropout), num_outputs=opt.embed_size,
                                   biases_initializer=biasInit, activation_fn=None, scope=prefix + 'dis_1',
                                   reuse=is_reuse)
    H_dis_0n = tf.nn.relu(H_dis_0)                               
    H_dis_1 = layers.fully_connected(tf.nn.dropout(H_dis_0n, keep_prob=dropout), num_outputs=opt.embed_size,
                                   biases_initializer=biasInit, activation_fn=None, scope=prefix + 'dis_2',
                                   reuse=is_reuse)
    H_dis_1n = tf.nn.relu(H_dis_1) + H_dis_0
    H_dis_2 = layers.fully_connected(tf.nn.dropout(H_dis_1n, keep_prob=dropout), num_outputs=opt.embed_size,
                                   biases_initializer=biasInit, activation_fn=None, scope=prefix + 'dis_3',
                                   reuse=is_reuse)
    H_dis_2n = tf.nn.relu(H_dis_2) + H_dis_1
    H_dis_3 = layers.fully_connected(tf.nn.dropout(H_dis_2n, keep_prob=dropout), num_outputs=opt.embed_size,
                                   biases_initializer=biasInit, activation_fn=None, scope=prefix + 'dis_4',
                                   reuse=is_reuse)

    logits = layers.linear(tf.nn.dropout(H_dis_3, keep_prob=dropout), num_outputs=num_outputs,
                           biases_initializer=biasInit, scope=prefix + 'dis_10', reuse=is_reuse)
    return logits 

def _get_filter(self, data, grid, scope=None):
        """ Generate an attention filter """
        with tf.variable_scope(scope, 'filter', [data]):
            x_offset, y_offset, log_stride, log_scale, log_gamma = tf.split(
                layers.linear(data, 5, scope='parameters'), 5, axis=1)

            center = self._get_center(grid, (x_offset, y_offset), tf.exp(log_stride))

            scale = tf.expand_dims(tf.maximum(tf.exp(log_scale), self.epsilon), -1)
            filter_x = 1 + tf.square((self.data_x - center[0]) / tf.maximum(scale, self.epsilon))
            filter_y = 1 + tf.square((self.data_y - center[1]) / tf.maximum(scale, self.epsilon))

            filter_x = tf.reciprocal(tf.maximum(pi * scale * filter_x, self.epsilon))
            filter_y = tf.reciprocal(tf.maximum(pi * scale * filter_y, self.epsilon))

            return filter_x, filter_y, tf.exp(log_gamma) 

def approximate_posterior(self, tensor, scope='posterior'):
        """ Calculate the approximate posterior given the tensor """
        # Generate mu and sigma of the Gaussian for the approximate posterior
        with tf.variable_scope(scope, 'posterior', [tensor]):
            mean = layers.linear(tensor, self.sample_size, scope='mean')

            # Use the log of sigma for numerical stability
            log_sigma = layers.linear(tensor, self.sample_size, scope='log_sigma')

            # Create the Gaussian distribution
            sigma = tf.exp(log_sigma)
            posterior = distributions.Normal(mean, sigma, name='posterior')

            self.collect_named_outputs(posterior.loc)
            self.collect_named_outputs(posterior.scale)
            self.posteriors.append(posterior)

            return posterior 

def approximate_posterior(self, tensor, scope='posterior'):
        """ Calculate the approximate posterior given the tensor """
        # Generate mu and sigma of the Gaussian for the approximate posterior
        sample_size = self.prior.batch_shape.as_list()[-1]
        with tf.variable_scope(scope, 'posterior', [tensor]):
            mean = layers.linear(tensor, sample_size, scope='mean')

            # Use the log of sigma for numerical stability
            log_variance = layers.linear(tensor, sample_size, scope='log_variance')

            # Create the Uniform distribution
            variance = tf.exp(log_variance)
            delta = tf.sqrt(3.0 * variance)
            posterior = distributions.Uniform(mean - delta, mean + delta, name='posterior')

            self.collect_named_outputs(posterior.low)
            self.collect_named_outputs(posterior.high)
            self.posteriors.append(posterior)

            return posterior 

def discriminator_1layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    # last layer must be linear
    H = tf.squeeze(H)
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    H_dis = layers.fully_connected(tf.nn.dropout(H, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_1',
                                   reuse=is_reuse)
    return H_dis 

def _inference(self):
        logging.info('...create inference')

        fw_state_tuple = self.unstack_fw_states(self.fw_state)

        fw_cells   = list()
        for i in range(0, self.num_layers):
            if (self.cell_type == 'lstm'):
                cell = rnn.LSTMCell(num_units=self.cell_sizes[i], state_is_tuple=True)
            elif (self.cell_type == 'gru'):
                # change to GRU
                cell = rnn.GRUCell(num_units=self.cell_sizes[i])
            else:
                cell = rnn.BasicRNNCell(num_units=self.cell_sizes[i])

            cell = rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob)
            fw_cells.append(cell)
        self.fw_cells = rnn.MultiRNNCell(fw_cells, state_is_tuple=True)

        rnn_outputs, states = tf.nn.dynamic_rnn(
            self.fw_cells, 
            self.inputs, 
            initial_state=fw_state_tuple,
            sequence_length=self.seq_lengths,
            dtype=tf.float32, time_major=True)

        # project output from rnn output size to OUTPUT_SIZE. Sometimes it is worth adding
        # an extra layer here.
        self.projection = lambda x: layers.linear(x, 
            num_outputs=self.label_classes, activation_fn=tf.nn.sigmoid)

        self.logits = tf.map_fn(self.projection, rnn_outputs, name="logits")
        self.probs  = tf.nn.softmax(self.logits, name="probs")
        self.states = states

        tf.add_to_collection('probs',  self.probs) 

def discriminator_3layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    # last layer must be linear
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    H_dis = layers.fully_connected(tf.nn.dropout(H, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_1',
                                   reuse=is_reuse)
    H_dis = layers.fully_connected(tf.nn.dropout(H_dis, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_2',
                                   reuse=is_reuse)
    logits = layers.linear(tf.nn.dropout(H_dis, keep_prob=dropout), num_outputs=num_outputs,
                           biases_initializer=biasInit, scope=prefix + 'dis_3', reuse=is_reuse)
    return logits 

def discriminator_2layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    # last layer must be linear
    print(num_outputs, "===num outputs===")
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    H_dis = layers.fully_connected(tf.nn.dropout(H, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_1',
                                   reuse=is_reuse)
    logits = layers.linear(tf.nn.dropout(H_dis, keep_prob=dropout), num_outputs=num_outputs,
                           biases_initializer=biasInit, scope=prefix + 'dis_2', reuse=is_reuse)
    return logits 

def discriminator_0layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    H = tf.squeeze(H)
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    logits = layers.linear(tf.nn.dropout(H, keep_prob=dropout), num_outputs=num_outputs, biases_initializer=biasInit,
                           scope=prefix + 'dis', reuse=is_reuse)
    return logits 

def discriminator_3layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    # last layer must be linear
    # H = tf.squeeze(H, [1,2])
    # pdb.set_trace()
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    H_dis = layers.fully_connected(tf.nn.dropout(H, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_1',
                                   reuse=is_reuse)
    H_dis = layers.fully_connected(tf.nn.dropout(H_dis, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_2',
                                   reuse=is_reuse)
    logits = layers.linear(tf.nn.dropout(H_dis, keep_prob=dropout), num_outputs=num_outputs,
                           biases_initializer=biasInit, scope=prefix + 'dis_3', reuse=is_reuse)
    return logits 

def discriminator_2layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    # last layer must be linear
    # H = tf.squeeze(H, [1,2])
    # pdb.set_trace()
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    H_dis = layers.fully_connected(tf.nn.dropout(H, keep_prob=dropout), num_outputs=opt.H_dis,
                                   biases_initializer=biasInit, activation_fn=tf.nn.relu, scope=prefix + 'dis_1',
                                   reuse=is_reuse)
    logits = layers.linear(tf.nn.dropout(H_dis, keep_prob=dropout), num_outputs=num_outputs,
                           biases_initializer=biasInit, scope=prefix + 'dis_2', reuse=is_reuse)
    return logits 

def discriminator_0layer(H, opt, dropout, prefix='', num_outputs=1, is_reuse=None):
    H = tf.squeeze(H)
    biasInit = tf.constant_initializer(0.001, dtype=tf.float32)
    logits = layers.linear(tf.nn.dropout(H, keep_prob=dropout), num_outputs=num_outputs, biases_initializer=biasInit,
                           scope=prefix + 'dis', reuse=is_reuse)
    return logits 

def write(self, data):
        """ Do a write given the data """
        return layers.linear(data, self.output_size, scope='write') 

def _logits(logits_input, logits, logits_dimension):
  """Validate logits args, and create `logits` if necessary.

  Exactly one of `logits_input` and `logits` must be provided.

  Args:
    logits_input: `Tensor` input to `logits`.
    logits: `Tensor` output.
    logits_dimension: Integer, last dimension of `logits`. This is used to
      create `logits` from `logits_input` if `logits` is `None`; otherwise, it's
      used to validate `logits`.

  Returns:
    `logits` `Tensor`.

  Raises:
    ValueError: if neither or both of `logits` and `logits_input` are supplied.
  """
  if (logits_dimension is None) or (logits_dimension < 1):
    raise ValueError("Invalid logits_dimension %s." % logits_dimension)

  # If not provided, create logits.
  if logits is None:
    if logits_input is None:
      raise ValueError("Neither logits nor logits_input supplied.")
    return layers_lib.linear(logits_input, logits_dimension, scope="logits")

  if logits_input is not None:
    raise ValueError("Both logits and logits_input supplied.")

  logits = ops.convert_to_tensor(logits, name="logits")
  logits_dims = logits.get_shape().dims
  if logits_dims is not None:
    logits_dims[-1].assert_is_compatible_with(logits_dimension)

  return logits 

def regression_head(label_name=None,
                    weight_column_name=None,
                    label_dimension=1,
                    enable_centered_bias=False,
                    head_name=None):
  """Creates a `Head` for linear regression.

  Args:
    label_name: String, name of the key in label dict. Can be null if label
        is a tensor (single headed models).
    weight_column_name: A string defining feature column name representing
      weights. It is used to down weight or boost examples during training. It
      will be multiplied by the loss of the example.
    label_dimension: Number of regression labels per example. This is the size
      of the last dimension of the labels `Tensor` (typically, this has shape
      `[batch_size, label_dimension]`).
    enable_centered_bias: A bool. If True, estimator will learn a centered
      bias variable for each class. Rest of the model structure learns the
      residual after centered bias.
    head_name: name of the head. If provided, predictions, summary and metrics
      keys will be suffixed by `"/" + head_name` and the default variable scope
      will be `head_name`.

  Returns:
    An instance of `Head` for linear regression.
  """
  return _RegressionHead(
      label_name=label_name,
      weight_column_name=weight_column_name,
      label_dimension=label_dimension,
      enable_centered_bias=enable_centered_bias,
      head_name=head_name,
      loss_fn=_mean_squared_loss,
      link_fn=array_ops.identity) 

def approximate_posterior(self, tensor, scope='posterior'):
        """ Calculate the approximate posterior given the tensor """
        # Generate the minimum chi squared divergence distribution 'f' from the prior 'g'
        with tf.variable_scope(scope, 'posterior', [tensor]):
            mean = layers.linear(tensor, self.sample_size, scope='mean')

            # Create the Gaussian distribution
            posterior = MinChiSquaredDistribution(mean, name='posterior')

            self.collect_named_outputs(posterior.mean())
            self.collect_named_outputs(posterior.variance())
            self.posteriors.append(posterior)

            return posterior 

def _get_filter(self, data, grid, scope=None):
        """ Generate an attention filter """
        with tf.variable_scope(scope, 'filter', [data]):
            x_offset, y_offset, log_stride, log_variance, log_gamma = tf.split(
                layers.linear(data, 5, scope='parameters'), 5, axis=1)

            center = self._get_center(grid, (x_offset, y_offset), tf.exp(log_stride))

            scale = 2.0 * tf.square(tf.exp(log_variance / 2.0))
            scale = tf.expand_dims(tf.maximum(scale, self.epsilon), -1)

            filter_x = tf.exp(-tf.square(self.data_x - center[0]) / scale)
            filter_y = tf.exp(-tf.square(self.data_y - center[1]) / scale)

            return filter_x, filter_y, tf.exp(log_gamma) 

def _get_filter(self, data, grid, scope=None):
        """ Generate an attention filter """
        with tf.variable_scope(scope, 'filter', [data]):
            x_offset, y_offset, log_stride, scale, log_gamma = tf.split(
                layers.linear(data, 5, scope='parameters'), 5, axis=1)

            center = self._get_center(grid, (x_offset, y_offset), tf.exp(log_stride))

            scale = tf.expand_dims(scale, -1)
            filter_x = scale * (self.data_x - center[0])
            filter_y = scale * (self.data_y - center[1])

            return filter_x, filter_y, tf.exp(log_gamma) 

def _get_key(self, focus):
        """ Get the key for the data """
        beta = layers.linear(focus, 1)
        key = layers.linear(focus, self.shape[1])

        return beta, tf.expand_dims(tf.nn.l2_normalize(key, -1), -1) 

def read_multiple(self, data_list, focus):
        """ Do a filtered read for multiple tensors using the same focus """
        focus = layers.linear(focus, data_list[0].get_shape().as_list()[-1])
        focused = tf.expand_dims(self.focus_fn(focus, name='focus'), 1)

        focus_list = []
        for data in data_list:
            focus_list.append(layers.flatten(focused * data))

        return tf.concat(focus_list, 1) 

def read(self, data, focus):
        """ Do a read given the data """
        focus = layers.linear(focus, data.get_shape().as_list()[-1])
        focused = tf.expand_dims(self.focus_fn(focus, name='focus'), 1)

        return layers.flatten(focused * data) 

def model_fn(mode, inputs, reuse=False):
    samples = tf.reshape(inputs['images'], (-1, SEQUENCE_LENGTH, 1))
    ground_truth = tf.cast(inputs['labels'], tf.int64)

    is_training = (mode == 'train')

    with tf.variable_scope('model', reuse=reuse):
        cell, initial_state = create_model(model=FLAGS.model,
                                           num_cells=[FLAGS.rnn_cells] * FLAGS.rnn_layers,
                                           batch_size=FLAGS.batch_size)

        rnn_outputs, rnn_states = tf.nn.dynamic_rnn(cell, samples, dtype=tf.float32, initial_state=initial_state)

        # Split the outputs of the RNN into the actual outputs and the state update gate
        rnn_outputs, updated_states = split_rnn_outputs(FLAGS.model, rnn_outputs)

        logits = layers.linear(inputs=rnn_outputs[:, -1, :], num_outputs=OUTPUT_SIZE)
        predictions = tf.argmax(logits, 1)

    # Compute cross-entropy loss
    cross_entropy_per_sample = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=ground_truth)
    cross_entropy = tf.reduce_mean(cross_entropy_per_sample)

    # Compute accuracy
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predictions, ground_truth), tf.float32))

    # Compute loss for each updated state
    budget_loss = compute_budget_loss(FLAGS.model, cross_entropy, updated_states, FLAGS.cost_per_sample)

    # Combine all losses
    loss = cross_entropy + budget_loss
    loss = tf.reshape(loss, [])

    if is_training:
        # Optimizer
        opt, grads_and_vars = compute_gradients(loss, FLAGS.learning_rate, FLAGS.grad_clip)
        train_fn = opt.apply_gradients(grads_and_vars)

    model_spec = inputs
    model_spec['variable_init_op'] = tf.global_variables_initializer()
    model_spec['samples'] = samples
    model_spec['labels'] = ground_truth
    model_spec['loss'] = loss
    model_spec['accuracy'] = accuracy
    model_spec['updated_states'] = updated_states

    if is_training:
        model_spec['train_fn'] = train_fn

    return model_spec 

def __init__(self, params):
        """
        
        :param params: dictionary with fields:
            "N_HIDDEN": number of hidden states
            "N_BINS": number of bins on input/ output
            "LEARNING_RATE": learning rate in optimizer
        """
        self.params = params

        tf.reset_default_graph()

        self.session = tf.Session()

        self.inputs = tf.placeholder(tf.float32, (None, None, params['N_BINS']))

        self.cell = tf.contrib.rnn.LSTMCell(params['N_HIDDEN'], state_is_tuple=True)
        self.batch_size = tf.shape(self.inputs)[1]

        self.h_init = tf.Variable(tf.zeros([1, params['N_HIDDEN']]), trainable=True)
        self.h_init_til = tf.tile(self.h_init, [self.batch_size, 1])

        self.c_init = tf.Variable(tf.zeros([1, params['N_HIDDEN']]), trainable=True)
        self.c_init_til = tf.tile(self.c_init, [self.batch_size, 1])

        self.initial_state = LSTMStateTuple(self.c_init_til, self.h_init_til)

        self.rnn_outputs, self.rnn_states = \
            tf.nn.dynamic_rnn(self.cell,
                              self.inputs,
                              initial_state=self.initial_state,
                              time_major=True)

        with tf.variable_scope("output"):

            self.intermediate_projection = \
                lambda x: layers.fully_connected(x, num_outputs=params['N_HIDDEN'])

            self.final_projection = \
                lambda x: layers.linear(x, num_outputs=params['N_BINS'])

            self.intermediate_features = tf.map_fn(self.intermediate_projection, self.rnn_outputs)
            self.final_features = tf.map_fn(self.final_projection, self.intermediate_features)
            self.predicted_outputs = layers.softmax(self.final_features)

        with tf.variable_scope("train"):
            self.outputs = \
                tf.placeholder(tf.float32, (None, None, params['N_BINS']))

            self.mask = tf.placeholder(tf.float32, (None, None, params['N_BINS']))

            self.all_errors = losses.categorical_crossentropy(self.outputs * self.mask, self.predicted_outputs)

            self.error = tf.reduce_mean(self.all_errors)

            self.train_fn = \
                tf.train.AdamOptimizer(learning_rate=params['LEARNING_RATE']) \
                    .minimize(self.error) 

def __init__(self, params):
        """

        :param params: dictionary with fields:
            "N_HIDDEN": number of hidden states
            "N_BINS": number of bins on input/ output
            "WINDOW_LENGTH": number of time-steps in training window
        """
        tf.reset_default_graph()

        self.session = tf.Session()

        self.inputs = tf.placeholder(tf.float32, (None, None, params['N_BINS']))

        self.cell = tf.contrib.rnn.LSTMCell(params['N_HIDDEN'], state_is_tuple=True)
        self.batch_size = tf.shape(self.inputs)[1]

        self.h_init = tf.Variable(tf.zeros([1, params['N_HIDDEN']]), trainable=True)
        self.h_init_til = tf.tile(self.h_init, [self.batch_size, 1])

        self.c_init = tf.Variable(tf.zeros([1, params['N_HIDDEN']]), trainable=True)
        self.c_init_til = tf.tile(self.c_init, [self.batch_size, 1])

        self.initial_state = LSTMStateTuple(self.c_init_til, self.h_init_til)

        self.rnn_outputs, self.rnn_states = \
            tf.nn.dynamic_rnn(self.cell,
                              self.inputs,
                              initial_state=self.initial_state,
                              time_major=True)

        self.intermediate_projection = layers.fully_connected(self.rnn_outputs[params["WINDOW_LENGTH"] - 2, :, :], num_outputs=params['N_HIDDEN'])
        self.final_features = layers.linear(self.intermediate_projection, num_outputs=params["N_BINS"])

        self.predicted_outputs = layers.softmax(self.final_features)

        with tf.variable_scope("train"):
            # get element-wise error
            self.outputs = \
                tf.placeholder(tf.float32, (None, params['N_BINS']))

            self.all_errors = losses.categorical_crossentropy(self.outputs, self.predicted_outputs)

            # measure error
            self.error = tf.reduce_mean(self.all_errors)

            self.train_fn = \
                tf.train.AdamOptimizer(learning_rate=params['LEARNING_RATE']) \
                    .minimize(self.error)