import numpy as np import matplotlib.pyplot as plt from scipy.stats import norm import keras from keras.layers import Input, Dense, Lambda, Flatten, Reshape, Layer from keras.layers import Conv2D, Conv2DTranspose from keras.models import Model from keras import backend as K from keras import metrics from keras.datasets import cifar10 import cPickle # import parameters from caltech101_128_params import * """ loading vae model back is not a straight-forward task because of custom loss layer. we have to define some architecture back again to specify custom loss layer and hence to load model back again. """ # tensorflow or theano if K.image_data_format() == 'channels_first': original_img_size = (img_chns, img_rows, img_cols) else: original_img_size = (img_rows, img_cols, img_chns) # encoder architecture x = Input(shape=original_img_size) conv_1 = Conv2D(img_chns, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')(x) conv_2 = Conv2D(filters, kernel_size=(3, 3), padding='same', activation='relu', strides=(2, 2))(conv_1) conv_3 = Conv2D(filters, kernel_size=num_conv, padding='same', activation='relu', strides=(2, 2))(conv_2) conv_4 = Conv2D(filters, kernel_size=num_conv, padding='same', activation='relu', strides=(2, 2))(conv_3) flat = Flatten()(conv_4) hidden = Dense(intermediate_dim, activation='relu')(flat) z_mean = Dense(latent_dim)(hidden) z_log_var = Dense(latent_dim)(hidden) # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): self.is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean_squash): x = K.flatten(x) x_decoded_mean_squash = K.flatten(x_decoded_mean_squash) xent_loss = img_rows * img_cols * metrics.binary_crossentropy(x, x_decoded_mean_squash) kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return K.mean(xent_loss + kl_loss) def call(self, inputs): x = inputs[0] x_decoded_mean_squash = inputs[1] loss = self.vae_loss(x, x_decoded_mean_squash) self.add_loss(loss, inputs=inputs) # We don't use this output. return x # load saved models vae = keras.models.load_model('../models/object101_ld_%d_conv_%d_id_%d_e_%d_vae.h5' % (latent_dim, num_conv, intermediate_dim, epochs), custom_objects={'latent_dim':latent_dim, 'epsilon_std':epsilon_std, 'CustomVariationalLayer':CustomVariationalLayer}) encoder = keras.models.load_model('../models/object101_ld_%d_conv_%d_id_%d_e_%d_encoder.h5' % (latent_dim, num_conv, intermediate_dim, epochs), custom_objects={'latent_dim':latent_dim, 'epsilon_std':epsilon_std, 'CustomVariationalLayer':CustomVariationalLayer}) generator = keras.models.load_model('../models/object101_ld_%d_conv_%d_id_%d_e_%d_generator.h5' % (latent_dim, num_conv, intermediate_dim, epochs), custom_objects={'latent_dim':latent_dim, 'epsilon_std':epsilon_std, 'CustomVariationalLayer':CustomVariationalLayer}) # load history if saved fname = '../models/object101_ld_%d_conv_%d_id_%d_e_%d_history.pkl' % (latent_dim, num_conv, intermediate_dim, epochs) try: with open(fname, 'rb') as fo: history = cPickle.load(fo) print history except: print "training history not saved" """ with open('../datasets/101_ObjectCategories.pkl') as f: dic = cPickle.load(f) x_train = dic['all_images'] print "dataset loaded" """ """ # display a 2D plot of the classes in the latent space x_test_encoded = encoder.predict(x_test, batch_size=batch_size) plt.figure(figsize=(6, 6)) plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test) plt.colorbar() plt.show() # display a 2D manifold of the images n = 15 # figure with 15x15 images digit_size = 32 figure = np.zeros((digit_size * n, digit_size * n, img_chns)) # linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian # to produce values of the latent variables z, since the prior of the latent space is Gaussian grid_x = norm.ppf(np.linspace(0.05, 0.95, n)) grid_y = norm.ppf(np.linspace(0.05, 0.95, n)) for i, yi in enumerate(grid_x): for j, xi in enumerate(grid_y): z_sample = np.array([[xi, yi]]) z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2) x_decoded = generator.predict(z_sample, batch_size=batch_size) digit = x_decoded[0].reshape(digit_size, digit_size, img_chns) figure[i * digit_size: (i + 1) * digit_size, j * digit_size: (j + 1) * digit_size] = digit plt.figure(figsize=(10, 10)) plt.imshow(figure, cmap='Greys_r') plt.show() """ # display images generated from randomly sampled latent vector # make a 2D manifold of generated images n = 10 digit_size = 128 figure = np.zeros((digit_size * n, digit_size * n, img_chns)) for i in range(n): for j in range(n): z_sample = np.array([np.random.uniform(-1, 1,size=latent_dim)]) x_decoded = generator.predict(z_sample) digit = x_decoded[0].reshape(digit_size, digit_size, img_chns) figure[i * digit_size: (i + 1) * digit_size,j * digit_size: (j + 1) * digit_size] = digit plt.figure(figsize=(20, 20)) plt.imshow(figure, cmap='Greys_r') plt.show()