"""
Created on Thu Oct 21 11:09:09 2017
@author: Utku Ozbulak - github.com/utkuozbulak
"""
import os
import copy
import cv2
import numpy as np
import torch
from torch.autograd import Variable
from torchvision import models
def convert_to_grayscale(cv2im):
"""
Converts 3d image to grayscale
Args:
cv2im (numpy arr): RGB image with shape (D,W,H)
returns:
grayscale_im (numpy_arr): Grayscale image with shape (1,W,D)
"""
grayscale_im = np.sum(np.abs(cv2im), axis=0)
im_max = np.percentile(grayscale_im, 99)
im_min = np.min(grayscale_im)
grayscale_im = (np.clip((grayscale_im - im_min) / (im_max - im_min), 0, 1))
grayscale_im = np.expand_dims(grayscale_im, axis=0)
return grayscale_im
def save_gradient_images(gradient, file_name):
"""
Exports the original gradient image
Args:
gradient (np arr): Numpy array of the gradient with shape (3, 224, 224)
file_name (str): File name to be exported
"""
gradient = gradient - gradient.min()
gradient /= gradient.max()
gradient = np.uint8(gradient * 255).transpose(1, 2, 0)
path_to_file = os.path.join('../results', file_name + '.jpg')
# Convert RBG to GBR
gradient = gradient[..., ::-1]
cv2.imwrite(path_to_file, gradient)
def save_class_activation_on_image(org_img, activation_map, file_name):
"""
Saves cam activation map and activation map on the original image
Args:
org_img (PIL img): Original image
activation_map (numpy arr): activation map (grayscale) 0-255
file_name (str): File name of the exported image
"""
# Grayscale activation map
path_to_file = os.path.join('../results', file_name+'_Cam_Grayscale.jpg')
cv2.imwrite(path_to_file, activation_map)
# Heatmap of activation map
activation_heatmap = cv2.applyColorMap(activation_map, cv2.COLORMAP_HSV)
path_to_file = os.path.join('../results', file_name+'_Cam_Heatmap.jpg')
cv2.imwrite(path_to_file, activation_heatmap)
# Heatmap on picture
org_img = cv2.resize(org_img, (224, 224))
img_with_heatmap = np.float32(activation_heatmap) + np.float32(org_img)
img_with_heatmap = img_with_heatmap / np.max(img_with_heatmap)
path_to_file = os.path.join('../results', file_name+'_Cam_On_Image.jpg')
cv2.imwrite(path_to_file, np.uint8(255 * img_with_heatmap))
def preprocess_image(cv2im, resize_im=True):
"""
Processes image for CNNs
Args:
PIL_img (PIL_img): Image to process
resize_im (bool): Resize to 224 or not
returns:
im_as_var (Pytorch variable): Variable that contains processed float tensor
"""
# mean and std list for channels (Imagenet)
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
# Resize image
if resize_im:
cv2im = cv2.resize(cv2im, (224, 224))
im_as_arr = np.float32(cv2im)
im_as_arr = np.ascontiguousarray(im_as_arr[..., ::-1])
im_as_arr = im_as_arr.transpose(2, 0, 1) # Convert array to D,W,H
# Normalize the channels
for channel, _ in enumerate(im_as_arr):
im_as_arr[channel] /= 255
im_as_arr[channel] -= mean[channel]
im_as_arr[channel] /= std[channel]
# Convert to float tensor
im_as_ten = torch.from_numpy(im_as_arr).float()
# Add one more channel to the beginning. Tensor shape = 1,3,224,224
im_as_ten.unsqueeze_(0)
# Convert to Pytorch variable
im_as_var = Variable(im_as_ten, requires_grad=True)
return im_as_var
<<<<<<< HEAD
# def save_image(tensor, filename, imtype=np.uint8, mode='RGB'):
# image_numpy = tensor[0].cpu().float().numpy()
# # [-1,1] -- [0,255]
# image_numpy = np.transpose(image_numpy, (1, 2, 0))
# if np.uint16:
# from math import pow
# mult = pow(2, 16) - 1
# else:
# mult = 255
# image_numpy = (image_numpy + 1) / 2.0 * mult
# image_numpy = image_numpy.astype(imtype)
# image_save = Image.fromarray(np.squeeze(image_numpy), mode=mode).convert(mode='I')
# if use_padding:
# image_width, image_height = self.opt.imageSize if len(self.opt.imageSize) == 2 else self.opt.imageSize * 2
# d_left, _, d_upper, _ = self.opt.padding
# # left, upper, right, lower
# image_save = image_save.crop((d_left, d_upper, image_width + d_left, image_height + d_upper))
# image_save.save(filename)
# def save_output_convolution(output):
# self.save_image(output.data, '{}/output/output_{:04}.png'.format(self.save_samples_path, index), imtype=np.uint16, mode='I;16')
=======
>>>>>>> 1050d9cd7bf9b36e200a8be4b70b581fa5876e22
def recreate_image(im_as_var):
"""
Recreates images from a torch variable, sort of reverse preprocessing
Args:
im_as_var (torch variable): Image to recreate
returns:
recreated_im (numpy arr): Recreated image in array
"""
<<<<<<< HEAD
# reverse_mean = [-0.485, -0.456, -0.406]
# reverse_std = [1/0.229, 1/0.224, 1/0.225]
reverse_mean = [-0.5, -0.5, -0.5]
reverse_std = [1 / 0.5, 1 / 0.5, 1 / 0.5]
recreated_im = copy.copy(im_as_var.data.cpu().numpy()[0])
=======
reverse_mean = [-0.485, -0.456, -0.406]
reverse_std = [1/0.229, 1/0.224, 1/0.225]
recreated_im = copy.copy(im_as_var.data.numpy()[0])
>>>>>>> 1050d9cd7bf9b36e200a8be4b70b581fa5876e22
for c in range(3):
recreated_im[c] /= reverse_std[c]
recreated_im[c] -= reverse_mean[c]
recreated_im[recreated_im > 1] = 1
recreated_im[recreated_im < 0] = 0
recreated_im = np.round(recreated_im * 255)
recreated_im = np.uint8(recreated_im).transpose(1, 2, 0)
# Convert RBG to GBR
recreated_im = recreated_im[..., ::-1]
return recreated_im
def get_positive_negative_saliency(gradient):
"""
Generates positive and negative saliency maps based on the gradient
Args:
gradient (numpy arr): Gradient of the operation to visualize
returns:
pos_saliency ( )
"""
pos_saliency = (np.maximum(0, gradient) / gradient.max())
neg_saliency = (np.maximum(0, -gradient) / -gradient.min())
return pos_saliency, neg_saliency
def get_params(example_index):
"""
Gets used variables for almost all visualizations, like the image, model etc.
Args:
example_index (int): Image id to use from examples
returns:
original_image (numpy arr): Original image read from the file
prep_img (numpy_arr): Processed image
target_class (int): Target class for the image
file_name_to_export (string): File name to export the visualizations
pretrained_model(Pytorch model): Model to use for the operations
"""
# Pick one of the examples
example_list = [['../input_images/snake.jpg', 56],
['../input_images/cat_dog.png', 243],
['../input_images/spider.png', 72],
['../input_images/apple.JPEG', 948],
['../input_images/eel.JPEG', 390],
['../input_images/bird.JPEG', 13]]
selected_example = example_index
img_path = example_list[selected_example][0]
target_class = example_list[selected_example][1]
file_name_to_export = img_path[img_path.rfind('/')+1:img_path.rfind('.')]
# Read image
original_image = cv2.imread(img_path, 1)
# Process image
prep_img = preprocess_image(original_image)
# Define model
pretrained_model = models.vgg19(pretrained=True)
return (original_image,
prep_img,
target_class,
file_name_to_export,
pretrained_model)
