import os
import pickle
import itertools
from matplotlib import pyplot as plt
from sklearn import metrics
import configparser
import numpy as np
from bird import loader
import evaluate
import data_analysis
def chunks(l, n):
chunk_size = int(np.ceil(len(l)/n))
"""Yield n chunks from l."""
for i in range(0, len(l), chunk_size):
yield l[i:i + chunk_size]
def plot_all(experiment_path):
plot_confusion_matrix(experiment_path)
plot_training_history(experiment_path)
# plot_decending_training_samples_by_number_of_predictions(experiment_path,
# "datasets/birdClef2016Whole1/train/")
def load_predictions(experiment_path):
picke_file = os.path.join(experiment_path, "predictions.pkl")
print(picke_file)
with open(picke_file, 'rb') as input:
y_trues = pickle.load(input)
y_scores = pickle.load(input)
return y_trues, y_scores
def plot_decending_training_samples_by_number_of_predictions(experiment_path,
train_dir):
config_parser = configparser.ConfigParser()
config_parser.read(os.path.join(experiment_path, "conf.ini"))
model_name = config_parser['MODEL']['ModelName']
# order by number of training samples
# compute number of predictions per class
index_to_species = loader.build_class_index(train_dir)
species_to_index = {v: k for k, v in index_to_species.items()}
index_to_nb_training_segments = {}
classes = os.listdir(train_dir)
for c in classes:
class_dir = os.path.join(train_dir, c)
nb_training_segments = len(os.listdir(class_dir))
index_to_nb_training_segments[species_to_index[c]] = nb_training_segments
y_trues, y_scores = load_predictions(experiment_path)
y_true = [np.argmax(y_t) for y_t in y_trues]
y_pred = [np.argmax(y_s) for y_s in y_scores]
confusion_matrix = metrics.confusion_matrix(y_true, y_pred)
print(confusion_matrix)
decending_by_training = []
for key, value in index_to_nb_training_segments.items():
decending_by_training.append((key, value))
decending_by_training.sort(key=lambda x: x[1], reverse=True)
ys1 = []
ys2 = []
ys3 = []
for (key, value) in decending_by_training:
actual_nb_predictions = np.sum(confusion_matrix[:,key])
expected_nb_predictions = np.sum(confusion_matrix[key,:])
ys1.append(actual_nb_predictions)
ys2.append(expected_nb_predictions)
ys3.append(value)
ys1 = chunks(ys1, 20)
ys1 = [np.mean(y) for y in ys1]
ys2 = chunks(ys2, 20)
ys2 = [np.mean(y) for y in ys2]
ys3 = chunks(ys3, 20)
ys3 = [np.mean(y) for y in ys3]
title = "#Predictions and Training Segments ranked by #Training Segments"
fig, ax1 = plt.subplots()
plt.title(title)
ax1.set_ylim([0, 10])
ax1.plot(ys1, 'ro-', label="actual predictions")
ax1.plot(ys2, 'go-', label="expected predictions")
ax1.set_ylabel("Average Number of Predictions")
ax1.set_xlabel("5% Chunks of Classes Ranked by Number of Training Segments (decreasing)")
ax1.legend(loc="lower left")
ax2 = ax1.twinx()
ax2.set_ylim([0, 600])
ax2.set_ylabel("Numer of Training Segments")
ax2.plot(ys3, 'bo-', label="training segments")
ax2.legend(loc="upper right")
fig.savefig(os.path.join(experiment_path,
"training_samples_by_number_of_predictions.png"))
fig.clf()
plt.close(fig)
def plot_sound_class_by_decending_accuracy(experiment_path):
config_parser = configparser.ConfigParser()
config_parser.read(os.path.join(experiment_path, "conf.ini"))
model_name = config_parser['MODEL']['ModelName']
y_trues, y_scores = load_predictions(experiment_path)
y_true = [np.argmax(y_t) for y_t in y_trues]
y_pred = [np.argmax(y_s) for y_s in y_scores]
confusion_matrix = metrics.confusion_matrix(y_true, y_pred)
accuracies = []
(nb_rows, nb_cols) = confusion_matrix.shape
for i in range(nb_rows):
accuracy = confusion_matrix[i][i] / np.sum(confusion_matrix[i,:])
accuracies.append(accuracy)
fig = plt.figure()
plt.title("Sound Class ranked by Accuracy ({})".format(model_name))
plt.plot(sorted(accuracies, reverse=True))
plt.ylabel("Accuracy")
plt.xlabel("Rank")
# plt.pcolormesh(confusion_matrix, cmap=cmap)
fig.savefig(os.path.join(experiment_path, "descending_accuracy.png"))
def print_top_n_confusions(top_n, experiment_path, train_dir):
config_parser = configparser.ConfigParser()
config_parser.read(os.path.join(experiment_path, "conf.ini"))
model_name = config_parser['MODEL']['ModelName']
index_to_species = loader.build_class_index(train_dir)
y_trues, y_scores = load_predictions(experiment_path)
y_true = [np.argmax(y_t) for y_t in y_trues]
y_pred = [np.argmax(y_s) for y_s in y_scores]
confusion_matrix = metrics.confusion_matrix(y_true, y_pred)
# confusion_matrix = np.flip(confusion_matrix, 0)
(nb_rows, nb_cols) = confusion_matrix.shape
confusions = []
for i in range(nb_rows):
confused_predictions = np.sum(confusion_matrix[:,i]) - confusion_matrix[i][i]
confusions.append((i, confused_predictions))
sorted_confusions = sorted(confusions, key=lambda x: x[1], reverse=True)
for (i, confusion) in sorted_confusions[:top_n]:
accuracy = confusion_matrix[i][i] / np.sum(confusion_matrix[i,:])
print("Class {} ({}): {} (accuracy : {})".format(index_to_species[i],
i, confusion, accuracy))
def plot_confusion_matrix(experiment_path):
config_parser = configparser.ConfigParser()
config_parser.read(os.path.join(experiment_path, "conf.ini"))
model_name = config_parser['MODEL']['ModelName']
y_trues, y_scores = load_predictions(experiment_path)
y_true = [np.argmax(y_t) for y_t in y_trues]
y_pred = [np.argmax(y_s) for y_s in y_scores]
confusion_matrix = metrics.confusion_matrix(y_true, y_pred)
# confusion_matrix = np.flip(confusion_matrix, 0)
title = "Confusion Matrix ({})".format(model_name)
cmap = plt.cm.get_cmap('jet')
fig = plt.figure()
plt.imshow(confusion_matrix, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
plt.ylabel("True class")
plt.xlabel("Predicted class")
# plt.pcolormesh(confusion_matrix, cmap=cmap)
fig.savefig(os.path.join(experiment_path, "confusion.png"))
fig.clf()
plt.close(fig)
(nb_rows, nb_cols) = confusion_matrix.shape
confusions = []
for i in range(nb_rows):
confused_predictions = np.sum(confusion_matrix[:,i]) - confusion_matrix[i][i]
confusions.append((i, confused_predictions))
sorted_confusions = sorted(confusions, key=lambda x: x[1], reverse=True)
for (i, confusion) in sorted_confusions:
print("Class {}: {}".format(i, confusion))
def plot_training_history(experiment_path):
pickle_file = os.path.join(experiment_path, "history.pkl")
with open(pickle_file, 'rb') as input:
trainLoss = pickle.load(input)
validLoss = pickle.load(input)
trainAcc = pickle.load(input)
validAcc = pickle.load(input)
x = range(len(validAcc))
y = validAcc
z = np.polyfit(x, y, 3)
p = np.poly1d(z)
fig = plt.figure(1)
plt.subplot(211)
axes = plt.gca()
axes.set_ylim([0, 10])
plt.ylabel("Loss")
plt.plot(trainLoss, 'o-', label="train")
plt.plot(validLoss, 'o-', label="valid")
plt.legend(loc="upper right")
plt.subplot(212)
axes = plt.gca()
axes.set_ylim([0, 1])
plt.ylabel("Accuracy")
plt.xlabel("Epoch")
plt.plot(trainAcc, 'o-', label="train")
plt.plot(validAcc, 'o-', label="valid")
# plt.plot(p(x), 'o-', label="trend")
plt.legend(loc="lower right")
fig.savefig(os.path.join(experiment_path, "history.png"))
plt.clf()
plt.close()
# def plot_decending_accuracy_by_sound_class(experiment_path):
# y_trues, y_scores = load_predictions(experiment_path)
# y_labels = [np.argmax(y_t) for y_t in y_trues]
def main():
plot_all()
if __name__ == "__main__":
main()
