```########################################################################
# Scores a Machine Learning Problem Dataset
# Suitable for assigning to a student for a homework assignment
#
#
########################################################################

import argparse
from math import sqrt

import pandas as pd
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
import sys

def parse_args(args):
"""
Returns arguments passed at the command line as a dict
:return: configuration dictionary

"""
parser = argparse.ArgumentParser(description='Scores a ML dataset solution.')
dest='pred')
dest='key')
return vars(parser.parse_args(args))

"""
Opens file names and returns dataframes

:return: y, y_hat  as a tuple of dataframes
"""
# our test key has the features and the answer in it, we only need the answer to score
y = y_df['y']
return y, y_hat

def guess_problem_type(key):
"""
Infers the problem type, using the key dataframe
:return: Inferred Problem Type
"""
num_values = len(key.unique())
if num_values == 2:
return "binary"
elif (num_values > 2) and (num_values < 100):  # assumptions that will burn me later probably
return "multiclass"
else:
return "regression"

def score_binary_classification(y, y_hat, report=True):
"""
Create binary classification output
:param y: true value
:param y_hat: class 1 probabilities
:param report:
:return:
"""
y_hat_class = [1 if x >= 0.5 else 0 for x in y_hat]  # convert probability to class for classification report

report_string = "---Binary Classification Score--- \n"
report_string += classification_report(y, y_hat_class)
score = roc_auc_score(y, y_hat)
report_string += "\nAUC = " + str(score)

if report:
print(report_string)

return score, report_string

def score_multiclass_classification(y, y_hat, report=True):
"""
Create multiclass classification score
:param y:
:param y_hat:
:return:
"""
report_string = "---Multiclass Classification Score--- \n"
report_string += classification_report(y, y_hat)
score = accuracy_score(y, y_hat)
report_string += "\nAccuracy = " + str(score)

if report:
print(report_string)

return score, report_string

def score_regression(y, y_hat, report=True):
"""
Create regression score
:param y:
:param y_hat:
:return:
"""
r2 = r2_score(y, y_hat)
rmse = sqrt(mean_squared_error(y, y_hat))
mae = mean_absolute_error(y, y_hat)

report_string = "---Regression Score--- \n"
report_string += "R2 = " + str(r2) + "\n"
report_string += "RMSE = " + str(rmse) + "\n"
report_string += "MAE = " + str(mae) + "\n"

if report:
print(report_string)

return mae, report_string

def score_dataset(y_file=None, y_hat_file=None):
"""
1 Reads in key file and prediction file (students predictions)
2 guesses problem type
3 scores problem

:return:
"""

report_output = True
if y_file is None and y_hat_file is None:
# called from the command line so parse configuration
args = parse_args(sys.argv[1:])
else:
report_output = False

problem_type = guess_problem_type(y)
print("Problem Type Detection: " + problem_type)
print("y shape: " + str(y.shape) + " y hat shape: " + str(y_hat.shape))
if problem_type == 'binary':
results = score_binary_classification(y, y_hat[0], report=report_output)
elif problem_type == 'multiclass':
results = score_multiclass_classification(y, y_hat[0], report=report_output)
else:
results = score_regression(y, y_hat)

if not report_output:
return results

if __name__ == "__main__":
score_dataset()
```