import gzip
import math
import pickle
import zlib
import io
import pandas as pd
import numpy as np
# import scipy.stats
from sklearn.preprocessing import LabelEncoder
import engines
from utils import *
np.random.seed(2016)
transformers = {}
def assert_uniq(series, name):
uniq = np.unique(series, return_counts=True)
print("assert_uniq", name, uniq)
def custom_one_hot(df, features, name, names, dtype=np.int8, check=False):
for n, val in names.items():
new_name = "%s_%s" % (name, n)
print(name, new_name)
df[new_name] = df[name].map(lambda x: 1 if x == val else 0).astype(dtype)
if check:
assert_uniq(df[new_name], new_name)
features.append(new_name)
def label_encode(df, features, name):
df[name] = df[name].astype('str')
if name in transformers: # test
df[name] = transformers[name].transform(df[name])
else: # train
transformers[name] = LabelEncoder()
df[name] = transformers[name].fit_transform(df[name])
features.append(name)
def encode_top(s, count=100, dtype=np.int8):
uniqs, freqs = np.unique(s, return_counts=True)
top = sorted(zip(uniqs,freqs), key=lambda vk: vk[1], reverse = True)[:count]
top_map = {uf[0]: l+1 for uf, l in zip(top, range(len(top)))}
return s.map(lambda x: top_map.get(x, 0)).astype(dtype)
def apply_transforms(train_df):
features = []
with Timer("apply transforms"):
label_encode(train_df, features, "canal_entrada")
# label_encode(train_df, features, "nomprov") # use cod_prov only
label_encode(train_df, features, "pais_residencia")
train_df["age"] = train_df["age"].fillna(0.0).astype(np.int16)
features.append("age")
train_df["renta"].fillna(1.0, inplace=True)
train_df["renta_top"] = encode_top(train_df["renta"])
assert_uniq(train_df["renta_top"], "renta_top")
features.append("renta_top")
train_df["renta"] = train_df["renta"].map(math.log)
features.append("renta")
train_df["antiguedad"] = train_df["antiguedad"].map(lambda x: 0.0 if x < 0 or math.isnan(x) else x+1.0).astype(np.int16)
features.append("antiguedad")
train_df["tipodom"] = train_df["tipodom"].fillna(0.0).astype(np.int8)
features.append("tipodom")
train_df["cod_prov"] = train_df["cod_prov"].fillna(0.0).astype(np.int8)
features.append("cod_prov")
train_df["fecha_dato_month"] = train_df["fecha_dato"].map(lambda x: int(x.split("-")[1])).astype(np.int8)
features.append("fecha_dato_month")
train_df["fecha_dato_year"] = train_df["fecha_dato"].map(lambda x: float(x.split("-")[0])).astype(np.int16)
features.append("fecha_dato_year")
train_df["fecha_alta_month"] = train_df["fecha_alta"].map(lambda x: 0.0 if x.__class__ is float else float(x.split("-")[1])).astype(np.int8)
features.append("fecha_alta_month")
train_df["fecha_alta_year"] = train_df["fecha_alta"].map(lambda x: 0.0 if x.__class__ is float else float(x.split("-")[0])).astype(np.int16)
features.append("fecha_alta_year")
train_df["fecha_dato_float"] = train_df["fecha_dato"].map(date_to_float)
train_df["fecha_alta_float"] = train_df["fecha_alta"].map(date_to_float)
train_df["dato_minus_alta"] = train_df["fecha_dato_float"] - train_df["fecha_alta_float"]
features.append("dato_minus_alta")
train_df["int_date"] = train_df["fecha_dato"].map(date_to_int).astype(np.int8)
custom_one_hot(train_df, features, "indresi", {"n":"N"})
custom_one_hot(train_df, features, "indext", {"s":"S"})
custom_one_hot(train_df, features, "conyuemp", {"n":"N"})
custom_one_hot(train_df, features, "sexo", {"h":"H", "v":"V"})
custom_one_hot(train_df, features, "ind_empleado", {"a":"A", "b":"B", "f":"F", "n":"N"})
custom_one_hot(train_df, features, "ind_nuevo", {"new":1})
custom_one_hot(train_df, features, "segmento", {"top":"01 - TOP", "particulares":"02 - PARTICULARES", "universitario":"03 - UNIVERSITARIO"})
custom_one_hot(train_df, features, "indfall", {"s":"S"})
train_df["ind_actividad_cliente"] = train_df["ind_actividad_cliente"].map(lambda x: 0.0 if math.isnan(x) else x+1.0).astype(np.int8)
features.append("ind_actividad_cliente")
custom_one_hot(train_df, features, "indrel", {"1":1, "99":99})
train_df["indrel_1mes"] = train_df["indrel_1mes"].map(lambda x: 5.0 if x == "P" else x).astype(float).fillna(0.0).astype(np.int8)
assert_uniq(train_df["indrel_1mes"], "indrel_1mes")
features.append("indrel_1mes")
custom_one_hot(train_df, features, "tiprel_1mes", {"a":"A", "i":"I", "p":"P", "r":"R"}, check=True)
return train_df, tuple(features)
def make_prev_df(train_df, step):
with Timer("make prev%s DF" % step):
prev_df = pd.DataFrame()
prev_df["ncodpers"] = train_df["ncodpers"]
prev_df["int_date"] = train_df["int_date"].map(lambda x: x+step).astype(np.int8)
prod_features = ["%s_prev%s" % (prod, step) for prod in products]
for prod, prev in zip(products, prod_features):
prev_df[prev] = train_df[prod]
return prev_df, tuple(prod_features)
def load_data(fname="../input/all_clean.csv"):
with Timer("load train csv"):
train_df = pd.read_csv(fname, dtype=dtypes)
with Timer("fill products NA"):
for prod in products:
train_df[prod] = train_df[prod].fillna(0.0).astype(np.int8)
train_df, features = apply_transforms(train_df)
prev_dfs = []
prod_features = None
use_features = frozenset([1,2])
for step in range(1,6):
prev1_train_df, prod1_features = make_prev_df(train_df, step)
prev_dfs.append(prev1_train_df)
if step in use_features:
features += prod1_features
if step == 1:
prod_features = prod1_features
return train_df, prev_dfs, features, prod_features
def join_with_prev(df, prev_df, how):
with Timer("join %s" % how):
assert set(df.columns.values.tolist()) & set(prev_df.columns.values.tolist()) == set(["ncodpers", "int_date"])
print("before join", len(df))
df = df.merge(prev_df, on=["ncodpers", "int_date"], how=how)
for f in set(prev_df.columns.values.tolist()) - set(["ncodpers", "int_date"]):
df[f] = df[f].astype(np.float16)
print("after join", len(df))
return df
def make_data():
train_df, prev_dfs, features, prod_features = load_data()
for i, prev_df in enumerate(prev_dfs):
with Timer("join train with prev%s" % (i+1)):
how = "inner" if i == 0 else "left"
train_df = join_with_prev(train_df, prev_df, how=how)
# Various aggregates to try
# for prod in products:
# print()
# print(prod)
# #prev1_bin = (train_df[prod + "_prev1"] != 1).astype(np.int8)
# for begin, end in [(2,5),(1,4)]:
# prods = ["%s_prev%s" % (prod, i) for i in range(begin,end+1)]
# mp_df = train_df.as_matrix(columns=prods)
# print(prods)
#
# stdf = "%s_std_%s_%s" % (prod,begin,end)
# train_df[stdf] = np.nanstd(mp_df, axis=1) # * prev1_bin
#
# maxf = "%s_max_%s_%s"%(prod,begin,end)
# train_df[maxf] = np.nanmax(mp_df, axis=1).astype(np.int8)
#
# # minf = "%s_min_%s_%s"%(prod,begin,end)
# # train_df[minf] = np.nanmin(mp_df, axis=1).astype(np.int8)
#
# chf = "%s_ch_%s_%s"%(prod,begin,end)
# train_df[chf] = np.sum(np.invert(np.isclose(mp_df[:,1:], mp_df[:,:-1], equal_nan=True)), axis=1, dtype=np.int8)
#
# sumf = "%s_sum_%s_%s"%(prod,begin,end)
# train_df[sumf] = np.nansum(mp_df, axis=1, dtype=np.int8)
#
# skewf = "%s_skew_%s_%s"%(prod,begin,end)
# train_df[skewf] = scipy.stats.skew(mp_df, axis=1)
#
# features += (stdf,maxf,chf,sumf,skewf)
for prod in products:
print()
print(prod)
for begin, end in [(1,3),(1,5),(2,5)]:
prods = ["%s_prev%s" % (prod, i) for i in range(begin,end+1)]
mp_df = train_df.as_matrix(columns=prods)
print(prods)
stdf = "%s_std_%s_%s" % (prod,begin,end)
train_df[stdf] = np.nanstd(mp_df, axis=1) # * prev1_bin
features += (stdf,)
for prod in products:
print()
print(prod)
for begin, end in [(2,3),(2,5)]:
prods = ["%s_prev%s" % (prod, i) for i in range(begin,end+1)]
mp_df = train_df.as_matrix(columns=prods)
print(prods)
minf = "%s_min_%s_%s"%(prod,begin,end)
train_df[minf] = np.nanmin(mp_df, axis=1).astype(np.int8)
maxf = "%s_max_%s_%s"%(prod,begin,end)
train_df[maxf] = np.nanmax(mp_df, axis=1).astype(np.int8)
features += (minf,maxf,)
with Timer("Remove unused columns"):
leave_columns = ["ncodpers", "int_date", "fecha_dato"] + list(products) + list(features)
assert len(leave_columns) == len(set(leave_columns))
train_df = train_df[leave_columns]
return train_df, features, prod_features
def make_submission(f, Y_test, C):
Y_ret = []
with Timer("make submission"):
f.write("ncodpers,added_products\n".encode('utf-8'))
for c, y_test in zip(C, Y_test):
y_prods = [(y,p,ip) for y,p,ip in zip(y_test, products, range(len(products)))]
y_prods = sorted(y_prods, key=lambda a: a[0], reverse=True)[:7]
Y_ret.append([ip for y,p,ip in y_prods])
y_prods = [p for y,p,ip in y_prods]
f.write(("%s,%s\n" % (int(c), " ".join(y_prods))).encode('utf-8'))
return Y_ret
def train_predict(all_df, features, prod_features, str_date, cv):
test_date = date_to_int(str_date)
train_df = all_df[all_df.int_date < test_date]
test_df = pd.DataFrame(all_df[all_df.int_date == test_date])
print(sorted(set(train_df.columns.values.tolist())))
print(len(train_df.columns.values.tolist()), len(set(train_df.columns.values.tolist())))
print(len(features),len(set(features)))
X = []
Y = []
for i, prod in enumerate(products):
prev = prod + "_prev1"
prX = train_df[(train_df[prod] == 1) & (train_df[prev] == 0)]
prY = np.zeros(prX.shape[0], dtype=np.int8) + i
X.append(prX)
Y.append(prY)
print(prod, prX.shape)
XY = pd.concat(X)
Y = np.hstack(Y)
XY["y"] = Y
XY["url"] = np.zeros(len(XY), dtype=np.int8)
del train_df
del all_df
XY["ncodepers_fecha_dato"] = XY["ncodpers"].astype(str) + XY["fecha_dato"]
uniqs, counts = np.unique(XY["ncodepers_fecha_dato"], return_counts=True)
weights = np.exp(1/counts - 1)
print(np.unique(counts, return_counts=True))
print(np.unique(weights, return_counts=True))
wdf = pd.DataFrame()
wdf["ncodepers_fecha_dato"] = uniqs
wdf["counts"] = counts
wdf["weight"] = weights
print("before merge", len(XY))
XY = XY.merge(wdf, on="ncodepers_fecha_dato")
print("after merge", len(XY))
print(XY.shape)
mask = np.random.rand(len(XY)) < 0.8
XY_train = XY[mask]
XY_validate = XY[~mask]
with Timer("prepare test data"):
test_df["y"] = test_df["ncodpers"]
test_df["url"] = np.zeros(len(test_df), dtype=np.int8)
test_df["weight"] = np.ones(len(test_df), dtype=np.int8)
Y_prev = test_df.as_matrix(columns=prod_features)
C = test_df.as_matrix(columns=["ncodpers"])
for prod in products:
prev = prod + "_prev1"
padd = prod + "_add"
test_df[padd] = test_df[prod] - test_df[prev]
test_add_mat = test_df.as_matrix(columns=[prod + "_add" for prod in products])
test_add_list = [list() for i in range(len(C))]
assert test_add_mat.shape == (len(C), len(products))
count = 0
for c in range(len(C)):
for p in range(len(products)):
if test_add_mat[c,p] > 0:
test_add_list[c].append(p)
count += 1
if cv:
max_map7 = mapk(test_add_list, test_add_list, 7, 0.0)
map7coef = float(len(test_add_list)) / float(sum([int(bool(a)) for a in test_add_list]))
print("Max MAP@7", str_date, max_map7, max_map7*map7coef)
with Timer("LightGBM"):
Y_test_lgbm = engines.lightgbm(XY_train, XY_validate, test_df, features, XY_all = XY,
restore = (str_date == "2016-06-28")
)
test_add_list_lightgbm = make_submission(io.BytesIO() if cv else gzip.open("tmp/%s.lightgbm.csv.gz" % str_date, "wb"),
Y_test_lgbm - Y_prev, C)
if cv:
map7lightgbm = mapk(test_add_list, test_add_list_lightgbm, 7, 0.0)
print("LightGBMlib MAP@7", str_date, map7lightgbm, map7lightgbm*map7coef)
with Timer("XGBoost"):
Y_test_xgb = engines.xgboost(XY_train, XY_validate, test_df, features, XY_all = XY,
restore = (str_date == "2016-06-28")
)
test_add_list_xgboost = make_submission(io.BytesIO() if cv else gzip.open("tmp/%s.xgboost.csv.gz" % str_date, "wb"),
Y_test_xgb - Y_prev, C)
if cv:
map7xgboost = mapk(test_add_list, test_add_list_xgboost, 7, 0.0)
print("XGBoost MAP@7", str_date, map7xgboost, map7xgboost*map7coef)
Y_test = np.sqrt(np.multiply(Y_test_xgb, Y_test_lgbm))
test_add_list_xl = make_submission(io.BytesIO() if cv else gzip.open("tmp/%s.xgboost-lightgbm.csv.gz" % str_date, "wb"),
Y_test - Y_prev, C)
if cv:
map7xl = mapk(test_add_list, test_add_list_xl, 7, 0.0)
print("XGBoost+LightGBM MAP@7", str_date, map7xl, map7xl*map7coef)
if __name__ == "__main__":
if True:
all_df, features, prod_features = make_data()
with Timer("save data"):
all_df.to_pickle("tmp/cv_data.pickle")
pickle.dump((features, prod_features), open("tmp/cv_meta.pickle", "wb"))
else:
with Timer("restore data"):
all_df = pd.read_pickle("tmp/cv_data.pickle")
(features, prod_features) = pickle.load(open("tmp/cv_meta.pickle", "rb"))
train_predict(all_df, features, prod_features, "2016-05-28", cv=True)
train_predict(all_df, features, prod_features, "2016-06-28", cv=False)
