# -*- encoding:utf-8 -*-
from __future__ import print_function
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd
# import warnings
# noinspection PyUnresolvedReferences
import abu_local_env
import abupy
from abupy import abu
from abupy import ABuSymbolPd
import sklearn.preprocessing as preprocessing
# warnings.filterwarnings('ignore')
sns.set_context(rc={'figure.figsize': (14, 7)})
# 使用沙盒数据，目的是和书中一样的数据环境
abupy.env.enable_example_env_ipython()
"""
第10章 量化系统——机器学习•猪老三
abu量化系统github地址：https://github.com/bbfamily/abu (您的star是我的动力！)
abu量化文档教程ipython notebook：https://github.com/bbfamily/abu/tree/master/abupy_lecture
"""
"""
10.2 猪老三世界中的量化环境
"""
"""
是否开启date_week噪音, 开启这个的目的是让分类结果正确率降低，接近真实
"""
g_with_date_week_noise = False
def _gen_another_word_price(kl_another_word):
"""
生成股票在另一个世界中的价格
:param kl_another_word:
:return:
"""
for ind in np.arange(2, kl_another_word.shape[0]):
# 前天数据
bf_yesterday = kl_another_word.iloc[ind - 2]
# 昨天
yesterday = kl_another_word.iloc[ind - 1]
# 今天
today = kl_another_word.iloc[ind]
# 生成今天的收盘价格
kl_another_word.close[ind] = _gen_another_word_price_rule(
yesterday.close, yesterday.volume,
bf_yesterday.close, bf_yesterday.volume,
today.volume, today.date_week)
def _gen_another_word_price_rule(yesterday_close, yesterday_volume,
bf_yesterday_close,
bf_yesterday_volume,
today_volume, date_week):
"""
通过前天收盘量价，昨天收盘量价，今天的量，构建另一个世界中的价格模型
"""
# 昨天收盘价格与前天收盘价格的价格差
price_change = yesterday_close - bf_yesterday_close
# 昨天成交量与前天成交量的量差
volume_change = yesterday_volume - bf_yesterday_volume
# 如果量和价变动一致，今天价格涨，否则跌
# 即量价齐涨－>涨, 量价齐跌－>涨，量价不一致－>跌
sign = 1.0 if price_change * volume_change > 0 else -1.0
# 通过date_week生成噪音，否则之后分类100%分对
if g_with_date_week_noise:
# 针对sign生成噪音，噪音的生效的先决条件是今天的量是这三天最大的
gen_noise = today_volume > np.max(
[yesterday_volume, bf_yesterday_volume])
# 如果量是这三天最大 且是周五，下跌
if gen_noise and date_week == 4:
sign = -1.0
# 如果量是这三天最大，如果是周一，上涨
elif gen_noise and date_week == 0:
sign = 1.0
# 今天的涨跌幅度基础是price_change（昨天前天的价格变动）
price_base = abs(price_change)
# 今天的涨跌幅度变动因素：量比，
# 今天的成交量/昨天的成交量 和 今天的成交量/前天的成交量 的均值
price_factor = np.mean([today_volume / yesterday_volume,
today_volume / bf_yesterday_volume])
if abs(price_base * price_factor) < yesterday_close * 0.10:
# 如果 量比 * price_base 没超过10%，今天价格计算
today_price = yesterday_close + \
sign * price_base * price_factor
else:
# 如果涨跌幅度超过10%，限制上限，下限为10%
today_price = yesterday_close + sign * yesterday_close * 0.10
return today_price
def change_real_to_another_word(symbol):
"""
将原始真正的股票数据价格列只保留前两天数据，成交量，周几列完全保留
价格列其他数据使用_gen_another_word_price变成另一个世界价格
:param symbol:
:return:
"""
kl_pd = ABuSymbolPd.make_kl_df(symbol)
if kl_pd is not None:
# 原始股票数据也只保留价格，周几，成交量
kl_pig_three = kl_pd.filter(['close', 'date_week', 'volume'])
# 只保留原始头两天的交易收盘价格，其他的的都赋予nan
kl_pig_three['close'][2:] = np.nan
# 将其他nan价格变成猪老三世界中价格使用_gen_another_word_price
_gen_another_word_price(kl_pig_three)
return kl_pig_three
def sample_102(show=True):
"""
10.2 生成猪老三的世界中的映射股票数据
:return:
"""
choice_symbols = ['usNOAH', 'usSFUN', 'usBIDU', 'usAAPL', 'usGOOG',
'usTSLA', 'usWUBA', 'usVIPS']
another_word_dict = {}
real_dict = {}
for symbol in choice_symbols:
# 猪老三世界的股票走势字典
another_word_dict[symbol] = change_real_to_another_word(symbol)
# 真实世界的股票走势字典，这里不考虑运行效率问题
real_dict[symbol] = ABuSymbolPd.make_kl_df(symbol)
if show:
# 表10-1所示
print('another_word_dict[usNOAH].head():\n', another_word_dict['usNOAH'].head())
print('real_dict[usNOAH].head():\n', real_dict['usNOAH'].head().filter(['close', 'date_week', 'volume']))
import itertools
# 4 ＊ 2
_, axs = plt.subplots(nrows=4, ncols=2, figsize=(20, 15))
# 将画布序列拉平
axs_list = list(itertools.chain.from_iterable(axs))
for symbol, ax in zip(choice_symbols, axs_list):
# 绘制猪老三世界的股价走势
another_word_dict[symbol].close.plot(ax=ax)
# 同样的股票在真实世界的股价走势
real_dict[symbol].close.plot(ax=ax)
ax.set_title(symbol)
plt.show()
return another_word_dict
"""
10.3 有监督机器学习
"""
def gen_pig_three_feature(kl_another_word):
"""
猪老三构建特征模型函数
:param kl_another_word: 即上一节使用_gen_another_word_price
生成的dataframe有收盘价，周几，成交量列
:return:
"""
# y值使用close.pct_change即涨跌幅度
kl_another_word['regress_y'] = kl_another_word.close.pct_change()
# 前天收盘价格
kl_another_word['bf_yesterday_close'] = 0
# 昨天收盘价格
kl_another_word['yesterday_close'] = 0
# 昨天收盘成交量
kl_another_word['yesterday_volume'] = 0
# 前天收盘成交量
kl_another_word['bf_yesterday_volume'] = 0
# 对齐特征，前天收盘价格即与今天的收盘错2个时间单位，[2:] = [:-2]
kl_another_word['bf_yesterday_close'][2:] = \
kl_another_word['close'][:-2]
# 对齐特征，前天成交量
kl_another_word['bf_yesterday_volume'][2:] = \
kl_another_word['volume'][:-2]
# 对齐特征，昨天收盘价与今天的收盘错1个时间单位，[1:] = [:-1]
kl_another_word['yesterday_close'][1:] = \
kl_another_word['close'][:-1]
# 对齐特征，昨天成交量
kl_another_word['yesterday_volume'][1:] = \
kl_another_word['volume'][:-1]
# 特征1: 价格差
kl_another_word['feature_price_change'] = \
kl_another_word['yesterday_close'] - \
kl_another_word['bf_yesterday_close']
# 特征2: 成交量差
kl_another_word['feature_volume_Change'] = \
kl_another_word['yesterday_volume'] - \
kl_another_word['bf_yesterday_volume']
# 特征3: 涨跌sign
kl_another_word['feature_sign'] = np.sign(
kl_another_word['feature_price_change'] * kl_another_word[
'feature_volume_Change'])
# 特征4: 周几
kl_another_word['feature_date_week'] = kl_another_word[
'date_week']
"""
构建噪音特征, 因为猪老三也不可能全部分析正确真实的特征因素
这里引入一些噪音特征
"""
# 成交量乘积
kl_another_word['feature_volume_noise'] = \
kl_another_word['yesterday_volume'] * \
kl_another_word['bf_yesterday_volume']
# 价格乘积
kl_another_word['feature_price_noise'] = \
kl_another_word['yesterday_close'] * \
kl_another_word['bf_yesterday_close']
# 将数据标准化
scaler = preprocessing.StandardScaler()
kl_another_word['feature_price_change'] = scaler.fit_transform(
kl_another_word['feature_price_change'].values.reshape(-1, 1))
kl_another_word['feature_volume_Change'] = scaler.fit_transform(
kl_another_word['feature_volume_Change'].values.reshape(-1, 1))
kl_another_word['feature_volume_noise'] = scaler.fit_transform(
kl_another_word['feature_volume_noise'].values.reshape(-1, 1))
kl_another_word['feature_price_noise'] = scaler.fit_transform(
kl_another_word['feature_price_noise'].values.reshape(-1, 1))
# 只筛选feature_开头的特征和regress_y，抛弃前两天数据，即[2:]
kl_pig_three_feature = kl_another_word.filter(
regex='regress_y|feature_*')[2:]
return kl_pig_three_feature
def sample_103_0(show=True):
"""
10.3 生成猪老三的训练集特征示例
:return:
"""
another_word_dict = sample_102(show=False)
pig_three_feature = None
for symbol in another_word_dict:
# 首先拿出对应的走势数据
kl_another_word = another_word_dict[symbol]
# 通过走势数据生成训练集特征通过gen_pig_three_feature
kl_feature = gen_pig_three_feature(kl_another_word)
# 将每个股票的特征数据都拼接起来，形成训练集
pig_three_feature = kl_feature if pig_three_feature is None \
else pig_three_feature.append(kl_feature)
# Dataframe -> matrix
feature_np = pig_three_feature.as_matrix()
# x特征矩阵
train_x = feature_np[:, 1:]
# 回归训练的连续值y
train_y_regress = feature_np[:, 0]
# 分类训练的离散值y，之后分类技术使用
# noinspection PyTypeChecker
train_y_classification = np.where(train_y_regress > 0, 1, 0)
if show:
print('pig_three_feature.shape:', pig_three_feature.shape)
print('pig_three_feature.tail():\n', pig_three_feature.tail())
print('train_x[:5], train_y_regress[:5], train_y_classification[:5]:\n', train_x[:5], train_y_regress[:5],
train_y_classification[:5])
return train_x, train_y_regress, train_y_classification, pig_three_feature
"""
猪老三使用回归预测股价
"""
def sample_1031_1():
"""
10.3.1_1 猪老三使用回归预测股价：生成训练集数据和测试集数据
:return:
"""
# noinspection PyShadowingNames
def gen_feature_from_symbol(symbol):
"""
封装由一个symbol转换为特征矩阵序列函数
:param symbol:
:return:
"""
# 真实世界走势数据转换到老三的世界
kl_another_word = change_real_to_another_word(symbol)
# 由走势转换为特征dataframe通过gen_pig_three_feature
kl_another_word_feature_test = gen_pig_three_feature(kl_another_word)
# 转换为matrix
feature_np_test = kl_another_word_feature_test.as_matrix()
# 从matrix抽取y回归
test_y_regress = feature_np_test[:, 0]
# y回归 －> y分类
# noinspection PyTypeChecker
test_y_classification = np.where(test_y_regress > 0, 1, 0)
# 从matrix抽取x特征矩阵
test_x = feature_np_test[:, 1:]
return test_x, test_y_regress, test_y_classification, kl_another_word_feature_test
# 生成训练集数据
train_x, train_y_regress, train_y_classification, pig_three_feature = sample_103_0(show=False)
# 生成测试集数据
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = gen_feature_from_symbol('usFB')
print('训练集：{}, 测试集：{}'.format(pig_three_feature.shape[0], kl_another_word_feature_test.shape[0]))
return train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test
def regress_process(estimator, train_x, train_y_regress, test_x,
test_y_regress):
# 训练训练集数据
estimator.fit(train_x, train_y_regress)
# 使用训练好的模型预测测试集对应的y，即根据usFB的走势特征预测股价涨跌幅度
test_y_prdict_regress = estimator.predict(test_x)
# 绘制usFB实际股价涨跌幅度
plt.plot(test_y_regress.cumsum())
# 绘制通过模型预测的usFB股价涨跌幅度
plt.plot(test_y_prdict_regress.cumsum())
# 针对训练集数据做交叉验证
from abupy import cross_val_score
from abupy.CoreBu.ABuFixes import mean_squared_error_scorer
scores = cross_val_score(estimator, train_x,
train_y_regress, cv=10,
scoring=mean_squared_error_scorer)
# mse开方 -> rmse
mean_sc = -np.mean(np.sqrt(-scores))
print('{} RMSE: {}'.format(estimator.__class__.__name__, mean_sc))
def sample_1031_2():
"""
10.3.1_2 猪老三使用回归预测股价：LinearRegressio
:return:
"""
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
# 实例化线性回归对象estimator
from sklearn.linear_model import LinearRegression
estimator = LinearRegression()
# 将回归模型对象，训练集x，训练集连续y值，测试集x，测试集连续y传入
regress_process(estimator, train_x, train_y_regress, test_x,
test_y_regress)
plt.show()
from abupy import ABuMLExecute
ABuMLExecute.plot_learning_curve(estimator, train_x, train_y_regress, cv=10)
def sample_1031_3():
"""
10.3.1_3 猪老三使用回归预测股价：PolynomialFeatures
:return:
"""
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
# pipeline套上 degree=3 ＋ LinearRegression
estimator = make_pipeline(PolynomialFeatures(degree=3),
LinearRegression())
# 继续使用regress_process，区别是estimator变了
regress_process(estimator, train_x, train_y_regress, test_x,
test_y_regress)
plt.show()
def sample_1031_4():
"""
10.3.1_4 猪老三使用回归预测股价：使用集成学习算法预测股价AdaBoost与RandomForest
:return:
"""
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
# AdaBoost
from sklearn.ensemble import AdaBoostRegressor
estimator = AdaBoostRegressor(n_estimators=100)
regress_process(estimator, train_x, train_y_regress, test_x,
test_y_regress)
plt.show()
# RandomForest
from sklearn.ensemble import RandomForestRegressor
estimator = RandomForestRegressor(n_estimators=100)
regress_process(estimator, train_x, train_y_regress, test_x, test_y_regress)
plt.show()
"""
10.3.2 猪老三使用分类预测股票涨跌
"""
def classification_process(estimator, train_x, train_y_classification,
test_x, test_y_classification):
from sklearn import metrics
# 训练数据，这里分类要所以要使用y_classification
estimator.fit(train_x, train_y_classification)
# 使用训练好的分类模型预测测试集对应的y，即根据usFB的走势特征预测涨跌
test_y_prdict_classification = estimator.predict(test_x)
# 通过metrics.accuracy_score度量预测涨跌的准确率
print("{} accuracy = {:.2f}".format(
estimator.__class__.__name__,
metrics.accuracy_score(test_y_classification,
test_y_prdict_classification)))
from abupy import cross_val_score
# 针对训练集数据做交叉验证scoring='accuracy'，cv＝10
scores = cross_val_score(estimator, train_x,
train_y_classification,
cv=10,
scoring='accuracy')
# 所有交叉验证的分数取平均值
mean_sc = np.mean(scores)
print('cross validation accuracy mean: {:.2f}'.format(mean_sc))
def sample_1032_1():
"""
10.3.2_1 猪老三使用分类预测股票涨跌：LogisticRegression
:return:
"""
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
# 无噪音分类正确100%
from sklearn.linear_model import LogisticRegression
estimator = LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
# 将分类器，训练集x，训练集y分类，测试集，测试集y分别传入函数
classification_process(estimator, train_x, train_y_classification,
test_x, test_y_classification)
# 开启噪音，再来一遍，有噪音正确率93%, 之后的都开启g_with_date_week_noise
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
classification_process(estimator, train_x, train_y_classification,
test_x, test_y_classification)
def sample_1032_2():
"""
10.3.2_2 猪老三使用分类预测股票涨跌：svm
:return:
"""
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
from sklearn.svm import SVC
estimator = SVC(kernel='rbf')
classification_process(estimator, train_x, train_y_classification,
test_x, test_y_classification)
def sample_1032_3():
"""
10.3.2_3 猪老三使用分类预测股票涨跌：RandomForestClassifier
:return:
"""
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
from sklearn.ensemble import RandomForestClassifier
estimator = RandomForestClassifier(n_estimators=100)
classification_process(estimator, train_x, train_y_classification,
test_x, test_y_classification)
def sample_1032_4(show=True):
"""
10.3.2_4 猪老三使用分类预测股票涨跌：train_test_split
:return:
"""
from sklearn import metrics
from abupy import train_test_split
# noinspection PyShadowingNames
def train_test_split_xy(estimator, x, y, test_size=0.5,
random_state=0):
# 通过train_test_split将原始训练集随机切割为新训练集与测试集
train_x, test_x, train_y, test_y = \
train_test_split(x, y, test_size=test_size,
random_state=random_state)
if show:
print(x.shape, y.shape)
print(train_x.shape, train_y.shape)
print(test_x.shape, test_y.shape)
clf = estimator.fit(train_x, train_y)
predictions = clf.predict(test_x)
if show:
# 度量准确率
print("accuracy = %.2f" %
(metrics.accuracy_score(test_y, predictions)))
# 度量查准率
print("precision_score = %.2f" %
(metrics.precision_score(test_y, predictions)))
# 度量回收率
print("recall_score = %.2f" %
(metrics.recall_score(test_y, predictions)))
return test_y, predictions
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
from sklearn.ensemble import RandomForestClassifier
estimator = RandomForestClassifier(n_estimators=100)
test_y, predictions = train_test_split_xy(estimator, train_x, train_y_classification)
return estimator, train_x, train_y_classification, test_y, predictions
def sample_1032_5():
"""
10.3.2_5 猪老三使用分类预测股票涨跌：混淆矩阵和roc曲线
:return:
"""
from sklearn import metrics
# noinspection PyShadowingNames
def confusion_matrix_with_report(test_y, predictions):
confusion_matrix = metrics.confusion_matrix(test_y, predictions)
# print("Confusion Matrix ", confusion_matrix)
print(" Predicted")
print(" | 0 | 1 |")
print(" |-----|-----|")
print(" 0 | %3d | %3d |" % (confusion_matrix[0, 0],
confusion_matrix[0, 1]))
print("Actual |-----|-----|")
print(" 1 | %3d | %3d |" % (confusion_matrix[1, 0],
confusion_matrix[1, 1]))
print(" |-----|-----|")
print(metrics.classification_report(test_y, predictions))
estimator, train_x, train_y_classification, test_y, predictions = sample_1032_4(show=False)
confusion_matrix_with_report(test_y, predictions)
from abupy import ABuMLExecute
ABuMLExecute.plot_roc_estimator(estimator, train_x, train_y_classification)
def sample_1033_1():
"""
10.3.3 通过决策树分类，绘制出决策图
这里需要安装dot graphviz，才能通过os.system("dot -T png graphviz.dot -o graphviz.png")生成png
:return:
"""
from sklearn.tree import DecisionTreeClassifier
from sklearn import tree
import os
estimator = DecisionTreeClassifier(max_depth=2, random_state=1)
# noinspection PyShadowingNames
def graphviz_tree(estimator, features, x, y):
if not hasattr(estimator, 'tree_'):
print('only tree can graphviz!')
return
estimator.fit(x, y)
# 将决策模型导出graphviz.dot文件
tree.export_graphviz(estimator.tree_, out_file='graphviz.dot',
feature_names=features)
# 通过dot将模型绘制决策图，保存png
os.system("dot -T png graphviz.dot -o graphviz.png")
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
# 这里会使用到特征的名称列pig_three_feature.columns[1:]
graphviz_tree(estimator, pig_three_feature.columns[1:], train_x,
train_y_classification)
import PIL.Image
PIL.Image.open('graphviz.png').show()
def sample_1033_2():
"""
10.3.3 特征的重要性排序及支持度评级
:return:
"""
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
# noinspection PyShadowingNames
def importances_coef_pd(estimator):
"""
特征的重要性
"""
if hasattr(estimator, 'feature_importances_'):
# 有feature_importances_的通过sort_values排序
return pd.DataFrame(
{'feature': list(pig_three_feature.columns[1:]),
'importance': estimator.feature_importances_}).sort_values('importance')
elif hasattr(estimator, 'coef_'):
# 有coef_的通过coef排序
return pd.DataFrame(
{"columns": list(pig_three_feature.columns)[1:], "coef": list(estimator.coef_.T)}).sort_values('coef')
else:
print('estimator not hasattr feature_importances_ or coef_!')
# 使用随机森林分类器
from sklearn.ensemble import RandomForestClassifier
estimator = RandomForestClassifier(n_estimators=100)
# 训练数据模型
estimator.fit(train_x, train_y_classification)
# 对训练后的模型特征的重要度进行判定，重要程度由小到大，表10-4所示
print('importances_coef_pd(estimator):\n', importances_coef_pd(estimator))
from sklearn.feature_selection import RFE
# noinspection PyShadowingNames
def feature_selection(estimator, x, y):
"""
支持度评级
"""
selector = RFE(estimator)
selector.fit(x, y)
print('RFE selection')
print(pd.DataFrame(
{'support': selector.support_, 'ranking': selector.ranking_},
index=pig_three_feature.columns[1:]))
print('feature_selection(estimator, train_x, train_y_classification):\n',
feature_selection(estimator, train_x, train_y_classification))
"""
10.4 无监督机器学习
"""
def sample_1041():
"""
10.4.1 使用降维可视化数据
:return:
"""
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
from sklearn.decomposition import PCA
from abupy import ABuMLExecute
# noinspection PyShadowingNames
def plot_decision_function(estimator, x, y):
# pca进行降维，只保留2个特征序列
pca_2n = PCA(n_components=2)
x = pca_2n.fit_transform(x)
# 进行训练
estimator.fit(x, y)
plt.scatter(x[:, 0], x[:, 1], c=y, s=50, cmap='spring')
ABuMLExecute.plot_decision_boundary(
lambda p_x: estimator.predict(p_x), x, y)
from sklearn.ensemble import RandomForestClassifier
estimator = RandomForestClassifier(n_estimators=100)
plot_decision_function(estimator, train_x, train_y_classification)
# noinspection PyTypeChecker
def sample_1042():
"""
10.4.2 猪老三使用聚类算法提高正确率
:return:
"""
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
# 使用随机森林作为分类器
from sklearn.ensemble import RandomForestClassifier
estimator = RandomForestClassifier(n_estimators=100)
estimator.fit(train_x, train_y_classification)
test_y_prdict_classification = estimator.predict(test_x)
from sklearn import metrics
print("accuracy = %.2f" % (
metrics.accuracy_score(test_y_classification,
test_y_prdict_classification)))
# 测试集feature即usFB的kl feature
pig_three_kmean_feature = kl_another_word_feature_test
# 测试集真实的涨跌结果test_y_classification
pig_three_kmean_feature['y'] = test_y_classification
# 使用刚刚的随机森林作为分类器的预测涨跌结果test_y_prdict_classification
pig_three_kmean_feature['y_prdict'] = test_y_prdict_classification
# 即生成一列新数据记录预测是否正确
pig_three_kmean_feature['y_same'] = np.where(
pig_three_kmean_feature['y'] ==
pig_three_kmean_feature['y_prdict'], 1, 0)
# 将feature中只保留刚刚得到的y_same
pig_three_kmean_feature = pig_three_kmean_feature.filter(['y_same'])
from sklearn.cluster import KMeans
# 使用刚刚得到的只有y_same列的数据赋值x_kmean
x_kmean = pig_three_kmean_feature.values
# n_clusters=2, 即只聚两类数据
kmean = KMeans(n_clusters=2)
kmean.fit(x_kmean)
# 将聚类标签赋予新的一列cluster
pig_three_kmean_feature['cluster'] = kmean.predict(x_kmean)
# 将周几这个特征合并过来
pig_three_kmean_feature['feature_date_week'] = \
kl_another_word_feature_test['feature_date_week']
# 表10-5所示
print('pig_three_kmean_feature.tail():\n', pig_three_kmean_feature.tail())
# 表10-6所示
print('pd.crosstab(pig_three_kmean_feature.feature_date_week, pig_three_kmean_feature.cluster):\n',
pd.crosstab(pig_three_kmean_feature.feature_date_week, pig_three_kmean_feature.cluster))
"""
10.5 梦醒时分
"""
def sample_105_0():
"""
10.5 AbuML
:return:
"""
global g_with_date_week_noise
g_with_date_week_noise = True
train_x, train_y_regress, train_y_classification, pig_three_feature, \
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
from abupy import AbuML
# 通过x, y矩阵和特征的DataFrame对象组成AbuML
ml = AbuML(train_x, train_y_classification, pig_three_feature)
# 使用随机森林作为分类器
_ = ml.estimator.random_forest_classifier()
# 交织验证结果的正确率
print('ml.cross_val_accuracy_score():\n', ml.cross_val_accuracy_score())
# 特征的选择
print('ml.feature_selection():\n', ml.feature_selection())
"""
如下内容不能使用沙盒环境, 建议对照阅读：
abu量化文档－第十九节 数据源
第20节 美股交易UMP决策
"""
def sample_1051_0():
"""
10.5.1 回测中生成特征，切分训练测试集，成交买单快照: 数据准备
如果没有运行过abu量化文档－第十九节 数据源：中使用腾讯数据源进行数据更新，需要运行
如果运行过就不要重复运行了：
"""
from abupy import EMarketTargetType, EMarketSourceType, EDataCacheType
# 关闭沙盒数据环境
abupy.env.disable_example_env_ipython()
abupy.env.g_market_source = EMarketSourceType.E_MARKET_SOURCE_tx
abupy.env.g_data_cache_type = EDataCacheType.E_DATA_CACHE_CSV
# 首选这里预下载市场中所有股票的6年数据(做5年回测，需要预先下载6年数据)
abu.run_kl_update(start='2011-08-08', end='2017-08-08', market=EMarketTargetType.E_MARKET_TARGET_US)
def sample_1051_1(from_cache=False, show=True):
"""
10.5.1 回测中生成特征，切分训练测试集，成交买单快照: 数据准备
:return:
"""
from abupy import AbuMetricsBase
from abupy import AbuFactorBuyBreak
from abupy import AbuFactorAtrNStop
from abupy import AbuFactorPreAtrNStop
from abupy import AbuFactorCloseAtrNStop
# 关闭沙盒数据环境
abupy.env.disable_example_env_ipython()
from abupy import EMarketDataFetchMode
# 因为sample_94_1下载了预先数据，使用缓存，设置E_DATA_FETCH_FORCE_LOCAL，实际上run_kl_update最后会把设置set到FORCE_LOCAL
abupy.env.g_data_fetch_mode = EMarketDataFetchMode.E_DATA_FETCH_FORCE_LOCAL
# 设置选股因子，None为不使用选股因子
stock_pickers = None
# 买入因子依然延用向上突破因子
buy_factors = [{'xd': 60, 'class': AbuFactorBuyBreak},
{'xd': 42, 'class': AbuFactorBuyBreak}]
# 卖出因子继续使用上一章使用的因子
sell_factors = [
{'stop_loss_n': 1.0, 'stop_win_n': 3.0,
'class': AbuFactorAtrNStop},
{'class': AbuFactorPreAtrNStop, 'pre_atr_n': 1.5},
{'class': AbuFactorCloseAtrNStop, 'close_atr_n': 1.5}
]
# 回测生成买入时刻特征
abupy.env.g_enable_ml_feature = True
# 回测将symbols切割分为训练集数据和测试集数据
abupy.env.g_enable_train_test_split = True
# 下面设置回测时切割训练集，测试集使用的切割比例参数，默认为10，即切割为10份，9份做为训练，1份做为测试，
# 由于美股股票数量多，所以切割分为4份，3份做为训练集，1份做为测试集
abupy.env.g_split_tt_n_folds = 4
from abupy import EStoreAbu
if from_cache:
abu_result_tuple = \
abu.load_abu_result_tuple(n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
custom_name='train_us')
else:
# 初始化资金500万，资金管理依然使用默认atr
read_cash = 5000000
# 每笔交易的买入基数资金设置为万分之15
abupy.beta.atr.g_atr_pos_base = 0.0015
# 使用run_loop_back运行策略，因子使用和之前一样，
# choice_symbols=None为全市场回测，5年历史数据回测
abu_result_tuple, _ = abu.run_loop_back(read_cash,
buy_factors, sell_factors,
stock_pickers,
choice_symbols=None,
start='2012-08-08', end='2017-08-08')
# 把运行的结果保存在本地，以便之后分析回测使用，保存回测结果数据代码如下所示
abu.store_abu_result_tuple(abu_result_tuple, n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
custom_name='train_us')
if show:
metrics = AbuMetricsBase(*abu_result_tuple)
metrics.fit_metrics()
metrics.plot_returns_cmp(only_show_returns=True)
"＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊"
abupy.env.g_enable_train_test_split = False
# 使用切割好的测试数据
abupy.env.g_enable_last_split_test = True
from abupy import EStoreAbu
if from_cache:
abu_result_tuple_test = \
abu.load_abu_result_tuple(n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
custom_name='test_us')
else:
read_cash = 5000000
abupy.beta.atr.g_atr_pos_base = 0.007
choice_symbols = None
abu_result_tuple_test, kl_pd_manager_test = abu.run_loop_back(read_cash,
buy_factors, sell_factors, stock_pickers,
choice_symbols=choice_symbols, start='2012-08-08',
end='2017-08-08')
abu.store_abu_result_tuple(abu_result_tuple_test, n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
custom_name='test_us')
if show:
metrics = AbuMetricsBase(*abu_result_tuple_test)
metrics.fit_metrics()
metrics.plot_returns_cmp(only_show_returns=True)
print(abu_result_tuple.orders_pd[abu_result_tuple.orders_pd.result != 0].head())
return abu_result_tuple, abu_result_tuple_test
# noinspection PyUnresolvedReferences
def sample_1052():
"""
10.5.2 基于特征的交易预测
:return:
"""
# 需要在有缓存的情况下运行
abu_result_tuple, _ = sample_1051_1(from_cache=True, show=False)
from abupy.UmpBu.ABuUmpMainMul import UmpMulFiter
mul = UmpMulFiter(orders_pd=abu_result_tuple.orders_pd, scaler=False)
print('mul.df.head():\n', mul.df.head())
# 默认使用svm作为分类器
print('decision_tree_classifier cv please wait...')
mul.estimator.decision_tree_classifier()
mul.cross_val_accuracy_score()
# 默认使用svm作为分类器
print('knn_classifier cv please wait...')
# 默认使用svm作为分类器, 改分类器knn
mul.estimator.knn_classifier()
mul.cross_val_accuracy_score()
from abupy.UmpBu.ABuUmpMainBase import UmpDegFiter
deg = UmpDegFiter(orders_pd=abu_result_tuple.orders_pd)
print('deg.df.head():\n', deg.df.head())
print('xgb_classifier cv please wait...')
# 分类器使用GradientBoosting
deg.estimator.xgb_classifier()
deg.cross_val_accuracy_score()
print('adaboost_classifier cv please wait...')
# 分类器使用adaboost
deg.estimator.adaboost_classifier(base_estimator=None)
deg.cross_val_accuracy_score()
print('train_test_split_xy please wait...')
deg.train_test_split_xy()
if __name__ == "__main__":
sample_102()
# sample_103_0()
# sample_1031_1()
# sample_1031_2()
# sample_1031_3()
# sample_1031_4()
# sample_1032_1()
# sample_1032_2()
# sample_1032_3()
# sample_1032_4()
# sample_1032_5()
# sample_1033_1()
# sample_1033_2()
# sample_1041()
# sample_1042()
# sample_105_0()
# sample_1051_0()
# sample_1051_1(from_cache=True)
# sample_1051_1(from_cache=False)
# sample_1052()
