From 183fcb3c5b0972abb56d3e9942af2a6bc5b34a8f Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Fri, 11 Sep 2020 03:23:44 -0600 Subject: [PATCH 01/22] Revert "Revert "Vectorized operations"" This reverts commit b09e227af92c63d7680f72b6d091b50902cbfe11. --- backtester.py | 71 ++++++++------- eiten.py | 36 ++++---- simulator.py | 246 ++++++++++++++++++++++++++------------------------ 3 files changed, 187 insertions(+), 166 deletions(-) diff --git a/backtester.py b/backtester.py index b8a68fd..419102f 100644 --- a/backtester.py +++ b/backtester.py @@ -21,14 +21,16 @@ class BackTester: """ Backtester module that does both backward and forward testing for our portfolios. """ + def __init__(self): print("\n--# Backtester has been initialized") - def calculate_percentage_change(self, old, new): + def price_delta(self, prices): """ Percentage change """ - return ((new - old) * 100) / old + + return ((prices - prices.shift()) * 100 / prices.shift())[1:] def portfolio_weight_manager(self, weight, is_long_only): """ @@ -46,37 +48,42 @@ def back_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data_d """ # Get market returns during the backtesting time - historical_price_market = list(historical_price_market["Close"]) - market_returns = [self.calculate_percentage_change(historical_price_market[i - 1], historical_price_market[i]) for i in range(1, len(historical_price_market))] + historical_prices = historical_price_market["Close"] + market_returns = self.price_delta(historical_prices) market_returns_cumulative = np.cumsum(market_returns) # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] for symbol in symbol_names: - symbol_historical_prices = list(portfolio_data_dictionary[symbol]["historical_prices"]["Close"]) - symbol_historical_returns = [self.calculate_percentage_change(symbol_historical_prices[i - 1], symbol_historical_prices[i]) for i in range(1, len(symbol_historical_prices))] + symbol_historical_prices = portfolio_data_dictionary[symbol]["historical"]["Close"] + symbol_historical_returns = self.price_delta( + symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns normal_returns_matrix = np.array(normal_returns_matrix).transpose() - portfolio_weights_vector = np.array([self.portfolio_weight_manager(portfolio_weights_dictionary[symbol], is_long_only) for symbol in portfolio_weights_dictionary]).transpose() - portfolio_returns = np.dot(normal_returns_matrix, portfolio_weights_vector) + portfolio_weights_vector = np.array([self.portfolio_weight_manager( + portfolio_weights_dictionary[symbol], is_long_only) for symbol in portfolio_weights_dictionary]).transpose() + portfolio_returns = np.dot( + normal_returns_matrix, portfolio_weights_vector) portfolio_returns_cumulative = np.cumsum(portfolio_returns) # Plot returns x = np.arange(len(portfolio_returns_cumulative)) - plt.plot(x, portfolio_returns_cumulative, linewidth = 2.0, label = strategy_name) - plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') + plt.plot(x, portfolio_returns_cumulative, + linewidth=2.0, label=strategy_name) + plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') if market_chart: x = np.arange(len(market_returns_cumulative)) - plt.plot(x, market_returns_cumulative, linewidth = 2.0, color = '#282828', label = 'Market Index', linestyle = '--') + plt.plot(x, market_returns_cumulative, linewidth=2.0, + color='#282828', label='Market Index', linestyle='--') # Plotting styles - plt.title("Backtest Results", fontsize = 14) - plt.xlabel("Bars (Time Sorted)", fontsize = 14) - plt.ylabel("Cumulative Percentage Return", fontsize = 14) - plt.xticks(fontsize = 14) - plt.yticks(fontsize = 14) + plt.title("Backtest Results", fontsize=14) + plt.xlabel("Bars (Time Sorted)", fontsize=14) + plt.ylabel("Cumulative Percentage Return", fontsize=14) + plt.xticks(fontsize=14) + plt.yticks(fontsize=14) def future_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_price_market, is_long_only, market_chart, strategy_name): """ @@ -84,35 +91,37 @@ def future_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data """ # Get future prices - print(future_price_market) - future_price_market = [item[4] for item in list(future_price_market)] - market_returns = [self.calculate_percentage_change(future_price_market[i - 1], future_price_market[i]) for i in range(1, len(future_price_market))] + market_returns = self.price_delta(future_price_market["Close"]) market_returns_cumulative = np.cumsum(market_returns) # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] for symbol in symbol_names: - symbol_historical_prices = [item[4] for item in list(portfolio_data_dictionary[symbol]["future_prices"])] - symbol_historical_returns = [self.calculate_percentage_change(symbol_historical_prices[i - 1], symbol_historical_prices[i]) for i in range(1, len(symbol_historical_prices))] + symbol_historical_prices = portfolio_data_dictionary[symbol]["future"]["Close"] + symbol_historical_returns = self.price_delta(symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns normal_returns_matrix = np.array(normal_returns_matrix).transpose() - portfolio_weights_vector = np.array([self.portfolio_weight_manager(portfolio_weights_dictionary[symbol], is_long_only) for symbol in portfolio_weights_dictionary]).transpose() - portfolio_returns = np.dot(normal_returns_matrix, portfolio_weights_vector) + portfolio_weights_vector = np.array([self.portfolio_weight_manager( + portfolio_weights_dictionary[symbol], is_long_only) for symbol in portfolio_weights_dictionary]).transpose() + portfolio_returns = np.dot( + normal_returns_matrix, portfolio_weights_vector) portfolio_returns_cumulative = np.cumsum(portfolio_returns) # Plot x = np.arange(len(portfolio_returns_cumulative)) - plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') - plt.plot(x, portfolio_returns_cumulative, linewidth = 2.0, label = strategy_name) + plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') + plt.plot(x, portfolio_returns_cumulative, + linewidth=2.0, label=strategy_name) if market_chart: x = np.arange(len(market_returns_cumulative)) - plt.plot(x, market_returns_cumulative, linewidth = 2.0, color = '#282828', label = 'Market Index', linestyle = '--') + plt.plot(x, market_returns_cumulative, linewidth=2.0, + color='#282828', label='Market Index', linestyle='--') # Plotting styles - plt.title("Future Test Results", fontsize = 14) - plt.xlabel("Bars (Time Sorted)", fontsize = 14) - plt.ylabel("Cumulative Percentage Return", fontsize = 14) - plt.xticks(fontsize = 14) - plt.yticks(fontsize = 14) + plt.title("Future Test Results", fontsize=14) + plt.xlabel("Bars (Time Sorted)", fontsize=14) + plt.ylabel("Cumulative Percentage Return", fontsize=14) + plt.xticks(fontsize=14) + plt.yticks(fontsize=14) diff --git a/eiten.py b/eiten.py index abeb764..e13509a 100644 --- a/eiten.py +++ b/eiten.py @@ -11,10 +11,7 @@ class Eiten: def __init__(self, args): - plt.style.use('seaborn-white') - plt.rc('grid', linestyle="dotted", color='#a0a0a0') - plt.rcParams['axes.edgecolor'] = "#04383F" - plt.rcParams['figure.figsize'] = (12, 6) + print("\n--* Eiten has been initialized...") self.args = args @@ -36,11 +33,11 @@ def __init__(self, args): print('\n') - def calculate_percentage_change(self, old, new): + def price_delta(self, prices): """ Calculate percentage change """ - return ((new - old) * 100) / old + return ((prices - prices.shift()) * 100 / prices.shift())[1:] def create_returns(self, historical_price_info): """ @@ -51,18 +48,16 @@ def create_returns(self, historical_price_info): returns_matrix = [] returns_matrix_percentages = [] predicted_return_vectors = [] + for i in range(0, len(historical_price_info)): - close_prices = list(historical_price_info[i]["Close"]) - log_returns = [math.log(close_prices[i] / close_prices[i - 1]) - for i in range(1, len(close_prices))] - percentage_returns = [self.calculate_percentage_change( - close_prices[i - 1], close_prices[i]) for i in range(1, len(close_prices))] + close_prices = historical_price_info[i]["Close"] + log_returns = np.log((close_prices / close_prices.shift())[1:]) + percentage_returns = self.price_delta(close_prices) - total_data = len(close_prices) + total_data = close_prices.shape[0] # Expected returns in future. We can either use historical returns as future returns on try to simulate future returns and take the mean. For simulation, you can modify the functions in simulator to use here. - future_expected_returns = np.mean([(self.calculate_percentage_change(close_prices[i - 1], close_prices[i])) / ( - total_data - i) for i in range(1, len(close_prices))]) # More focus on recent returns + future_expected_returns = np.mean((self.price_delta(close_prices)) / (total_data - i)) # More focus on recent returns # Add to matrices returns_matrix.append(log_returns) @@ -92,8 +87,8 @@ def load_data(self): historical_price_info, future_prices = [], [] for symbol in symbol_names: historical_price_info.append( - self.data_dictionary[symbol]["historical_prices"]) - future_prices.append(self.data_dictionary[symbol]["future_prices"]) + self.data_dictionary[symbol]["historical"]) + future_prices.append(self.data_dictionary[symbol]["future"]) # Get return matrices and vectors predicted_return_vectors, returns_matrix, returns_matrix_percentages = self.create_returns( @@ -238,6 +233,10 @@ def draw_plot(self, filename="output/graph.png"): Draw plots """ # Styling for plots + plt.style.use('seaborn-white') + plt.rc('grid', linestyle="dotted", color='#a0a0a0') + plt.rcParams['axes.edgecolor'] = "#04383F" + plt.rcParams['figure.figsize'] = (12, 6) plt.grid() plt.legend(fontsize=14) @@ -248,6 +247,11 @@ def draw_plot(self, filename="output/graph.png"): plt.show() def print_and_plot_portfolio_weights(self, weights_dictionary: dict, strategy, plot_num: int) -> None: + plt.style.use('seaborn-white') + plt.rc('grid', linestyle="dotted", color='#a0a0a0') + plt.rcParams['axes.edgecolor'] = "#04383F" + plt.rcParams['figure.figsize'] = (12, 6) + print("\n-------- Weights for %s --------" % strategy) symbols = list(sorted(weights_dictionary.keys())) symbol_weights = [] diff --git a/simulator.py b/simulator.py index d77244c..ba61fc6 100644 --- a/simulator.py +++ b/simulator.py @@ -19,122 +19,130 @@ class MontoCarloSimulator: - """ - Monto carlo simulator that calculates the historical returns distribution and uses it to predict the future returns - """ - - def __init__(self): - print("\n--$ Simulator has been initialized") - - def calculate_percentage_change(self, old, new): - """ - Percentage change - """ - return ((new - old) * 100) / old - - def draw_portfolio_performance_chart(self, returns_matrix, portfolio_weights_dictionary, strategy_name): - """ - Draw returns chart for portfolio performance - """ - - # Get portfolio returns - returns_matrix = np.array(returns_matrix).transpose() - portfolio_weights_vector = np.array([portfolio_weights_dictionary[symbol] for symbol in portfolio_weights_dictionary]).transpose() - portfolio_returns = np.dot(returns_matrix, portfolio_weights_vector) - portfolio_returns_cumulative = np.cumsum(portfolio_returns) - - # Plot - x = np.arange(len(portfolio_returns_cumulative)) - plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') - plt.plot(x, portfolio_returns_cumulative, linewidth = 2.0, label = "Projected Returns from " + str(strategy_name)) - plt.title("Simulated Future Returns", fontsize = 14) - plt.xlabel("Bars (Time Sorted)", fontsize = 14) - plt.ylabel("Cumulative Percentage Return", fontsize = 14) - plt.xticks(fontsize = 14) - plt.yticks(fontsize = 14) - - def draw_market_performance_chart(self, actual_returns, strategy_name): - """ - Draw actual market returns if future data is available - """ - - # Get market returns - cumulative_returns = np.cumsum(actual_returns) - - # Plot - x = np.arange(len(cumulative_returns)) - plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') - plt.plot(x, cumulative_returns, linewidth = 2.0, color = '#282828', linestyle = '--', label = "Market Index Returns") - plt.title("Simulated Future Returns", fontsize = 14) - plt.xlabel("Bars (Time Sorted)", fontsize = 14) - plt.ylabel("Cumulative Percentage Return", fontsize = 14) - plt.xticks(fontsize = 14) - plt.yticks(fontsize = 14) - - def simulate_portfolio(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_prices_market, test_or_predict, market_chart, strategy_name, simulation_timesteps = 25): - """ - Simulate portfolio returns in the future - """ - returns_matrix = [] - actual_returns_matrix = [] - - # Iterate over each symbol to get their returns - for symbol in symbol_names: - - # Get symbol returns using monte carlo - historical_close_prices = list(portfolio_data_dictionary[symbol]["historical_prices"]["Close"]) - future_price_predictions, _ = self.simulate_and_get_future_prices(historical_close_prices, simulation_timesteps = max(simulation_timesteps, len(list(portfolio_data_dictionary[symbol]["future_prices"])))) - predicted_future_returns = [self.calculate_percentage_change(future_price_predictions[i - 1], future_price_predictions[i]) for i in range(1, len(future_price_predictions))] - returns_matrix.append(predicted_future_returns) - - - # Get portfolio returns - self.draw_portfolio_performance_chart(returns_matrix, portfolio_weights_dictionary, strategy_name) - - # Check whether we have actual future data available or not - if test_or_predict == 1: - future_prices_market = [item[4] for item in list(future_prices_market)] - actual_future_prices_returns = [self.calculate_percentage_change(future_prices_market[i - 1], future_prices_market[i]) for i in range(1, len(future_prices_market))] - if market_chart == True: - # Also draw the actual future returns - self.draw_market_performance_chart(actual_future_prices_returns, strategy_name) - - def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps = 25): - - # Get log returns from historical data - close_prices = historical_prices - returns = [math.log(close_prices[i] / close_prices[i - 1]) for i in range(1, len(close_prices))] - - # Get distribution of returns - hist = np.histogram(returns, bins = 32) - hist_dist = scipy.stats.rv_histogram(hist) # Distribution function - - predicted_prices = [] - # Do 25 iterations to simulate prices - for iteration in range(25): - new_close_prices = [close_prices[-1]] - new_close_prices_percentages = [] - for i in range(simulation_timesteps): - random_value = random.uniform(0, 1) - return_value = round(np.exp(hist_dist.ppf(random_value)), 5) # Get simulated return - price_last_point = new_close_prices[-1] - price_next_point = price_last_point * return_value - percentage_change = self.calculate_percentage_change(price_last_point, price_next_point) - - # Add to list - new_close_prices.append(price_next_point) - - predicted_prices.append(new_close_prices) - - # Calculate confidence intervals and average future returns. Conf intervals are not being used right now - conf_intervals = st.t.interval(0.95, len(predicted_prices), loc=np.mean(predicted_prices, axis = 0), scale=st.sem(predicted_prices, axis = 0)) - predicted_prices_mean = np.mean(predicted_prices, axis = 0) - return predicted_prices_mean, conf_intervals - - def is_nan(self, object): - """ - Check if object is null - """ - return object != object - - + """ + Monto carlo simulator that calculates the historical returns distribution and uses it to predict the future returns + """ + + def __init__(self): + print("\n--$ Simulator has been initialized") + + def price_delta(self, prices): + """ + Percentage change + """ + + return ((prices - prices.shift()) * 100 / prices.shift())[1:] + + def draw_portfolio_performance_chart(self, returns_matrix, portfolio_weights_dictionary, strategy_name): + """ + Draw returns chart for portfolio performance + """ + + # Get portfolio returns + returns_matrix = np.array(returns_matrix).transpose() + portfolio_weights_vector = np.array( + [portfolio_weights_dictionary[symbol] for symbol in portfolio_weights_dictionary]).transpose() + portfolio_returns = np.dot(returns_matrix, portfolio_weights_vector) + portfolio_returns_cumulative = np.cumsum(portfolio_returns) + + # Plot + x = np.arange(len(portfolio_returns_cumulative)) + plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') + plt.plot(x, portfolio_returns_cumulative, linewidth=2.0, + label="Projected Returns from " + str(strategy_name)) + plt.title("Simulated Future Returns", fontsize=14) + plt.xlabel("Bars (Time Sorted)", fontsize=14) + plt.ylabel("Cumulative Percentage Return", fontsize=14) + plt.xticks(fontsize=14) + plt.yticks(fontsize=14) + plt.show() + + def draw_market_performance_chart(self, actual_returns, strategy_name): + plt.style.use('seaborn-white') + plt.rc('grid', linestyle="dotted", color='#a0a0a0') + plt.rcParams['axes.edgecolor'] = "#04383F" + plt.rcParams['figure.figsize'] = (12, 6) + """ + Draw actual market returns if future data is available + """ + + # Get market returns + cumulative_returns = np.cumsum(actual_returns) + + # Plot + x = np.arange(len(cumulative_returns)) + plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') + plt.plot(x, cumulative_returns, linewidth=2.0, color='#282828', + linestyle='--', label="Market Index Returns") + plt.title("Simulated Future Returns", fontsize=14) + plt.xlabel("Bars (Time Sorted)", fontsize=14) + plt.ylabel("Cumulative Percentage Return", fontsize=14) + plt.xticks(fontsize=14) + plt.yticks(fontsize=14) + plt.show() + + def simulate_portfolio(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_prices_market, test_or_predict, market_chart, strategy_name, simulation_timesteps=25): + """ + Simulate portfolio returns in the future + """ + returns_matrix = [] + actual_returns_matrix = [] + + # Iterate over each symbol to get their returns + for symbol in symbol_names: + + # Get symbol returns using monte carlo + historical_close_prices = portfolio_data_dictionary[symbol]["historical"]["Close"] + future_price_predictions = self.simulate_and_get_future_prices(historical_close_prices, simulation_timesteps=max( + simulation_timesteps, portfolio_data_dictionary[symbol]["future"].shape[0])) + predicted_future_returns = self.price_delta(future_price_predictions) + returns_matrix.append(predicted_future_returns) + + # Get portfolio returns + self.draw_portfolio_performance_chart( + returns_matrix, portfolio_weights_dictionary, strategy_name) + + # Check whether we have actual future data available or not + if test_or_predict == 1: + actual_future_prices_returns = self.price_delta(future_prices_market.Close) + if market_chart: + # Also draw the actual future returns + self.draw_market_performance_chart( + actual_future_prices_returns, strategy_name) + + def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps=25): + + # Get log returns from historical data + close_prices = historical_prices + returns = np.log((close_prices / close_prices.shift())[1:]) + + # Get distribution of returns + hist = np.histogram(returns, bins=32) + hist_dist = scipy.stats.rv_histogram(hist) # Distribution function + + predicted_prices = [] + # Do 25 iterations to simulate prices + for iteration in range(25): + new_close_prices = [close_prices.values[-1]] + for i in range(simulation_timesteps): + random_value = random.uniform(0, 1) + # Get simulated return + return_value = round(np.exp(hist_dist.ppf(random_value)), 5) + price_next_point = new_close_prices[-1] * return_value + + # Add to list + new_close_prices.append(price_next_point) + + predicted_prices.append(new_close_prices) + + # Calculate confidence intervals and average future returns. Conf intervals are not being used right now + # conf_intervals = st.t.interval(0.95, len(predicted_prices), loc=np.mean( + # predicted_prices, axis=0), scale=st.sem(predicted_prices, axis=0)) + + return pd.DataFrame(columns=["Close"], data = np.mean(predicted_prices, axis=0)) + + def is_nan(self, object): + """ + Check if object is null + """ + return object != object From 673c071904c2caf01fea207bbfc0e288ec174e34 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 04:23:52 -0600 Subject: [PATCH 02/22] Refactor Code now uses vectorized operations remove some retundancy integrated pandas update strategies to make them compliant to new paradigm --- backtester.py | 38 ++-- data_loader.py | 100 +++------ eiten.py | 231 ++++++++++---------- simulator.py | 63 +++--- strategies/eigen_portfolio_strategy.py | 31 ++- strategies/genetic_algo_strategy.py | 217 +++++++++--------- strategies/maximum_sharpe_ratio_strategy.py | 38 ++-- strategies/strategy_helper_functions.py | 74 ++++--- strategy_manager.py | 107 ++++----- 9 files changed, 455 insertions(+), 444 deletions(-) diff --git a/backtester.py b/backtester.py index 419102f..8fa0051 100644 --- a/backtester.py +++ b/backtester.py @@ -34,7 +34,8 @@ def price_delta(self, prices): def portfolio_weight_manager(self, weight, is_long_only): """ - Manage portfolio weights. If portfolio is long only, set the negative weights to zero. + Manage portfolio weights. If portfolio is long only, + set the negative weights to zero. """ if is_long_only == 1: weight = max(weight, 0) @@ -42,20 +43,23 @@ def portfolio_weight_manager(self, weight, is_long_only): weight = weight return weight - def back_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, historical_price_market, is_long_only, market_chart, strategy_name): + def back_test(self, p_weights, data, market_data, long_only: bool, + market_chart: bool, strategy_name: str): """ - Main backtest function. Takes in the portfolio weights and compares the portfolio returns with a market index of your choice. + Main backtest function. Takes in the portfolio weights and compares + the portfolio returns with a market index of your choice. """ - + historical_data = market_data # Get market returns during the backtesting time - historical_prices = historical_price_market["Close"] + symbol_names = list(p_weights.keys()) + historical_prices = historical_data["historical"]["Close"] market_returns = self.price_delta(historical_prices) market_returns_cumulative = np.cumsum(market_returns) # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] for symbol in symbol_names: - symbol_historical_prices = portfolio_data_dictionary[symbol]["historical"]["Close"] + symbol_historical_prices = data[symbol]["historical"]["Close"] symbol_historical_returns = self.price_delta( symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) @@ -63,7 +67,7 @@ def back_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data_d # Get portfolio returns normal_returns_matrix = np.array(normal_returns_matrix).transpose() portfolio_weights_vector = np.array([self.portfolio_weight_manager( - portfolio_weights_dictionary[symbol], is_long_only) for symbol in portfolio_weights_dictionary]).transpose() + p_weights[symbol], long_only) for symbol in p_weights]).transpose() portfolio_returns = np.dot( normal_returns_matrix, portfolio_weights_vector) portfolio_returns_cumulative = np.cumsum(portfolio_returns) @@ -85,26 +89,34 @@ def back_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data_d plt.xticks(fontsize=14) plt.yticks(fontsize=14) - def future_test(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_price_market, is_long_only, market_chart, strategy_name): + def future_test(self, p_weights, data, market_data, long_only: bool, + market_chart: bool, strategy_name: str): """ - Main future test function. If future data is available i.e is_test is set to 1 and future_bars set to > 0, this takes in the portfolio weights and compares the portfolio returns with a market index of your choice in the future. + Main future test function. If future data is available i.e is_test + is set to 1 and future_bars set to > 0, this takes in the portfolio + weights and compares the portfolio returns with a market index of1 + your choice in the future. """ + symbol_names = list(p_weights.keys()) + #future_data = data_dict[symbol]["future"].Close # Get future prices - market_returns = self.price_delta(future_price_market["Close"]) + future_price_market = market_data["future"].Close + market_returns = self.price_delta(future_price_market) market_returns_cumulative = np.cumsum(market_returns) # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] for symbol in symbol_names: - symbol_historical_prices = portfolio_data_dictionary[symbol]["future"]["Close"] - symbol_historical_returns = self.price_delta(symbol_historical_prices) + symbol_historical_prices = data[symbol]["future"].Close + symbol_historical_returns = self.price_delta( + symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns normal_returns_matrix = np.array(normal_returns_matrix).transpose() portfolio_weights_vector = np.array([self.portfolio_weight_manager( - portfolio_weights_dictionary[symbol], is_long_only) for symbol in portfolio_weights_dictionary]).transpose() + p_weights[symbol], long_only) for symbol in p_weights]).transpose() portfolio_returns = np.dot( normal_returns_matrix, portfolio_weights_vector) portfolio_returns_cumulative = np.cumsum(portfolio_returns) diff --git a/data_loader.py b/data_loader.py index aca708e..ee3d60e 100644 --- a/data_loader.py +++ b/data_loader.py @@ -15,13 +15,15 @@ def __init__(self, args): # Stocks list self.directory_path = str(os.path.dirname(os.path.abspath(__file__))) - self.stocks_file_path = f"{self.directory_path}/{self.args.stocks_file_path}" + str_path = f"{self.directory_path}/{self.args.stocks_file_path}" + self.stocks_file_path = str_path self.stocks_list = [] # Load stock names in a list self.load_stocks_from_file() - # Dictionary to store data. This will only store and save data if the argument is_save_dictionary is 1. + # Dictionary to store data. This will only store and save data if + # the argument is_save_dictionary is 1. self.data_dictionary = {} # Data length @@ -32,18 +34,24 @@ def load_stocks_from_file(self): Load stock names from the file """ print("Loading all stocks from file...") - stocks_list = open(self.stocks_file_path, "r").readlines() - stocks_list = [str(item).strip("\n") for item in stocks_list] + stocks_list = [] + with open(self.stocks_file_path, "r") as f: + stocks_list = [str(item).strip() for item in f] # Load symbols stocks_list = list(sorted(set(stocks_list))) print("Total number of stocks: %d" % len(stocks_list)) self.stocks_list = stocks_list - def get_most_frequent_key(self, input_list): + def get_most_frequent_count(self, input_list): counter = collections.Counter(input_list) return list(counter.keys())[0] + def _split_data(self, data): + if self.args.is_test: + return (data.iloc[-self.args.future_bars:], + data.iloc[:-self.args.future_bars]) + def get_data(self, symbol): """ Get stock data from yahoo finance. @@ -79,16 +87,6 @@ def get_data(self, symbol): data_length = stock_prices.shape[0] self.stock_data_length.append(data_length) - # After getting some data, ignore partial data from yfinance - # based on number of data samples - if len(self.stock_data_length) > 5: - most_frequent_key = self.get_most_frequent_key( - self.stock_data_length) - if (data_length != most_frequent_key and - data_length != self.args.history_to_use and - symbol != self.args.market_index): # Needs index - return [], [], True - if self.args.history_to_use == "all": # For some reason, yfinance gives some 0 # values in the first index @@ -96,30 +94,12 @@ def get_data(self, symbol): else: stock_prices = stock_prices.iloc[-self.args.history_to_use:] - if self.args.is_test == 1: - future_prices = stock_prices.iloc[-self.args.future_bars:] - historical_prices = stock_prices.iloc[:-self.args.future_bars] - else: - historical_prices = stock_prices + historical_prices, future_prices = self._split_data(stock_prices) - if stock_prices.shape[0] == 0: - return [], [], True except Exception as e: print("Exception", e) - return [], [], True - return historical_prices, future_prices, False - - def get_market_index_price(self): - """ - Gets market index price e.g SPY. One can change it to some other index - """ - symbol = self.args.market_index - stock_price_data, future_prices, not_found = self.get_data(symbol) - if not_found: - return None, None - - return stock_price_data, future_prices + return historical_prices, future_prices def collect_data_for_all_tickers(self): """ @@ -127,54 +107,34 @@ def collect_data_for_all_tickers(self): """ print("Loading data for all stocks...") - symbol_names = [] - historical_price = [] - future_price = [] + data_dict = {} # Any stock with very low volatility is ignored. # You can change this line to address that. for i in tqdm(range(len(self.stocks_list))): symbol = self.stocks_list[i] try: - stock_price_data, future_prices, not_found = self.get_data( - symbol) - if not not_found: - # Add to lists - symbol_names.append(symbol) - historical_price.append(stock_price_data) - future_price.append(future_prices) + historical_data, future_data = self.get_data(symbol) + if historical_data is not None: + data_dict[symbol] = { + "historical": historical_data, + "future": future_data + } except Exception as e: print("Exception", e) continue - # Sometimes, there are some errors in feature generation or price - # extraction, let us remove that stuff - historical_price_info, future_price_info, symbol_names = self.remove_bad_data( - historical_price, future_price, symbol_names) - for i in range(0, len(symbol_names)): - self.data_dictionary[symbol_names[i]] = { - "historical": historical_price_info[i], - "future": future_price_info[i]} + return self.clean_data(data_dict) - return self.data_dictionary - - def remove_bad_data(self, historical_price, future_price, symbol_names): + def clean_data(self, data_dict): """ - Remove bad data i.e data that had some errors while scraping or feature generation + Remove bad data i.e data that had some errors while scraping or + feature generation - *** This can be much more improved with dicts and filter function. """ length_dictionary = collections.Counter( - [i.shape[0] for i in historical_price]) - length_dictionary = list(length_dictionary.keys()) - most_common_length = length_dictionary[0] - - filtered_historical_price, filtered_future_prices, filtered_symbols = [], [], [], - for i in range(len(future_price)): - if historical_price[i].shape[0] == most_common_length: - filtered_symbols.append(symbol_names[i]) - filtered_historical_price.append(historical_price[i]) - filtered_future_prices.append(future_price[i]) - - return filtered_historical_price, filtered_future_prices, filtered_symbols + [data_dict[i]["historical"].shape[0] for i in data_dict]) + std_len = length_dictionary.most_common(1)[0][0] + + return {i: data_dict[i] for i in data_dict if data_dict[i]["historical"].shape[0] == std_len} diff --git a/eiten.py b/eiten.py index e13509a..015de87 100644 --- a/eiten.py +++ b/eiten.py @@ -1,6 +1,7 @@ -import math import numpy as np +import pandas as pd import matplotlib.pyplot as plt +import json # Load our modules from data_loader import DataEngine @@ -9,9 +10,20 @@ from strategy_manager import StrategyManager +class _dotdict(dict): + """dot.notation access to dictionary attributes""" + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + class Eiten: - def __init__(self, args): - + def __init__(self, args: dict = None): + if args is None: + arg_types = {"str": str, "int": int, "bool": bool} + x = json.load(open("commands.json", "r")) + args = _dotdict( + {i["comm"][2:]: arg_types[i["type"]](i["default"]) for i in x}) print("\n--* Eiten has been initialized...") self.args = args @@ -19,7 +31,7 @@ def __init__(self, args): # Create data engine self.dataEngine = DataEngine(args) - # Monto carlo simulator + # Monte carlo simulator self.simulator = MontoCarloSimulator() # Strategy manager @@ -29,98 +41,92 @@ def __init__(self, args): self.backTester = BackTester() # Data dictionary - self.data_dictionary = {} + self.data_dict = {} # {"market": args.market_index} + self.market_data = {} print('\n') - def price_delta(self, prices): + def _get_price_delta(self, prices: pd.DataFrame): """ Calculate percentage change """ return ((prices - prices.shift()) * 100 / prices.shift())[1:] - def create_returns(self, historical_price_info): - """ - Create log return matrix, percentage return matrix, and mean return - vector - """ - - returns_matrix = [] - returns_matrix_percentages = [] - predicted_return_vectors = [] + def _get_abstract_returns(self, f): + """Abstract form of getting returns - for i in range(0, len(historical_price_info)): - close_prices = historical_price_info[i]["Close"] - log_returns = np.log((close_prices / close_prices.shift())[1:]) - percentage_returns = self.price_delta(close_prices) + This function takes a function f as parameter and applies the function + to the closing prices of the current data dictionary - total_data = close_prices.shape[0] + :param f: a function that takes a dictionary and returns some returns + :type f: function + :returns: A Pandas Dataframe of returns for each asset + :rtype: pd.Dataframe + """ + res = pd.DataFrame(pd.DataFrame(columns=list(self.data_dict.keys()))) + for i in self.data_dict: + c = self.data_dict[i]["historical"].Close + res[i] = f(c) + return res - # Expected returns in future. We can either use historical returns as future returns on try to simulate future returns and take the mean. For simulation, you can modify the functions in simulator to use here. - future_expected_returns = np.mean((self.price_delta(close_prices)) / (total_data - i)) # More focus on recent returns + def _get_perc_returns(self): + return self._get_abstract_returns(self._get_price_delta) - # Add to matrices - returns_matrix.append(log_returns) - returns_matrix_percentages.append(percentage_returns) + def _get_log_return(self, data: pd.DataFrame) -> np.ndarray: + return np.log((data / data.shift())[1:]) - # Add returns to vector - # Assuming that future returns are similar to past returns - predicted_return_vectors.append(future_expected_returns) + def _get_log_returns(self): + return self._get_abstract_returns(self._get_log_return) - # Convert to numpy arrays for one liner calculations - predicted_return_vectors = np.array(predicted_return_vectors) - returns_matrix = np.array(returns_matrix) - returns_matrix_percentages = np.array(returns_matrix_percentages) + def _get_predicted_return(self, data: pd.DataFrame) -> np.ndarray: + return np.array([np.mean(self._get_price_delta(data) / + np.array(np.arange(len(data) - 1, 0, -1)))]) - return predicted_return_vectors, returns_matrix, returns_matrix_percentages + def _get_predicted_returns(self): + return self._get_abstract_returns(self._get_predicted_return) def load_data(self): """ Loads data needed for analysis """ # Gather data for all stocks in a dictionary format - # Dictionary keys will be -> historical_prices, future_prices - self.data_dictionary = self.dataEngine.collect_data_for_all_tickers() - - # Add data to lists - symbol_names = list(sorted(self.data_dictionary.keys())) - historical_price_info, future_prices = [], [] - for symbol in symbol_names: - historical_price_info.append( - self.data_dictionary[symbol]["historical"]) - future_prices.append(self.data_dictionary[symbol]["future"]) - + # Dictionary keys will be -> historical, future + self.data_dict = self.dataEngine.collect_data_for_all_tickers() + p, f = self.dataEngine.get_data(self.args.market_index) + self.market_data["historical"], self.market_data["future"] = p, f # Get return matrices and vectors - predicted_return_vectors, returns_matrix, returns_matrix_percentages = self.create_returns( - historical_price_info) - return historical_price_info, future_prices, symbol_names, predicted_return_vectors, returns_matrix, returns_matrix_percentages + return self.data_dict def run_strategies(self): """ Run strategies, back and future test them, and simulate the returns. """ - historical_price_info, future_prices, symbol_names, predicted_return_vectors, returns_matrix, returns_matrix_percentages = self.load_data() - historical_price_market, future_prices_market = self.dataEngine.get_market_index_price() + self.load_data() # Calculate covariance matrix - covariance_matrix = np.cov(returns_matrix) + log_returns = self._get_log_returns() + cov_matrix = log_returns.cov() # Use random matrix theory to filter out the noisy eigen values if self.args.apply_noise_filtering: print( "\n** Applying random matrix theory to filter out noise in the covariance matrix...\n") - covariance_matrix = self.strategyManager.random_matrix_theory_based_cov( - returns_matrix) + cov_matrix = self.strategyManager.random_matrix_theory_based_cov( + log_returns) + + symbol_names = list(self.data_dict.keys()) + pred_returns = self._get_predicted_returns() + perc_returns = self._get_perc_returns() # Get weights for the portfolio eigen_portfolio_weights_dictionary = self.strategyManager.calculate_eigen_portfolio( - symbol_names, covariance_matrix, self.args.eigen_portfolio_number) + cov_matrix, self.args.eigen_portfolio_number) mvp_portfolio_weights_dictionary = self.strategyManager.calculate_minimum_variance_portfolio( - symbol_names, covariance_matrix) + symbol_names, cov_matrix) msr_portfolio_weights_dictionary = self.strategyManager.calculate_maximum_sharpe_portfolio( - symbol_names, covariance_matrix, predicted_return_vectors) + cov_matrix, pred_returns.T) ga_portfolio_weights_dictionary = self.strategyManager.calculate_genetic_algo_portfolio( - symbol_names, returns_matrix_percentages) + symbol_names, perc_returns) # Print weights print("\n*% Printing portfolio weights...") @@ -136,28 +142,31 @@ def run_strategies(self): # Back test print("\n*& Backtesting the portfolios...") - self.backTester.back_test(symbol_names, eigen_portfolio_weights_dictionary, - self.data_dictionary, - historical_price_market, + + self.backTester.back_test(eigen_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=True, strategy_name='Eigen Portfolio') - self.backTester.back_test(symbol_names, - mvp_portfolio_weights_dictionary, - self.data_dictionary, historical_price_market, - self.args.only_long, - market_chart=False, - strategy_name='Minimum Variance Portfolio (MVP)') - self.backTester.back_test(symbol_names, msr_portfolio_weights_dictionary, - self.data_dictionary, - historical_price_market, + + self.backTester.back_test( + mvp_portfolio_weights_dictionary, + self.data_dict, + self.market_data, + self.args.only_long, + market_chart=False, + strategy_name='Minimum Variance Portfolio (MVP)') + + self.backTester.back_test(msr_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=False, strategy_name='Maximum Sharpe Portfolio (MSR)') - self.backTester.back_test(symbol_names, - ga_portfolio_weights_dictionary, - self.data_dictionary, - historical_price_market, + self.backTester.back_test(ga_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=False, strategy_name='Genetic Algo (GA)') @@ -166,31 +175,28 @@ def run_strategies(self): if self.args.is_test: print("\n#^ Future testing the portfolios...") # Future test - self.backTester.future_test(symbol_names, - eigen_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.backTester.future_test(eigen_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=True, strategy_name='Eigen Portfolio') - self.backTester.future_test(symbol_names, - mvp_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.backTester.future_test(mvp_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=False, strategy_name='Minimum Variance Portfolio (MVP)') - self.backTester.future_test(symbol_names, - msr_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + + self.backTester.future_test(msr_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=False, strategy_name='Maximum Sharpe Portfolio (MSR)') - self.backTester.future_test(symbol_names, - ga_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.backTester.future_test(ga_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.only_long, market_chart=False, strategy_name='Genetic Algo (GA)') @@ -198,34 +204,31 @@ def run_strategies(self): # Simulation print("\n+$ Simulating future prices using monte carlo...") - self.simulator.simulate_portfolio(symbol_names, - eigen_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.simulator.simulate_portfolio(eigen_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.is_test, market_chart=True, strategy_name='Eigen Portfolio') - self.simulator.simulate_portfolio(symbol_names, - eigen_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.simulator.simulate_portfolio(eigen_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.is_test, market_chart=False, strategy_name='Minimum Variance Portfolio (MVP)') - self.simulator.simulate_portfolio(symbol_names, - eigen_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.simulator.simulate_portfolio(eigen_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.is_test, market_chart=False, strategy_name='Maximum Sharpe Portfolio (MSR)') - self.simulator.simulate_portfolio(symbol_names, - ga_portfolio_weights_dictionary, - self.data_dictionary, - future_prices_market, + self.simulator.simulate_portfolio(ga_portfolio_weights_dictionary, + self.data_dict, + self.market_data, self.args.is_test, market_chart=False, strategy_name='Genetic Algo (GA)') + self.draw_plot("output/monte_carlo.png") def draw_plot(self, filename="output/graph.png"): @@ -245,26 +248,26 @@ def draw_plot(self, filename="output/graph.png"): else: plt.tight_layout() plt.show() + plt.clf() + plt.cla() - def print_and_plot_portfolio_weights(self, weights_dictionary: dict, strategy, plot_num: int) -> None: + def print_and_plot_portfolio_weights(self, weights: dict, + strategy: str, plot_num: int) -> None: plt.style.use('seaborn-white') plt.rc('grid', linestyle="dotted", color='#a0a0a0') plt.rcParams['axes.edgecolor'] = "#04383F" plt.rcParams['figure.figsize'] = (12, 6) - + print("\n-------- Weights for %s --------" % strategy) - symbols = list(sorted(weights_dictionary.keys())) - symbol_weights = [] - for symbol in symbols: - print("Symbol: %s, Weight: %.4f" % - (symbol, weights_dictionary[symbol])) - symbol_weights.append(weights_dictionary[symbol]) + symbols = list(weights.keys()) + for k, v in weights.items(): + print(f"Symbol: {k}, Weight: {v:.4f}") # Plot width = 0.1 - x = np.arange(len(symbol_weights)) + x = np.arange(len(weights)) plt.bar(x + (width * (plot_num - 1)) + 0.05, - symbol_weights, label=strategy, width=width) + list(weights.values()), label=strategy, width=width) plt.xticks(x, symbols, fontsize=14) plt.yticks(fontsize=14) plt.xlabel("Symbols", fontsize=14) diff --git a/simulator.py b/simulator.py index ba61fc6..074f6b5 100644 --- a/simulator.py +++ b/simulator.py @@ -1,7 +1,4 @@ # Basic libraries -import os -import sys -import math import scipy import random import collections @@ -20,7 +17,8 @@ class MontoCarloSimulator: """ - Monto carlo simulator that calculates the historical returns distribution and uses it to predict the future returns + Monto carlo simulator that calculates the historical returns distribution + and uses it to predict the future returns """ def __init__(self): @@ -33,16 +31,17 @@ def price_delta(self, prices): return ((prices - prices.shift()) * 100 / prices.shift())[1:] - def draw_portfolio_performance_chart(self, returns_matrix, portfolio_weights_dictionary, strategy_name): + def draw_portfolio_performance_chart(self, returns_matrix, + p_weights: dict, strategy_name: str): """ Draw returns chart for portfolio performance """ # Get portfolio returns - returns_matrix = np.array(returns_matrix).transpose() - portfolio_weights_vector = np.array( - [portfolio_weights_dictionary[symbol] for symbol in portfolio_weights_dictionary]).transpose() - portfolio_returns = np.dot(returns_matrix, portfolio_weights_vector) + returns_matrix = np.array(returns_matrix).T + p_vector = np.array(list(p_weights.values())).T + print(p_weights) + portfolio_returns = np.dot(returns_matrix, p_vector) portfolio_returns_cumulative = np.cumsum(portfolio_returns) # Plot @@ -55,7 +54,6 @@ def draw_portfolio_performance_chart(self, returns_matrix, portfolio_weights_dic plt.ylabel("Cumulative Percentage Return", fontsize=14) plt.xticks(fontsize=14) plt.yticks(fontsize=14) - plt.show() def draw_market_performance_chart(self, actual_returns, strategy_name): plt.style.use('seaborn-white') @@ -80,37 +78,47 @@ def draw_market_performance_chart(self, actual_returns, strategy_name): plt.xticks(fontsize=14) plt.yticks(fontsize=14) plt.show() - - def simulate_portfolio(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_prices_market, test_or_predict, market_chart, strategy_name, simulation_timesteps=25): + plt.clf() + plt.cla() + plt.close() + + def simulate_portfolio(self, p_weights, + data, market_data, test_or_predict, + market_chart: bool, strategy_name: str, + simulation_timesteps: int = 25): """ Simulate portfolio returns in the future """ + symbol_names = list(p_weights.keys()) returns_matrix = [] - actual_returns_matrix = [] + future_price_market = market_data["future"].Close # Iterate over each symbol to get their returns for symbol in symbol_names: # Get symbol returns using monte carlo - historical_close_prices = portfolio_data_dictionary[symbol]["historical"]["Close"] + historical_close_prices = data[symbol]["historical"]["Close"] future_price_predictions = self.simulate_and_get_future_prices(historical_close_prices, simulation_timesteps=max( - simulation_timesteps, portfolio_data_dictionary[symbol]["future"].shape[0])) - predicted_future_returns = self.price_delta(future_price_predictions) + simulation_timesteps, data[symbol]["future"].shape[0])) + predicted_future_returns = self.price_delta( + future_price_predictions) returns_matrix.append(predicted_future_returns) # Get portfolio returns self.draw_portfolio_performance_chart( - returns_matrix, portfolio_weights_dictionary, strategy_name) + returns_matrix, p_weights, strategy_name) # Check whether we have actual future data available or not if test_or_predict == 1: - actual_future_prices_returns = self.price_delta(future_prices_market.Close) + actual_future_prices_returns = self.price_delta( + future_price_market) if market_chart: # Also draw the actual future returns self.draw_market_performance_chart( actual_future_prices_returns, strategy_name) - def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps=25): + def simulate_and_get_future_prices(self, historical_prices, + simulation_timesteps=25): # Get log returns from historical data close_prices = historical_prices @@ -118,7 +126,7 @@ def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps # Get distribution of returns hist = np.histogram(returns, bins=32) - hist_dist = scipy.stats.rv_histogram(hist) # Distribution function + hist_dist = st.rv_histogram(hist) # Distribution function predicted_prices = [] # Do 25 iterations to simulate prices @@ -127,7 +135,7 @@ def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps for i in range(simulation_timesteps): random_value = random.uniform(0, 1) # Get simulated return - return_value = round(np.exp(hist_dist.ppf(random_value)), 5) + return_value = np.round(np.exp(hist_dist.ppf(random_value)), 5) price_next_point = new_close_prices[-1] * return_value # Add to list @@ -135,11 +143,14 @@ def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps predicted_prices.append(new_close_prices) - # Calculate confidence intervals and average future returns. Conf intervals are not being used right now - # conf_intervals = st.t.interval(0.95, len(predicted_prices), loc=np.mean( - # predicted_prices, axis=0), scale=st.sem(predicted_prices, axis=0)) - - return pd.DataFrame(columns=["Close"], data = np.mean(predicted_prices, axis=0)) + # Calculate confidence intervals and average future returns. + # Conf intervals are not being used right now + # conf_intervals = st.t.interval(0.95, len(predicted_prices), + # loc=np.mean(predicted_prices, axis=0), + # scale=st.sem(predicted_prices, axis=0)) + + return pd.DataFrame(columns=["Close"], + data=np.mean(predicted_prices, axis=0)) def is_nan(self, object): """ diff --git a/strategies/eigen_portfolio_strategy.py b/strategies/eigen_portfolio_strategy.py index 51f5af0..0080046 100644 --- a/strategies/eigen_portfolio_strategy.py +++ b/strategies/eigen_portfolio_strategy.py @@ -4,17 +4,24 @@ import numpy as np warnings.filterwarnings("ignore") + class EigenPortfolioStrategy: - def __init__(self): - print("Eigen portfolio strategy has been created") - - def generate_portfolio(self, symbols, covariance_matrix, eigen_portfolio_number): - """ - Inspired by: https://srome.github.io/Eigenvesting-I-Linear-Algebra-Can-Help-You-Choose-Your-Stock-Portfolio/ - """ - eig_values, eig_vectors = np.linalg.eigh(covariance_matrix) - market_eigen_portfolio = eig_vectors[:,-1] / np.sum(eig_vectors[:,-1]) # We don't need this but in case someone wants to analyze - eigen_portfolio = eig_vectors[:,-eigen_portfolio_number] / np.sum(eig_vectors[:,-eigen_portfolio_number]) # This is a portfolio that is uncorrelated to market and still yields good returns + def __init__(self): + print("Eigen portfolio strategy has been created") + + def generate_portfolio(self, cov_matrix, eigen_portfolio_number): + """ + Inspired by: https://srome.github.io/Eigenvesting-I-Linear-Algebra-Can-Help-You-Choose-Your-Stock-Portfolio/ + """ + eigh_values, eigh_vectors = np.linalg.eigh(cov_matrix) + print("eig_values", eigh_values.shape, "eig_vectors", eigh_vectors.shape) + # We don't need this but in case someone wants to analyze + # market_eigen_portfolio = eig_vectors[:, -1] / np.sum(eig_vectors[:, -1]) + # This is a portfolio that is uncorrelated to market and still yields good returns + eigen_portfolio = eigh_vectors[:, -eigen_portfolio_number] / \ + np.sum(eigh_vectors[:, -eigen_portfolio_number]) + - portfolio_weights_dictionary = dict([(symbols[x], eigen_portfolio[x]) for x in range(0, len(eigen_portfolio))]) - return portfolio_weights_dictionary + weights = {cov_matrix.columns[i]: eigen_portfolio[i] + for i in range(eigen_portfolio.shape[0])} + return weights diff --git a/strategies/genetic_algo_strategy.py b/strategies/genetic_algo_strategy.py index 124409a..c1961d8 100644 --- a/strategies/genetic_algo_strategy.py +++ b/strategies/genetic_algo_strategy.py @@ -5,114 +5,111 @@ import numpy as np warnings.filterwarnings("ignore") + class GeneticAlgoStrategy: - """ - My own custom implementation of genetic algorithms for portfolio - """ - def __init__(self): - print("Genetic algo strategy has been created") - self.initial_genes = 100 - self.selection_top = 25 - self.mutation_iterations = 50 - self.weight_update_factor = 0.1 - self.gene_length = None - self.genes_in_each_iteration = 250 - self.iterations = 50 - self.crossover_probability = 0.05 - - def generate_portfolio(self, symbols, return_matrix): - self.gene_length = len(symbols) - - # Create initial genes - initial_genes = self.generate_initial_genes(symbols) - - for i in range(self.iterations): - # Select - top_genes = self.select(return_matrix, initial_genes) - #print("Iteration %d Best Sharpe Ratio: %.3f" % (i, top_genes[0][0])) - top_genes = [item[1] for item in top_genes] - - # Mutate - mutated_genes = self.mutate(top_genes) - initial_genes = mutated_genes - - top_genes = self.select(return_matrix, initial_genes) - best_gene = top_genes[0][1] - transposed_gene = np.array(best_gene).transpose() # Gene is a distribution of weights for different stocks - return_matrix_transposed = return_matrix.transpose() - returns = np.dot(return_matrix_transposed, transposed_gene) - returns_cumsum = np.cumsum(returns) - - ga_portfolio_weights = best_gene - ga_portfolio_weights = dict([(symbols[x], ga_portfolio_weights[x]) for x in range(0, len(ga_portfolio_weights))]) - return ga_portfolio_weights - - def generate_initial_genes(self, symbols): - total_symbols = len(symbols) - - genes = [] - for i in range(self.initial_genes): - gene = [random.uniform(-1, 1) for _ in range(0, total_symbols)] - genes.append(gene) - - return genes - - def mutate(self, genes): - new_genes = [] - - for gene in genes: - for x in range(0, self.mutation_iterations): - mutation = gene + (self.weight_update_factor * np.random.uniform(-1, 1, self.gene_length)) - mutation = list(mutation) - new_genes.append(mutation) - - new_genes = genes + new_genes - random.shuffle(new_genes) - genes_to_keep = new_genes[:self.genes_in_each_iteration] - - # Add crossovers - crossovers = self.crossover(new_genes) - genes_to_keep = genes_to_keep + crossovers - - return genes_to_keep - - def select(self, return_matrix, genes): - genes_with_scores = [] - for gene in genes: - transposed_gene = np.array(gene).transpose() # Gene is a distribution of weights for different stocks - return_matrix_transposed = return_matrix.transpose() - returns = np.dot(return_matrix_transposed, transposed_gene) - returns_cumsum = np.cumsum(returns) - - # Get fitness score - fitness = self.fitness_score(returns) - genes_with_scores.append([fitness, gene]) - - # Sort - random_genes = [self.generate_a_gene() for _ in range(5)] - genes_with_scores = list(reversed(sorted(genes_with_scores))) - genes_with_scores = genes_with_scores[:self.selection_top] + random_genes - return genes_with_scores - - def fitness_score(self, returns): - sharpe_returns = np.mean(returns) / np.std(returns) - return sharpe_returns - - def generate_a_gene(self): - gene = [random.uniform(-1, 1) for _ in range(self.gene_length)] - return gene - - def crossover(self, population): - crossover_population = [] - for z in range(0, len(population)): - if random.uniform(0, 1) < self.crossover_probability: - try: - random_gene_first = list(random.sample(population, 1)[0]) - random_gene_second = list(random.sample(population, 1)[0]) - random_split = random.randrange(1, len(random_gene_first) - 1) - crossover_gene = random_gene_first[:random_split] + random_gene_second[random_split:] - crossover_population.append(crossover_gene) - except Exception as e: - continue - - return crossover_population \ No newline at end of file + """ + My own custom implementation of genetic algorithms for portfolio + """ + + def __init__(self): + print("Genetic algo strategy has been created") + self.initial_genes = 100 + self.selection_top = 25 + self.mutation_iterations = 50 + self.weight_update_factor = 0.1 + self.gene_length = None + self.genes_in_each_iteration = 250 + self.iterations = 50 + self.crossover_probability = 0.05 + + def generate_portfolio(self, symbols, return_matrix): + self.gene_length = len(symbols) + + # Create initial genes + initial_genes = self.generate_initial_genes(symbols) + + for i in range(self.iterations): + # Select + top_genes = self.select(return_matrix, initial_genes) + # print("Iteration %d Best Sharpe Ratio: %.3f" % (i, top_genes[0][0])) + top_genes = [item[1] for item in top_genes] + + # Mutate + mutated_genes = self.mutate(top_genes) + initial_genes = mutated_genes + + top_genes = self.select(return_matrix, initial_genes) + best_gene = top_genes[0][1] + # Gene is a distribution of weights for different stocks + # transposed_gene = np.array(best_gene).transpose() + # returns = np.dot(return_matrix, transposed_gene) + # returns_cumsum = np.cumsum(returns) + + weights = {symbols[x]: best_gene[x] for x in range(0, len(best_gene))} + return weights + + def generate_initial_genes(self, symbols): + return np.array( + [self.generate_gene() for _ in range(self.gene_length)]) + + def mutate(self, genes): + new_genes = [] + + for gene in genes: + for x in range(0, self.mutation_iterations): + mutation = gene + (self.weight_update_factor * + np.random.uniform(-1, 1, self.gene_length)) + new_genes.append(mutation) + + new_genes = genes + new_genes + np.random.shuffle(new_genes) + genes_to_keep = new_genes[:self.genes_in_each_iteration] + + # Add crossovers + crossovers = self.crossover(new_genes) + genes_to_keep = genes_to_keep + crossovers + + return genes_to_keep + + def select(self, return_matrix, genes): + genes_with_scores = [] + for gene in genes: + # Gene is a distribution of weights for different stocks + transposed_gene = gene.transpose() + returns = np.dot(return_matrix, transposed_gene) + # returns_cumsum = np.cumsum(returns) + + # Get fitness score + fitness = self.fitness_score(returns) + genes_with_scores.append([fitness, gene]) + + # Sort + random_genes = [self.generate_gene() for _ in range(5)] + genes_with_scores = sorted( + genes_with_scores, reverse=True, key=lambda x: x[0]) + genes_with_scores = (genes_with_scores[:self.selection_top] + + random_genes) + return genes_with_scores + + def fitness_score(self, returns): + sharpe_returns = np.mean(returns) / np.std(returns) + return sharpe_returns + + def generate_gene(self): + return np.random.uniform(-1, 1, self.gene_length) + + def crossover(self, population): + rng = np.random.default_rng() + crossover_population = [] + + population = np.array( + list(filter(lambda x: type(x) == np.ndarray, population))) + for z in range(0, len(population)): + if np.random.uniform(0, 1) < self.crossover_probability: + a, b = rng.choice(population, 2) + random_split = np.random.randint(1, len(a) - 1) + ab = np.concatenate( + (a[:random_split], b[random_split:]), axis=0) + crossover_population.append(ab) + + return crossover_population diff --git a/strategies/maximum_sharpe_ratio_strategy.py b/strategies/maximum_sharpe_ratio_strategy.py index f47c251..361da64 100644 --- a/strategies/maximum_sharpe_ratio_strategy.py +++ b/strategies/maximum_sharpe_ratio_strategy.py @@ -1,24 +1,28 @@ # Basic libraries -import os -import random import warnings import numpy as np warnings.filterwarnings("ignore") + class MaximumSharpeRatioStrategy: - def __init__(self): - print("Maximum sharpe ratio strategy has been created") - - def generate_portfolio(self, symbols, covariance_matrix, returns_vector): - """ - Inspired by: Eigen Portfolio Selection: A Robust Approach to Sharpe Ratio Maximization, https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3070416 - """ - inverse_cov_matrix = np.linalg.pinv(covariance_matrix) - ones = np.ones(len(inverse_cov_matrix)) + def __init__(self): + print("Maximum sharpe ratio strategy has been created") + + def generate_portfolio(self, cov_matrix, returns_vector): + """ + Inspired by: Eigen Portfolio Selection: + A Robust Approach to Sharpe Ratio Maximization, + https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3070416 + """ + inverse_cov_matrix = np.linalg.pinv(cov_matrix) + ones = np.ones(len(inverse_cov_matrix)) + + numerator = np.dot(inverse_cov_matrix, returns_vector) + denominator = np.dot( + np.dot(ones.transpose(), inverse_cov_matrix), returns_vector) + msr_portfolio_weights = numerator / denominator + + weights = {cov_matrix.columns[i]: msr_portfolio_weights[i][0] + for i in range(len(msr_portfolio_weights))} - numerator = np.dot(inverse_cov_matrix, returns_vector) - denominator = np.dot(np.dot(ones.transpose(), inverse_cov_matrix), returns_vector) - msr_portfolio_weights = numerator / denominator - - portfolio_weights_dictionary = dict([(symbols[x], msr_portfolio_weights[x]) for x in range(0, len(msr_portfolio_weights))]) - return portfolio_weights_dictionary \ No newline at end of file + return weights diff --git a/strategies/strategy_helper_functions.py b/strategies/strategy_helper_functions.py index dd32694..048e126 100644 --- a/strategies/strategy_helper_functions.py +++ b/strategies/strategy_helper_functions.py @@ -3,39 +3,47 @@ import random import warnings import numpy as np +import pandas as pd warnings.filterwarnings("ignore") + class StrategyHelperFunctions: - def __init__(self): - print("Helper functions have been created") - - def random_matrix_theory_based_cov(self, returns_matrix): - """ - This is inspired by the excellent post @ https://srome.github.io/Eigenvesting-III-Random-Matrix-Filtering-In-Finance/ - """ - - # Calculate variance and std, will come in handy during reconstruction - variances = np.diag(np.cov(returns_matrix)) - standard_deviations = np.sqrt(variances) - - # Get correlation matrix and compute eigen vectors and values - correlation_matrix = np.corrcoef(returns_matrix) - eig_values, eig_vectors = np.linalg.eigh(correlation_matrix) - - # Get maximum theoretical eigen value for a random matrix - sigma = 1 # The variance for all of the standardized log returns is 1 - Q = len(returns_matrix[0]) / len(returns_matrix) - max_theoretical_eval = np.power(sigma*(1 + np.sqrt(1/Q)),2) - - # Prune random eigen values - eig_values_pruned = eig_values[eig_values > max_theoretical_eval] - eig_values[eig_values <= max_theoretical_eval] = 0 - - # Reconstruct the covariance matrix from the correlation matrix and filtered eigen values - temp = np.dot(eig_vectors, np.dot(np.diag(eig_values), np.transpose(eig_vectors))) - np.fill_diagonal(temp, 1) - filtered_matrix = temp - filtered_cov_matrix = np.dot(np.diag(standard_deviations), - np.dot(filtered_matrix,np.diag(standard_deviations))) - - return filtered_cov_matrix \ No newline at end of file + def __init__(self): + print("Helper functions have been created") + + def random_matrix_theory_based_cov(self, log_returns): + """ + This is inspired by the excellent post @ + https://srome.github.io/Eigenvesting-III-Random-Matrix-Filtering-In-Finance/ + """ + returns_matrix = log_returns.T + + # Calculate variance and std, will come in handy during reconstruction + variances = np.diag(np.cov(returns_matrix)) + standard_deviations = np.sqrt(variances) + + # Get correlation matrix and compute eigen vectors and values + correlation_matrix = np.corrcoef(returns_matrix) + eig_values, eig_vectors = np.linalg.eigh(correlation_matrix) + + # Get maximum theoretical eigen value for a random matrix + sigma = 1 # The variance for all of the standardized log returns is 1 + Q = returns_matrix.shape[1] / returns_matrix.shape[0] + max_theoretical_eval = np.power(sigma*(1 + np.sqrt(1/Q)), 2) + + # Prune random eigen values + # eig_values_pruned = eig_values[eig_values > max_theoretical_eval] + eig_values[eig_values <= max_theoretical_eval] = 0 + + # Reconstruct the covariance matrix from the correlation matrix + # and filtered eigen values + temp = np.dot(eig_vectors, np.dot( + np.diag(eig_values), np.transpose(eig_vectors))) + np.fill_diagonal(temp, 1) + filtered_matrix = temp + filtered_cov_matrix = np.dot(np.diag(standard_deviations), + np.dot(filtered_matrix, + np.diag(standard_deviations))) + return pd.DataFrame(columns=log_returns.columns, + data=filtered_cov_matrix) + diff --git a/strategy_manager.py b/strategy_manager.py index 8fb7774..3ccf78b 100644 --- a/strategy_manager.py +++ b/strategy_manager.py @@ -8,53 +8,62 @@ from strategies.strategy_helper_functions import StrategyHelperFunctions warnings.filterwarnings("ignore") + class StrategyManager: - """ - Runs and manages all strategies - """ - def __init__(self): - print("\n--= Strategy manager has been created...") - self.geneticAlgoStrategy = GeneticAlgoStrategy() - self.minimumVarianceStrategy = MinimumVarianceStrategy() - self.eigenPortfolioStrategy = EigenPortfolioStrategy() - self.maximumSharpeRatioStrategy = MaximumSharpeRatioStrategy() - self.strategyHelperFunctions = StrategyHelperFunctions() - - def calculate_genetic_algo_portfolio(self, symbols, returns_matrix_percentages): - """ - Genetic algorithm based portfolio that maximizes sharpe ratio. This is my own implementation - """ - print("-* Calculating portfolio weights using genetic algorithm...") - portfolio_weights_dictionary = self.geneticAlgoStrategy.generate_portfolio(symbols, returns_matrix_percentages) - return portfolio_weights_dictionary - - def calculate_eigen_portfolio(self, symbols, covariance_matrix, eigen_portfolio_number): - """ - 2nd Eigen Portfolio - """ - print("-$ Calculating portfolio weights using eigen values...") - portfolio_weights_dictionary = self.eigenPortfolioStrategy.generate_portfolio(symbols, covariance_matrix, eigen_portfolio_number) - return portfolio_weights_dictionary - - def calculate_minimum_variance_portfolio(self, symbols, covariance_matrix): - """ - Minimum variance portfolio - """ - print("-! Calculating portfolio weights using minimum variance portfolio algorithm...") - portfolio_weights_dictionary = self.minimumVarianceStrategy.generate_portfolio(symbols, covariance_matrix) - return portfolio_weights_dictionary - - def calculate_maximum_sharpe_portfolio(self, symbols, covariance_matrix, returns_vector): - """ - Maximum sharpe portfolio - """ - print("-# Calculating portfolio weights using maximum sharpe portfolio algorithm...") - portfolio_weights_dictionary = self.maximumSharpeRatioStrategy.generate_portfolio(symbols, covariance_matrix, returns_vector) - return portfolio_weights_dictionary - - def random_matrix_theory_based_cov(self, returns_matrix): - """ - Covariance matrix filtering using random matrix theory - """ - filtered_covariance_matrix = self.strategyHelperFunctions.random_matrix_theory_based_cov(returns_matrix) - return filtered_covariance_matrix \ No newline at end of file + """ + Runs and manages all strategies + """ + + def __init__(self): + print("\n--= Strategy manager has been created...") + self.geneticAlgoStrategy = GeneticAlgoStrategy() + self.minimumVarianceStrategy = MinimumVarianceStrategy() + self.eigenPortfolioStrategy = EigenPortfolioStrategy() + self.maximumSharpeRatioStrategy = MaximumSharpeRatioStrategy() + self.strategyHelperFunctions = StrategyHelperFunctions() + + def calculate_genetic_algo_portfolio(self, symbols, returns_matrix_percentages): + """ + Genetic algorithm based portfolio that maximizes sharpe ratio. This is my own implementation + """ + print("-* Calculating portfolio weights using genetic algorithm...") + portfolio_weights_dictionary = self.geneticAlgoStrategy.generate_portfolio( + symbols, returns_matrix_percentages) + return portfolio_weights_dictionary + + def calculate_eigen_portfolio(self, cov_matrix, eigen_portfolio_number): + """ + 2nd Eigen Portfolio + """ + print("-$ Calculating portfolio weights using eigen values...") + portfolio_weights_dictionary = self.eigenPortfolioStrategy.generate_portfolio( + cov_matrix, eigen_portfolio_number) + return portfolio_weights_dictionary + + def calculate_minimum_variance_portfolio(self, symbols, covariance_matrix): + """ + Minimum variance portfolio + """ + print( + "-! Calculating portfolio weights using minimum variance portfolio algorithm...") + portfolio_weights_dictionary = self.minimumVarianceStrategy.generate_portfolio( + symbols, covariance_matrix) + return portfolio_weights_dictionary + + def calculate_maximum_sharpe_portfolio(self, cov_matrix, returns_vector): + """ + Maximum sharpe portfolio + """ + print( + "-# Calculating portfolio weights using maximum sharpe portfolio algorithm...") + portfolio_weights_dictionary = self.maximumSharpeRatioStrategy.generate_portfolio( + cov_matrix, returns_vector) + return portfolio_weights_dictionary + + def random_matrix_theory_based_cov(self, returns_matrix): + """ + Covariance matrix filtering using random matrix theory + """ + filtered_covariance_matrix = self.strategyHelperFunctions.random_matrix_theory_based_cov( + returns_matrix) + return filtered_covariance_matrix From 17b67f2b44a92578486292c74728b4c0d3bc537b Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 04:24:28 -0600 Subject: [PATCH 03/22] Notebooks Created a demo notebook to show that now Eiten can be used as a library XD --- notebooks/Demo.ipynb | 208 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 notebooks/Demo.ipynb diff --git a/notebooks/Demo.ipynb b/notebooks/Demo.ipynb new file mode 100644 index 0000000..20afe42 --- /dev/null +++ b/notebooks/Demo.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Populating the interactive namespace from numpy and matplotlib\n" + ] + } + ], + "source": [ + "%pylab inline\n", + "import sys, os\n", + "sys.path.insert(1, os.path.join(sys.path[0], '..'))\n", + "from eiten import Eiten " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--* Eiten has been initialized...\n", + "\n", + "--> Data engine has been initialized...\n", + "Loading all stocks from file...\n", + "Total number of stocks: 9\n", + "\n", + "--$ Simulator has been initialized\n", + "\n", + "--= Strategy manager has been created...\n", + "Genetic algo strategy has been created\n", + "Minimum Variance strategy has been created\n", + "Eigen portfolio strategy has been created\n", + "Maximum sharpe ratio strategy has been created\n", + "Helper functions have been created\n", + "\n", + "--# Backtester has been initialized\n", + "\n", + "\n" + ] + } + ], + "source": [ + "et = Eiten()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/9 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", + "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", + "{'AAPL': -0.4369306238899776, 'AMD': -1.0115784282008393, 'AMZN': -2.3091148432564372, 'FB': 0.5058836309304608, 'MSFT': 0.009511216792968975, 'NFLX': 0.33246631319412956, 'NVDA': -0.18129905403714677, 'SQQQ': -0.680171626027902, 'TSLA': 0.721905802094419}\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsQAAAFkCAYAAAAnoS3wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAATXUlEQVR4nO3dUWiX97348Y9JTNTGKp56egZtPJgab8o5mnozPGG2Wyit3UDD+lM37YUgvRqMjK03BhGr2ezFOHbdoYOJE1Yj4oUR2kGqRQjtRcRYZKjgXKDjHCxHpU2ymqa/539R/PEPPSe/6skvaft5va7yPN/n4fe5+BLeefjxZF5RFEUAAEBSdXM9AAAAzCVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACk9qWC+OLFi7F9+/YvnD9z5kx0dXVFqVSK48ePz/hwAABQaw3VLvjd734Xp06dioULF045/+mnn8aBAwfixIkTsXDhwti6dWs8+eSTsXz58poNCwAAM61qELe0tMShQ4fi5z//+ZTz165di5aWlliyZElERDzxxBMxNDQUzzzzzJTrPvnkTgxduRLfWrYs6hvqZ3B0AAD43GeTn8V/3rwZ61avjgULmu7p3qpB/PTTT8cHH3zwhfOjo6OxePHiyvEDDzwQo6OjX7hu6MqVePLFn9zTUAAAcD/O/se/x7/967/c0z1Vg/h/09zcHGNjY5XjsbGxKYF81z/9w7LKcI/8o69TAAAw8z648WE8+eJPKu15L+47iFtbW2NkZCRu374dixYtiqGhodi5c+cXP6D+869JPPKPy+Ofv/Wt+/04AACo6m573tM993pDf39/jI+PR6lUipdeeil27twZRVFEV1dXPPzww/c8AAAAzKUvFcSPPPJI5bVq3//+9yvnn3rqqXjqqadqMxkAAMwC/5gDAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNSqBnG5XI6enp4olUqxffv2GBkZmbJ+6tSp2LRpU3R1dcUf//jHmg0KAAC10FDtgoGBgZiYmIi+vr4YHh6O3t7e+O1vf1tZ/9WvfhWnT5+ORYsWxcaNG2Pjxo2xZMmSmg4NAAAzpWoQnz9/Pjo6OiIiYs2aNXHp0qUp66tXr46PP/44GhoaoiiKmDdvXm0mBQCAGqgaxKOjo9Hc3Fw5rq+vj8nJyWho+PzWVatWRVdXVyxcuDA6OzvjwQcfrN20AAAww6p+h7i5uTnGxsYqx+VyuRLDly9fjnfeeSfefvvtOHPmTNy8eTPefPPN2k0LAAAzrGoQt7e3x7lz5yIiYnh4ONra2iprixcvjgULFkRTU1PU19fHsmXL4qOPPqrdtAAAMMOqfmWis7MzBgcHY8uWLVEURezfvz/6+/tjfHw8SqVSlEql2LZtW8yfPz9aWlpi06ZNszE3AADMiKpBXFdXF3v37p1yrrW1tfLz1q1bY+vWrTM/GQAAzAL/mAMAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpNVS7oFwux549e+LKlSvR2NgY+/btixUrVlTW33///ejt7Y2iKGL58uVx8ODBaGpqqunQAAAwU6o+IR4YGIiJiYno6+uL7u7u6O3trawVRRG7d++OAwcOxBtvvBEdHR3xt7/9raYDAwDATKr6hPj8+fPR0dERERFr1qyJS5cuVdauX78eS5cujSNHjsTVq1fjO9/5TqxcubJ20wIAwAyr+oR4dHQ0mpubK8f19fUxOTkZERG3bt2KCxcuxLZt2+Lw4cPx3nvvxbvvvlu7aQEAYIZVDeLm5uYYGxurHJfL5Who+PzB8tKlS2PFihXx2GOPxfz586Ojo2PKE2QAAPiqqxrE7e3tce7cuYiIGB4ejra2tsrao48+GmNjYzEyMhIREUNDQ7Fq1aoajQoAADOv6neIOzs7Y3BwMLZs2RJFUcT+/fujv78/xsfHo1Qqxcsvvxzd3d1RFEWsXbs2NmzYMAtjAwDAzKgaxHV1dbF3794p51pbWys/f/vb344TJ07M/GQAADAL/GMOAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFKrGsTlcjl6enqiVCrF9u3bY2Rk5H+8bvfu3fHKK6/M+IAAAFBLVYN4YGAgJiYmoq+vL7q7u6O3t/cL1xw7diyuXr1akwEBAKCWqgbx+fPno6OjIyIi1qxZE5cuXZqyfuHChbh48WKUSqXaTAgAADVUNYhHR0ejubm5clxfXx+Tk5MREXHjxo149dVXo6enp3YTAgBADTVUu6C5uTnGxsYqx+VyORoaPr/trbfeilu3bsWuXbviww8/jE8++SRWrlwZmzdvrt3EAAAwg6oGcXt7e5w9ezaeffbZGB4ejra2tsrajh07YseOHRERcfLkyfjLX/4ihgEA+FqpGsSdnZ0xODgYW7ZsiaIoYv/+/dHf3x/j4+O+NwwAwNde1SCuq6uLvXv3TjnX2tr6hes8GQYA4OvIP+YAACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAILWGaheUy+XYs2dPXLlyJRobG2Pfvn2xYsWKyvrp06fjyJEjUV9fH21tbbFnz56oq9PZAAB8PVQt14GBgZiYmIi+vr7o7u6O3t7eytonn3wSv/71r+MPf/hDHDt2LEZHR+Ps2bM1HRgAAGZS1SA+f/58dHR0RETEmjVr4tKlS5W1xsbGOHbsWCxcuDAiIiYnJ6OpqalGowIAwMyrGsSjo6PR3NxcOa6vr4/JycnPb66ri4ceeigiIo4ePRrj4+Oxfv36Go0KAAAzr+p3iJubm2NsbKxyXC6Xo6GhYcrxwYMH4/r163Ho0KGYN29ebSYFAIAaqPqEuL29Pc6dOxcREcPDw9HW1jZlvaenJ+7cuROvvfZa5asTAADwdVH1CXFnZ2cMDg7Gli1boiiK2L9/f/T398f4+Hg8/vjjceLEiVi3bl288MILERGxY8eO6OzsrPngAAAwE6oGcV1dXezdu3fKudbW1srPly9fnvmpAABglnhhMAAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpVQ3icrkcPT09USqVYvv27TEyMjJl/cyZM9HV1RWlUimOHz9es0EBAKAWqgbxwMBATExMRF9fX3R3d0dvb29l7dNPP40DBw7E73//+zh69Gj09fXFhx9+WNOBAQBgJjVUu+D8+fPR0dERERFr1qyJS5cuVdauXbsWLS0tsWTJkoiIeOKJJ2JoaCieeeaZyjWTn30WEREf3BDKAADUxt3WvNue96JqEI+OjkZzc3PluL6+PiYnJ6OhoSFGR0dj8eLFlbUHHnggRkdHp9z/X/99MyIinnzxJ/c8HAAA3Iv/+u+b8dgjj9zTPVWDuLm5OcbGxirH5XI5Ghoa/se1sbGxKYEcEbFu9eo4+x//Ht9atizqG+rvaTgAAPgyPpv8LP7z5s1Yt3r1Pd9bNYjb29vj7Nmz8eyzz8bw8HC0tbVV1lpbW2NkZCRu374dixYtiqGhodi5c+eU+xcsaIp/+9d/uefBAADgXrQ+em9Phu+aVxRFMd0F5XI59uzZE1evXo2iKGL//v3x5z//OcbHx6NUKsWZM2fiN7/5TRRFEV1dXfGjH/3ovgYBAIC5UDWIv6y74XzlypVobGyMffv2xYoVKyrrd8O5oaEhurq64vnnn5+Jj+Urrtq+OH36dBw5ciTq6+ujra0t9uzZE3V1Xo/9TVdtX9y1e/fuWLJkSfzsZz+bgymZbdX2xfvvvx+9vb1RFEUsX748Dh48GE1NTXM4MbOh2r44depUHD58OOrq6qKrqyu2bds2h9Mymy5evBivvPJKHD16dMr5+2rOYob86U9/Kn7xi18URVEUFy5cKF588cXK2sTERPG9732vuH37dnHnzp1i8+bNxY0bN2bqo/kKm25f/P3vfy+++93vFuPj40VRFMVPf/rTYmBgYE7mZHZNty/ueuONN4rnn3++OHjw4GyPxxyZbl+Uy+XiBz/4QfHXv/61KIqiOH78eHHt2rU5mZPZVe33xfr164tbt24Vd+7cqbQG33yvv/568dxzzxU//OEPp5y/3+acsUdxX/b1bI2NjZXXs/HNN92+aGxsjGPHjsXChQsjImJyctLTniSm2xcRERcuXIiLFy9GqVSai/GYI9Pti+vXr8fSpUvjyJEj8eMf/zhu374dK1eunKtRmUXVfl+sXr06Pv7445iYmIiiKGLevHlzMSazrKWlJQ4dOvSF8/fbnDMWxP/b69nurlV7PRvfTNPti7q6unjooYciIuLo0aMxPj4e69evn5M5mV3T7YsbN27Eq6++Gj09PXM1HnNkun1x69atuHDhQmzbti0OHz4c7733Xrz77rtzNSqzaLp9ERGxatWq6Orqio0bN8aGDRviwQcfnIsxmWVPP/105a1n/7/7bc4ZC+L/6+vZ+Gaabl/cPf7lL38Zg4ODcejQIX/ZJzHdvnjrrbfi1q1bsWvXrnj99dfj9OnTcfLkybkalVk03b5YunRprFixIh577LGYP39+dHR0fOFJId9M0+2Ly5cvxzvvvBNvv/12nDlzJm7evBlvvvnmXI3KV8D9NueMBXF7e3ucO3cuImLa17NNTEzE0NBQrF27dqY+mq+w6fZFRERPT0/cuXMnXnvttcpXJ/jmm25f7NixI06ePBlHjx6NXbt2xXPPPRebN2+eq1GZRdPti0cffTTGxsZiZGQkIiKGhoZi1apVczIns2u6fbF48eJYsGBBNDU1RX19fSxbtiw++uijuRqVr4D7bc6q7yH+sjo7O2NwcDC2bNlSeT1bf39/5fVsL730UuzcubPyeraHH354pj6ar7Dp9sXjjz8eJ06ciHXr1sULL7wQEZ/HUGdn5xxPTa1V+31BTtX2xcsvvxzd3d1RFEWsXbs2NmzYMNcjMwuq7YtSqRTbtm2L+fPnR0tLS2zatGmuR2YO/F+bc8ZeuwYAAF9HXvgKAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACC1/we/joqoI4LRIgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "et.run_strategies()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 0f3c439a309273fe8321a8326324c59a5b4ddf3c Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 12:46:15 -0600 Subject: [PATCH 04/22] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2cbcfa0..c3e1ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.png +notebooks/commands.json From c319459f29d193df880b820e71639b3c9ab10f80 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 12:52:18 -0600 Subject: [PATCH 05/22] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2cbcfa0..c3e1ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.png +notebooks/commands.json From db91d904e791359c2ec39f8a38c829e14ee53844 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 12:55:06 -0600 Subject: [PATCH 06/22] \t for space --- simulator.py | 238 ++++++++++++++++++++++---------------------- strategy_manager.py | 88 ++++++++-------- 2 files changed, 163 insertions(+), 163 deletions(-) diff --git a/simulator.py b/simulator.py index da1d08b..e43b45e 100644 --- a/simulator.py +++ b/simulator.py @@ -19,122 +19,122 @@ class MontoCarloSimulator: - """ - Monto carlo simulator that calculates the historical returns distribution and uses it to predict the future returns - """ - - def __init__(self): - print("\n--$ Simulator has been initialized") - - def calculate_percentage_change(self, old, new): - """ - Percentage change - """ - return ((new - old) * 100) / old - - def draw_portfolio_performance_chart(self, returns_matrix, portfolio_weights_dictionary, strategy_name): - """ - Draw returns chart for portfolio performance - """ - - # Get portfolio returns - returns_matrix = np.array(returns_matrix).transpose() - portfolio_weights_vector = np.array([portfolio_weights_dictionary[symbol] for symbol in portfolio_weights_dictionary]).transpose() - portfolio_returns = np.dot(returns_matrix, portfolio_weights_vector) - portfolio_returns_cumulative = np.cumsum(portfolio_returns) - - # Plot - x = np.arange(len(portfolio_returns_cumulative)) - plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') - plt.plot(x, portfolio_returns_cumulative, linewidth = 2.0, label = "Projected Returns from " + str(strategy_name)) - plt.title("Simulated Future Returns", fontsize = 14) - plt.xlabel("Bars (Time Sorted)", fontsize = 14) - plt.ylabel("Cumulative Percentage Return", fontsize = 14) - plt.xticks(fontsize = 14) - plt.yticks(fontsize = 14) - - def draw_market_performance_chart(self, actual_returns, strategy_name): - """ - Draw actual market returns if future data is available - """ - - # Get market returns - cumulative_returns = np.cumsum(actual_returns) - - # Plot - x = np.arange(len(cumulative_returns)) - plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') - plt.plot(x, cumulative_returns, linewidth = 2.0, color = '#282828', linestyle = '--', label = "Market Index Returns") - plt.title("Simulated Future Returns", fontsize = 14) - plt.xlabel("Bars (Time Sorted)", fontsize = 14) - plt.ylabel("Cumulative Percentage Return", fontsize = 14) - plt.xticks(fontsize = 14) - plt.yticks(fontsize = 14) - - def simulate_portfolio(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_prices_market, test_or_predict, market_chart, strategy_name, simulation_timesteps = 25): - """ - Simulate portfolio returns in the future - """ - returns_matrix = [] - actual_returns_matrix = [] - - # Iterate over each symbol to get their returns - for symbol in symbol_names: - - # Get symbol returns using monte carlo - historical_close_prices = list(portfolio_data_dictionary[symbol]["historical_prices"]["Close"]) - future_price_predictions, _ = self.simulate_and_get_future_prices(historical_close_prices, simulation_timesteps = max(simulation_timesteps, len(list(portfolio_data_dictionary[symbol]["future_prices"])))) - predicted_future_returns = [self.calculate_percentage_change(future_price_predictions[i - 1], future_price_predictions[i]) for i in range(1, len(future_price_predictions))] - returns_matrix.append(predicted_future_returns) - - - # Get portfolio returns - self.draw_portfolio_performance_chart(returns_matrix, portfolio_weights_dictionary, strategy_name) - - # Check whether we have actual future data available or not - if test_or_predict == 1: - future_prices_market = [item[4] for item in list(future_prices_market)] - actual_future_prices_returns = [self.calculate_percentage_change(future_prices_market[i - 1], future_prices_market[i]) for i in range(1, len(future_prices_market))] - if market_chart == True: - # Also draw the actual future returns - self.draw_market_performance_chart(actual_future_prices_returns, strategy_name) - - def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps = 25): - - # Get log returns from historical data - close_prices = historical_prices - returns = [math.log(close_prices[i] / close_prices[i - 1]) for i in range(1, len(close_prices))] - - # Get distribution of returns - hist = np.histogram(returns, bins = 32) - hist_dist = scipy.stats.rv_histogram(hist) # Distribution function - - predicted_prices = [] - # Do 25 iterations to simulate prices - for iteration in range(25): - new_close_prices = [close_prices[-1]] - new_close_prices_percentages = [] - for i in range(simulation_timesteps): - random_value = random.uniform(0, 1) - return_value = round(np.exp(hist_dist.ppf(random_value)), 5) # Get simulated return - price_last_point = new_close_prices[-1] - price_next_point = price_last_point * return_value - percentage_change = self.calculate_percentage_change(price_last_point, price_next_point) - - # Add to list - new_close_prices.append(price_next_point) - - predicted_prices.append(new_close_prices) - - # Calculate confidence intervals and average future returns. Conf intervals are not being used right now - conf_intervals = st.t.interval(0.95, len(predicted_prices), loc=np.mean(predicted_prices, axis = 0), scale=st.sem(predicted_prices, axis = 0)) - predicted_prices_mean = np.mean(predicted_prices, axis = 0) - return predicted_prices_mean, conf_intervals - - def is_nan(self, object): - """ - Check if object is null - """ - return object != object - - + """ + Monto carlo simulator that calculates the historical returns distribution and uses it to predict the future returns + """ + + def __init__(self): + print("\n--$ Simulator has been initialized") + + def calculate_percentage_change(self, old, new): + """ + Percentage change + """ + return ((new - old) * 100) / old + + def draw_portfolio_performance_chart(self, returns_matrix, portfolio_weights_dictionary, strategy_name): + """ + Draw returns chart for portfolio performance + """ + + # Get portfolio returns + returns_matrix = np.array(returns_matrix).transpose() + portfolio_weights_vector = np.array([portfolio_weights_dictionary[symbol] for symbol in portfolio_weights_dictionary]).transpose() + portfolio_returns = np.dot(returns_matrix, portfolio_weights_vector) + portfolio_returns_cumulative = np.cumsum(portfolio_returns) + + # Plot + x = np.arange(len(portfolio_returns_cumulative)) + plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') + plt.plot(x, portfolio_returns_cumulative, linewidth = 2.0, label = "Projected Returns from " + str(strategy_name)) + plt.title("Simulated Future Returns", fontsize = 14) + plt.xlabel("Bars (Time Sorted)", fontsize = 14) + plt.ylabel("Cumulative Percentage Return", fontsize = 14) + plt.xticks(fontsize = 14) + plt.yticks(fontsize = 14) + + def draw_market_performance_chart(self, actual_returns, strategy_name): + """ + Draw actual market returns if future data is available + """ + + # Get market returns + cumulative_returns = np.cumsum(actual_returns) + + # Plot + x = np.arange(len(cumulative_returns)) + plt.axhline(y = 0, linestyle = 'dotted', alpha = 0.3, color = 'black') + plt.plot(x, cumulative_returns, linewidth = 2.0, color = '#282828', linestyle = '--', label = "Market Index Returns") + plt.title("Simulated Future Returns", fontsize = 14) + plt.xlabel("Bars (Time Sorted)", fontsize = 14) + plt.ylabel("Cumulative Percentage Return", fontsize = 14) + plt.xticks(fontsize = 14) + plt.yticks(fontsize = 14) + + def simulate_portfolio(self, symbol_names, portfolio_weights_dictionary, portfolio_data_dictionary, future_prices_market, test_or_predict, market_chart, strategy_name, simulation_timesteps = 25): + """ + Simulate portfolio returns in the future + """ + returns_matrix = [] + actual_returns_matrix = [] + + # Iterate over each symbol to get their returns + for symbol in symbol_names: + + # Get symbol returns using monte carlo + historical_close_prices = list(portfolio_data_dictionary[symbol]["historical_prices"]["Close"]) + future_price_predictions, _ = self.simulate_and_get_future_prices(historical_close_prices, simulation_timesteps = max(simulation_timesteps, len(list(portfolio_data_dictionary[symbol]["future_prices"])))) + predicted_future_returns = [self.calculate_percentage_change(future_price_predictions[i - 1], future_price_predictions[i]) for i in range(1, len(future_price_predictions))] + returns_matrix.append(predicted_future_returns) + + + # Get portfolio returns + self.draw_portfolio_performance_chart(returns_matrix, portfolio_weights_dictionary, strategy_name) + + # Check whether we have actual future data available or not + if test_or_predict == 1: + future_prices_market = [item[4] for item in list(future_prices_market)] + actual_future_prices_returns = [self.calculate_percentage_change(future_prices_market[i - 1], future_prices_market[i]) for i in range(1, len(future_prices_market))] + if market_chart == True: + # Also draw the actual future returns + self.draw_market_performance_chart(actual_future_prices_returns, strategy_name) + + def simulate_and_get_future_prices(self, historical_prices, simulation_timesteps = 25): + + # Get log returns from historical data + close_prices = historical_prices + returns = [math.log(close_prices[i] / close_prices[i - 1]) for i in range(1, len(close_prices))] + + # Get distribution of returns + hist = np.histogram(returns, bins = 32) + hist_dist = scipy.stats.rv_histogram(hist) # Distribution function + + predicted_prices = [] + # Do 25 iterations to simulate prices + for iteration in range(25): + new_close_prices = [close_prices[-1]] + new_close_prices_percentages = [] + for i in range(simulation_timesteps): + random_value = random.uniform(0, 1) + return_value = round(np.exp(hist_dist.ppf(random_value)), 5) # Get simulated return + price_last_point = new_close_prices[-1] + price_next_point = price_last_point * return_value + percentage_change = self.calculate_percentage_change(price_last_point, price_next_point) + + # Add to list + new_close_prices.append(price_next_point) + + predicted_prices.append(new_close_prices) + + # Calculate confidence intervals and average future returns. Conf intervals are not being used right now + conf_intervals = st.t.interval(0.95, len(predicted_prices), loc=np.mean(predicted_prices, axis = 0), scale=st.sem(predicted_prices, axis = 0)) + predicted_prices_mean = np.mean(predicted_prices, axis = 0) + return predicted_prices_mean, conf_intervals + + def is_nan(self, object): + """ + Check if object is null + """ + return object != object + + diff --git a/strategy_manager.py b/strategy_manager.py index d32b3d2..efb6c04 100644 --- a/strategy_manager.py +++ b/strategy_manager.py @@ -9,52 +9,52 @@ warnings.filterwarnings("ignore") class StrategyManager: - """ - Runs and manages all strategies - """ - def __init__(self): - print("\n--= Strategy manager has been created...") - self.geneticAlgoStrategy = GeneticAlgoStrategy() - self.minimumVarianceStrategy = MinimumVarianceStrategy() - self.eigenPortfolioStrategy = EigenPortfolioStrategy() - self.maximumSharpeRatioStrategy = MaximumSharpeRatioStrategy() - self.strategyHelperFunctions = StrategyHelperFunctions() + """ + Runs and manages all strategies + """ + def __init__(self): + print("\n--= Strategy manager has been created...") + self.geneticAlgoStrategy = GeneticAlgoStrategy() + self.minimumVarianceStrategy = MinimumVarianceStrategy() + self.eigenPortfolioStrategy = EigenPortfolioStrategy() + self.maximumSharpeRatioStrategy = MaximumSharpeRatioStrategy() + self.strategyHelperFunctions = StrategyHelperFunctions() - def calculate_genetic_algo_portfolio(self, symbols, returns_matrix_percentages): - """ - Genetic algorithm based portfolio that maximizes sharpe ratio. This is my own implementation - """ - print("-* Calculating portfolio weights using genetic algorithm...") - portfolio_weights_dictionary = self.geneticAlgoStrategy.generate_portfolio(symbols, returns_matrix_percentages) - return portfolio_weights_dictionary + def calculate_genetic_algo_portfolio(self, symbols, returns_matrix_percentages): + """ + Genetic algorithm based portfolio that maximizes sharpe ratio. This is my own implementation + """ + print("-* Calculating portfolio weights using genetic algorithm...") + portfolio_weights_dictionary = self.geneticAlgoStrategy.generate_portfolio(symbols, returns_matrix_percentages) + return portfolio_weights_dictionary - def calculate_eigen_portfolio(self, symbols, covariance_matrix, eigen_portfolio_number): - """ - 2nd Eigen Portfolio - """ - print("-$ Calculating portfolio weights using eigen values...") - portfolio_weights_dictionary = self.eigenPortfolioStrategy.generate_portfolio(symbols, covariance_matrix, eigen_portfolio_number) - return portfolio_weights_dictionary + def calculate_eigen_portfolio(self, symbols, covariance_matrix, eigen_portfolio_number): + """ + 2nd Eigen Portfolio + """ + print("-$ Calculating portfolio weights using eigen values...") + portfolio_weights_dictionary = self.eigenPortfolioStrategy.generate_portfolio(symbols, covariance_matrix, eigen_portfolio_number) + return portfolio_weights_dictionary - def calculate_minimum_variance_portfolio(self, symbols, covariance_matrix): - """ - Minimum variance portfolio - """ - print("-! Calculating portfolio weights using minimum variance portfolio algorithm...") - portfolio_weights_dictionary = self.minimumVarianceStrategy.generate_portfolio(symbols, covariance_matrix) - return portfolio_weights_dictionary + def calculate_minimum_variance_portfolio(self, symbols, covariance_matrix): + """ + Minimum variance portfolio + """ + print("-! Calculating portfolio weights using minimum variance portfolio algorithm...") + portfolio_weights_dictionary = self.minimumVarianceStrategy.generate_portfolio(symbols, covariance_matrix) + return portfolio_weights_dictionary - def calculate_maximum_sharpe_portfolio(self, symbols, covariance_matrix, returns_vector): - """ - Maximum sharpe portfolio - """ - print("-# Calculating portfolio weights using maximum sharpe portfolio algorithm...") - portfolio_weights_dictionary = self.maximumSharpeRatioStrategy.generate_portfolio(symbols, covariance_matrix, returns_vector) - return portfolio_weights_dictionary + def calculate_maximum_sharpe_portfolio(self, symbols, covariance_matrix, returns_vector): + """ + Maximum sharpe portfolio + """ + print("-# Calculating portfolio weights using maximum sharpe portfolio algorithm...") + portfolio_weights_dictionary = self.maximumSharpeRatioStrategy.generate_portfolio(symbols, covariance_matrix, returns_vector) + return portfolio_weights_dictionary - def random_matrix_theory_based_cov(self, returns_matrix): - """ - Covariance matrix filtering using random matrix theory - """ - filtered_covariance_matrix = self.strategyHelperFunctions.random_matrix_theory_based_cov(returns_matrix) - return filtered_covariance_matrix \ No newline at end of file + def random_matrix_theory_based_cov(self, returns_matrix): + """ + Covariance matrix filtering using random matrix theory + """ + filtered_covariance_matrix = self.strategyHelperFunctions.random_matrix_theory_based_cov(returns_matrix) + return filtered_covariance_matrix \ No newline at end of file From 1f9f90a9517928058c77ebcc3204d0c7aed98595 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 13:07:19 -0600 Subject: [PATCH 07/22] added save_plot back --- commands.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/commands.json b/commands.json index 7dc1ca4..71f48ce 100644 --- a/commands.json +++ b/commands.json @@ -51,5 +51,11 @@ "type": "str", "default": "stocks/stocks.txt", "help": "Stocks file that contains the list of stocks you want to build your portfolio with." + }, + { + "comm": "--save_plot", + "type": "bool", + "default": "False", + "help": "Save plot instead of rendering it immediately." } -] \ No newline at end of file +] \ No newline at end of file From a8967b7670d75ff29ec359cc1659086ab8255c23 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 13:24:39 -0600 Subject: [PATCH 08/22] re-organization moved random matrix factor to util, since its not a strategy. --- backtester.py | 2 +- eiten.py | 4 +- strategies/strategy_helper_functions.py | 49 ------------------------- strategy_manager.py | 11 ------ utils.py | 42 +++++++++++++++++++++ 5 files changed, 45 insertions(+), 63 deletions(-) delete mode 100644 strategies/strategy_helper_functions.py create mode 100644 utils.py diff --git a/backtester.py b/backtester.py index 8fa0051..5c63dc9 100644 --- a/backtester.py +++ b/backtester.py @@ -98,7 +98,7 @@ def future_test(self, p_weights, data, market_data, long_only: bool, your choice in the future. """ symbol_names = list(p_weights.keys()) - #future_data = data_dict[symbol]["future"].Close + # future_data = data_dict[symbol]["future"].Close # Get future prices future_price_market = market_data["future"].Close diff --git a/eiten.py b/eiten.py index 8792fe1..ea82e77 100644 --- a/eiten.py +++ b/eiten.py @@ -8,6 +8,7 @@ from simulator import MontoCarloSimulator from backtester import BackTester from strategy_manager import StrategyManager +from utils import random_matrix_theory_based_cov class _dotdict(dict): @@ -111,8 +112,7 @@ def run_strategies(self): if self.args.apply_noise_filtering: print( "\n** Applying random matrix theory to filter out noise in the covariance matrix...\n") - cov_matrix = self.strategyManager.random_matrix_theory_based_cov( - log_returns) + cov_matrix = random_matrix_theory_based_cov(log_returns) symbol_names = list(self.data_dict.keys()) pred_returns = self._get_predicted_returns() diff --git a/strategies/strategy_helper_functions.py b/strategies/strategy_helper_functions.py deleted file mode 100644 index 048e126..0000000 --- a/strategies/strategy_helper_functions.py +++ /dev/null @@ -1,49 +0,0 @@ -# Basic libraries -import os -import random -import warnings -import numpy as np -import pandas as pd -warnings.filterwarnings("ignore") - - -class StrategyHelperFunctions: - def __init__(self): - print("Helper functions have been created") - - def random_matrix_theory_based_cov(self, log_returns): - """ - This is inspired by the excellent post @ - https://srome.github.io/Eigenvesting-III-Random-Matrix-Filtering-In-Finance/ - """ - returns_matrix = log_returns.T - - # Calculate variance and std, will come in handy during reconstruction - variances = np.diag(np.cov(returns_matrix)) - standard_deviations = np.sqrt(variances) - - # Get correlation matrix and compute eigen vectors and values - correlation_matrix = np.corrcoef(returns_matrix) - eig_values, eig_vectors = np.linalg.eigh(correlation_matrix) - - # Get maximum theoretical eigen value for a random matrix - sigma = 1 # The variance for all of the standardized log returns is 1 - Q = returns_matrix.shape[1] / returns_matrix.shape[0] - max_theoretical_eval = np.power(sigma*(1 + np.sqrt(1/Q)), 2) - - # Prune random eigen values - # eig_values_pruned = eig_values[eig_values > max_theoretical_eval] - eig_values[eig_values <= max_theoretical_eval] = 0 - - # Reconstruct the covariance matrix from the correlation matrix - # and filtered eigen values - temp = np.dot(eig_vectors, np.dot( - np.diag(eig_values), np.transpose(eig_vectors))) - np.fill_diagonal(temp, 1) - filtered_matrix = temp - filtered_cov_matrix = np.dot(np.diag(standard_deviations), - np.dot(filtered_matrix, - np.diag(standard_deviations))) - return pd.DataFrame(columns=log_returns.columns, - data=filtered_cov_matrix) - diff --git a/strategy_manager.py b/strategy_manager.py index 4d5f58b..37a4411 100644 --- a/strategy_manager.py +++ b/strategy_manager.py @@ -1,11 +1,9 @@ # Basic libraries -import os import warnings from strategies.genetic_algo_strategy import GeneticAlgoStrategy from strategies.maximum_sharpe_ratio_strategy import MaximumSharpeRatioStrategy from strategies.eigen_portfolio_strategy import EigenPortfolioStrategy from strategies.minimum_variance_strategy import MinimumVarianceStrategy -from strategies.strategy_helper_functions import StrategyHelperFunctions warnings.filterwarnings("ignore") @@ -20,7 +18,6 @@ def __init__(self): self.minimumVarianceStrategy = MinimumVarianceStrategy() self.eigenPortfolioStrategy = EigenPortfolioStrategy() self.maximumSharpeRatioStrategy = MaximumSharpeRatioStrategy() - self.strategyHelperFunctions = StrategyHelperFunctions() def calculate_genetic_algo_portfolio(self, symbols, returns_matrix_percentages): """ @@ -60,11 +57,3 @@ def calculate_maximum_sharpe_portfolio(self, cov_matrix, returns_vector): cov_matrix, returns_vector) return portfolio_weights_dictionary - def random_matrix_theory_based_cov(self, returns_matrix): - """ - Covariance matrix filtering using random matrix theory - """ - filtered_covariance_matrix = self.strategyHelperFunctions.random_matrix_theory_based_cov( - returns_matrix) - return filtered_covariance_matrix - diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..46c993c --- /dev/null +++ b/utils.py @@ -0,0 +1,42 @@ +# Basic libraries +import warnings +import numpy as np +import pandas as pd +warnings.filterwarnings("ignore") + + +def random_matrix_theory_based_cov(log_returns): + """ + This is inspired by the excellent post @ + https://srome.github.io/Eigenvesting-III-Random-Matrix-Filtering-In-Finance/ + """ + returns_matrix = log_returns.T + + # Calculate variance and std, will come in handy during reconstruction + variances = np.diag(np.cov(returns_matrix)) + standard_deviations = np.sqrt(variances) + + # Get correlation matrix and compute eigen vectors and values + correlation_matrix = np.corrcoef(returns_matrix) + eig_values, eig_vectors = np.linalg.eigh(correlation_matrix) + + # Get maximum theoretical eigen value for a random matrix + sigma = 1 # The variance for all of the standardized log returns is 1 + Q = returns_matrix.shape[1] / returns_matrix.shape[0] + max_theoretical_eval = np.power(sigma*(1 + np.sqrt(1/Q)), 2) + + # Prune random eigen values + # eig_values_pruned = eig_values[eig_values > max_theoretical_eval] + eig_values[eig_values <= max_theoretical_eval] = 0 + + # Reconstruct the covariance matrix from the correlation matrix + # and filtered eigen values + temp = np.dot(eig_vectors, np.dot( + np.diag(eig_values), np.transpose(eig_vectors))) + np.fill_diagonal(temp, 1) + filtered_matrix = temp + filtered_cov_matrix = np.dot(np.diag(standard_deviations), + np.dot(filtered_matrix, + np.diag(standard_deviations))) + return pd.DataFrame(columns=log_returns.columns, + data=filtered_cov_matrix) From dc7f7817fca17304c3f702c9ed205035f99fb340 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 17:21:03 -0600 Subject: [PATCH 09/22] Improved pipeline Started removing hardcoding now all strategies in the strategy folder are considered. used for loops in eiten removed strategy manager --- eiten.py | 147 ++++++-------------- strategies/__init__.py | 14 ++ strategies/eigen_portfolio_strategy.py | 17 +-- strategies/genetic_algo_strategy.py | 13 +- strategies/maximum_sharpe_ratio_strategy.py | 14 +- strategies/minimum_variance_strategy.py | 33 +++-- strategy_manager.py | 59 -------- utils.py | 5 + 8 files changed, 101 insertions(+), 201 deletions(-) create mode 100644 strategies/__init__.py delete mode 100644 strategy_manager.py diff --git a/eiten.py b/eiten.py index ea82e77..28136be 100644 --- a/eiten.py +++ b/eiten.py @@ -7,15 +7,9 @@ from data_loader import DataEngine from simulator import MontoCarloSimulator from backtester import BackTester -from strategy_manager import StrategyManager from utils import random_matrix_theory_based_cov - - -class _dotdict(dict): - """dot.notation access to dictionary attributes""" - __getattr__ = dict.get - __setattr__ = dict.__setitem__ - __delattr__ = dict.__delitem__ +from utils import dotdict +from strategies import portfolios class Eiten: @@ -23,7 +17,7 @@ def __init__(self, args: dict = None): if args is None: arg_types = {"str": str, "int": int, "bool": bool} x = json.load(open("commands.json", "r")) - args = _dotdict( + args = dotdict( {i["comm"][2:]: arg_types[i["type"]](i["default"]) for i in x}) print("\n--* Eiten has been initialized...") @@ -35,9 +29,6 @@ def __init__(self, args: dict = None): # Monte carlo simulator self.simulator = MontoCarloSimulator() - # Strategy manager - self.strategyManager = StrategyManager() - # Back tester self.backTester = BackTester() @@ -114,121 +105,64 @@ def run_strategies(self): "\n** Applying random matrix theory to filter out noise in the covariance matrix...\n") cov_matrix = random_matrix_theory_based_cov(log_returns) - symbol_names = list(self.data_dict.keys()) pred_returns = self._get_predicted_returns() perc_returns = self._get_perc_returns() + self.portfolios = {} # Get weights for the portfolio - eigen_portfolio_weights_dictionary = self.strategyManager.calculate_eigen_portfolio( - cov_matrix, self.args.eigen_portfolio_number) - mvp_portfolio_weights_dictionary = self.strategyManager.calculate_minimum_variance_portfolio( - symbol_names, cov_matrix) - msr_portfolio_weights_dictionary = self.strategyManager.calculate_maximum_sharpe_portfolio( - cov_matrix, pred_returns.T) - ga_portfolio_weights_dictionary = self.strategyManager.calculate_genetic_algo_portfolio( - symbol_names, perc_returns) + for p in portfolios: + name = p.name + weights = p.generate_portfolio( + cov_matrix=cov_matrix, p_number=self.args.eigen_portfolio_number, + pred_returns=pred_returns.T, + perc_returns=perc_returns) + self.portfolios[name] = weights # Print weights print("\n*% Printing portfolio weights...") - self.print_and_plot_portfolio_weights( - eigen_portfolio_weights_dictionary, 'Eigen Portfolio', plot_num=1) - self.print_and_plot_portfolio_weights( - mvp_portfolio_weights_dictionary, 'Minimum Variance Portfolio (MVP)', plot_num=2) - self.print_and_plot_portfolio_weights( - msr_portfolio_weights_dictionary, 'Maximum Sharpe Portfolio (MSR)', plot_num=3) - self.print_and_plot_portfolio_weights( - ga_portfolio_weights_dictionary, 'Genetic Algo (GA)', plot_num=4) + p_count = 1 + for i in self.portfolios: + self.print_and_plot_portfolio_weights( + self.portfolios[i], i, plot_num=p_count) + p_count += 1 self.draw_plot("output/weights.png") # Back test print("\n*& Backtesting the portfolios...") - self.backTester.back_test(eigen_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=True, - strategy_name='Eigen Portfolio') - - self.backTester.back_test( - mvp_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=False, - strategy_name='Minimum Variance Portfolio (MVP)') - - self.backTester.back_test(msr_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=False, - strategy_name='Maximum Sharpe Portfolio (MSR)') - self.backTester.back_test(ga_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=False, - strategy_name='Genetic Algo (GA)') + for i in self.portfolios: + + self.backTester.back_test(self.portfolios[i], + self.data_dict, + self.market_data, + self.args.only_long, + market_chart=True, + strategy_name=i) self.draw_plot("output/backtest.png") if self.args.is_test: print("\n#^ Future testing the portfolios...") # Future test - self.backTester.future_test(eigen_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=True, - strategy_name='Eigen Portfolio') - self.backTester.future_test(mvp_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=False, - strategy_name='Minimum Variance Portfolio (MVP)') - - self.backTester.future_test(msr_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=False, - strategy_name='Maximum Sharpe Portfolio (MSR)') - self.backTester.future_test(ga_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=False, - strategy_name='Genetic Algo (GA)') + for i in self.portfolios: + + self.backTester.future_test(self.portfolios[i], + self.data_dict, + self.market_data, + self.args.only_long, + market_chart=True, + strategy_name=i) + self.draw_plot("output/future_tests.png") # Simulation print("\n+$ Simulating future prices using monte carlo...") - self.simulator.simulate_portfolio(eigen_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.is_test, - market_chart=True, - strategy_name='Eigen Portfolio') - self.simulator.simulate_portfolio(eigen_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.is_test, - market_chart=False, - strategy_name='Minimum Variance Portfolio (MVP)') - self.simulator.simulate_portfolio(eigen_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.is_test, - market_chart=False, - strategy_name='Maximum Sharpe Portfolio (MSR)') - self.simulator.simulate_portfolio(ga_portfolio_weights_dictionary, - self.data_dict, - self.market_data, - self.args.is_test, - market_chart=False, - strategy_name='Genetic Algo (GA)') - + for i in self.portfolios: + self.simulator.simulate_portfolio(self.portfolios[i], + self.data_dict, + self.market_data, + self.args.is_test, + market_chart=True, + strategy_name=i) self.draw_plot("output/monte_carlo.png") def draw_plot(self, filename="output/graph.png"): @@ -273,4 +207,3 @@ def print_and_plot_portfolio_weights(self, weights: dict, plt.xlabel("Symbols", fontsize=14) plt.ylabel("Weight in Portfolio", fontsize=14) plt.title("Portfolio Weights for Different Strategies", fontsize=14) - diff --git a/strategies/__init__.py b/strategies/__init__.py new file mode 100644 index 0000000..5039f0a --- /dev/null +++ b/strategies/__init__.py @@ -0,0 +1,14 @@ +from os.path import dirname, basename, isfile, join +import glob +import importlib.util +import inspect + +modules = glob.glob(join(dirname(__file__), "*.py")) +__names = [basename(f)[:-3] for f in modules if isfile(f) + and not f.endswith('__init__.py')] +portfolios = [] +for i in __names: + spec = importlib.util.spec_from_file_location("", f"./strategies/{i}.py") + foo = importlib.util.module_from_spec(spec) + spec.loader.exec_module(foo) + portfolios.append(inspect.getmembers(foo, inspect.isclass)[0][1]()) diff --git a/strategies/eigen_portfolio_strategy.py b/strategies/eigen_portfolio_strategy.py index 0080046..da23ab0 100644 --- a/strategies/eigen_portfolio_strategy.py +++ b/strategies/eigen_portfolio_strategy.py @@ -2,26 +2,27 @@ import os import warnings import numpy as np +from utils import dotdict warnings.filterwarnings("ignore") class EigenPortfolioStrategy: def __init__(self): - print("Eigen portfolio strategy has been created") + self.name = "Eigen Portfolio" - def generate_portfolio(self, cov_matrix, eigen_portfolio_number): + def generate_portfolio(self, **kwargs): """ Inspired by: https://srome.github.io/Eigenvesting-I-Linear-Algebra-Can-Help-You-Choose-Your-Stock-Portfolio/ """ - eigh_values, eigh_vectors = np.linalg.eigh(cov_matrix) - print("eig_values", eigh_values.shape, "eig_vectors", eigh_vectors.shape) + kwargs = dotdict(kwargs) + eigh_values, eigh_vectors = np.linalg.eigh(kwargs.cov_matrix) # We don't need this but in case someone wants to analyze # market_eigen_portfolio = eig_vectors[:, -1] / np.sum(eig_vectors[:, -1]) # This is a portfolio that is uncorrelated to market and still yields good returns - eigen_portfolio = eigh_vectors[:, -eigen_portfolio_number] / \ - np.sum(eigh_vectors[:, -eigen_portfolio_number]) + eigen_portfolio = eigh_vectors[:, -kwargs.p_number] / \ + np.sum(eigh_vectors[:, -kwargs.p_number]) - - weights = {cov_matrix.columns[i]: eigen_portfolio[i] + weights = {kwargs.cov_matrix.columns[i]: eigen_portfolio[i] for i in range(eigen_portfolio.shape[0])} return weights + diff --git a/strategies/genetic_algo_strategy.py b/strategies/genetic_algo_strategy.py index c1961d8..4b8c345 100644 --- a/strategies/genetic_algo_strategy.py +++ b/strategies/genetic_algo_strategy.py @@ -1,8 +1,7 @@ # Basic libraries -import os -import random import warnings import numpy as np +from utils import dotdict warnings.filterwarnings("ignore") @@ -12,7 +11,7 @@ class GeneticAlgoStrategy: """ def __init__(self): - print("Genetic algo strategy has been created") + self.name = "Genetic Algo" self.initial_genes = 100 self.selection_top = 25 self.mutation_iterations = 50 @@ -22,7 +21,9 @@ def __init__(self): self.iterations = 50 self.crossover_probability = 0.05 - def generate_portfolio(self, symbols, return_matrix): + def generate_portfolio(self, **kwargs): + kwargs = dotdict(kwargs) + symbols = list(kwargs.cov_matrix.columns) self.gene_length = len(symbols) # Create initial genes @@ -30,7 +31,7 @@ def generate_portfolio(self, symbols, return_matrix): for i in range(self.iterations): # Select - top_genes = self.select(return_matrix, initial_genes) + top_genes = self.select(kwargs.perc_returns, initial_genes) # print("Iteration %d Best Sharpe Ratio: %.3f" % (i, top_genes[0][0])) top_genes = [item[1] for item in top_genes] @@ -38,7 +39,7 @@ def generate_portfolio(self, symbols, return_matrix): mutated_genes = self.mutate(top_genes) initial_genes = mutated_genes - top_genes = self.select(return_matrix, initial_genes) + top_genes = self.select(kwargs.perc_returns, initial_genes) best_gene = top_genes[0][1] # Gene is a distribution of weights for different stocks # transposed_gene = np.array(best_gene).transpose() diff --git a/strategies/maximum_sharpe_ratio_strategy.py b/strategies/maximum_sharpe_ratio_strategy.py index 361da64..bf68519 100644 --- a/strategies/maximum_sharpe_ratio_strategy.py +++ b/strategies/maximum_sharpe_ratio_strategy.py @@ -1,28 +1,30 @@ # Basic libraries import warnings import numpy as np +from utils import dotdict warnings.filterwarnings("ignore") class MaximumSharpeRatioStrategy: def __init__(self): - print("Maximum sharpe ratio strategy has been created") + self.name = 'Maximum Sharpe Portfolio (MSR)' - def generate_portfolio(self, cov_matrix, returns_vector): + def generate_portfolio(self, **kwargs): """ Inspired by: Eigen Portfolio Selection: A Robust Approach to Sharpe Ratio Maximization, https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3070416 """ - inverse_cov_matrix = np.linalg.pinv(cov_matrix) + kwargs = dotdict(kwargs) + inverse_cov_matrix = np.linalg.pinv(kwargs.cov_matrix) ones = np.ones(len(inverse_cov_matrix)) - numerator = np.dot(inverse_cov_matrix, returns_vector) + numerator = np.dot(inverse_cov_matrix, kwargs.pred_returns) denominator = np.dot( - np.dot(ones.transpose(), inverse_cov_matrix), returns_vector) + np.dot(ones.transpose(), inverse_cov_matrix), kwargs.pred_returns) msr_portfolio_weights = numerator / denominator - weights = {cov_matrix.columns[i]: msr_portfolio_weights[i][0] + weights = {kwargs.cov_matrix.columns[i]: msr_portfolio_weights[i][0] for i in range(len(msr_portfolio_weights))} return weights diff --git a/strategies/minimum_variance_strategy.py b/strategies/minimum_variance_strategy.py index 53ab176..62f3219 100644 --- a/strategies/minimum_variance_strategy.py +++ b/strategies/minimum_variance_strategy.py @@ -1,21 +1,24 @@ # Basic libraries -import os -import random import warnings import numpy as np +from utils import dotdict warnings.filterwarnings("ignore") + class MinimumVarianceStrategy: - def __init__(self): - print("Minimum Variance strategy has been created") - - def generate_portfolio(self, symbols, covariance_matrix): - """ - Inspired by: https://srome.github.io/Eigenvesting-II-Optimize-Your-Portfolio-With-Optimization/ - """ - inverse_cov_matrix = np.linalg.pinv(covariance_matrix) - ones = np.ones(len(inverse_cov_matrix)) - inverse_dot_ones = np.dot(inverse_cov_matrix, ones) - min_var_weights = inverse_dot_ones / np.dot( inverse_dot_ones, ones) - portfolio_weights_dictionary = dict([(symbols[x], min_var_weights[x]) for x in range(0, len(min_var_weights))]) - return portfolio_weights_dictionary + def __init__(self): + self.name = "Minimum Variance Portfolio (MVP)" + + def generate_portfolio(self, **kwargs): + """ + Inspired by: https://srome.github.io/Eigenvesting-II-Optimize-Your-Portfolio-With-Optimization/ + """ + kwargs = dotdict(kwargs) + + inverse_cov_matrix = np.linalg.pinv(kwargs.cov_matrix) + ones = np.ones(len(inverse_cov_matrix)) + inverse_dot_ones = np.dot(inverse_cov_matrix, ones) + min_var_weights = inverse_dot_ones / np.dot(inverse_dot_ones, ones) + weights = {kwargs.cov_matrix.columns[i]: min_var_weights[i] + for i in range(min_var_weights.shape[0])} + return weights diff --git a/strategy_manager.py b/strategy_manager.py deleted file mode 100644 index 37a4411..0000000 --- a/strategy_manager.py +++ /dev/null @@ -1,59 +0,0 @@ -# Basic libraries -import warnings -from strategies.genetic_algo_strategy import GeneticAlgoStrategy -from strategies.maximum_sharpe_ratio_strategy import MaximumSharpeRatioStrategy -from strategies.eigen_portfolio_strategy import EigenPortfolioStrategy -from strategies.minimum_variance_strategy import MinimumVarianceStrategy -warnings.filterwarnings("ignore") - - -class StrategyManager: - """ - Runs and manages all strategies - """ - - def __init__(self): - print("\n--= Strategy manager has been created...") - self.geneticAlgoStrategy = GeneticAlgoStrategy() - self.minimumVarianceStrategy = MinimumVarianceStrategy() - self.eigenPortfolioStrategy = EigenPortfolioStrategy() - self.maximumSharpeRatioStrategy = MaximumSharpeRatioStrategy() - - def calculate_genetic_algo_portfolio(self, symbols, returns_matrix_percentages): - """ - Genetic algorithm based portfolio that maximizes sharpe ratio. This is my own implementation - """ - print("-* Calculating portfolio weights using genetic algorithm...") - portfolio_weights_dictionary = self.geneticAlgoStrategy.generate_portfolio( - symbols, returns_matrix_percentages) - return portfolio_weights_dictionary - - def calculate_eigen_portfolio(self, cov_matrix, eigen_portfolio_number): - """ - 2nd Eigen Portfolio - """ - print("-$ Calculating portfolio weights using eigen values...") - portfolio_weights_dictionary = self.eigenPortfolioStrategy.generate_portfolio( - cov_matrix, eigen_portfolio_number) - return portfolio_weights_dictionary - - def calculate_minimum_variance_portfolio(self, symbols, covariance_matrix): - """ - Minimum variance portfolio - """ - print( - "-! Calculating portfolio weights using minimum variance portfolio algorithm...") - portfolio_weights_dictionary = self.minimumVarianceStrategy.generate_portfolio( - symbols, covariance_matrix) - return portfolio_weights_dictionary - - def calculate_maximum_sharpe_portfolio(self, cov_matrix, returns_vector): - """ - Maximum sharpe portfolio - """ - print( - "-# Calculating portfolio weights using maximum sharpe portfolio algorithm...") - portfolio_weights_dictionary = self.maximumSharpeRatioStrategy.generate_portfolio( - cov_matrix, returns_vector) - return portfolio_weights_dictionary - diff --git a/utils.py b/utils.py index 46c993c..98a5e2c 100644 --- a/utils.py +++ b/utils.py @@ -4,6 +4,11 @@ import pandas as pd warnings.filterwarnings("ignore") +class dotdict(dict): + """dot.notation access to dictionary attributes""" + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ def random_matrix_theory_based_cov(log_returns): """ From 514f6b2d1eb2e038a45fe9f5cc27f580e6f7ddcc Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 17:45:13 -0600 Subject: [PATCH 10/22] fixed Monto -> Monte --- simulator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator.py b/simulator.py index aae1ce1..546e294 100644 --- a/simulator.py +++ b/simulator.py @@ -15,9 +15,9 @@ plt.rcParams['axes.edgecolor'] = "#04383F" -class MontoCarloSimulator: +class MonteCarloSimulator: """ - Monto carlo simulator that calculates the historical returns distribution + Monte carlo simulator that calculates the historical returns distribution and uses it to predict the future returns """ From 7aa1c276623fd481991cc66e3c308ba5a0339215 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 17:59:37 -0600 Subject: [PATCH 11/22] re-structure --- backtester.py | 127 ++++++++++++++++++++++++++------------------------ eiten.py | 29 ++++-------- 2 files changed, 75 insertions(+), 81 deletions(-) diff --git a/backtester.py b/backtester.py index 5c63dc9..d64225a 100644 --- a/backtester.py +++ b/backtester.py @@ -9,22 +9,16 @@ import pandas as pd import matplotlib.pyplot as plt import warnings +from utils import dotdict warnings.filterwarnings("ignore") -# Styling for plots -plt.style.use('seaborn-white') -plt.rc('grid', linestyle="dotted", color='#a0a0a0') -plt.rcParams['axes.edgecolor'] = "#04383F" - class BackTester: """ Backtester module that does both backward and forward testing for our portfolios. """ - def __init__(self): - print("\n--# Backtester has been initialized") - + @classmethod def price_delta(self, prices): """ Percentage change @@ -32,6 +26,7 @@ def price_delta(self, prices): return ((prices - prices.shift()) * 100 / prices.shift())[1:] + @classmethod def portfolio_weight_manager(self, weight, is_long_only): """ Manage portfolio weights. If portfolio is long only, @@ -43,97 +38,105 @@ def portfolio_weight_manager(self, weight, is_long_only): weight = weight return weight - def back_test(self, p_weights, data, market_data, long_only: bool, - market_chart: bool, strategy_name: str): + @classmethod + def plot_market(self, **kwargs): + kwargs = dotdict(kwargs) + x = np.arange(len(kwargs.market_returns_cumulative)) + plt.plot(x, kwargs.market_returns_cumulative, linewidth=2.0, + color='#282828', label='Market Index', linestyle='--') + + @classmethod + def plot_test(self, **kwargs): + # Styling for plots + plt.style.use('seaborn-white') + plt.rc('grid', linestyle="dotted", color='#a0a0a0') + plt.rcParams['axes.edgecolor'] = "#04383F" + # Plot + x = np.arange(len(kwargs.portfolio_returns_cumulative)) + plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') + plt.plot(x, kwargs.portfolio_returns_cumulative, + linewidth=2.0, label=kwargs.strategy_name) + + # Plotting styles + plt.title(kwargs.title, fontsize=14) + plt.xlabel(kwargs.xlabel, fontsize=14) + plt.ylabel(kwargs.ylabel, fontsize=14) + plt.xticks(fontsize=14) + plt.yticks(fontsize=14) + + @classmethod + def get_historical_test(self, p_weights, data, long_only: bool): """ Main backtest function. Takes in the portfolio weights and compares the portfolio returns with a market index of your choice. """ - historical_data = market_data - # Get market returns during the backtesting time symbol_names = list(p_weights.keys()) - historical_prices = historical_data["historical"]["Close"] - market_returns = self.price_delta(historical_prices) - market_returns_cumulative = np.cumsum(market_returns) # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] for symbol in symbol_names: symbol_historical_prices = data[symbol]["historical"]["Close"] - symbol_historical_returns = self.price_delta( + symbol_historical_returns = BackTester.price_delta( symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns normal_returns_matrix = np.array(normal_returns_matrix).transpose() - portfolio_weights_vector = np.array([self.portfolio_weight_manager( + portfolio_weights_vector = np.array([BackTester.portfolio_weight_manager( p_weights[symbol], long_only) for symbol in p_weights]).transpose() portfolio_returns = np.dot( normal_returns_matrix, portfolio_weights_vector) - portfolio_returns_cumulative = np.cumsum(portfolio_returns) - - # Plot returns - x = np.arange(len(portfolio_returns_cumulative)) - plt.plot(x, portfolio_returns_cumulative, - linewidth=2.0, label=strategy_name) - plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') - if market_chart: - x = np.arange(len(market_returns_cumulative)) - plt.plot(x, market_returns_cumulative, linewidth=2.0, - color='#282828', label='Market Index', linestyle='--') + return np.cumsum(portfolio_returns) + + # # Plot returns + # x = np.arange(len(portfolio_returns_cumulative)) + # plt.plot(x, portfolio_returns_cumulative, + # linewidth=2.0, label=strategy_name) + # plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') + + # # Plotting styles + # plt.title("Backtest Results", fontsize=14) + # plt.xlabel("Bars (Time Sorted)", fontsize=14) + # plt.ylabel("Cumulative Percentage Return", fontsize=14) + # plt.xticks(fontsize=14) + # plt.yticks(fontsize=14) + + @classmethod + def get_future_market_returns(self, market_data): + # Get future prices + future_price_market = market_data["future"].Close + market_returns = self.price_delta(future_price_market) + return np.cumsum(market_returns) - # Plotting styles - plt.title("Backtest Results", fontsize=14) - plt.xlabel("Bars (Time Sorted)", fontsize=14) - plt.ylabel("Cumulative Percentage Return", fontsize=14) - plt.xticks(fontsize=14) - plt.yticks(fontsize=14) + def get_historical_market_returns(self, historical_data): + # Get market returns during the backtesting time + historical_prices = historical_data["historical"]["Close"] + market_returns = self.price_delta(historical_prices) + return np.cumsum(market_returns) - def future_test(self, p_weights, data, market_data, long_only: bool, - market_chart: bool, strategy_name: str): + def predict_future_returns(self, p_weights, data, + long_only: bool) -> np.ndarray: """ Main future test function. If future data is available i.e is_test is set to 1 and future_bars set to > 0, this takes in the portfolio - weights and compares the portfolio returns with a market index of1 + weights and compares the portfolio returns with a market index of your choice in the future. """ symbol_names = list(p_weights.keys()) # future_data = data_dict[symbol]["future"].Close - # Get future prices - future_price_market = market_data["future"].Close - market_returns = self.price_delta(future_price_market) - market_returns_cumulative = np.cumsum(market_returns) - # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] for symbol in symbol_names: symbol_historical_prices = data[symbol]["future"].Close - symbol_historical_returns = self.price_delta( + symbol_historical_returns = BackTester.price_delta( symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns normal_returns_matrix = np.array(normal_returns_matrix).transpose() - portfolio_weights_vector = np.array([self.portfolio_weight_manager( + portfolio_weights_vector = np.array([BackTester.portfolio_weight_manager( p_weights[symbol], long_only) for symbol in p_weights]).transpose() portfolio_returns = np.dot( normal_returns_matrix, portfolio_weights_vector) - portfolio_returns_cumulative = np.cumsum(portfolio_returns) - - # Plot - x = np.arange(len(portfolio_returns_cumulative)) - plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') - plt.plot(x, portfolio_returns_cumulative, - linewidth=2.0, label=strategy_name) - if market_chart: - x = np.arange(len(market_returns_cumulative)) - plt.plot(x, market_returns_cumulative, linewidth=2.0, - color='#282828', label='Market Index', linestyle='--') - - # Plotting styles - plt.title("Future Test Results", fontsize=14) - plt.xlabel("Bars (Time Sorted)", fontsize=14) - plt.ylabel("Cumulative Percentage Return", fontsize=14) - plt.xticks(fontsize=14) - plt.yticks(fontsize=14) + return np.cumsum(portfolio_returns) diff --git a/eiten.py b/eiten.py index 28136be..d9bb7a4 100644 --- a/eiten.py +++ b/eiten.py @@ -5,7 +5,7 @@ # Load our modules from data_loader import DataEngine -from simulator import MontoCarloSimulator +from simulator import MonteCarloSimulator from backtester import BackTester from utils import random_matrix_theory_based_cov from utils import dotdict @@ -23,15 +23,6 @@ def __init__(self, args: dict = None): print("\n--* Eiten has been initialized...") self.args = args - # Create data engine - self.dataEngine = DataEngine(args) - - # Monte carlo simulator - self.simulator = MontoCarloSimulator() - - # Back tester - self.backTester = BackTester() - # Data dictionary self.data_dict = {} # {"market": args.market_index} self.market_data = {} @@ -83,8 +74,9 @@ def load_data(self): """ # Gather data for all stocks in a dictionary format # Dictionary keys will be -> historical, future - self.data_dict = self.dataEngine.collect_data_for_all_tickers() - p, f = self.dataEngine.get_data(self.args.market_index) + de = DataEngine() + self.data_dict = de.collect_data_for_all_tickers() + p, f = de.get_data(self.args.market_index) self.market_data["historical"], self.market_data["future"] = p, f # Get return matrices and vectors return self.data_dict @@ -130,16 +122,15 @@ def run_strategies(self): # Back test print("\n*& Backtesting the portfolios...") + df_back = pd.Dataframe(columns=list(self.portfolios.keys())) for i in self.portfolios: - - self.backTester.back_test(self.portfolios[i], - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=True, - strategy_name=i) + df_back[i] = self.backTester.get_historical_test(self.portfolios[i], + self.data_dict, + self.args.only_long) self.draw_plot("output/backtest.png") + return + if self.args.is_test: print("\n#^ Future testing the portfolios...") # Future test From e4960ee128eec9e1d8c76ee25daccbc1590037c6 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 19:15:01 -0600 Subject: [PATCH 12/22] Separate plot and calculations separating calculation methods from plot methods in backtest. changed default data timeframe to 1 day. --- backtester.py | 47 +++++++++++-------------------------- commands.json | 6 ++--- data_loader.py | 6 +++-- eiten.py | 63 ++++++++++++++++++++++++++++++-------------------- 4 files changed, 59 insertions(+), 63 deletions(-) diff --git a/backtester.py b/backtester.py index d64225a..9fdc8d1 100644 --- a/backtester.py +++ b/backtester.py @@ -39,30 +39,27 @@ def portfolio_weight_manager(self, weight, is_long_only): return weight @classmethod - def plot_market(self, **kwargs): - kwargs = dotdict(kwargs) - x = np.arange(len(kwargs.market_returns_cumulative)) - plt.plot(x, kwargs.market_returns_cumulative, linewidth=2.0, + def plot_market(self, market_returns): + x = np.arange(len(market_returns)) + plt.plot(x, market_returns, linewidth=2.0, color='#282828', label='Market Index', linestyle='--') @classmethod def plot_test(self, **kwargs): # Styling for plots + kwargs = dotdict(kwargs) plt.style.use('seaborn-white') plt.rc('grid', linestyle="dotted", color='#a0a0a0') plt.rcParams['axes.edgecolor'] = "#04383F" + plt.rcParams['axes.titlesize'] = "large" + plt.rcParams['axes.labelsize'] = "medium" + plt.rcParams['lines.linewidth'] = 2 # Plot - x = np.arange(len(kwargs.portfolio_returns_cumulative)) plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') - plt.plot(x, kwargs.portfolio_returns_cumulative, - linewidth=2.0, label=kwargs.strategy_name) # Plotting styles - plt.title(kwargs.title, fontsize=14) - plt.xlabel(kwargs.xlabel, fontsize=14) - plt.ylabel(kwargs.ylabel, fontsize=14) - plt.xticks(fontsize=14) - plt.yticks(fontsize=14) + kwargs.df.plot(fontsize=14, title=kwargs.title, + xlabel=kwargs.xlabel, ylabel=kwargs.ylabel,) @classmethod def get_historical_test(self, p_weights, data, long_only: bool): @@ -88,32 +85,16 @@ def get_historical_test(self, p_weights, data, long_only: bool): normal_returns_matrix, portfolio_weights_vector) return np.cumsum(portfolio_returns) - # # Plot returns - # x = np.arange(len(portfolio_returns_cumulative)) - # plt.plot(x, portfolio_returns_cumulative, - # linewidth=2.0, label=strategy_name) - # plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') - - # # Plotting styles - # plt.title("Backtest Results", fontsize=14) - # plt.xlabel("Bars (Time Sorted)", fontsize=14) - # plt.ylabel("Cumulative Percentage Return", fontsize=14) - # plt.xticks(fontsize=14) - # plt.yticks(fontsize=14) - @classmethod - def get_future_market_returns(self, market_data): + def get_market_returns(self, market_data, direction): + assert (direction in ["historical", "future"] + ), "direction must be 'historical' or 'future'" # Get future prices - future_price_market = market_data["future"].Close + future_price_market = market_data[direction].Close market_returns = self.price_delta(future_price_market) return np.cumsum(market_returns) - def get_historical_market_returns(self, historical_data): - # Get market returns during the backtesting time - historical_prices = historical_data["historical"]["Close"] - market_returns = self.price_delta(historical_prices) - return np.cumsum(market_returns) - + @classmethod def predict_future_returns(self, p_weights, data, long_only: bool) -> np.ndarray: """ diff --git a/commands.json b/commands.json index 71f48ce..f0efcc1 100644 --- a/commands.json +++ b/commands.json @@ -2,12 +2,12 @@ "comm": "--history_to_use", "type": "str", "default": "all", - "help": "How many bars of 1 hour do you want to use for the anomaly detection model. Either an integer or all" + "help": "How many bars of do you want to use for the anomaly detection model. Either an integer or all" }, { "comm": "--data_granularity_minutes", "type": "int", - "default": "15", + "default": "3600", "help": "Minute level data granularity that you want to use. Default is 60 minute bars." }, { @@ -19,7 +19,7 @@ { "comm": "--future_bars", "type": "int", - "default": "30", + "default": "90", "help": "How many bars to keep for testing purposes." }, { diff --git a/data_loader.py b/data_loader.py index ee3d60e..57fa868 100644 --- a/data_loader.py +++ b/data_loader.py @@ -49,8 +49,9 @@ def get_most_frequent_count(self, input_list): def _split_data(self, data): if self.args.is_test: - return (data.iloc[-self.args.future_bars:], - data.iloc[:-self.args.future_bars]) + + return (data.iloc[:-self.args.future_bars], + data.iloc[-self.args.future_bars:]) def get_data(self, symbol): """ @@ -95,6 +96,7 @@ def get_data(self, symbol): stock_prices = stock_prices.iloc[-self.args.history_to_use:] historical_prices, future_prices = self._split_data(stock_prices) + print(f"Data separation\nH:{historical_prices.shape[0]}\nF:{future_prices.shape[0]}") except Exception as e: print("Exception", e) diff --git a/eiten.py b/eiten.py index d9bb7a4..fe43afe 100644 --- a/eiten.py +++ b/eiten.py @@ -74,13 +74,43 @@ def load_data(self): """ # Gather data for all stocks in a dictionary format # Dictionary keys will be -> historical, future - de = DataEngine() + de = DataEngine(self.args) self.data_dict = de.collect_data_for_all_tickers() p, f = de.get_data(self.args.market_index) self.market_data["historical"], self.market_data["future"] = p, f # Get return matrices and vectors return self.data_dict + def _backtest(self): + # Back test + print("\n*& Backtesting the portfolios...") + + df = pd.DataFrame(columns=list(self.portfolios.keys())) + for i in self.portfolios: + df[i] = BackTester.get_historical_test( + self.portfolios[i], + self.data_dict, + self.args.only_long) + mp = BackTester.get_market_returns(self.market_data, "historical") + BackTester.plot_test(title="Backtest Results", + xlabel="Bars (Time Sorted)", + ylabel="Cumulative Percentage Return", + df=df) + BackTester.plot_market(mp) + + def _futuretest(self): + print("\n#^ Future testing the portfolios...") + # Future test + df = pd.DataFrame(columns=list(self.portfolios.keys())) + for i in self.portfolios: + df[i] = BackTester.predict_future_returns(self.portfolios[i], + self.data_dict, + self.args.only_long) + BackTester.plot_test(title="Future Test Results", + xlabel="Bars (Time Sorted)", + ylabel="Cumulative Percentage Return", + df=df) + def run_strategies(self): """ Run strategies, back and future test them, and simulate the returns. @@ -119,32 +149,12 @@ def run_strategies(self): p_count += 1 self.draw_plot("output/weights.png") - # Back test - print("\n*& Backtesting the portfolios...") - - df_back = pd.Dataframe(columns=list(self.portfolios.keys())) - for i in self.portfolios: - df_back[i] = self.backTester.get_historical_test(self.portfolios[i], - self.data_dict, - self.args.only_long) - self.draw_plot("output/backtest.png") - - return + self._backtest() if self.args.is_test: - print("\n#^ Future testing the portfolios...") - # Future test - for i in self.portfolios: - - self.backTester.future_test(self.portfolios[i], - self.data_dict, - self.market_data, - self.args.only_long, - market_chart=True, - strategy_name=i) - - self.draw_plot("output/future_tests.png") - + self._futuretest() + self.draw_plot("output/future_tests.png") + return # Simulation print("\n+$ Simulating future prices using monte carlo...") for i in self.portfolios: @@ -164,6 +174,9 @@ def draw_plot(self, filename="output/graph.png"): plt.style.use('seaborn-white') plt.rc('grid', linestyle="dotted", color='#a0a0a0') plt.rcParams['axes.edgecolor'] = "#04383F" + plt.rcParams['axes.titlesize'] = "large" + plt.rcParams['axes.labelsize'] = "medium" + plt.rcParams['lines.linewidth'] = 2 plt.rcParams['figure.figsize'] = (12, 6) plt.grid() From 09fa03fbac02bace45d0bff47c773c7b06410deb Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 19:15:52 -0600 Subject: [PATCH 13/22] Hotfix Inversion of future and past --- data_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_loader.py b/data_loader.py index ee3d60e..e2fbb30 100644 --- a/data_loader.py +++ b/data_loader.py @@ -49,8 +49,8 @@ def get_most_frequent_count(self, input_list): def _split_data(self, data): if self.args.is_test: - return (data.iloc[-self.args.future_bars:], - data.iloc[:-self.args.future_bars]) + return (data.iloc[:-self.args.future_bars], + data.iloc[-self.args.future_bars:]) def get_data(self, symbol): """ From 9d7432dca1d8cd1b917f4d1e1bd420848dc71024 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 19:17:17 -0600 Subject: [PATCH 14/22] Monto -> Monte corrected typo --- eiten.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eiten.py b/eiten.py index 28136be..6aaa668 100644 --- a/eiten.py +++ b/eiten.py @@ -5,7 +5,7 @@ # Load our modules from data_loader import DataEngine -from simulator import MontoCarloSimulator +from simulator import MonteCarloSimulator from backtester import BackTester from utils import random_matrix_theory_based_cov from utils import dotdict @@ -27,7 +27,7 @@ def __init__(self, args: dict = None): self.dataEngine = DataEngine(args) # Monte carlo simulator - self.simulator = MontoCarloSimulator() + self.simulator = MonteCarloSimulator() # Back tester self.backTester = BackTester() From aa7d4c52a1ef4fd9a1080347a4751e8a1d74df12 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 22:51:01 -0600 Subject: [PATCH 15/22] Dataframes migrates all vectorization to dataframes drastically improves speed. --- .../Demo-checkpoint.ipynb | 181 +++- Demo.ipynb | 813 ++++++++++++++++++ backtester.py | 100 ++- data_loader.py | 37 +- eiten.py | 63 +- simulator.py | 160 ---- strategies/eigen_portfolio_strategy.py | 12 +- strategies/maximum_sharpe_ratio_strategy.py | 2 +- utils.py | 3 + 9 files changed, 1099 insertions(+), 272 deletions(-) rename notebooks/Demo.ipynb => .ipynb_checkpoints/Demo-checkpoint.ipynb (92%) create mode 100644 Demo.ipynb delete mode 100644 simulator.py diff --git a/notebooks/Demo.ipynb b/.ipynb_checkpoints/Demo-checkpoint.ipynb similarity index 92% rename from notebooks/Demo.ipynb rename to .ipynb_checkpoints/Demo-checkpoint.ipynb index 20afe42..39c03b7 100644 --- a/notebooks/Demo.ipynb +++ b/.ipynb_checkpoints/Demo-checkpoint.ipynb @@ -31,28 +31,183 @@ "text": [ "\n", "--* Eiten has been initialized...\n", + "\n", + "\n" + ] + } + ], + "source": [ + "et = Eiten()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/9 [00:00 Data engine has been initialized...\n", "Loading all stocks from file...\n", "Total number of stocks: 9\n", - "\n", - "--$ Simulator has been initialized\n", - "\n", - "--= Strategy manager has been created...\n", - "Genetic algo strategy has been created\n", - "Minimum Variance strategy has been created\n", - "Eigen portfolio strategy has been created\n", - "Maximum sharpe ratio strategy has been created\n", - "Helper functions have been created\n", - "\n", - "--# Backtester has been initialized\n", - "\n", + "Loading data for all stocks...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 11%|█████████▎ | 1/9 [00:00<00:04, 1.89it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 22%|██████████████████▋ | 2/9 [00:00<00:03, 2.28it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 33%|████████████████████████████ | 3/9 [00:00<00:02, 2.65it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 44%|█████████████████████████████████████▎ | 4/9 [00:01<00:01, 2.84it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 56%|██████████████████████████████████████████████▋ | 5/9 [00:01<00:01, 3.12it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 67%|████████████████████████████████████████████████████████ | 6/9 [00:01<00:00, 3.43it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 78%|█████████████████████████████████████████████████████████████████▎ | 7/9 [00:01<00:00, 3.69it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 89%|██████████████████████████████████████████████████████████████████████████▋ | 8/9 [00:02<00:00, 3.92it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:02<00:00, 3.73it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ "\n" ] } ], "source": [ - "et = Eiten()" + "data_dict = et.load_data()" ] }, { diff --git a/Demo.ipynb b/Demo.ipynb new file mode 100644 index 0000000..43c656b --- /dev/null +++ b/Demo.ipynb @@ -0,0 +1,813 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Populating the interactive namespace from numpy and matplotlib\n" + ] + } + ], + "source": [ + "%pylab inline\n", + "import sys, os\n", + "sys.path.insert(1, os.path.join(sys.path[0], '..'))\n", + "from eiten import Eiten\n", + "from utils import random_matrix_theory_based_cov" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--* Eiten has been initialized...\n", + "\n", + "\n" + ] + } + ], + "source": [ + "et = Eiten()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/9 [00:00 Data engine has been initialized...\n", + "Loading all stocks from file...\n", + "Total number of stocks: 9\n", + "Loading data for all stocks...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:02<00:00, 4.18it/s]\n" + ] + } + ], + "source": [ + "data_dict = et.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AAPLAMDAMZNFBMSFTNFLXNVDASQQQTSLA
029.0700001.860000522.36999592.90000243.98000099.16000422.9400011863.19995150.714001
129.1025011.890000527.39001593.44999744.299999104.08000223.1000001835.19995152.450001
228.4800001.890000538.86999594.33999644.250000104.20999923.3099991828.80004952.414001
328.3624991.870000540.26001094.40000243.480000102.62000323.2900011905.59997652.124001
428.8025001.810000548.39001595.55000344.110001100.30000323.5300011891.19995152.840000
..............................
116371.93250353.6600002372.709961194.190002177.429993411.890015298.45999158.299999160.102005
116473.44999752.3899992474.000000204.710007179.210007419.850006292.27999957.900002156.376007
116572.26750249.8800012286.040039202.270004174.570007415.269989282.77999963.099998140.264008
116673.29000152.5600012315.989990205.259995178.839996428.149994291.29000960.849998152.238007
116774.38999952.1899992317.800049207.070007180.759995424.679993293.73999058.799999153.641998
\n", + "

1168 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " AAPL AMD AMZN FB MSFT NFLX \\\n", + "0 29.070000 1.860000 522.369995 92.900002 43.980000 99.160004 \n", + "1 29.102501 1.890000 527.390015 93.449997 44.299999 104.080002 \n", + "2 28.480000 1.890000 538.869995 94.339996 44.250000 104.209999 \n", + "3 28.362499 1.870000 540.260010 94.400002 43.480000 102.620003 \n", + "4 28.802500 1.810000 548.390015 95.550003 44.110001 100.300003 \n", + "... ... ... ... ... ... ... \n", + "1163 71.932503 53.660000 2372.709961 194.190002 177.429993 411.890015 \n", + "1164 73.449997 52.389999 2474.000000 204.710007 179.210007 419.850006 \n", + "1165 72.267502 49.880001 2286.040039 202.270004 174.570007 415.269989 \n", + "1166 73.290001 52.560001 2315.989990 205.259995 178.839996 428.149994 \n", + "1167 74.389999 52.189999 2317.800049 207.070007 180.759995 424.679993 \n", + "\n", + " NVDA SQQQ TSLA \n", + "0 22.940001 1863.199951 50.714001 \n", + "1 23.100000 1835.199951 52.450001 \n", + "2 23.309999 1828.800049 52.414001 \n", + "3 23.290001 1905.599976 52.124001 \n", + "4 23.530001 1891.199951 52.840000 \n", + "... ... ... ... \n", + "1163 298.459991 58.299999 160.102005 \n", + "1164 292.279999 57.900002 156.376007 \n", + "1165 282.779999 63.099998 140.264008 \n", + "1166 291.290009 60.849998 152.238007 \n", + "1167 293.739990 58.799999 153.641998 \n", + "\n", + "[1168 rows x 9 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_dict[\"historical\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AAPLAMDAMZNFBMSFTNFLXNVDASQQQTSLA
AAPL0.0003290.0002970.0001920.0002040.0002220.0002070.000297-0.0006040.000228
AMD0.0002970.0016340.0002830.0002510.0002640.0003490.000635-0.0007880.000405
AMZN0.0001920.0002830.0003490.0002320.0002170.0002710.000277-0.0005630.000227
FB0.0002040.0002510.0002320.0004050.0002200.0002350.000290-0.0005850.000231
MSFT0.0002220.0002640.0002170.0002200.0002980.0002310.000305-0.0006200.000233
NFLX0.0002070.0003490.0002710.0002350.0002310.0006590.000332-0.0006150.000282
NVDA0.0002970.0006350.0002770.0002900.0003050.0003320.000886-0.0008120.000363
SQQQ-0.000604-0.000788-0.000563-0.000585-0.000620-0.000615-0.0008120.001605-0.000621
TSLA0.0002280.0004050.0002270.0002310.0002330.0002820.000363-0.0006210.001152
\n", + "
" + ], + "text/plain": [ + " AAPL AMD AMZN FB MSFT NFLX NVDA \\\n", + "AAPL 0.000329 0.000297 0.000192 0.000204 0.000222 0.000207 0.000297 \n", + "AMD 0.000297 0.001634 0.000283 0.000251 0.000264 0.000349 0.000635 \n", + "AMZN 0.000192 0.000283 0.000349 0.000232 0.000217 0.000271 0.000277 \n", + "FB 0.000204 0.000251 0.000232 0.000405 0.000220 0.000235 0.000290 \n", + "MSFT 0.000222 0.000264 0.000217 0.000220 0.000298 0.000231 0.000305 \n", + "NFLX 0.000207 0.000349 0.000271 0.000235 0.000231 0.000659 0.000332 \n", + "NVDA 0.000297 0.000635 0.000277 0.000290 0.000305 0.000332 0.000886 \n", + "SQQQ -0.000604 -0.000788 -0.000563 -0.000585 -0.000620 -0.000615 -0.000812 \n", + "TSLA 0.000228 0.000405 0.000227 0.000231 0.000233 0.000282 0.000363 \n", + "\n", + " SQQQ TSLA \n", + "AAPL -0.000604 0.000228 \n", + "AMD -0.000788 0.000405 \n", + "AMZN -0.000563 0.000227 \n", + "FB -0.000585 0.000231 \n", + "MSFT -0.000620 0.000233 \n", + "NFLX -0.000615 0.000282 \n", + "NVDA -0.000812 0.000363 \n", + "SQQQ 0.001605 -0.000621 \n", + "TSLA -0.000621 0.001152 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "log_returns = et._get_log_returns()\n", + "log_returns.cov()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AAPLAMDAMZNFBMSFTNFLXNVDASQQQTSLA
00.0003290.0003450.0002220.0002270.0002230.0002590.000331-0.0005680.000272
10.0003450.0016340.0003520.0003590.0003520.0004090.000523-0.0008990.000430
20.0002220.0003520.0003490.0002310.0002270.0002630.000337-0.0005790.000277
30.0002270.0003590.0002310.0004050.0002320.0002690.000344-0.0005910.000283
40.0002230.0003520.0002270.0002320.0002980.0002640.000337-0.0005790.000277
50.0002590.0004090.0002630.0002690.0002640.0006590.000392-0.0006730.000322
60.0003310.0005230.0003370.0003440.0003370.0003920.000886-0.0008610.000411
7-0.000568-0.000899-0.000579-0.000591-0.000579-0.000673-0.0008610.001605-0.000707
80.0002720.0004300.0002770.0002830.0002770.0003220.000411-0.0007070.001152
\n", + "
" + ], + "text/plain": [ + " AAPL AMD AMZN FB MSFT NFLX NVDA \\\n", + "0 0.000329 0.000345 0.000222 0.000227 0.000223 0.000259 0.000331 \n", + "1 0.000345 0.001634 0.000352 0.000359 0.000352 0.000409 0.000523 \n", + "2 0.000222 0.000352 0.000349 0.000231 0.000227 0.000263 0.000337 \n", + "3 0.000227 0.000359 0.000231 0.000405 0.000232 0.000269 0.000344 \n", + "4 0.000223 0.000352 0.000227 0.000232 0.000298 0.000264 0.000337 \n", + "5 0.000259 0.000409 0.000263 0.000269 0.000264 0.000659 0.000392 \n", + "6 0.000331 0.000523 0.000337 0.000344 0.000337 0.000392 0.000886 \n", + "7 -0.000568 -0.000899 -0.000579 -0.000591 -0.000579 -0.000673 -0.000861 \n", + "8 0.000272 0.000430 0.000277 0.000283 0.000277 0.000322 0.000411 \n", + "\n", + " SQQQ TSLA \n", + "0 -0.000568 0.000272 \n", + "1 -0.000899 0.000430 \n", + "2 -0.000579 0.000277 \n", + "3 -0.000591 0.000283 \n", + "4 -0.000579 0.000277 \n", + "5 -0.000673 0.000322 \n", + "6 -0.000861 0.000411 \n", + "7 0.001605 -0.000707 \n", + "8 -0.000707 0.001152 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "random_matrix_theory_based_cov(log_returns)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/9 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", + "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", + "{'AAPL': -0.4369306238899776, 'AMD': -1.0115784282008393, 'AMZN': -2.3091148432564372, 'FB': 0.5058836309304608, 'MSFT': 0.009511216792968975, 'NFLX': 0.33246631319412956, 'NVDA': -0.18129905403714677, 'SQQQ': -0.680171626027902, 'TSLA': 0.721905802094419}\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsQAAAFkCAYAAAAnoS3wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAATXUlEQVR4nO3dUWiX97348Y9JTNTGKp56egZtPJgab8o5mnozPGG2Wyit3UDD+lM37YUgvRqMjK03BhGr2ezFOHbdoYOJE1Yj4oUR2kGqRQjtRcRYZKjgXKDjHCxHpU2ymqa/539R/PEPPSe/6skvaft5va7yPN/n4fe5+BLeefjxZF5RFEUAAEBSdXM9AAAAzCVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACk9qWC+OLFi7F9+/YvnD9z5kx0dXVFqVSK48ePz/hwAABQaw3VLvjd734Xp06dioULF045/+mnn8aBAwfixIkTsXDhwti6dWs8+eSTsXz58poNCwAAM61qELe0tMShQ4fi5z//+ZTz165di5aWlliyZElERDzxxBMxNDQUzzzzzJTrPvnkTgxduRLfWrYs6hvqZ3B0AAD43GeTn8V/3rwZ61avjgULmu7p3qpB/PTTT8cHH3zwhfOjo6OxePHiyvEDDzwQo6OjX7hu6MqVePLFn9zTUAAAcD/O/se/x7/967/c0z1Vg/h/09zcHGNjY5XjsbGxKYF81z/9w7LKcI/8o69TAAAw8z648WE8+eJPKu15L+47iFtbW2NkZCRu374dixYtiqGhodi5c+cXP6D+869JPPKPy+Ofv/Wt+/04AACo6m573tM993pDf39/jI+PR6lUipdeeil27twZRVFEV1dXPPzww/c8AAAAzKUvFcSPPPJI5bVq3//+9yvnn3rqqXjqqadqMxkAAMwC/5gDAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNSqBnG5XI6enp4olUqxffv2GBkZmbJ+6tSp2LRpU3R1dcUf//jHmg0KAAC10FDtgoGBgZiYmIi+vr4YHh6O3t7e+O1vf1tZ/9WvfhWnT5+ORYsWxcaNG2Pjxo2xZMmSmg4NAAAzpWoQnz9/Pjo6OiIiYs2aNXHp0qUp66tXr46PP/44GhoaoiiKmDdvXm0mBQCAGqgaxKOjo9Hc3Fw5rq+vj8nJyWho+PzWVatWRVdXVyxcuDA6OzvjwQcfrN20AAAww6p+h7i5uTnGxsYqx+VyuRLDly9fjnfeeSfefvvtOHPmTNy8eTPefPPN2k0LAAAzrGoQt7e3x7lz5yIiYnh4ONra2iprixcvjgULFkRTU1PU19fHsmXL4qOPPqrdtAAAMMOqfmWis7MzBgcHY8uWLVEURezfvz/6+/tjfHw8SqVSlEql2LZtW8yfPz9aWlpi06ZNszE3AADMiKpBXFdXF3v37p1yrrW1tfLz1q1bY+vWrTM/GQAAzAL/mAMAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpNVS7oFwux549e+LKlSvR2NgY+/btixUrVlTW33///ejt7Y2iKGL58uVx8ODBaGpqqunQAAAwU6o+IR4YGIiJiYno6+uL7u7u6O3trawVRRG7d++OAwcOxBtvvBEdHR3xt7/9raYDAwDATKr6hPj8+fPR0dERERFr1qyJS5cuVdauX78eS5cujSNHjsTVq1fjO9/5TqxcubJ20wIAwAyr+oR4dHQ0mpubK8f19fUxOTkZERG3bt2KCxcuxLZt2+Lw4cPx3nvvxbvvvlu7aQEAYIZVDeLm5uYYGxurHJfL5Who+PzB8tKlS2PFihXx2GOPxfz586Ojo2PKE2QAAPiqqxrE7e3tce7cuYiIGB4ejra2tsrao48+GmNjYzEyMhIREUNDQ7Fq1aoajQoAADOv6neIOzs7Y3BwMLZs2RJFUcT+/fujv78/xsfHo1Qqxcsvvxzd3d1RFEWsXbs2NmzYMAtjAwDAzKgaxHV1dbF3794p51pbWys/f/vb344TJ07M/GQAADAL/GMOAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFKrGsTlcjl6enqiVCrF9u3bY2Rk5H+8bvfu3fHKK6/M+IAAAFBLVYN4YGAgJiYmoq+vL7q7u6O3t/cL1xw7diyuXr1akwEBAKCWqgbx+fPno6OjIyIi1qxZE5cuXZqyfuHChbh48WKUSqXaTAgAADVUNYhHR0ejubm5clxfXx+Tk5MREXHjxo149dVXo6enp3YTAgBADTVUu6C5uTnGxsYqx+VyORoaPr/trbfeilu3bsWuXbviww8/jE8++SRWrlwZmzdvrt3EAAAwg6oGcXt7e5w9ezaeffbZGB4ejra2tsrajh07YseOHRERcfLkyfjLX/4ihgEA+FqpGsSdnZ0xODgYW7ZsiaIoYv/+/dHf3x/j4+O+NwwAwNde1SCuq6uLvXv3TjnX2tr6hes8GQYA4OvIP+YAACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAILWGaheUy+XYs2dPXLlyJRobG2Pfvn2xYsWKyvrp06fjyJEjUV9fH21tbbFnz56oq9PZAAB8PVQt14GBgZiYmIi+vr7o7u6O3t7eytonn3wSv/71r+MPf/hDHDt2LEZHR+Ps2bM1HRgAAGZS1SA+f/58dHR0RETEmjVr4tKlS5W1xsbGOHbsWCxcuDAiIiYnJ6OpqalGowIAwMyrGsSjo6PR3NxcOa6vr4/JycnPb66ri4ceeigiIo4ePRrj4+Oxfv36Go0KAAAzr+p3iJubm2NsbKxyXC6Xo6GhYcrxwYMH4/r163Ho0KGYN29ebSYFAIAaqPqEuL29Pc6dOxcREcPDw9HW1jZlvaenJ+7cuROvvfZa5asTAADwdVH1CXFnZ2cMDg7Gli1boiiK2L9/f/T398f4+Hg8/vjjceLEiVi3bl288MILERGxY8eO6OzsrPngAAAwE6oGcV1dXezdu3fKudbW1srPly9fnvmpAABglnhhMAAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpVQ3icrkcPT09USqVYvv27TEyMjJl/cyZM9HV1RWlUimOHz9es0EBAKAWqgbxwMBATExMRF9fX3R3d0dvb29l7dNPP40DBw7E73//+zh69Gj09fXFhx9+WNOBAQBgJjVUu+D8+fPR0dERERFr1qyJS5cuVdauXbsWLS0tsWTJkoiIeOKJJ2JoaCieeeaZyjWTn30WEREf3BDKAADUxt3WvNue96JqEI+OjkZzc3PluL6+PiYnJ6OhoSFGR0dj8eLFlbUHHnggRkdHp9z/X/99MyIinnzxJ/c8HAAA3Iv/+u+b8dgjj9zTPVWDuLm5OcbGxirH5XI5Ghoa/se1sbGxKYEcEbFu9eo4+x//Ht9atizqG+rvaTgAAPgyPpv8LP7z5s1Yt3r1Pd9bNYjb29vj7Nmz8eyzz8bw8HC0tbVV1lpbW2NkZCRu374dixYtiqGhodi5c+eU+xcsaIp/+9d/uefBAADgXrQ+em9Phu+aVxRFMd0F5XI59uzZE1evXo2iKGL//v3x5z//OcbHx6NUKsWZM2fiN7/5TRRFEV1dXfGjH/3ovgYBAIC5UDWIv6y74XzlypVobGyMffv2xYoVKyrrd8O5oaEhurq64vnnn5+Jj+Urrtq+OH36dBw5ciTq6+ujra0t9uzZE3V1Xo/9TVdtX9y1e/fuWLJkSfzsZz+bgymZbdX2xfvvvx+9vb1RFEUsX748Dh48GE1NTXM4MbOh2r44depUHD58OOrq6qKrqyu2bds2h9Mymy5evBivvPJKHD16dMr5+2rOYob86U9/Kn7xi18URVEUFy5cKF588cXK2sTERPG9732vuH37dnHnzp1i8+bNxY0bN2bqo/kKm25f/P3vfy+++93vFuPj40VRFMVPf/rTYmBgYE7mZHZNty/ueuONN4rnn3++OHjw4GyPxxyZbl+Uy+XiBz/4QfHXv/61KIqiOH78eHHt2rU5mZPZVe33xfr164tbt24Vd+7cqbQG33yvv/568dxzzxU//OEPp5y/3+acsUdxX/b1bI2NjZXXs/HNN92+aGxsjGPHjsXChQsjImJyctLTniSm2xcRERcuXIiLFy9GqVSai/GYI9Pti+vXr8fSpUvjyJEj8eMf/zhu374dK1eunKtRmUXVfl+sXr06Pv7445iYmIiiKGLevHlzMSazrKWlJQ4dOvSF8/fbnDMWxP/b69nurlV7PRvfTNPti7q6unjooYciIuLo0aMxPj4e69evn5M5mV3T7YsbN27Eq6++Gj09PXM1HnNkun1x69atuHDhQmzbti0OHz4c7733Xrz77rtzNSqzaLp9ERGxatWq6Orqio0bN8aGDRviwQcfnIsxmWVPP/105a1n/7/7bc4ZC+L/6+vZ+Gaabl/cPf7lL38Zg4ODcejQIX/ZJzHdvnjrrbfi1q1bsWvXrnj99dfj9OnTcfLkybkalVk03b5YunRprFixIh577LGYP39+dHR0fOFJId9M0+2Ly5cvxzvvvBNvv/12nDlzJm7evBlvvvnmXI3KV8D9NueMBXF7e3ucO3cuImLa17NNTEzE0NBQrF27dqY+mq+w6fZFRERPT0/cuXMnXnvttcpXJ/jmm25f7NixI06ePBlHjx6NXbt2xXPPPRebN2+eq1GZRdPti0cffTTGxsZiZGQkIiKGhoZi1apVczIns2u6fbF48eJYsGBBNDU1RX19fSxbtiw++uijuRqVr4D7bc6q7yH+sjo7O2NwcDC2bNlSeT1bf39/5fVsL730UuzcubPyeraHH354pj6ar7Dp9sXjjz8eJ06ciHXr1sULL7wQEZ/HUGdn5xxPTa1V+31BTtX2xcsvvxzd3d1RFEWsXbs2NmzYMNcjMwuq7YtSqRTbtm2L+fPnR0tLS2zatGmuR2YO/F+bc8ZeuwYAAF9HXvgKAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACC1/we/joqoI4LRIgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "et.run_strategies()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/backtester.py b/backtester.py index 9fdc8d1..dbe0cb6 100644 --- a/backtester.py +++ b/backtester.py @@ -7,6 +7,7 @@ import collections import numpy as np import pandas as pd +import scipy.stats as st import matplotlib.pyplot as plt import warnings from utils import dotdict @@ -27,16 +28,14 @@ def price_delta(self, prices): return ((prices - prices.shift()) * 100 / prices.shift())[1:] @classmethod - def portfolio_weight_manager(self, weight, is_long_only): + def filter_short(self, weights, long_only): """ Manage portfolio weights. If portfolio is long only, set the negative weights to zero. """ - if is_long_only == 1: - weight = max(weight, 0) - else: - weight = weight - return weight + if long_only: + return np.array([max(i, 0) for i in weights]) + return weights @classmethod def plot_market(self, market_returns): @@ -62,62 +61,77 @@ def plot_test(self, **kwargs): xlabel=kwargs.xlabel, ylabel=kwargs.ylabel,) @classmethod - def get_historical_test(self, p_weights, data, long_only: bool): + def get_test(self, p_weights, data, direction: str, long_only: bool): """ Main backtest function. Takes in the portfolio weights and compares the portfolio returns with a market index of your choice. """ - symbol_names = list(p_weights.keys()) + + assert (direction in ["historical", "future", "sim"] + ), "direction must be 'historical', 'future' or 'sim'" # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] - for symbol in symbol_names: - symbol_historical_prices = data[symbol]["historical"]["Close"] - symbol_historical_returns = BackTester.price_delta( - symbol_historical_prices) - normal_returns_matrix.append(symbol_historical_returns) + symbol_historical_prices = data[direction] + symbol_historical_returns = BackTester.price_delta( + symbol_historical_prices) + normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns - normal_returns_matrix = np.array(normal_returns_matrix).transpose() - portfolio_weights_vector = np.array([BackTester.portfolio_weight_manager( - p_weights[symbol], long_only) for symbol in p_weights]).transpose() + normal_returns_matrix = np.array(normal_returns_matrix) portfolio_returns = np.dot( - normal_returns_matrix, portfolio_weights_vector) + normal_returns_matrix, list(p_weights.values())) return np.cumsum(portfolio_returns) @classmethod def get_market_returns(self, market_data, direction): - assert (direction in ["historical", "future"] - ), "direction must be 'historical' or 'future'" + assert (direction in ["historical", "future", "sim"] + ), "direction must be 'historical', 'future' or 'sim'" # Get future prices - future_price_market = market_data[direction].Close + future_price_market = market_data[direction] market_returns = self.price_delta(future_price_market) return np.cumsum(market_returns) @classmethod - def predict_future_returns(self, p_weights, data, - long_only: bool) -> np.ndarray: - """ - Main future test function. If future data is available i.e is_test - is set to 1 and future_bars set to > 0, this takes in the portfolio - weights and compares the portfolio returns with a market index of - your choice in the future. + def simulate_future_prices(self, data: dict, + simulation_timesteps: int = 30) ->pd.DataFrame: + """Simulates the price of a collection of stocks in the future + + [description] + :param data: a dictionary with the loaded data + :type data: dict + :param simulation_timesteps: number of steps, defaults to 30 + :type simulation_timesteps: number, optional + :returns: A dataframe of simulated prices + :rtype: pd.Dataframe """ - symbol_names = list(p_weights.keys()) - # future_data = data_dict[symbol]["future"].Close + - # Get invidiual returns for each stock in our portfolio - normal_returns_matrix = [] - for symbol in symbol_names: - symbol_historical_prices = data[symbol]["future"].Close - symbol_historical_returns = BackTester.price_delta( - symbol_historical_prices) - normal_returns_matrix.append(symbol_historical_returns) + # Get log returns from historical data + close_prices = data["historical"] + returns = np.log((close_prices / close_prices.shift())[1:]) + symbol_simulations = [] + for col in returns.columns: + # Get distribution of returns + hist = np.histogram(returns[col], bins=32) + hist_dist = st.rv_histogram(hist) # Distribution function - # Get portfolio returns - normal_returns_matrix = np.array(normal_returns_matrix).transpose() - portfolio_weights_vector = np.array([BackTester.portfolio_weight_manager( - p_weights[symbol], long_only) for symbol in p_weights]).transpose() - portfolio_returns = np.dot( - normal_returns_matrix, portfolio_weights_vector) - return np.cumsum(portfolio_returns) + simulations = [] + # Do 25 iterations to simulate prices + for iteration in range(25): + timeseries = [close_prices[col].values[-1]] + for i in range(simulation_timesteps): + # Get simulated return + return_value = np.round( + np.exp(hist_dist.ppf(random.uniform(0, 1))), 5) + data_point = timeseries[-1] * return_value + + # Add to list + timeseries.append(data_point) + # print(timeseries) + simulations.append(np.array(timeseries)) + symbol_simulations.append(np.mean(np.array(simulations), axis=0)) + + df = pd.DataFrame(columns=returns.columns, + data=np.array(symbol_simulations).T) + return df diff --git a/data_loader.py b/data_loader.py index 57fa868..28f5221 100644 --- a/data_loader.py +++ b/data_loader.py @@ -50,8 +50,8 @@ def get_most_frequent_count(self, input_list): def _split_data(self, data): if self.args.is_test: - return (data.iloc[:-self.args.future_bars], - data.iloc[-self.args.future_bars:]) + return (data.iloc[:-self.args.future_bars]["Close"].values, + data.iloc[-self.args.future_bars:]["Close"].values) def get_data(self, symbol): """ @@ -80,10 +80,6 @@ def get_data(self, symbol): auto_adjust=False, progress=False) stock_prices = stock_prices.reset_index() - try: - stock_prices = stock_prices.drop(columns=["Adj Close"]) - except Exception as e: - print("Exception", e) data_length = stock_prices.shape[0] self.stock_data_length.append(data_length) @@ -96,7 +92,6 @@ def get_data(self, symbol): stock_prices = stock_prices.iloc[-self.args.history_to_use:] historical_prices, future_prices = self._split_data(stock_prices) - print(f"Data separation\nH:{historical_prices.shape[0]}\nF:{future_prices.shape[0]}") except Exception as e: print("Exception", e) @@ -109,7 +104,9 @@ def collect_data_for_all_tickers(self): """ print("Loading data for all stocks...") - data_dict = {} + data_dict = {"historical": pd.DataFrame(columns=self.stocks_list), + "future": pd.DataFrame(columns=self.stocks_list) + } # Any stock with very low volatility is ignored. # You can change this line to address that. @@ -118,25 +115,13 @@ def collect_data_for_all_tickers(self): try: historical_data, future_data = self.get_data(symbol) if historical_data is not None: - data_dict[symbol] = { - "historical": historical_data, - "future": future_data - } + data_dict["historical"][symbol] = historical_data + if future_data is not None: + data_dict["future"][symbol] = future_data except Exception as e: print("Exception", e) continue + data_dict["historical"].fillna(1) + data_dict["future"].fillna(1) - return self.clean_data(data_dict) - - def clean_data(self, data_dict): - """ - Remove bad data i.e data that had some errors while scraping or - feature generation - - """ - - length_dictionary = collections.Counter( - [data_dict[i]["historical"].shape[0] for i in data_dict]) - std_len = length_dictionary.most_common(1)[0][0] - - return {i: data_dict[i] for i in data_dict if data_dict[i]["historical"].shape[0] == std_len} + return data_dict diff --git a/eiten.py b/eiten.py index fe43afe..4427f46 100644 --- a/eiten.py +++ b/eiten.py @@ -5,7 +5,6 @@ # Load our modules from data_loader import DataEngine -from simulator import MonteCarloSimulator from backtester import BackTester from utils import random_matrix_theory_based_cov from utils import dotdict @@ -46,11 +45,7 @@ def _get_abstract_returns(self, f): :returns: A Pandas Dataframe of returns for each asset :rtype: pd.Dataframe """ - res = pd.DataFrame(pd.DataFrame(columns=list(self.data_dict.keys()))) - for i in self.data_dict: - c = self.data_dict[i]["historical"].Close - res[i] = f(c) - return res + return f(self.data_dict["historical"]) def _get_perc_returns(self): return self._get_abstract_returns(self._get_price_delta) @@ -62,8 +57,8 @@ def _get_log_returns(self): return self._get_abstract_returns(self._get_log_return) def _get_predicted_return(self, data: pd.DataFrame) -> np.ndarray: - return np.array([np.mean(self._get_price_delta(data) / - np.array(np.arange(len(data) - 1, 0, -1)))]) + return self._get_price_delta(data).div( + np.array(np.arange(len(data) - 1, 0, -1)), axis=0).mean(axis=0) def _get_predicted_returns(self): return self._get_abstract_returns(self._get_predicted_return) @@ -77,7 +72,12 @@ def load_data(self): de = DataEngine(self.args) self.data_dict = de.collect_data_for_all_tickers() p, f = de.get_data(self.args.market_index) - self.market_data["historical"], self.market_data["future"] = p, f + + print("Market", p.shape, f.shape) + self.market_data["historical"] = pd.DataFrame( + columns=[self.args.market_index], data=p) + self.market_data["future"] = pd.DataFrame( + columns=[self.args.market_index], data=f) # Get return matrices and vectors return self.data_dict @@ -87,9 +87,10 @@ def _backtest(self): df = pd.DataFrame(columns=list(self.portfolios.keys())) for i in self.portfolios: - df[i] = BackTester.get_historical_test( + df[i] = BackTester.get_test( self.portfolios[i], self.data_dict, + "historical", self.args.only_long) mp = BackTester.get_market_returns(self.market_data, "historical") BackTester.plot_test(title="Backtest Results", @@ -103,10 +104,27 @@ def _futuretest(self): # Future test df = pd.DataFrame(columns=list(self.portfolios.keys())) for i in self.portfolios: - df[i] = BackTester.predict_future_returns(self.portfolios[i], - self.data_dict, - self.args.only_long) - BackTester.plot_test(title="Future Test Results", + df[i] = BackTester.get_test(self.portfolios[i], + self.data_dict, + "future", + self.args.only_long) + mp = BackTester.get_market_returns(self.market_data, "future") + BackTester.plot_test(title="Future Test Results", + xlabel="Bars (Time Sorted)", + ylabel="Cumulative Percentage Return", + df=df) + BackTester.plot_market(mp) + + def _monte_carlo(self): + df = pd.DataFrame(columns=list(self.portfolios.keys())) + self.data_dict["sim"] = BackTester.simulate_future_prices( + self.data_dict, 30) + for i in self.portfolios: + df[i] = BackTester.get_test(self.portfolios[i], + self.data_dict, + "sim", + self.args.only_long) + BackTester.plot_test(title="Simulated Future Returns", xlabel="Bars (Time Sorted)", ylabel="Cumulative Percentage Return", df=df) @@ -116,6 +134,8 @@ def run_strategies(self): Run strategies, back and future test them, and simulate the returns. """ self.load_data() + print("historical", self.data_dict["historical"].shape) + print("future", self.data_dict["future"].shape) # Calculate covariance matrix log_returns = self._get_log_returns() @@ -137,7 +157,8 @@ def run_strategies(self): weights = p.generate_portfolio( cov_matrix=cov_matrix, p_number=self.args.eigen_portfolio_number, pred_returns=pred_returns.T, - perc_returns=perc_returns) + perc_returns=perc_returns, + long_only=self.args.only_long) self.portfolios[name] = weights # Print weights @@ -148,23 +169,17 @@ def run_strategies(self): self.portfolios[i], i, plot_num=p_count) p_count += 1 self.draw_plot("output/weights.png") - self._backtest() if self.args.is_test: self._futuretest() self.draw_plot("output/future_tests.png") - return + # Simulation print("\n+$ Simulating future prices using monte carlo...") - for i in self.portfolios: - self.simulator.simulate_portfolio(self.portfolios[i], - self.data_dict, - self.market_data, - self.args.is_test, - market_chart=True, - strategy_name=i) + self._monte_carlo() self.draw_plot("output/monte_carlo.png") + return def draw_plot(self, filename="output/graph.png"): """ diff --git a/simulator.py b/simulator.py deleted file mode 100644 index 546e294..0000000 --- a/simulator.py +++ /dev/null @@ -1,160 +0,0 @@ -# Basic libraries -import scipy -import random -import collections -import numpy as np -import pandas as pd -import scipy.stats as st -import matplotlib.pyplot as plt -import warnings -warnings.filterwarnings("ignore") - -# Styling for plots -plt.style.use('seaborn-white') -plt.rc('grid', linestyle="dotted", color='#a0a0a0') -plt.rcParams['axes.edgecolor'] = "#04383F" - - -class MonteCarloSimulator: - """ - Monte carlo simulator that calculates the historical returns distribution - and uses it to predict the future returns - """ - - def __init__(self): - print("\n--$ Simulator has been initialized") - - def price_delta(self, prices): - """ - Percentage change - """ - - return ((prices - prices.shift()) * 100 / prices.shift())[1:] - - def draw_portfolio_performance_chart(self, returns_matrix, - p_weights: dict, strategy_name: str): - """ - Draw returns chart for portfolio performance - """ - - # Get portfolio returns - returns_matrix = np.array(returns_matrix).T - p_vector = np.array(list(p_weights.values())).T - print(p_weights) - portfolio_returns = np.dot(returns_matrix, p_vector) - portfolio_returns_cumulative = np.cumsum(portfolio_returns) - - # Plot - x = np.arange(len(portfolio_returns_cumulative)) - plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') - plt.plot(x, portfolio_returns_cumulative, linewidth=2.0, - label="Projected Returns from " + str(strategy_name)) - plt.title("Simulated Future Returns", fontsize=14) - plt.xlabel("Bars (Time Sorted)", fontsize=14) - plt.ylabel("Cumulative Percentage Return", fontsize=14) - plt.xticks(fontsize=14) - plt.yticks(fontsize=14) - - def draw_market_performance_chart(self, actual_returns, strategy_name): - plt.style.use('seaborn-white') - plt.rc('grid', linestyle="dotted", color='#a0a0a0') - plt.rcParams['axes.edgecolor'] = "#04383F" - plt.rcParams['figure.figsize'] = (12, 6) - """ - Draw actual market returns if future data is available - """ - - # Get market returns - cumulative_returns = np.cumsum(actual_returns) - - # Plot - x = np.arange(len(cumulative_returns)) - plt.axhline(y=0, linestyle='dotted', alpha=0.3, color='black') - plt.plot(x, cumulative_returns, linewidth=2.0, color='#282828', - linestyle='--', label="Market Index Returns") - plt.title("Simulated Future Returns", fontsize=14) - plt.xlabel("Bars (Time Sorted)", fontsize=14) - plt.ylabel("Cumulative Percentage Return", fontsize=14) - plt.xticks(fontsize=14) - plt.yticks(fontsize=14) - plt.show() - plt.clf() - plt.cla() - plt.close() - - def simulate_portfolio(self, p_weights, - data, market_data, test_or_predict, - market_chart: bool, strategy_name: str, - simulation_timesteps: int = 25): - """ - Simulate portfolio returns in the future - """ - symbol_names = list(p_weights.keys()) - returns_matrix = [] - future_price_market = market_data["future"].Close - - # Iterate over each symbol to get their returns - for symbol in symbol_names: - - # Get symbol returns using monte carlo - historical_close_prices = data[symbol]["historical"]["Close"] - future_price_predictions = self.simulate_and_get_future_prices(historical_close_prices, simulation_timesteps=max( - simulation_timesteps, data[symbol]["future"].shape[0])) - predicted_future_returns = self.price_delta( - future_price_predictions) - returns_matrix.append(predicted_future_returns) - - # Get portfolio returns - self.draw_portfolio_performance_chart( - returns_matrix, p_weights, strategy_name) - - # Check whether we have actual future data available or not - if test_or_predict == 1: - actual_future_prices_returns = self.price_delta( - future_price_market) - if market_chart: - # Also draw the actual future returns - self.draw_market_performance_chart( - actual_future_prices_returns, strategy_name) - - def simulate_and_get_future_prices(self, historical_prices, - simulation_timesteps=25): - - # Get log returns from historical data - close_prices = historical_prices - returns = np.log((close_prices / close_prices.shift())[1:]) - - # Get distribution of returns - hist = np.histogram(returns, bins=32) - hist_dist = st.rv_histogram(hist) # Distribution function - - predicted_prices = [] - # Do 25 iterations to simulate prices - for iteration in range(25): - new_close_prices = [close_prices.values[-1]] - for i in range(simulation_timesteps): - random_value = random.uniform(0, 1) - # Get simulated return - return_value = np.round(np.exp(hist_dist.ppf(random_value)), 5) - price_next_point = new_close_prices[-1] * return_value - - # Add to list - new_close_prices.append(price_next_point) - - predicted_prices.append(new_close_prices) - - # Calculate confidence intervals and average future returns. - # Conf intervals are not being used right now - # conf_intervals = st.t.interval(0.95, len(predicted_prices), - # loc=np.mean(predicted_prices, axis=0), - # scale=st.sem(predicted_prices, axis=0)) - - return pd.DataFrame(columns=["Close"], - data=np.mean(predicted_prices, axis=0)) - - def is_nan(self, object): - """ - Check if object is null - """ - return object != object - diff --git a/strategies/eigen_portfolio_strategy.py b/strategies/eigen_portfolio_strategy.py index da23ab0..2621bfb 100644 --- a/strategies/eigen_portfolio_strategy.py +++ b/strategies/eigen_portfolio_strategy.py @@ -15,14 +15,16 @@ def generate_portfolio(self, **kwargs): Inspired by: https://srome.github.io/Eigenvesting-I-Linear-Algebra-Can-Help-You-Choose-Your-Stock-Portfolio/ """ kwargs = dotdict(kwargs) - eigh_values, eigh_vectors = np.linalg.eigh(kwargs.cov_matrix) + eigh_values, eigh_vectors = np.linalg.eigh(kwargs.cov_matrix.T) # We don't need this but in case someone wants to analyze # market_eigen_portfolio = eig_vectors[:, -1] / np.sum(eig_vectors[:, -1]) # This is a portfolio that is uncorrelated to market and still yields good returns eigen_portfolio = eigh_vectors[:, -kwargs.p_number] / \ np.sum(eigh_vectors[:, -kwargs.p_number]) - - weights = {kwargs.cov_matrix.columns[i]: eigen_portfolio[i] - for i in range(eigen_portfolio.shape[0])} + if kwargs.long_only: + weights = {kwargs.cov_matrix.columns[i]: max(0, eigen_portfolio[i]) + for i in range(eigen_portfolio.shape[0])} + else: + weights = {kwargs.cov_matrix.columns[i]: max(0, eigen_portfolio[i]) + for i in range(eigen_portfolio.shape[0])} return weights - diff --git a/strategies/maximum_sharpe_ratio_strategy.py b/strategies/maximum_sharpe_ratio_strategy.py index bf68519..cf429b6 100644 --- a/strategies/maximum_sharpe_ratio_strategy.py +++ b/strategies/maximum_sharpe_ratio_strategy.py @@ -24,7 +24,7 @@ def generate_portfolio(self, **kwargs): np.dot(ones.transpose(), inverse_cov_matrix), kwargs.pred_returns) msr_portfolio_weights = numerator / denominator - weights = {kwargs.cov_matrix.columns[i]: msr_portfolio_weights[i][0] + weights = {kwargs.cov_matrix.columns[i]: msr_portfolio_weights[i] for i in range(len(msr_portfolio_weights))} return weights diff --git a/utils.py b/utils.py index 98a5e2c..c66a297 100644 --- a/utils.py +++ b/utils.py @@ -2,14 +2,17 @@ import warnings import numpy as np import pandas as pd + warnings.filterwarnings("ignore") + class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ + def random_matrix_theory_based_cov(log_returns): """ This is inspired by the excellent post @ From 737ec5b1395d58791e933c93221772e154744e2d Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 22:57:33 -0600 Subject: [PATCH 16/22] Delete Demo-checkpoint.ipynb --- .ipynb_checkpoints/Demo-checkpoint.ipynb | 363 ----------------------- 1 file changed, 363 deletions(-) delete mode 100644 .ipynb_checkpoints/Demo-checkpoint.ipynb diff --git a/.ipynb_checkpoints/Demo-checkpoint.ipynb b/.ipynb_checkpoints/Demo-checkpoint.ipynb deleted file mode 100644 index 39c03b7..0000000 --- a/.ipynb_checkpoints/Demo-checkpoint.ipynb +++ /dev/null @@ -1,363 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Populating the interactive namespace from numpy and matplotlib\n" - ] - } - ], - "source": [ - "%pylab inline\n", - "import sys, os\n", - "sys.path.insert(1, os.path.join(sys.path[0], '..'))\n", - "from eiten import Eiten " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--* Eiten has been initialized...\n", - "\n", - "\n" - ] - } - ], - "source": [ - "et = Eiten()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 0%| | 0/9 [00:00 Data engine has been initialized...\n", - "Loading all stocks from file...\n", - "Total number of stocks: 9\n", - "Loading data for all stocks...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 11%|█████████▎ | 1/9 [00:00<00:04, 1.89it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 22%|██████████████████▋ | 2/9 [00:00<00:03, 2.28it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 33%|████████████████████████████ | 3/9 [00:00<00:02, 2.65it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 44%|█████████████████████████████████████▎ | 4/9 [00:01<00:01, 2.84it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 56%|██████████████████████████████████████████████▋ | 5/9 [00:01<00:01, 3.12it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 67%|████████████████████████████████████████████████████████ | 6/9 [00:01<00:00, 3.43it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 78%|█████████████████████████████████████████████████████████████████▎ | 7/9 [00:01<00:00, 3.69it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 89%|██████████████████████████████████████████████████████████████████████████▋ | 8/9 [00:02<00:00, 3.92it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:02<00:00, 3.73it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exception 'numpy.ndarray' object has no attribute 'fillna'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "data_dict = et.load_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 0%| | 0/9 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", - "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", - "{'AAPL': -0.4369306238899776, 'AMD': -1.0115784282008393, 'AMZN': -2.3091148432564372, 'FB': 0.5058836309304608, 'MSFT': 0.009511216792968975, 'NFLX': 0.33246631319412956, 'NVDA': -0.18129905403714677, 'SQQQ': -0.680171626027902, 'TSLA': 0.721905802094419}\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsQAAAFkCAYAAAAnoS3wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAATXUlEQVR4nO3dUWiX97348Y9JTNTGKp56egZtPJgab8o5mnozPGG2Wyit3UDD+lM37YUgvRqMjK03BhGr2ezFOHbdoYOJE1Yj4oUR2kGqRQjtRcRYZKjgXKDjHCxHpU2ymqa/539R/PEPPSe/6skvaft5va7yPN/n4fe5+BLeefjxZF5RFEUAAEBSdXM9AAAAzCVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACk9qWC+OLFi7F9+/YvnD9z5kx0dXVFqVSK48ePz/hwAABQaw3VLvjd734Xp06dioULF045/+mnn8aBAwfixIkTsXDhwti6dWs8+eSTsXz58poNCwAAM61qELe0tMShQ4fi5z//+ZTz165di5aWlliyZElERDzxxBMxNDQUzzzzzJTrPvnkTgxduRLfWrYs6hvqZ3B0AAD43GeTn8V/3rwZ61avjgULmu7p3qpB/PTTT8cHH3zwhfOjo6OxePHiyvEDDzwQo6OjX7hu6MqVePLFn9zTUAAAcD/O/se/x7/967/c0z1Vg/h/09zcHGNjY5XjsbGxKYF81z/9w7LKcI/8o69TAAAw8z648WE8+eJPKu15L+47iFtbW2NkZCRu374dixYtiqGhodi5c+cXP6D+869JPPKPy+Ofv/Wt+/04AACo6m573tM993pDf39/jI+PR6lUipdeeil27twZRVFEV1dXPPzww/c8AAAAzKUvFcSPPPJI5bVq3//+9yvnn3rqqXjqqadqMxkAAMwC/5gDAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNSqBnG5XI6enp4olUqxffv2GBkZmbJ+6tSp2LRpU3R1dcUf//jHmg0KAAC10FDtgoGBgZiYmIi+vr4YHh6O3t7e+O1vf1tZ/9WvfhWnT5+ORYsWxcaNG2Pjxo2xZMmSmg4NAAAzpWoQnz9/Pjo6OiIiYs2aNXHp0qUp66tXr46PP/44GhoaoiiKmDdvXm0mBQCAGqgaxKOjo9Hc3Fw5rq+vj8nJyWho+PzWVatWRVdXVyxcuDA6OzvjwQcfrN20AAAww6p+h7i5uTnGxsYqx+VyuRLDly9fjnfeeSfefvvtOHPmTNy8eTPefPPN2k0LAAAzrGoQt7e3x7lz5yIiYnh4ONra2iprixcvjgULFkRTU1PU19fHsmXL4qOPPqrdtAAAMMOqfmWis7MzBgcHY8uWLVEURezfvz/6+/tjfHw8SqVSlEql2LZtW8yfPz9aWlpi06ZNszE3AADMiKpBXFdXF3v37p1yrrW1tfLz1q1bY+vWrTM/GQAAzAL/mAMAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpNVS7oFwux549e+LKlSvR2NgY+/btixUrVlTW33///ejt7Y2iKGL58uVx8ODBaGpqqunQAAAwU6o+IR4YGIiJiYno6+uL7u7u6O3trawVRRG7d++OAwcOxBtvvBEdHR3xt7/9raYDAwDATKr6hPj8+fPR0dERERFr1qyJS5cuVdauX78eS5cujSNHjsTVq1fjO9/5TqxcubJ20wIAwAyr+oR4dHQ0mpubK8f19fUxOTkZERG3bt2KCxcuxLZt2+Lw4cPx3nvvxbvvvlu7aQEAYIZVDeLm5uYYGxurHJfL5Who+PzB8tKlS2PFihXx2GOPxfz586Ojo2PKE2QAAPiqqxrE7e3tce7cuYiIGB4ejra2tsrao48+GmNjYzEyMhIREUNDQ7Fq1aoajQoAADOv6neIOzs7Y3BwMLZs2RJFUcT+/fujv78/xsfHo1Qqxcsvvxzd3d1RFEWsXbs2NmzYMAtjAwDAzKgaxHV1dbF3794p51pbWys/f/vb344TJ07M/GQAADAL/GMOAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFKrGsTlcjl6enqiVCrF9u3bY2Rk5H+8bvfu3fHKK6/M+IAAAFBLVYN4YGAgJiYmoq+vL7q7u6O3t/cL1xw7diyuXr1akwEBAKCWqgbx+fPno6OjIyIi1qxZE5cuXZqyfuHChbh48WKUSqXaTAgAADVUNYhHR0ejubm5clxfXx+Tk5MREXHjxo149dVXo6enp3YTAgBADTVUu6C5uTnGxsYqx+VyORoaPr/trbfeilu3bsWuXbviww8/jE8++SRWrlwZmzdvrt3EAAAwg6oGcXt7e5w9ezaeffbZGB4ejra2tsrajh07YseOHRERcfLkyfjLX/4ihgEA+FqpGsSdnZ0xODgYW7ZsiaIoYv/+/dHf3x/j4+O+NwwAwNde1SCuq6uLvXv3TjnX2tr6hes8GQYA4OvIP+YAACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAILWGaheUy+XYs2dPXLlyJRobG2Pfvn2xYsWKyvrp06fjyJEjUV9fH21tbbFnz56oq9PZAAB8PVQt14GBgZiYmIi+vr7o7u6O3t7eytonn3wSv/71r+MPf/hDHDt2LEZHR+Ps2bM1HRgAAGZS1SA+f/58dHR0RETEmjVr4tKlS5W1xsbGOHbsWCxcuDAiIiYnJ6OpqalGowIAwMyrGsSjo6PR3NxcOa6vr4/JycnPb66ri4ceeigiIo4ePRrj4+Oxfv36Go0KAAAzr+p3iJubm2NsbKxyXC6Xo6GhYcrxwYMH4/r163Ho0KGYN29ebSYFAIAaqPqEuL29Pc6dOxcREcPDw9HW1jZlvaenJ+7cuROvvfZa5asTAADwdVH1CXFnZ2cMDg7Gli1boiiK2L9/f/T398f4+Hg8/vjjceLEiVi3bl288MILERGxY8eO6OzsrPngAAAwE6oGcV1dXezdu3fKudbW1srPly9fnvmpAABglnhhMAAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpVQ3icrkcPT09USqVYvv27TEyMjJl/cyZM9HV1RWlUimOHz9es0EBAKAWqgbxwMBATExMRF9fX3R3d0dvb29l7dNPP40DBw7E73//+zh69Gj09fXFhx9+WNOBAQBgJjVUu+D8+fPR0dERERFr1qyJS5cuVdauXbsWLS0tsWTJkoiIeOKJJ2JoaCieeeaZyjWTn30WEREf3BDKAADUxt3WvNue96JqEI+OjkZzc3PluL6+PiYnJ6OhoSFGR0dj8eLFlbUHHnggRkdHp9z/X/99MyIinnzxJ/c8HAAA3Iv/+u+b8dgjj9zTPVWDuLm5OcbGxirH5XI5Ghoa/se1sbGxKYEcEbFu9eo4+x//Ht9atizqG+rvaTgAAPgyPpv8LP7z5s1Yt3r1Pd9bNYjb29vj7Nmz8eyzz8bw8HC0tbVV1lpbW2NkZCRu374dixYtiqGhodi5c+eU+xcsaIp/+9d/uefBAADgXrQ+em9Phu+aVxRFMd0F5XI59uzZE1evXo2iKGL//v3x5z//OcbHx6NUKsWZM2fiN7/5TRRFEV1dXfGjH/3ovgYBAIC5UDWIv6y74XzlypVobGyMffv2xYoVKyrrd8O5oaEhurq64vnnn5+Jj+Urrtq+OH36dBw5ciTq6+ujra0t9uzZE3V1Xo/9TVdtX9y1e/fuWLJkSfzsZz+bgymZbdX2xfvvvx+9vb1RFEUsX748Dh48GE1NTXM4MbOh2r44depUHD58OOrq6qKrqyu2bds2h9Mymy5evBivvPJKHD16dMr5+2rOYob86U9/Kn7xi18URVEUFy5cKF588cXK2sTERPG9732vuH37dnHnzp1i8+bNxY0bN2bqo/kKm25f/P3vfy+++93vFuPj40VRFMVPf/rTYmBgYE7mZHZNty/ueuONN4rnn3++OHjw4GyPxxyZbl+Uy+XiBz/4QfHXv/61KIqiOH78eHHt2rU5mZPZVe33xfr164tbt24Vd+7cqbQG33yvv/568dxzzxU//OEPp5y/3+acsUdxX/b1bI2NjZXXs/HNN92+aGxsjGPHjsXChQsjImJyctLTniSm2xcRERcuXIiLFy9GqVSai/GYI9Pti+vXr8fSpUvjyJEj8eMf/zhu374dK1eunKtRmUXVfl+sXr06Pv7445iYmIiiKGLevHlzMSazrKWlJQ4dOvSF8/fbnDMWxP/b69nurlV7PRvfTNPti7q6unjooYciIuLo0aMxPj4e69evn5M5mV3T7YsbN27Eq6++Gj09PXM1HnNkun1x69atuHDhQmzbti0OHz4c7733Xrz77rtzNSqzaLp9ERGxatWq6Orqio0bN8aGDRviwQcfnIsxmWVPP/105a1n/7/7bc4ZC+L/6+vZ+Gaabl/cPf7lL38Zg4ODcejQIX/ZJzHdvnjrrbfi1q1bsWvXrnj99dfj9OnTcfLkybkalVk03b5YunRprFixIh577LGYP39+dHR0fOFJId9M0+2Ly5cvxzvvvBNvv/12nDlzJm7evBlvvvnmXI3KV8D9NueMBXF7e3ucO3cuImLa17NNTEzE0NBQrF27dqY+mq+w6fZFRERPT0/cuXMnXnvttcpXJ/jmm25f7NixI06ePBlHjx6NXbt2xXPPPRebN2+eq1GZRdPti0cffTTGxsZiZGQkIiKGhoZi1apVczIns2u6fbF48eJYsGBBNDU1RX19fSxbtiw++uijuRqVr4D7bc6q7yH+sjo7O2NwcDC2bNlSeT1bf39/5fVsL730UuzcubPyeraHH354pj6ar7Dp9sXjjz8eJ06ciHXr1sULL7wQEZ/HUGdn5xxPTa1V+31BTtX2xcsvvxzd3d1RFEWsXbs2NmzYMNcjMwuq7YtSqRTbtm2L+fPnR0tLS2zatGmuR2YO/F+bc8ZeuwYAAF9HXvgKAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACC1/we/joqoI4LRIgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "et.run_strategies()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} From e84feb9fe77d6cb46d308729f27fe463c7909b27 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 22:58:14 -0600 Subject: [PATCH 17/22] move --- Demo.ipynb => notebooks/Demo.ipynb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Demo.ipynb => notebooks/Demo.ipynb (100%) diff --git a/Demo.ipynb b/notebooks/Demo.ipynb similarity index 100% rename from Demo.ipynb rename to notebooks/Demo.ipynb From 39619218766c4a813db27070f67bce7a1d40aa3d Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sat, 12 Sep 2020 22:58:53 -0600 Subject: [PATCH 18/22] Delete Demo.ipynb --- notebooks/Demo.ipynb | 813 ------------------------------------------- 1 file changed, 813 deletions(-) delete mode 100644 notebooks/Demo.ipynb diff --git a/notebooks/Demo.ipynb b/notebooks/Demo.ipynb deleted file mode 100644 index 43c656b..0000000 --- a/notebooks/Demo.ipynb +++ /dev/null @@ -1,813 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Populating the interactive namespace from numpy and matplotlib\n" - ] - } - ], - "source": [ - "%pylab inline\n", - "import sys, os\n", - "sys.path.insert(1, os.path.join(sys.path[0], '..'))\n", - "from eiten import Eiten\n", - "from utils import random_matrix_theory_based_cov" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--* Eiten has been initialized...\n", - "\n", - "\n" - ] - } - ], - "source": [ - "et = Eiten()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 0%| | 0/9 [00:00 Data engine has been initialized...\n", - "Loading all stocks from file...\n", - "Total number of stocks: 9\n", - "Loading data for all stocks...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:02<00:00, 4.18it/s]\n" - ] - } - ], - "source": [ - "data_dict = et.load_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AAPLAMDAMZNFBMSFTNFLXNVDASQQQTSLA
029.0700001.860000522.36999592.90000243.98000099.16000422.9400011863.19995150.714001
129.1025011.890000527.39001593.44999744.299999104.08000223.1000001835.19995152.450001
228.4800001.890000538.86999594.33999644.250000104.20999923.3099991828.80004952.414001
328.3624991.870000540.26001094.40000243.480000102.62000323.2900011905.59997652.124001
428.8025001.810000548.39001595.55000344.110001100.30000323.5300011891.19995152.840000
..............................
116371.93250353.6600002372.709961194.190002177.429993411.890015298.45999158.299999160.102005
116473.44999752.3899992474.000000204.710007179.210007419.850006292.27999957.900002156.376007
116572.26750249.8800012286.040039202.270004174.570007415.269989282.77999963.099998140.264008
116673.29000152.5600012315.989990205.259995178.839996428.149994291.29000960.849998152.238007
116774.38999952.1899992317.800049207.070007180.759995424.679993293.73999058.799999153.641998
\n", - "

1168 rows × 9 columns

\n", - "
" - ], - "text/plain": [ - " AAPL AMD AMZN FB MSFT NFLX \\\n", - "0 29.070000 1.860000 522.369995 92.900002 43.980000 99.160004 \n", - "1 29.102501 1.890000 527.390015 93.449997 44.299999 104.080002 \n", - "2 28.480000 1.890000 538.869995 94.339996 44.250000 104.209999 \n", - "3 28.362499 1.870000 540.260010 94.400002 43.480000 102.620003 \n", - "4 28.802500 1.810000 548.390015 95.550003 44.110001 100.300003 \n", - "... ... ... ... ... ... ... \n", - "1163 71.932503 53.660000 2372.709961 194.190002 177.429993 411.890015 \n", - "1164 73.449997 52.389999 2474.000000 204.710007 179.210007 419.850006 \n", - "1165 72.267502 49.880001 2286.040039 202.270004 174.570007 415.269989 \n", - "1166 73.290001 52.560001 2315.989990 205.259995 178.839996 428.149994 \n", - "1167 74.389999 52.189999 2317.800049 207.070007 180.759995 424.679993 \n", - "\n", - " NVDA SQQQ TSLA \n", - "0 22.940001 1863.199951 50.714001 \n", - "1 23.100000 1835.199951 52.450001 \n", - "2 23.309999 1828.800049 52.414001 \n", - "3 23.290001 1905.599976 52.124001 \n", - "4 23.530001 1891.199951 52.840000 \n", - "... ... ... ... \n", - "1163 298.459991 58.299999 160.102005 \n", - "1164 292.279999 57.900002 156.376007 \n", - "1165 282.779999 63.099998 140.264008 \n", - "1166 291.290009 60.849998 152.238007 \n", - "1167 293.739990 58.799999 153.641998 \n", - "\n", - "[1168 rows x 9 columns]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "data_dict[\"historical\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AAPLAMDAMZNFBMSFTNFLXNVDASQQQTSLA
AAPL0.0003290.0002970.0001920.0002040.0002220.0002070.000297-0.0006040.000228
AMD0.0002970.0016340.0002830.0002510.0002640.0003490.000635-0.0007880.000405
AMZN0.0001920.0002830.0003490.0002320.0002170.0002710.000277-0.0005630.000227
FB0.0002040.0002510.0002320.0004050.0002200.0002350.000290-0.0005850.000231
MSFT0.0002220.0002640.0002170.0002200.0002980.0002310.000305-0.0006200.000233
NFLX0.0002070.0003490.0002710.0002350.0002310.0006590.000332-0.0006150.000282
NVDA0.0002970.0006350.0002770.0002900.0003050.0003320.000886-0.0008120.000363
SQQQ-0.000604-0.000788-0.000563-0.000585-0.000620-0.000615-0.0008120.001605-0.000621
TSLA0.0002280.0004050.0002270.0002310.0002330.0002820.000363-0.0006210.001152
\n", - "
" - ], - "text/plain": [ - " AAPL AMD AMZN FB MSFT NFLX NVDA \\\n", - "AAPL 0.000329 0.000297 0.000192 0.000204 0.000222 0.000207 0.000297 \n", - "AMD 0.000297 0.001634 0.000283 0.000251 0.000264 0.000349 0.000635 \n", - "AMZN 0.000192 0.000283 0.000349 0.000232 0.000217 0.000271 0.000277 \n", - "FB 0.000204 0.000251 0.000232 0.000405 0.000220 0.000235 0.000290 \n", - "MSFT 0.000222 0.000264 0.000217 0.000220 0.000298 0.000231 0.000305 \n", - "NFLX 0.000207 0.000349 0.000271 0.000235 0.000231 0.000659 0.000332 \n", - "NVDA 0.000297 0.000635 0.000277 0.000290 0.000305 0.000332 0.000886 \n", - "SQQQ -0.000604 -0.000788 -0.000563 -0.000585 -0.000620 -0.000615 -0.000812 \n", - "TSLA 0.000228 0.000405 0.000227 0.000231 0.000233 0.000282 0.000363 \n", - "\n", - " SQQQ TSLA \n", - "AAPL -0.000604 0.000228 \n", - "AMD -0.000788 0.000405 \n", - "AMZN -0.000563 0.000227 \n", - "FB -0.000585 0.000231 \n", - "MSFT -0.000620 0.000233 \n", - "NFLX -0.000615 0.000282 \n", - "NVDA -0.000812 0.000363 \n", - "SQQQ 0.001605 -0.000621 \n", - "TSLA -0.000621 0.001152 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "log_returns = et._get_log_returns()\n", - "log_returns.cov()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AAPLAMDAMZNFBMSFTNFLXNVDASQQQTSLA
00.0003290.0003450.0002220.0002270.0002230.0002590.000331-0.0005680.000272
10.0003450.0016340.0003520.0003590.0003520.0004090.000523-0.0008990.000430
20.0002220.0003520.0003490.0002310.0002270.0002630.000337-0.0005790.000277
30.0002270.0003590.0002310.0004050.0002320.0002690.000344-0.0005910.000283
40.0002230.0003520.0002270.0002320.0002980.0002640.000337-0.0005790.000277
50.0002590.0004090.0002630.0002690.0002640.0006590.000392-0.0006730.000322
60.0003310.0005230.0003370.0003440.0003370.0003920.000886-0.0008610.000411
7-0.000568-0.000899-0.000579-0.000591-0.000579-0.000673-0.0008610.001605-0.000707
80.0002720.0004300.0002770.0002830.0002770.0003220.000411-0.0007070.001152
\n", - "
" - ], - "text/plain": [ - " AAPL AMD AMZN FB MSFT NFLX NVDA \\\n", - "0 0.000329 0.000345 0.000222 0.000227 0.000223 0.000259 0.000331 \n", - "1 0.000345 0.001634 0.000352 0.000359 0.000352 0.000409 0.000523 \n", - "2 0.000222 0.000352 0.000349 0.000231 0.000227 0.000263 0.000337 \n", - "3 0.000227 0.000359 0.000231 0.000405 0.000232 0.000269 0.000344 \n", - "4 0.000223 0.000352 0.000227 0.000232 0.000298 0.000264 0.000337 \n", - "5 0.000259 0.000409 0.000263 0.000269 0.000264 0.000659 0.000392 \n", - "6 0.000331 0.000523 0.000337 0.000344 0.000337 0.000392 0.000886 \n", - "7 -0.000568 -0.000899 -0.000579 -0.000591 -0.000579 -0.000673 -0.000861 \n", - "8 0.000272 0.000430 0.000277 0.000283 0.000277 0.000322 0.000411 \n", - "\n", - " SQQQ TSLA \n", - "0 -0.000568 0.000272 \n", - "1 -0.000899 0.000430 \n", - "2 -0.000579 0.000277 \n", - "3 -0.000591 0.000283 \n", - "4 -0.000579 0.000277 \n", - "5 -0.000673 0.000322 \n", - "6 -0.000861 0.000411 \n", - "7 0.001605 -0.000707 \n", - "8 -0.000707 0.001152 " - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "random_matrix_theory_based_cov(log_returns)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 0%| | 0/9 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", - "{'AAPL': -0.8587993994766128, 'AMD': -0.6378536513178515, 'AMZN': -0.431210585800188, 'FB': -0.511639960273437, 'MSFT': -0.43735588414386617, 'NFLX': -1.1136164804406894, 'NVDA': -1.1735042343611668, 'SQQQ': 1.159765050860694, 'TSLA': 5.004215144953117}\n", - "{'AAPL': -0.4369306238899776, 'AMD': -1.0115784282008393, 'AMZN': -2.3091148432564372, 'FB': 0.5058836309304608, 'MSFT': 0.009511216792968975, 'NFLX': 0.33246631319412956, 'NVDA': -0.18129905403714677, 'SQQQ': -0.680171626027902, 'TSLA': 0.721905802094419}\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsQAAAFkCAYAAAAnoS3wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAATXUlEQVR4nO3dUWiX97348Y9JTNTGKp56egZtPJgab8o5mnozPGG2Wyit3UDD+lM37YUgvRqMjK03BhGr2ezFOHbdoYOJE1Yj4oUR2kGqRQjtRcRYZKjgXKDjHCxHpU2ymqa/539R/PEPPSe/6skvaft5va7yPN/n4fe5+BLeefjxZF5RFEUAAEBSdXM9AAAAzCVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACk9qWC+OLFi7F9+/YvnD9z5kx0dXVFqVSK48ePz/hwAABQaw3VLvjd734Xp06dioULF045/+mnn8aBAwfixIkTsXDhwti6dWs8+eSTsXz58poNCwAAM61qELe0tMShQ4fi5z//+ZTz165di5aWlliyZElERDzxxBMxNDQUzzzzzJTrPvnkTgxduRLfWrYs6hvqZ3B0AAD43GeTn8V/3rwZ61avjgULmu7p3qpB/PTTT8cHH3zwhfOjo6OxePHiyvEDDzwQo6OjX7hu6MqVePLFn9zTUAAAcD/O/se/x7/967/c0z1Vg/h/09zcHGNjY5XjsbGxKYF81z/9w7LKcI/8o69TAAAw8z648WE8+eJPKu15L+47iFtbW2NkZCRu374dixYtiqGhodi5c+cXP6D+869JPPKPy+Ofv/Wt+/04AACo6m573tM993pDf39/jI+PR6lUipdeeil27twZRVFEV1dXPPzww/c8AAAAzKUvFcSPPPJI5bVq3//+9yvnn3rqqXjqqadqMxkAAMwC/5gDAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNSqBnG5XI6enp4olUqxffv2GBkZmbJ+6tSp2LRpU3R1dcUf//jHmg0KAAC10FDtgoGBgZiYmIi+vr4YHh6O3t7e+O1vf1tZ/9WvfhWnT5+ORYsWxcaNG2Pjxo2xZMmSmg4NAAAzpWoQnz9/Pjo6OiIiYs2aNXHp0qUp66tXr46PP/44GhoaoiiKmDdvXm0mBQCAGqgaxKOjo9Hc3Fw5rq+vj8nJyWho+PzWVatWRVdXVyxcuDA6OzvjwQcfrN20AAAww6p+h7i5uTnGxsYqx+VyuRLDly9fjnfeeSfefvvtOHPmTNy8eTPefPPN2k0LAAAzrGoQt7e3x7lz5yIiYnh4ONra2iprixcvjgULFkRTU1PU19fHsmXL4qOPPqrdtAAAMMOqfmWis7MzBgcHY8uWLVEURezfvz/6+/tjfHw8SqVSlEql2LZtW8yfPz9aWlpi06ZNszE3AADMiKpBXFdXF3v37p1yrrW1tfLz1q1bY+vWrTM/GQAAzAL/mAMAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpNVS7oFwux549e+LKlSvR2NgY+/btixUrVlTW33///ejt7Y2iKGL58uVx8ODBaGpqqunQAAAwU6o+IR4YGIiJiYno6+uL7u7u6O3trawVRRG7d++OAwcOxBtvvBEdHR3xt7/9raYDAwDATKr6hPj8+fPR0dERERFr1qyJS5cuVdauX78eS5cujSNHjsTVq1fjO9/5TqxcubJ20wIAwAyr+oR4dHQ0mpubK8f19fUxOTkZERG3bt2KCxcuxLZt2+Lw4cPx3nvvxbvvvlu7aQEAYIZVDeLm5uYYGxurHJfL5Who+PzB8tKlS2PFihXx2GOPxfz586Ojo2PKE2QAAPiqqxrE7e3tce7cuYiIGB4ejra2tsrao48+GmNjYzEyMhIREUNDQ7Fq1aoajQoAADOv6neIOzs7Y3BwMLZs2RJFUcT+/fujv78/xsfHo1Qqxcsvvxzd3d1RFEWsXbs2NmzYMAtjAwDAzKgaxHV1dbF3794p51pbWys/f/vb344TJ07M/GQAADAL/GMOAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFKrGsTlcjl6enqiVCrF9u3bY2Rk5H+8bvfu3fHKK6/M+IAAAFBLVYN4YGAgJiYmoq+vL7q7u6O3t/cL1xw7diyuXr1akwEBAKCWqgbx+fPno6OjIyIi1qxZE5cuXZqyfuHChbh48WKUSqXaTAgAADVUNYhHR0ejubm5clxfXx+Tk5MREXHjxo149dVXo6enp3YTAgBADTVUu6C5uTnGxsYqx+VyORoaPr/trbfeilu3bsWuXbviww8/jE8++SRWrlwZmzdvrt3EAAAwg6oGcXt7e5w9ezaeffbZGB4ejra2tsrajh07YseOHRERcfLkyfjLX/4ihgEA+FqpGsSdnZ0xODgYW7ZsiaIoYv/+/dHf3x/j4+O+NwwAwNde1SCuq6uLvXv3TjnX2tr6hes8GQYA4OvIP+YAACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAILWGaheUy+XYs2dPXLlyJRobG2Pfvn2xYsWKyvrp06fjyJEjUV9fH21tbbFnz56oq9PZAAB8PVQt14GBgZiYmIi+vr7o7u6O3t7eytonn3wSv/71r+MPf/hDHDt2LEZHR+Ps2bM1HRgAAGZS1SA+f/58dHR0RETEmjVr4tKlS5W1xsbGOHbsWCxcuDAiIiYnJ6OpqalGowIAwMyrGsSjo6PR3NxcOa6vr4/JycnPb66ri4ceeigiIo4ePRrj4+Oxfv36Go0KAAAzr+p3iJubm2NsbKxyXC6Xo6GhYcrxwYMH4/r163Ho0KGYN29ebSYFAIAaqPqEuL29Pc6dOxcREcPDw9HW1jZlvaenJ+7cuROvvfZa5asTAADwdVH1CXFnZ2cMDg7Gli1boiiK2L9/f/T398f4+Hg8/vjjceLEiVi3bl288MILERGxY8eO6OzsrPngAAAwE6oGcV1dXezdu3fKudbW1srPly9fnvmpAABglnhhMAAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpCWIAAFITxAAApCaIAQBITRADAJCaIAYAIDVBDABAaoIYAIDUBDEAAKkJYgAAUhPEAACkJogBAEhNEAMAkJogBgAgNUEMAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACA1QQwAQGqCGACA1AQxAACpVQ3icrkcPT09USqVYvv27TEyMjJl/cyZM9HV1RWlUimOHz9es0EBAKAWqgbxwMBATExMRF9fX3R3d0dvb29l7dNPP40DBw7E73//+zh69Gj09fXFhx9+WNOBAQBgJjVUu+D8+fPR0dERERFr1qyJS5cuVdauXbsWLS0tsWTJkoiIeOKJJ2JoaCieeeaZyjWTn30WEREf3BDKAADUxt3WvNue96JqEI+OjkZzc3PluL6+PiYnJ6OhoSFGR0dj8eLFlbUHHnggRkdHp9z/X/99MyIinnzxJ/c8HAAA3Iv/+u+b8dgjj9zTPVWDuLm5OcbGxirH5XI5Ghoa/se1sbGxKYEcEbFu9eo4+x//Ht9atizqG+rvaTgAAPgyPpv8LP7z5s1Yt3r1Pd9bNYjb29vj7Nmz8eyzz8bw8HC0tbVV1lpbW2NkZCRu374dixYtiqGhodi5c+eU+xcsaIp/+9d/uefBAADgXrQ+em9Phu+aVxRFMd0F5XI59uzZE1evXo2iKGL//v3x5z//OcbHx6NUKsWZM2fiN7/5TRRFEV1dXfGjH/3ovgYBAIC5UDWIv6y74XzlypVobGyMffv2xYoVKyrrd8O5oaEhurq64vnnn5+Jj+Urrtq+OH36dBw5ciTq6+ujra0t9uzZE3V1Xo/9TVdtX9y1e/fuWLJkSfzsZz+bgymZbdX2xfvvvx+9vb1RFEUsX748Dh48GE1NTXM4MbOh2r44depUHD58OOrq6qKrqyu2bds2h9Mymy5evBivvPJKHD16dMr5+2rOYob86U9/Kn7xi18URVEUFy5cKF588cXK2sTERPG9732vuH37dnHnzp1i8+bNxY0bN2bqo/kKm25f/P3vfy+++93vFuPj40VRFMVPf/rTYmBgYE7mZHZNty/ueuONN4rnn3++OHjw4GyPxxyZbl+Uy+XiBz/4QfHXv/61KIqiOH78eHHt2rU5mZPZVe33xfr164tbt24Vd+7cqbQG33yvv/568dxzzxU//OEPp5y/3+acsUdxX/b1bI2NjZXXs/HNN92+aGxsjGPHjsXChQsjImJyctLTniSm2xcRERcuXIiLFy9GqVSai/GYI9Pti+vXr8fSpUvjyJEj8eMf/zhu374dK1eunKtRmUXVfl+sXr06Pv7445iYmIiiKGLevHlzMSazrKWlJQ4dOvSF8/fbnDMWxP/b69nurlV7PRvfTNPti7q6unjooYciIuLo0aMxPj4e69evn5M5mV3T7YsbN27Eq6++Gj09PXM1HnNkun1x69atuHDhQmzbti0OHz4c7733Xrz77rtzNSqzaLp9ERGxatWq6Orqio0bN8aGDRviwQcfnIsxmWVPP/105a1n/7/7bc4ZC+L/6+vZ+Gaabl/cPf7lL38Zg4ODcejQIX/ZJzHdvnjrrbfi1q1bsWvXrnj99dfj9OnTcfLkybkalVk03b5YunRprFixIh577LGYP39+dHR0fOFJId9M0+2Ly5cvxzvvvBNvv/12nDlzJm7evBlvvvnmXI3KV8D9NueMBXF7e3ucO3cuImLa17NNTEzE0NBQrF27dqY+mq+w6fZFRERPT0/cuXMnXnvttcpXJ/jmm25f7NixI06ePBlHjx6NXbt2xXPPPRebN2+eq1GZRdPti0cffTTGxsZiZGQkIiKGhoZi1apVczIns2u6fbF48eJYsGBBNDU1RX19fSxbtiw++uijuRqVr4D7bc6q7yH+sjo7O2NwcDC2bNlSeT1bf39/5fVsL730UuzcubPyeraHH354pj6ar7Dp9sXjjz8eJ06ciHXr1sULL7wQEZ/HUGdn5xxPTa1V+31BTtX2xcsvvxzd3d1RFEWsXbs2NmzYMNcjMwuq7YtSqRTbtm2L+fPnR0tLS2zatGmuR2YO/F+bc8ZeuwYAAF9HXvgKAEBqghgAgNQEMQAAqQliAABSE8QAAKQmiAEASE0QAwCQmiAGACC1/we/joqoI4LRIgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "et.run_strategies()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} From e8d327c12dd7a6bf75021f3c87590d22d42a9f73 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sun, 13 Sep 2020 10:38:00 -0600 Subject: [PATCH 19/22] treating Invalid stocks removed all invalid stocks from dataframe --- data_loader.py | 43 ++++++++++++++++++++------ eiten.py | 4 +-- strategies/eigen_portfolio_strategy.py | 12 +++---- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/data_loader.py b/data_loader.py index 9a2649b..f70c1b9 100644 --- a/data_loader.py +++ b/data_loader.py @@ -5,6 +5,7 @@ import yfinance as yf from tqdm import tqdm import warnings +import matplotlib.pyplot as plt warnings.filterwarnings("ignore") @@ -27,7 +28,7 @@ def __init__(self, args): self.data_dictionary = {} # Data length - self.stock_data_length = [] + self.stock_data_length = 0 def load_stocks_from_file(self): """ @@ -50,8 +51,8 @@ def get_most_frequent_count(self, input_list): def _split_data(self, data): if self.args.is_test: - return (data.iloc[:-self.args.future_bars]["Close"].values, - data.iloc[-self.args.future_bars:]["Close"].values) + return (data.iloc[:-self.args.future_bars]["Close"].values, + data.iloc[-self.args.future_bars:]["Close"].values) def get_data(self, symbol): """ @@ -79,10 +80,12 @@ def get_data(self, symbol): interval=interval, auto_adjust=False, progress=False) - stock_prices = stock_prices.reset_index() + # stock_prices = stock_prices.reset_index() - data_length = stock_prices.shape[0] - self.stock_data_length.append(data_length) + if self.stock_data_length == 0: + self.stock_data_length = stock_prices.shape[0] + elif stock_prices.shape[0] != self.stock_data_length: + raise Exception(f"{symbol}: Invalid Stock Length") if self.args.history_to_use == "all": # For some reason, yfinance gives some 0 @@ -95,6 +98,7 @@ def get_data(self, symbol): except Exception as e: print("Exception", e) + return None, None return historical_prices, future_prices @@ -104,8 +108,8 @@ def collect_data_for_all_tickers(self): """ print("Loading data for all stocks...") - data_dict = {"historical": pd.DataFrame(columns=self.stocks_list), - "future": pd.DataFrame(columns=self.stocks_list) + data_dict = {"historical": pd.DataFrame(), + "future": pd.DataFrame() } # Any stock with very low volatility is ignored. @@ -121,7 +125,26 @@ def collect_data_for_all_tickers(self): except Exception as e: print("Exception", e) continue - data_dict["historical"].fillna(1) - data_dict["future"].fillna(1) + data_dict["historical"] = data_dict["historical"].fillna(1) + data_dict["future"] = data_dict["future"].fillna(1) + + try: + data_dict["historical"].to_csv("historical.csv") + data_dict["future"].to_csv("future.csv") + + except Exception as e: + print("Exception: ", e) + + try: + plt.style.use('seaborn-white') + plt.rc('grid', linestyle="dotted", color='#a0a0a0') + plt.rcParams['axes.edgecolor'] = "#04383F" + plt.rcParams['figure.figsize'] = (16, 9) + data_dict["historical"].plot() + plt.savefig("./output/gt_historical.png") + data_dict["future"].plot() + plt.savefig("./output/gt_future.png") + except Exception as e: + print("Exception: ", e) return data_dict diff --git a/eiten.py b/eiten.py index 4ff02bc..69de634 100644 --- a/eiten.py +++ b/eiten.py @@ -171,6 +171,7 @@ def run_strategies(self): p_count += 1 self.draw_plot("output/weights.png") self._backtest() + self.draw_plot("output/back_test.png") if self.args.is_test: self._futuretest() @@ -202,7 +203,6 @@ def draw_plot(self, filename="output/graph.png"): else: plt.tight_layout() plt.show() - plt.clf() plt.cla() def print_and_plot_portfolio_weights(self, weights: dict, @@ -222,7 +222,7 @@ def print_and_plot_portfolio_weights(self, weights: dict, x = np.arange(len(weights)) plt.bar(x + (width * (plot_num - 1)) + 0.05, list(weights.values()), label=strategy, width=width) - plt.xticks(x, symbols, fontsize=14) + plt.xticks(x, symbols) plt.yticks(fontsize=14) plt.xlabel("Symbols", fontsize=14) plt.ylabel("Weight in Portfolio", fontsize=14) diff --git a/strategies/eigen_portfolio_strategy.py b/strategies/eigen_portfolio_strategy.py index 2621bfb..d565065 100644 --- a/strategies/eigen_portfolio_strategy.py +++ b/strategies/eigen_portfolio_strategy.py @@ -21,10 +21,10 @@ def generate_portfolio(self, **kwargs): # This is a portfolio that is uncorrelated to market and still yields good returns eigen_portfolio = eigh_vectors[:, -kwargs.p_number] / \ np.sum(eigh_vectors[:, -kwargs.p_number]) - if kwargs.long_only: - weights = {kwargs.cov_matrix.columns[i]: max(0, eigen_portfolio[i]) - for i in range(eigen_portfolio.shape[0])} - else: - weights = {kwargs.cov_matrix.columns[i]: max(0, eigen_portfolio[i]) - for i in range(eigen_portfolio.shape[0])} + # if kwargs.long_only: + # weights = {kwargs.cov_matrix.columns[i]: max(0, eigen_portfolio[i]) + # for i in range(eigen_portfolio.shape[0])} + # else: + weights = {kwargs.cov_matrix.columns[i]: eigen_portfolio[i] + for i in range(eigen_portfolio.shape[0])} return weights From 87e8eb3bf3f5304497ef891520ce6022595faa02 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Sun, 13 Sep 2020 10:38:22 -0600 Subject: [PATCH 20/22] Update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c3e1ec9..57069e7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ *.pyc *.png notebooks/commands.json +*.csv +*.txt +*.txt From 3c0d801193250cbf1c47476850c700a02d39e1f4 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Mon, 21 Sep 2020 21:06:07 -0600 Subject: [PATCH 21/22] Completely rewritten --- .gitignore | 6 +- argchecker.py | 3 +- backtester.py | 50 ++++----- data_loader.py | 39 ++++--- eiten.py | 116 +++++++------------- portfolio_manager.py | 3 +- strategies/eigen_portfolio_strategy.py | 5 +- strategies/genetic_algo_strategy.py | 18 +-- strategies/maximum_sharpe_ratio_strategy.py | 4 +- strategies/minimum_variance_strategy.py | 3 +- utils.py | 40 +++++++ 11 files changed, 152 insertions(+), 135 deletions(-) diff --git a/.gitignore b/.gitignore index 57069e7..3ce3c39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ - +.vscode *.pyc *.png -notebooks/commands.json +notebooks *.csv *.txt *.txt +*.ipynb + diff --git a/argchecker.py b/argchecker.py index c45ab25..c83de95 100644 --- a/argchecker.py +++ b/argchecker.py @@ -17,9 +17,10 @@ def check_arguments(self, args): assert not(args.is_test == 1 and args.future_bars < 2), "You want to test but the future bars are less than 2. That does not give us enough data to test the model properly. Please use a value larger than 2.\nExiting now..." - assert not(args.history_to_use != "all" and int(args.history_to_use_int) < + assert not(args.history_to_use != "all" and int(args.history_to_use) < args.future_bars), "It is a good idea to use more history and less future bars. Please change these two values and try again.\nExiting now..." args.market_index = str(args.market_index).upper() if args.history_to_use != "all": args.history_to_use = int(args.history_to_use) + return args diff --git a/backtester.py b/backtester.py index dbe0cb6..87d2615 100644 --- a/backtester.py +++ b/backtester.py @@ -10,7 +10,8 @@ import scipy.stats as st import matplotlib.pyplot as plt import warnings -from utils import dotdict +from utils import dotdict, get_price_deltas, get_log_returns +from utils import get_predicted_returns warnings.filterwarnings("ignore") @@ -19,16 +20,8 @@ class BackTester: Backtester module that does both backward and forward testing for our portfolios. """ - @classmethod - def price_delta(self, prices): - """ - Percentage change - """ - - return ((prices - prices.shift()) * 100 / prices.shift())[1:] - - @classmethod - def filter_short(self, weights, long_only): + @staticmethod + def filter_short(weights, long_only): """ Manage portfolio weights. If portfolio is long only, set the negative weights to zero. @@ -37,18 +30,19 @@ def filter_short(self, weights, long_only): return np.array([max(i, 0) for i in weights]) return weights - @classmethod - def plot_market(self, market_returns): + @staticmethod + def plot_market(market_returns): x = np.arange(len(market_returns)) plt.plot(x, market_returns, linewidth=2.0, color='#282828', label='Market Index', linestyle='--') - @classmethod - def plot_test(self, **kwargs): + @staticmethod + def plot_test(**kwargs): # Styling for plots kwargs = dotdict(kwargs) plt.style.use('seaborn-white') plt.rc('grid', linestyle="dotted", color='#a0a0a0') + plt.rcParams['figure.figsize'] = (18, 6) plt.rcParams['axes.edgecolor'] = "#04383F" plt.rcParams['axes.titlesize'] = "large" plt.rcParams['axes.labelsize'] = "medium" @@ -60,8 +54,8 @@ def plot_test(self, **kwargs): kwargs.df.plot(fontsize=14, title=kwargs.title, xlabel=kwargs.xlabel, ylabel=kwargs.ylabel,) - @classmethod - def get_test(self, p_weights, data, direction: str, long_only: bool): + @staticmethod + def get_test(p_weights, data, direction: str, long_only: bool): """ Main backtest function. Takes in the portfolio weights and compares the portfolio returns with a market index of your choice. @@ -73,7 +67,7 @@ def get_test(self, p_weights, data, direction: str, long_only: bool): # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] symbol_historical_prices = data[direction] - symbol_historical_returns = BackTester.price_delta( + symbol_historical_returns = get_price_deltas( symbol_historical_prices) normal_returns_matrix.append(symbol_historical_returns) @@ -83,17 +77,17 @@ def get_test(self, p_weights, data, direction: str, long_only: bool): normal_returns_matrix, list(p_weights.values())) return np.cumsum(portfolio_returns) - @classmethod - def get_market_returns(self, market_data, direction): + @staticmethod + def get_market_returns(market_data, direction): assert (direction in ["historical", "future", "sim"] ), "direction must be 'historical', 'future' or 'sim'" # Get future prices future_price_market = market_data[direction] - market_returns = self.price_delta(future_price_market) + market_returns = get_price_deltas(future_price_market) return np.cumsum(market_returns) - @classmethod - def simulate_future_prices(self, data: dict, + @staticmethod + def simulate_future_prices(data: dict, r_func, simulation_timesteps: int = 30) ->pd.DataFrame: """Simulates the price of a collection of stocks in the future @@ -105,11 +99,10 @@ def simulate_future_prices(self, data: dict, :returns: A dataframe of simulated prices :rtype: pd.Dataframe """ - # Get log returns from historical data close_prices = data["historical"] - returns = np.log((close_prices / close_prices.shift())[1:]) + returns = r_func(close_prices) symbol_simulations = [] for col in returns.columns: # Get distribution of returns @@ -118,9 +111,10 @@ def simulate_future_prices(self, data: dict, simulations = [] # Do 25 iterations to simulate prices - for iteration in range(25): + for _ in range(100): timeseries = [close_prices[col].values[-1]] - for i in range(simulation_timesteps): + for _ in range(min(simulation_timesteps, + data["future"].shape[0])): # Get simulated return return_value = np.round( np.exp(hist_dist.ppf(random.uniform(0, 1))), 5) @@ -131,7 +125,7 @@ def simulate_future_prices(self, data: dict, # print(timeseries) simulations.append(np.array(timeseries)) symbol_simulations.append(np.mean(np.array(simulations), axis=0)) - + df = pd.DataFrame(columns=returns.columns, data=np.array(symbol_simulations).T) return df diff --git a/data_loader.py b/data_loader.py index f70c1b9..3448875 100644 --- a/data_loader.py +++ b/data_loader.py @@ -5,7 +5,6 @@ import yfinance as yf from tqdm import tqdm import warnings -import matplotlib.pyplot as plt warnings.filterwarnings("ignore") @@ -51,13 +50,22 @@ def get_most_frequent_count(self, input_list): def _split_data(self, data): if self.args.is_test: - return (data.iloc[:-self.args.future_bars]["Close"].values, - data.iloc[-self.args.future_bars:]["Close"].values) + return (data.iloc[:-self.args.future_bars]["Adj Close"].values, + data.iloc[-self.args.future_bars:]["Adj Close"].values) + return data["Adj Close"].values, None - def get_data(self, symbol): + def _format_symbol(self, s): + x = s.upper() + x = x.replace(".VN", ".V") + if len(x.split(".")) > 2: + x = x.replace(".", "-", 1) + return x + + def get_data(self, symbol_raw): """ Get stock data from yahoo finance. """ + symbol = self._format_symbol(symbol_raw) future_prices = None historical_prices = None # Find period @@ -135,16 +143,17 @@ def collect_data_for_all_tickers(self): except Exception as e: print("Exception: ", e) - try: - plt.style.use('seaborn-white') - plt.rc('grid', linestyle="dotted", color='#a0a0a0') - plt.rcParams['axes.edgecolor'] = "#04383F" - plt.rcParams['figure.figsize'] = (16, 9) - data_dict["historical"].plot() - plt.savefig("./output/gt_historical.png") - data_dict["future"].plot() - plt.savefig("./output/gt_future.png") - except Exception as e: - print("Exception: ", e) + # try: + # plt.style.use('seaborn-white') + # plt.rc('grid', linestyle="dotted", color='#a0a0a0') + # plt.rcParams['axes.edgecolor'] = "#04383F" + # plt.rcParams['figure.figsize'] = (16, 9) + # data_dict["historical"].plot() + # plt.savefig("./output/gt_historical.png") + # data_dict["future"].plot() + # plt.savefig("./output/gt_future.png") + # plt.clf() + # except Exception as e: + # print("Exception: ", e) return data_dict diff --git a/eiten.py b/eiten.py index 69de634..221dd5f 100644 --- a/eiten.py +++ b/eiten.py @@ -1,3 +1,4 @@ +# !/usr/bin/env python import numpy as np import pandas as pd import matplotlib.pyplot as plt @@ -7,7 +8,8 @@ from data_loader import DataEngine from backtester import BackTester from utils import random_matrix_theory_based_cov -from utils import dotdict +from utils import dotdict, get_predicted_returns, get_exp_returns +from utils import get_price_deltas, get_log_returns from strategies import portfolios @@ -21,7 +23,8 @@ def __init__(self, args: dict = None): print("\n--* Eiten has been initialized...") self.args = args - + if self.args.history_to_use != "all": + self.args.history_to_use = int(self.args.history_to_use) # Data dictionary self.data_dict = {} # {"market": args.market_index} @@ -29,40 +32,8 @@ def __init__(self, args: dict = None): print('\n') - def _get_price_delta(self, prices: pd.DataFrame): - """ - Calculate percentage change - """ - return ((prices - prices.shift()) * 100 / prices.shift())[1:] - - def _get_abstract_returns(self, f): - """Abstract form of getting returns - - This function takes a function f as parameter and applies the function - to the closing prices of the current data dictionary - - :param f: a function that takes a dictionary and returns some returns - :type f: function - :returns: A Pandas Dataframe of returns for each asset - :rtype: pd.Dataframe - """ - return f(self.data_dict["historical"]) - - def _get_perc_returns(self): - return self._get_abstract_returns(self._get_price_delta) - - def _get_log_return(self, data: pd.DataFrame) -> np.ndarray: - return np.log((data / data.shift())[1:]) - - def _get_log_returns(self): - return self._get_abstract_returns(self._get_log_return) - - def _get_predicted_return(self, data: pd.DataFrame) -> np.ndarray: - return self._get_price_delta(data).div( - np.array(np.arange(len(data) - 1, 0, -1)), axis=0).mean(axis=0) + - def _get_predicted_returns(self): - return self._get_abstract_returns(self._get_predicted_return) def load_data(self): """ @@ -74,7 +45,6 @@ def load_data(self): self.data_dict = de.collect_data_for_all_tickers() p, f = de.get_data(self.args.market_index) - print("Market", p.shape, f.shape) self.market_data["historical"] = pd.DataFrame( columns=[self.args.market_index], data=p) self.market_data["future"] = pd.DataFrame( @@ -93,12 +63,13 @@ def _backtest(self): self.data_dict, "historical", self.args.only_long) - mp = BackTester.get_market_returns(self.market_data, "historical") - BackTester.plot_test(title="Backtest Results", - xlabel="Bars (Time Sorted)", - ylabel="Cumulative Percentage Return", - df=df) - BackTester.plot_market(mp) + return df + # mp = BackTester.get_market_returns(self.market_data, "historical") + # BackTester.plot_test(title="Backtest Results", + # xlabel="Bars (Time Sorted)", + # ylabel="Cumulative Percentage Return", + # df=df) + # BackTester.plot_market(mp) def _futuretest(self): print("\n#^ Future testing the portfolios...") @@ -109,47 +80,46 @@ def _futuretest(self): self.data_dict, "future", self.args.only_long) - mp = BackTester.get_market_returns(self.market_data, "future") - BackTester.plot_test(title="Future Test Results", - xlabel="Bars (Time Sorted)", - ylabel="Cumulative Percentage Return", - df=df) - BackTester.plot_market(mp) - - def _monte_carlo(self): + return df + # mp = BackTester.get_market_returns(self.market_data, "future") + # BackTester.plot_test(title="Future Test Results", + # xlabel="Bars (Time Sorted)", + # ylabel="Cumulative Percentage Return", + # df=df) + # BackTester.plot_market(mp) + + def _monte_carlo(self, span): df = pd.DataFrame(columns=list(self.portfolios.keys())) self.data_dict["sim"] = BackTester.simulate_future_prices( - self.data_dict, 30) + self.data_dict, get_predicted_returns, span) for i in self.portfolios: df[i] = BackTester.get_test(self.portfolios[i], self.data_dict, "sim", self.args.only_long) - BackTester.plot_test(title="Simulated Future Returns", - xlabel="Bars (Time Sorted)", - ylabel="Cumulative Percentage Return", - df=df) + return df + # BackTester.plot_test(title="Simulated Future Returns", + # xlabel="Bars (Time Sorted)", + # ylabel="Cumulative Percentage Return", + # df=df) def run_strategies(self): """ Run strategies, back and future test them, and simulate the returns. """ self.load_data() - print("historical", self.data_dict["historical"].shape) - print("future", self.data_dict["future"].shape) # Calculate covariance matrix - log_returns = self._get_log_returns() + log_returns = get_log_returns(self.data_dict["historical"]) cov_matrix = log_returns.cov() # Use random matrix theory to filter out the noisy eigen values if self.args.apply_noise_filtering: - print( - "\n** Applying random matrix theory to filter out noise in the covariance matrix...\n") + print("\nFiltering noise from cov matrix\n") cov_matrix = random_matrix_theory_based_cov(log_returns) - pred_returns = self._get_predicted_returns() - perc_returns = self._get_perc_returns() + pred_returns = get_predicted_returns(self.data_dict["historical"]) + perc_returns = get_price_deltas(self.data_dict["historical"]) self.portfolios = {} # Get weights for the portfolio @@ -169,7 +139,7 @@ def run_strategies(self): self.print_and_plot_portfolio_weights( self.portfolios[i], i, plot_num=p_count) p_count += 1 - self.draw_plot("output/weights.png") + self.draw_plot("output/weights.png", (p_count, 6)) self._backtest() self.draw_plot("output/back_test.png") @@ -179,11 +149,11 @@ def run_strategies(self): # Simulation print("\n+$ Simulating future prices using monte carlo...") - self._monte_carlo() + self._monte_carlo(self.args.future_bars) self.draw_plot("output/monte_carlo.png") return - def draw_plot(self, filename="output/graph.png"): + def draw_plot(self, filename="output/graph.png", figsize=(12, 6)): """ Draw plots """ @@ -194,7 +164,7 @@ def draw_plot(self, filename="output/graph.png"): plt.rcParams['axes.titlesize'] = "large" plt.rcParams['axes.labelsize'] = "medium" plt.rcParams['lines.linewidth'] = 2 - plt.rcParams['figure.figsize'] = (12, 6) + plt.rcParams['figure.figsize'] = figsize plt.grid() plt.legend(fontsize=14) @@ -203,14 +173,12 @@ def draw_plot(self, filename="output/graph.png"): else: plt.tight_layout() plt.show() - plt.cla() + # plt.cla() + plt.clf() - def print_and_plot_portfolio_weights(self, weights: dict, - strategy: str, plot_num: int) -> None: - plt.style.use('seaborn-white') - plt.rc('grid', linestyle="dotted", color='#a0a0a0') - plt.rcParams['axes.edgecolor'] = "#04383F" - plt.rcParams['figure.figsize'] = (12, 6) + def print_and_plot_portfolio_weights(self, + weights: dict, strategy: str, + plot_num: int, figsize=(12, 6)): print("\n-------- Weights for %s --------" % strategy) symbols = list(weights.keys()) @@ -222,7 +190,7 @@ def print_and_plot_portfolio_weights(self, weights: dict, x = np.arange(len(weights)) plt.bar(x + (width * (plot_num - 1)) + 0.05, list(weights.values()), label=strategy, width=width) - plt.xticks(x, symbols) + plt.xticks(x, symbols, rotation=90) plt.yticks(fontsize=14) plt.xlabel("Symbols", fontsize=14) plt.ylabel("Weight in Portfolio", fontsize=14) diff --git a/portfolio_manager.py b/portfolio_manager.py index 335cce8..8928b6a 100644 --- a/portfolio_manager.py +++ b/portfolio_manager.py @@ -25,8 +25,7 @@ def main(): # Get arguments args = argParser.parse_args() - # Check arguments - ArgChecker(args) + # Run strategies eiten = Eiten(args) diff --git a/strategies/eigen_portfolio_strategy.py b/strategies/eigen_portfolio_strategy.py index d565065..2ae527b 100644 --- a/strategies/eigen_portfolio_strategy.py +++ b/strategies/eigen_portfolio_strategy.py @@ -2,7 +2,7 @@ import os import warnings import numpy as np -from utils import dotdict +from utils import dotdict, normalize_weights warnings.filterwarnings("ignore") @@ -15,7 +15,7 @@ def generate_portfolio(self, **kwargs): Inspired by: https://srome.github.io/Eigenvesting-I-Linear-Algebra-Can-Help-You-Choose-Your-Stock-Portfolio/ """ kwargs = dotdict(kwargs) - eigh_values, eigh_vectors = np.linalg.eigh(kwargs.cov_matrix.T) + eigh_values, eigh_vectors = np.linalg.eigh(kwargs.cov_matrix) # We don't need this but in case someone wants to analyze # market_eigen_portfolio = eig_vectors[:, -1] / np.sum(eig_vectors[:, -1]) # This is a portfolio that is uncorrelated to market and still yields good returns @@ -25,6 +25,7 @@ def generate_portfolio(self, **kwargs): # weights = {kwargs.cov_matrix.columns[i]: max(0, eigen_portfolio[i]) # for i in range(eigen_portfolio.shape[0])} # else: + eigen_portfolio = normalize_weights(eigen_portfolio) weights = {kwargs.cov_matrix.columns[i]: eigen_portfolio[i] for i in range(eigen_portfolio.shape[0])} return weights diff --git a/strategies/genetic_algo_strategy.py b/strategies/genetic_algo_strategy.py index 4b8c345..1fdafdb 100644 --- a/strategies/genetic_algo_strategy.py +++ b/strategies/genetic_algo_strategy.py @@ -1,7 +1,7 @@ # Basic libraries import warnings import numpy as np -from utils import dotdict +from utils import dotdict, normalize_weights warnings.filterwarnings("ignore") @@ -13,13 +13,13 @@ class GeneticAlgoStrategy: def __init__(self): self.name = "Genetic Algo" self.initial_genes = 100 - self.selection_top = 25 + self.selection_top = 10 self.mutation_iterations = 50 - self.weight_update_factor = 0.1 + self.weight_update_factor = 0.01 self.gene_length = None self.genes_in_each_iteration = 250 - self.iterations = 50 - self.crossover_probability = 0.05 + self.iterations = 100 + self.crossover_probability = 0.1 def generate_portfolio(self, **kwargs): kwargs = dotdict(kwargs) @@ -31,7 +31,7 @@ def generate_portfolio(self, **kwargs): for i in range(self.iterations): # Select - top_genes = self.select(kwargs.perc_returns, initial_genes) + top_genes = self.select(kwargs.sample_returns, initial_genes) # print("Iteration %d Best Sharpe Ratio: %.3f" % (i, top_genes[0][0])) top_genes = [item[1] for item in top_genes] @@ -39,14 +39,14 @@ def generate_portfolio(self, **kwargs): mutated_genes = self.mutate(top_genes) initial_genes = mutated_genes - top_genes = self.select(kwargs.perc_returns, initial_genes) + top_genes = self.select(kwargs.sample_returns, initial_genes) best_gene = top_genes[0][1] # Gene is a distribution of weights for different stocks # transposed_gene = np.array(best_gene).transpose() # returns = np.dot(return_matrix, transposed_gene) # returns_cumsum = np.cumsum(returns) - - weights = {symbols[x]: best_gene[x] for x in range(0, len(best_gene))} + n_best = normalize_weights(best_gene) + weights = {symbols[x]: n_best[x] for x in range(0, len(best_gene))} return weights def generate_initial_genes(self, symbols): diff --git a/strategies/maximum_sharpe_ratio_strategy.py b/strategies/maximum_sharpe_ratio_strategy.py index cf429b6..23156af 100644 --- a/strategies/maximum_sharpe_ratio_strategy.py +++ b/strategies/maximum_sharpe_ratio_strategy.py @@ -1,7 +1,7 @@ # Basic libraries import warnings import numpy as np -from utils import dotdict +from utils import dotdict, normalize_weights warnings.filterwarnings("ignore") @@ -24,6 +24,8 @@ def generate_portfolio(self, **kwargs): np.dot(ones.transpose(), inverse_cov_matrix), kwargs.pred_returns) msr_portfolio_weights = numerator / denominator + msr_portfolio_weights = normalize_weights(msr_portfolio_weights) + weights = {kwargs.cov_matrix.columns[i]: msr_portfolio_weights[i] for i in range(len(msr_portfolio_weights))} diff --git a/strategies/minimum_variance_strategy.py b/strategies/minimum_variance_strategy.py index 62f3219..4f1b26c 100644 --- a/strategies/minimum_variance_strategy.py +++ b/strategies/minimum_variance_strategy.py @@ -1,7 +1,7 @@ # Basic libraries import warnings import numpy as np -from utils import dotdict +from utils import dotdict, normalize_weights warnings.filterwarnings("ignore") @@ -19,6 +19,7 @@ def generate_portfolio(self, **kwargs): ones = np.ones(len(inverse_cov_matrix)) inverse_dot_ones = np.dot(inverse_cov_matrix, ones) min_var_weights = inverse_dot_ones / np.dot(inverse_dot_ones, ones) + min_var_weights = normalize_weights(min_var_weights) weights = {kwargs.cov_matrix.columns[i]: min_var_weights[i] for i in range(min_var_weights.shape[0])} return weights diff --git a/utils.py b/utils.py index c66a297..3c9ed4a 100644 --- a/utils.py +++ b/utils.py @@ -12,6 +12,23 @@ class dotdict(dict): __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ +def normalize_weights(w): + pos_sum = 0 + neg_sum = 0 + for i in w: + if i > 0: + pos_sum += i + else: + neg_sum += i + neg_sum = abs(neg_sum) + for i in range(len(w)): + if w[i] > 0: + w[i] /= pos_sum + else: + w[i] /= neg_sum + return w + + def random_matrix_theory_based_cov(log_returns): """ @@ -48,3 +65,26 @@ def random_matrix_theory_based_cov(log_returns): np.diag(standard_deviations))) return pd.DataFrame(columns=log_returns.columns, data=filtered_cov_matrix) + + +def get_price_deltas(prices: pd.DataFrame): + """ + Calculate ratio of change + """ + return ((prices - prices.shift()) / prices.shift())[1:] + +def get_capm_returns(data:pd.DataFrame) -> pd.DataFrame: + #not correct + return data.std() * (get_price_deltas(data).mean() ) + +def get_log_returns(data: pd.DataFrame) -> pd.DataFrame: + return np.log((data / data.shift())[1:]) + + +def get_exp_returns(data: pd.DataFrame) -> pd.DataFrame: + return get_price_deltas(data).ewm(span=len(data)).mean() + + +def get_predicted_returns(data: pd.DataFrame) -> pd.DataFrame: + return get_price_deltas(data).div( + np.array(np.arange((len(data) - 1), 0, -1)), axis=0) From 47245bbf8ed71bbe588737814769c975008919d4 Mon Sep 17 00:00:00 2001 From: silvavn <37382997+silvavn@users.noreply.github.com> Date: Mon, 21 Sep 2020 23:54:12 -0600 Subject: [PATCH 22/22] Improvements better back/forward testing --- backtester.py | 14 +++++------ eiten.py | 66 ++++++++++++++------------------------------------- 2 files changed, 24 insertions(+), 56 deletions(-) diff --git a/backtester.py b/backtester.py index 87d2615..9270c34 100644 --- a/backtester.py +++ b/backtester.py @@ -57,7 +57,7 @@ def plot_test(**kwargs): @staticmethod def get_test(p_weights, data, direction: str, long_only: bool): """ - Main backtest function. Takes in the portfolio weights and compares + Main backtest function. Takes in the portfolio weights and compares the portfolio returns with a market index of your choice. """ @@ -67,15 +67,13 @@ def get_test(p_weights, data, direction: str, long_only: bool): # Get invidiual returns for each stock in our portfolio normal_returns_matrix = [] symbol_historical_prices = data[direction] - symbol_historical_returns = get_price_deltas( - symbol_historical_prices) - normal_returns_matrix.append(symbol_historical_returns) # Get portfolio returns - normal_returns_matrix = np.array(normal_returns_matrix) - portfolio_returns = np.dot( - normal_returns_matrix, list(p_weights.values())) - return np.cumsum(portfolio_returns) + normal_returns_matrix = get_price_deltas( + symbol_historical_prices).cumsum() + portfolio_returns = np.dot(normal_returns_matrix, p_weights) + + return portfolio_returns @staticmethod def get_market_returns(market_data, direction): diff --git a/eiten.py b/eiten.py index 221dd5f..4de7cfb 100644 --- a/eiten.py +++ b/eiten.py @@ -32,9 +32,6 @@ def __init__(self, args: dict = None): print('\n') - - - def load_data(self): """ Loads data needed for analysis @@ -52,52 +49,27 @@ def load_data(self): # Get return matrices and vectors return self.data_dict - def _backtest(self): + def _test(self, direction): # Back test print("\n*& Backtesting the portfolios...") + assert direction in ["historical", "future"], "Invalid direction!" - df = pd.DataFrame(columns=list(self.portfolios.keys())) - for i in self.portfolios: - df[i] = BackTester.get_test( - self.portfolios[i], - self.data_dict, - "historical", - self.args.only_long) - return df - # mp = BackTester.get_market_returns(self.market_data, "historical") - # BackTester.plot_test(title="Backtest Results", - # xlabel="Bars (Time Sorted)", - # ylabel="Cumulative Percentage Return", - # df=df) - # BackTester.plot_market(mp) - - def _futuretest(self): - print("\n#^ Future testing the portfolios...") - # Future test - df = pd.DataFrame(columns=list(self.portfolios.keys())) - for i in self.portfolios: - df[i] = BackTester.get_test(self.portfolios[i], - self.data_dict, - "future", - self.args.only_long) - return df - # mp = BackTester.get_market_returns(self.market_data, "future") - # BackTester.plot_test(title="Future Test Results", - # xlabel="Bars (Time Sorted)", - # ylabel="Cumulative Percentage Return", - # df=df) - # BackTester.plot_market(mp) + return pd.DataFrame(columns=self.portfolios.columns, + data=BackTester.get_test( + self.portfolios, + self.data_dict, + direction, + self.args.only_long)) def _monte_carlo(self, span): - df = pd.DataFrame(columns=list(self.portfolios.keys())) self.data_dict["sim"] = BackTester.simulate_future_prices( self.data_dict, get_predicted_returns, span) - for i in self.portfolios: - df[i] = BackTester.get_test(self.portfolios[i], - self.data_dict, - "sim", - self.args.only_long) - return df + return pd.DataFrame(columns=self.portfolios.columns, + data=BackTester.get_test( + self.portfolios, + self.data_dict, + "sim", + self.args.only_long)) # BackTester.plot_test(title="Simulated Future Returns", # xlabel="Bars (Time Sorted)", # ylabel="Cumulative Percentage Return", @@ -131,20 +103,18 @@ def run_strategies(self): perc_returns=perc_returns, long_only=self.args.only_long) self.portfolios[name] = weights + self.portfolios = pd.DataFrame.from_dict(self.portfolios) # Print weights print("\n*% Printing portfolio weights...") p_count = 1 - for i in self.portfolios: - self.print_and_plot_portfolio_weights( - self.portfolios[i], i, plot_num=p_count) - p_count += 1 + print(self.portfolios) self.draw_plot("output/weights.png", (p_count, 6)) - self._backtest() + self._test("historical") self.draw_plot("output/back_test.png") if self.args.is_test: - self._futuretest() + self._test("future") self.draw_plot("output/future_tests.png") # Simulation