import os
import warnings
import matplotlib
import numpy as np
import pandas as pd
import seaborn as sns
from itertools import product
import matplotlib.pyplot as plt
from scipy.stats import multinomial
from scipy.optimize import minimize
from IPython.display import display, HTML, Image
from wordcloud import WordCloud, STOPWORDS
from collections import Counter, namedtuple
from pandas.plotting import register_matplotlib_converters

register_matplotlib_converters()
matplotlib.font_manager._rebuild()
np.set_printoptions(linewidth=200, threshold=100, floatmode='fixed')
plt.rcParams['lines.linewidth'] = 2
plt.rcParams['font.family'] = 'serif'
plt.rcParams['font.serif'] = 'Palatino'
plt.rcParams['axes.facecolor'] = '#000000'
plt.rcParams['figure.figsize'] = (12, 5)
plt.rcParams['figure.dpi'] = 150
pd.set_option('max_row', 10)
pd.set_option('display.max_columns', 10)
pd.set_option('display.width', 1000)
warnings.filterwarnings('ignore')

df = pd.read_csv('data/posts.csv', index_col=0)
df.index = pd.to_datetime(df.index).date
display(HTML(df.to_html(max_rows=10)))

df_eod = df.copy()
df_eod['date'] = pd.to_datetime(df_eod.index).date
g = df_eod.groupby('date')
title = g.title.apply(' '.join)
abstract = g.abstract.apply(' '.join)
content = g.content.apply(lambda x: '\n\n'.join(map(str, x.values)))
df_eod = pd.concat([title, abstract, content], axis=1)
df_eod.index = pd.to_datetime(df_eod.index)
df_eod.index.name = None
display(HTML(df_eod.to_html(max_rows=10)))

print(df_eod.iloc[0].content)

df_eod.title = df_eod.title.str.lower()
df_eod.abstract = df_eod.abstract.str.lower()
df_eod.content = df_eod.content.str.lower()
display(HTML(df_eod.to_html(max_rows=10)))

print(df_eod.iloc[0].content)

remove_abbr = lambda x: ' '.join([_.split('’')[0] for _ in x.split()])
df_eod = df_eod.applymap(remove_abbr)
display(HTML(df_eod.to_html(max_rows=10)))

print(df_eod.iloc[0].content)

alph_space = lambda x: (ord('a') <= ord(x) <= ord('z')) or x == ' '
keep_alph = lambda x: ''.join(filter(alph_space, x))
df_eod = df_eod.applymap(keep_alph)
display(HTML(df_eod.to_html(max_rows=10)))

print(df_eod.iloc[0].content)

replace_stop_words = lambda x: ' '.join(_ for _ in x.split() if _ not in STOPWORDS)    
df_eod = df_eod.applymap(replace_stop_words)
display(HTML(df_eod.to_html(max_rows=10)))

print(df_eod.iloc[0].content)

df_eod = df_eod.applymap(str.split).applymap(sorted)
display(HTML(df_eod.to_html(max_rows=10)))

print(df_eod.iloc[0].content)

price = pd.read_csv('data/price.csv', index_col=0)
price.index = pd.to_datetime(price.index)
display(HTML(price.to_html(max_rows=10)))

mkcap = pd.read_csv('data/mkcap.csv', index_col=0)
mkcap.index = pd.to_datetime(mkcap.index)
display(HTML(mkcap.to_html(max_rows=10, float_format='%.4g')))

n, d = 3, 60

rtn = price.pct_change().fillna(0)
rtn.index.name = None
temp = (rtn * (mkcap.agg('argsort') < 10)).rolling(d).sum().T
coins = pd.DataFrame(
    [temp.nlargest(n, t).index.tolist() +
     temp.nsmallest(n, t).index.tolist() for t in rtn.index],
    index=rtn.index
).shift().dropna()  # shift to avoid forward-looking bias
display(HTML(coins.to_html(max_rows=10, header=False)))

coef = np.array([1] * n + [-1] * n) / n
rtn = pd.DataFrame([rtn.loc[t, coins.loc[t].values] @ coef for t in coins.index], index=coins.index)[0]
rtn.name = 'r'
print(f'Benchmark strategy has mean return {rtn.mean():.2%} with std {rtn.std():.2f}')

plt.figure()
plt.plot(rtn)
plt.ylabel('Daily Returns')
plt.axhline(y=0, color='w', ls='--')
plt.tight_layout()

plt.figure()
bins = np.arange(-.50, .55, .025)
heights, bins, patches = plt.hist(rtn, bins=bins, edgecolor='k', facecolor='deepskyblue', density=True)
for p in patches:
    x = abs(p.get_bbox().get_points().mean(axis=0)[0]) * 20
    p.set_alpha(1 if x < 1 else 1 / x)
plt.ylabel('Density')
plt.xlabel('Daily Returns')
plt.tight_layout()
plt.show()

df_total = df_eod[['content']].join(rtn, how='inner').dropna()
df_total['p'] = np.argsort(df_total.iloc[:, 1]) / (df_total.shape[0] - 1)
df_total.columns = ['words', 'r', 'p']

test_split = 0.3
len_total = len(df_total)
len_test = round(len_total * test_split)
len_train = len_total - len_test

df_train = df_total.iloc[:len_train, :]
df_test = df_total.iloc[-len_test:, :]
display(HTML(df_train.to_html(max_rows=10)))

upcloses = lambda x: x[x > 0].mean()
downcloses = lambda x: -x[x < 0].mean()
rsi = 100 - 100 / (1 + price.pct_change().fillna(0).rolling(60).apply(upcloses) / \
                   price.pct_change().fillna(0).rolling(60).apply(downcloses))
display(HTML(rsi.dropna().to_html(max_rows=10, header=True)))

fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_ylabel('Relative Strength Index')
for c in rsi.columns:
    rsi[c].dropna().plot(ax=ax, label=c, linewidth=1)
legend = plt.legend(frameon=False, ncol=7, loc='center left', bbox_to_anchor=(0, 1.08))
ax.axhline(y=70, ls='--', color='w')
ax.axhline(y=30, ls='--', color='w')

plt.setp(legend.get_texts(), color='k', size=10)
plt.xticks(rotation=0, ha='center')
plt.show()

words_all = sorted(list(set(w for words in df_train.words for w in words)))
n = len(words_all)

words_dict = {w: [] for w in words_all}

for i in range(df_train.shape[0]):
    words, r, _ = df_train.iloc[i]
    for w in words:
        words_dict[w].append(r > 0)

words_df = pd.DataFrame({
    'word': words_all,
    'r': [np.mean(words_dict[w]) for w in words_all],
    'k': [len(words_dict[w]) for w in words_all]
}, index=range(n)).sort_values(by='r')
words_df = words_df[words_df.k >= 50][words_df.word.apply(len) >= 5].reset_index(drop=True)
words_df['p'] = np.argsort(words_df.r) / (len(words_df) - 1)
display(HTML(words_df.set_index('word').T.to_html(max_cols=10)))

alpha = .1
pos_words = words_df[words_df.p > 1 - alpha].word.values.tolist()
neg_words = words_df[words_df.p < alpha].word.values.tolist()
print(f'There are {len(pos_words)} positive words, {len(neg_words)} negative words')

font_path = 'misc/Palatino.ttf'

fig = plt.figure(figsize=(12, 5))

ax = fig.add_subplot(121)
pos_cloud = WordCloud(
    width=800, height=800,
    background_color='white',
    colormap='Greens',
    font_path=font_path,
    min_font_size=10,
    random_state=123
).generate(' '.join(pos_words))
plt.imshow(pos_cloud)
plt.axis('off')

ax = fig.add_subplot(122)
neg_cloud = WordCloud(
    width=800, height=800,
    background_color='white',
    colormap='Reds',
    font_path=font_path,
    min_font_size=10,
    random_state=123
).generate(' '.join(neg_words))
plt.imshow(neg_cloud) 
plt.axis('off')

plt.tight_layout()
plt.show() 

keywords = pos_words + neg_words

def count_words(words):
    c = Counter(words)
    d = [c[w] for w in keywords]
    return np.array(d)

df_train['d'] = df_train.words.apply(count_words).values
display(HTML(df_train.to_html(max_rows=10)))

def print_array(array):
    print(np.array2string(array).replace('[[',' [').replace(']]',']'))

p = df_train.p.values
W = np.vstack([p, 1 - p]).T
print(f'W has shape {W.shape}:')
print_array(W)

D = np.vstack(df_train.d.values)
D = (D.T / D.sum(axis=1)).T
D[np.isnan(D)] = 0
print(f'D has shape {D.shape}:')
print_array(D)

O_pos, O_neg = np.maximum(np.linalg.pinv(W.T @ W) @ (W.T @ D), 0)
O_pos = (O_pos / O_pos.sum())
O_neg = (O_neg / O_neg.sum())
O_pm = np.hstack([O_pos.reshape(-1, 1), O_neg.reshape(-1, 1)])
print(f'O± has shape {O_pm.shape}:')
print_array(O_pm)

def pen_ll(p, d, lamda):
    return (d * np.log(p * O_pos + (1 - p) * O_neg)).sum() + lamda * np.log(p * (1 - p))

def predict(d, lamda):
    return minimize(lambda p: -pen_ll(p, d, lamda), x0=.5, method='Nelder-Mead').x[0]

def mse_p(lamda):
    p_hat = np.array([predict(d, lamda) for d in df_train.d])
    return ((df_train.p.values - p_hat)**2).mean()

best_lamda = minimize(mse_p, x0=0, method='Nelder-Mead', bounds=(0, None)).x[0]
best_lamda

df_test['d'] = df_test.words.apply(count_words).values
df_test['p_hat'] = [predict(d, best_lamda) for d in df_test.d]
df_test = df_test[['words', 'r', 'p', 'p_hat', 'd']]
display(HTML(df_test.to_html(max_rows=10)))

size = lambda x: x * 2 - 1  # actual size of position based on predicted p values
compound = lambda x: (1 + x).cumprod() - 1

r = df_test.r
size = df_test.p_hat.apply(size)

daily_rtn = r * size
cum_rtn = compound(daily_rtn)

daily_rtn_momentum = rtn[cum_rtn.index]
cum_rtn_momentum = compound(daily_rtn_momentum)

daily_rtn_momentum_rev = -daily_rtn_momentum
cum_rtn_momentum_rev = compound(daily_rtn_momentum_rev)

daily_rtn_buy_hold = price.BTC.pct_change().loc[cum_rtn.index]
cum_rtn_buy_hold = compound(daily_rtn_buy_hold)



fig1 = plt.figure()

ax1 = fig1.add_subplot(111)
ax1.set_ylabel('Cumulative Return')
cum_rtn_buy_hold.plot(ax=ax1, color='orange', label='Buy & Hold')
cum_rtn_momentum.plot(ax=ax1, color='r', label='Momentum')
cum_rtn_momentum_rev.plot(ax=ax1, color='darkred', label='Mom. Rev.')
cum_rtn.plot(ax=ax1, color='lime', label='Strategy')
ax1.axhline(y=0, ls='--', color='w')
legend1 = plt.legend(frameon=False, ncol=10, loc='center left', bbox_to_anchor=(0, 1.05))
plt.setp(legend1.get_texts(), color='k', size=12)
plt.xticks(rotation=0, ha='center')

ax1 = ax1.twinx()
ax1.grid(False)
ax1.set_ylabel('Position')
size.plot(ax=ax1, lw=1, alpha=.5, color='grey')
plt.tight_layout()

fig2 = plt.figure()

ax2 = fig2.add_subplot(111)
ax2.set_ylabel('Volatility')
daily_rtn_buy_hold.rolling(5).std().plot(ax=ax2, color='orange', label='Buy & Hold')
daily_rtn_momentum.rolling(5).std().plot(ax=ax2, color='r', label='Momentum')
daily_rtn_momentum_rev.rolling(5).std().plot(ax=ax2, color='darkred', label='Mom. Rev.')
daily_rtn.rolling(5).std().plot(ax=ax2, color='lime', label='Strategy')
ax2.axhline(y=0, ls='--', color='w')
legend2 = plt.legend(frameon=False, ncol=10, loc='center left', bbox_to_anchor=(0, 1.05))
plt.setp(legend2.get_texts(), color='k', size=12)
plt.xticks(rotation=0, ha='center')

ax2 = ax2.twinx()
ax2.grid(False)
ax2.set_ylabel('Position')
size.plot(ax=ax2, lw=1, alpha=.5, color='grey')

plt.tight_layout()
plt.show()



win_rate = lambda x: (x > 0).mean()
sharpe = lambda x: x.mean() / x.std() * np.sqrt(360)
mdd = lambda x: (1 - x / x.cummax()).max()

DAILY = [daily_rtn_buy_hold, daily_rtn_momentum, daily_rtn_momentum_rev, daily_rtn]
CUM = [cum_rtn_buy_hold, cum_rtn_momentum, cum_rtn_momentum_rev, cum_rtn]

cr = [x[-1] for x in CUM]
vol = [x.std() for x in DAILY]
wr = [win_rate(x) for x in DAILY]
sr = [sharpe(x) for x in DAILY]
md = [mdd(x + 1) for x in CUM]
result = pd.DataFrame({
    'Cum Return': cr,
    'Volatility': vol,
    'Win%': wr,
    'Sharpe': sr,
    'MDD': md
}, index=['Buy & Hold', 'Momentum', 'Mom. Rev.', 'Strategy'])
result.index.name = 'Statistics'
display(HTML(result.T.to_html(max_rows=10)))

Image('misc/lookback_window.png')

Image('misc/minimum_occurrence.png')

Image('misc/minimum_length.png')

Image('misc/alpha.png')

from misc.backtest import BacktestEngine

be = BacktestEngine()
be.run(
    lookback_window=60,
    minimum_occurrence=50,
    minimum_length=5,
    alpha=.1,
    size=lambda x: 2 * x - 1
)

import time
import random
import requests
import pandas as pd
from bs4 import BeautifulSoup
from datetime import datetime as dt
from randomheaders import LoadHeader as random_header


pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)


random.seed(123)
COLUMNS = ['time', 'title']
INFO = pd.DataFrame(columns=COLUMNS)
IP_LIST = []


def get_proxy():
    return requests.get('http://localhost:5555/get/').text

def delete_proxy(proxy):
    requests.get(f'http://localhost:5555/delete/?proxy={proxy}')

def get_soup(url, parser):
    while True:
        try:
            time.sleep(random.uniform(0, 1))
            proxy = get_proxy()
            html = requests.get(
                url, timeout=(3, 7), headers=random_header(),
                proxies={'http': proxy, 'https': proxy}).text
            return BeautifulSoup(html, parser)
        except Exception:
            delete_proxy(proxy)

def parse_time(timestr):
    raw_time = timestr[10:].split('/')[0].replace('on', '')
    time = dt.strptime(raw_time, '%H:%M  %B %d, %Y  ')
    return time

def get_content(url):
    soup = get_soup(url, 'lxml')
    paragraphs = soup.select('div[class="entry-content"]')[0].select('p')
    content = '\n'.join([para.text for para in paragraphs[:-1]])
    return content

def get_posts(soup):
    if '404 - PAGE NOT FOUND' in soup.text: return None
    posts = soup.select('a[class="stream-article"]')
    titles, times, abstracts, contents = [], [], [], []
    for p in posts:
        meta = p.select('div[class="meta"]')[0]
        titles.append(meta.select('h3')[0].text)
        times.append(meta.select('time')[0]['datetime'])
        abstracts.append(meta.select('p')[0].text)
        contents.append(get_content(p['href']))
    posts = pd.DataFrame({'title': titles, 'abstract': abstracts, 'content': contents}, index=times)
    posts.index = pd.to_datetime(posts.index)
    return posts


all_posts = pd.DataFrame(columns=['title', 'abstract', 'content'])

i = 0
while True:
    try:
        i += 1
        print(f'Crawling page {i}: ', end='')
        url = f'https://www.coindesk.com/category/markets-news/page/{i}'
        soup = get_soup(url, 'lxml')
        posts = get_posts(soup)
        if posts is None: raise EOFError
        all_posts = pd.concat([all_posts, posts])
        print(f'finished ({posts.shape[0]} posts)')
    except IndexError:
        print('failed, retrying', end='\r')
        i -= 1
    except (EOFError, KeyboardInterrupt):
        print('terminated')
        print(f'Total number of posts crawled: {all_posts.shape[0]}')
        all_posts.to_csv('../data/posts.csv')
        exit()

import os
import json
import sys
import requests
from bs4 import BeautifulSoup
import pandas as pd
from _datetime import datetime, date, timedelta


def save_crypto_coins_history(i_rank_start=1, i_rank_end=10, i_coin_file_path='crypto_coins', i_from_date=None, i_to_date=None, i_min_volume=100000, i_coin_markets=[]):
    from_date, to_date = get_from_to_dates(i_from_date, i_to_date)
    rank_range_from_start = i_rank_end - i_rank_start + 1
    coins_ranking_dict = get_coins_current_ranking(i_rank_start, rank_range_from_start, i_min_volume)
    df_coins = pd.DataFrame([])
    for rank, coin in coins_ranking_dict.items():
        if is_coin_in_markets(coin, set(i_coin_markets)):
            df_coins = df_coins.append(get_coins_historical_data(rank, coin, from_date, to_date))
            write_df_to_csv(df_coins, i_coin_file_path + '.csv')


def get_coins_current_ranking(i_start, i_limit, i_min_volume):
    url_coin_list_json = 'https://api.coinmarketcap.com/v1/ticker/?start={}&limit={}'.format(i_start - 1, i_limit)
    page = requests.get(url_coin_list_json)
    json_file = json.loads(page.text)
    coins_dict = {}
    for k in json_file:
        if float(k['24h_volume_usd']) >= i_min_volume:
            coins_dict[k['rank']] = k['id']
    return coins_dict


def get_coins_historical_data(i_rank, i_coin, i_from_date, i_to_date):
    df_coin = get_specific_coin_historical_data(i_coin, i_from_date, i_to_date)
    df_coin['Coin'] = i_coin
    df_coin['Cur. Rank'] = i_rank
    df_coin = pd.concat([df_coin.iloc[:, 7:], df_coin.iloc[:, 0:7]], axis=1, join_axes=[df_coin.index])
    return df_coin


def get_specific_coin_historical_data(i_coin, i_from_date, i_to_date):
    currencies = "https://coinmarketcap.com/currencies/"
    currencies_end = '/historical-data/'
    dates = '?start={}&end={}'.format(i_from_date, i_to_date)
    url = currencies + i_coin + currencies_end + dates
    page = requests.get(url)
    soup = BeautifulSoup(page.text, 'html.parser')
    table = soup.find('table')
    data = {'Date': [], 'Open': [], 'High': [], 'Low': [], 'Close': [], 'Volume': [], 'Market Cap': []}
    try:
        rows = table.findAll('tr')[1:]
        for row in rows:
            cols = row.findAll('td')
            data['Date'].append(cols[0].string)
            data['Open'].append(cols[1].string)
            data['High'].append(cols[2].string)
            data['Low'].append(cols[3].string)
            data['Close'].append(cols[4].string)
            data['Volume'].append(cols[5].string)
            data['Market Cap'].append(cols[6].string)
        coin_data = pd.DataFrame(data)
    except AttributeError:
        print('input parameters not valid')
        sys.exit(13)
    return coin_data


def write_df_to_csv(i_df, i_file):
    try:
        i_df.to_csv(i_file)
    except IOError as e:
        print(e)
        sys.exit(13)


def get_from_to_dates(i_from_date, i_to_date):
    try:
        if i_from_date is None:
            from_date = str(date.today() + timedelta(days=-30))
        else:
            from_date = i_from_date
        from_date = datetime.strptime(from_date, '%Y-%m-%d').strftime('%Y%m%d')

        if i_to_date is None:
            to_date = str(date.today() + timedelta(days=-1))
        else:
            to_date = i_to_date
        to_date = datetime.strptime(to_date, '%Y-%m-%d').strftime('%Y%m%d')

        return from_date, to_date
    except ValueError as e:
        print(e)
        sys.exit(13)


def is_coin_in_markets(i_coin, i_coin_markets_to_search):
    coin_in_markets = False
    coin_markets_url = 'https://coinmarketcap.com/currencies/{}/#markets'.format(i_coin)
    if not i_coin_markets_to_search:
        coin_in_markets = True
    else:
        page = requests.get(coin_markets_url)
        soup = BeautifulSoup(page.text, 'html.parser')
        table = soup.find('table')
        rows = table.findAll('tr')[1:]
        markets = set()
        for row in rows:
            cols = row.findAll('td')
            if cols[1].text is not None:
                markets.add(cols[1].text.upper())

            for market in i_coin_markets_to_search:
                if market.upper() in markets:
                    coin_in_markets = True
                    break

    return coin_in_markets


if __name__ == '__main__':
    symbols = [
        ('ANC', 'anoncoin'),
        ('BTB', 'bitbar'),
        ('BTC', 'bitcoin'),
        ('DGC', 'digitalcoin'),
        ('DOGE', 'dogecoin'),
        ('FRC', 'freicoin'),
        ('FTC', 'feathercoin'),
        ('IXC', 'ixcoin'),
        ('LTC', 'litecoin'),
        ('MEC', 'megacoin'),
        ('MNC', 'mincoin'),
        ('NMC', 'namecoin'),
        ('NVC', 'novacoin'),
        ('NXT', 'nxt'),
        ('OMNI', 'omni'),
        ('PPC', 'peercoin'),
        ('QRK', 'quark'),
        ('WDC', 'worldcoin'),
        ('XPM', 'primecoin'),
        ('XRP', 'ripple'),
        ('ZET', 'zetacoin'),
    ]
    price = dict()
    mkcap = dict()
    for sym, i_coin in symbols:
        if not os.path.isfile(f'{i_coin}.csv'):
            try:
                print(f'Downloading {i_coin:>12}', end=': ')
                df = get_specific_coin_historical_data(i_coin, '20120101', '20190520')[::-1]
                print(df.Date.iloc[0], df.Date.iloc[-1])
                df.to_csv(f'{i_coin}.csv')
            except:
                print('failed')
                pass

    for sym, i_coin in symbols:
        print(f'Loading {i_coin:>12}', end=': ')
        df = pd.read_csv(f'{i_coin}.csv').set_index('Date')
        df.index = pd.to_datetime(df.index)
        print(df.index[0].date(), df.index[-1].date())
        price[sym] = df['Close']
        mkcap[sym] = df['Market Cap']

    def convert(x):
        s = str(x).replace(',', '')
        try:
            return float(s)
        except:
            if s == '-':
                return float('nan')

    price = pd.DataFrame(price).dropna().applymap(convert).fillna(method='ffill')
    price.index.name = None
    price.to_csv('../data/price.csv')

    mkcap = pd.DataFrame(mkcap).dropna().applymap(convert).fillna(method='ffill')
    mkcap.index.name = None
    mkcap.to_csv('../data/mkcap.csv')

    print('Task finished')

import numpy as np
import pandas as pd
from collections import Counter
from scipy.optimize import minimize
from wordcloud import STOPWORDS


class BacktestEngine:
    def __init__(self):
        df = pd.read_csv('data/posts.csv', index_col=0)
        df.index = pd.to_datetime(df.index).date
        df_eod = df.copy()
        df_eod['date'] = pd.to_datetime(df_eod.index).date
        g = df_eod.groupby('date')
        title = g.title.apply(' '.join)
        abstract = g.abstract.apply(' '.join)
        content = g.content.apply(lambda x: '\n\n'.join(map(str, x.values)))
        df_eod = pd.concat([title, abstract, content], axis=1)
        df_eod.index = pd.to_datetime(df_eod.index)
        df_eod.index.name = None
        df_eod.title = df_eod.title.str.lower()
        df_eod.abstract = df_eod.abstract.str.lower()
        df_eod.content = df_eod.content.str.lower()
        remove_abbr = lambda x: ' '.join([_.split('’')[0] for _ in x.split()])
        df_eod = df_eod.applymap(remove_abbr)
        alph_space = lambda x: (ord('a') <= ord(x) <= ord('z')) or x == ' '
        keep_alph = lambda x: ''.join(filter(alph_space, x))
        df_eod = df_eod.applymap(keep_alph)
        replace_stop_words = lambda x: ' '.join(_ for _ in x.split() if _ not in STOPWORDS)
        df_eod = df_eod.applymap(replace_stop_words)
        df_eod = df_eod.applymap(str.split).applymap(sorted)
        price = pd.read_csv('data/price.csv', index_col=0)
        price.index = pd.to_datetime(price.index)
        mkcap = pd.read_csv('data/mkcap.csv', index_col=0)
        mkcap.index = pd.to_datetime(mkcap.index)
        self.data = (price, mkcap, df_eod)  # store data for later usage

    def run(self, lookback_window=60, minimum_occurrence=50, minimum_length=5, alpha=.1, size=lambda x: 2 * x - 1):
        n = 3  # number of coins to hold (long/short)
        d = lookback_window  # number of days to train
        price, mkcap, df_eod = self.data  # data stored for training (reusable)

        rtn = price.pct_change().fillna(0)
        rtn.index.name = None
        temp = (rtn * (mkcap.agg('argsort') < 10)).rolling(d).sum().T
        coins = pd.DataFrame(
            [temp.nlargest(n, t).index.tolist() + temp.nsmallest(n, t).index.tolist() for t in rtn.index],
            index=rtn.index).shift().dropna()  # shift to avoid forward-looking bias

        coef = np.array([1] * n + [-1] * n) / n
        rtn = pd.DataFrame([rtn.loc[t, coins.loc[t].values] @ coef for t in coins.index], index=coins.index)[0]
        rtn.name = 'r'

        df_total = df_eod[['content']].join(rtn, how='inner').dropna()
        df_total['p'] = np.argsort(df_total.iloc[:, 1]) / (df_total.shape[0] - 1)
        df_total.columns = ['words', 'r', 'p']

        test_split = 0.3
        len_total = len(df_total)
        len_test = round(len_total * test_split)
        len_train = len_total - len_test

        df_train = df_total.iloc[:len_train, :]
        df_test = df_total.iloc[-len_test:, :]

        words_all = sorted(list(set(w for words in df_train.words for w in words)))
        words_dict = {w: [] for w in words_all}

        for i in range(df_train.shape[0]):
            words, r, _ = df_train.iloc[i]
            for w in words: words_dict[w].append(r > 0)

        words_df = pd.DataFrame({
            'word': words_all,
            'r': [np.mean(words_dict[w]) for w in words_all],
            'k': [len(words_dict[w]) for w in words_all]
        }, index=range(len(words_all))).sort_values(by='r')

        words_df = words_df[words_df.k >= minimum_occurrence][words_df.word.apply(len) >= minimum_length].reset_index(drop=True)
        words_df['p'] = np.argsort(words_df.r) / (len(words_df) - 1)

        pos_words = words_df[words_df.p > 1 - alpha].word.values.tolist()
        neg_words = words_df[words_df.p < alpha].word.values.tolist()

        keywords = pos_words + neg_words

        def count_words(words):
            c = Counter(words)
            d = [c[w] for w in keywords]
            return np.array(d)

        df_train['d'] = df_train.words.apply(count_words).values

        p = df_train.p.values
        W = np.vstack([p, 1 - p]).T
        D = np.vstack(df_train.d.values)
        D = (D.T / D.sum(axis=1)).T
        D[np.isnan(D)] = 0
        O_pos, O_neg = np.maximum(np.linalg.pinv(W.T @ W) @ (W.T @ D), 0)
        O_pos = (O_pos / O_pos.sum())
        O_neg = (O_neg / O_neg.sum())

        def pen_ll(p, d, lamda):
            return (d * np.log(p * O_pos + (1 - p) * O_neg)).sum() + lamda * np.log(p * (1 - p))

        def predict(d, lamda):
            return minimize(lambda p: -pen_ll(p, d, lamda), x0=.5, method='Nelder-Mead').x[0]

        def mse_p(lamda):
            p_hat = np.array([predict(d, lamda) for d in df_train.d])
            return ((df_train.p.values - p_hat)**2).mean()

        best_lamda = minimize(mse_p, x0=0, method='Nelder-Mead', bounds=(0, None)).x[0]

        df_test['d'] = df_test.words.apply(count_words).values
        df_test['p_hat'] = [predict(d, best_lamda) for d in df_test.d]
        df_test = df_test[['words', 'r', 'p', 'p_hat', 'd']]

        compound = lambda x: (1 + x).cumprod() - 1

        r = df_test.r
        size = df_test.p_hat.apply(size)

        daily_rtn = r * size
        cum_rtn = compound(daily_rtn)
        daily_rtn_momentum = rtn[cum_rtn.index]
        cum_rtn_momentum = compound(daily_rtn_momentum)
        daily_rtn_momentum_rev = -daily_rtn_momentum
        cum_rtn_momentum_rev = compound(daily_rtn_momentum_rev)
        daily_rtn_buy_hold = price.BTC.pct_change().loc[cum_rtn.index]
        cum_rtn_buy_hold = compound(daily_rtn_buy_hold)

        win_rate = lambda x: (x > 0).mean()
        sharpe = lambda x: x.mean() / x.std() * np.sqrt(360)
        mdd = lambda x: (1 - x / x.cummax()).max()

        DAILY = [daily_rtn_buy_hold, daily_rtn_momentum, daily_rtn_momentum_rev, daily_rtn]
        CUM = [cum_rtn_buy_hold, cum_rtn_momentum, cum_rtn_momentum_rev, cum_rtn]
        wr = [win_rate(x) for x in DAILY]
        sr = [sharpe(x) for x in DAILY]
        md = [mdd(x + 1) for x in CUM]
        result = pd.DataFrame({'Win%': wr, 'Sharpe': sr, 'MDD': md}, index=['Buy & Hold', 'Momentum', 'Mom. Rev.', 'Strategy'])
        result.index.name = 'Statistics'

        return result

if not os.path.isfile('misc/results.csv'):
    from misc.backtest import BacktestEngine
    
    be = BacktestEngine()
    
    size_continuous = lambda x: 2 * x - 1
    size_binary = lambda x: 1 if x > .5 else -1
    size_trinary = lambda x: 1 if x > 2 / 3 else 0 if x > 1 / 3 else -1
    size_funcs = {
        'continuous': size_continuous,
        'binary': size_binary,
        'trinary': size_trinary
    }
    
    params = dict(
        lookback_window=[20, 60, 120],
        minimum_occurrence=[10, 50, 100],
        minimum_length=[1, 3, 5],
        alpha=[.05, .10, .20],
        size=['continuous', 'binary', 'trinary']
    )
    
    def grid(g):
        keys, values = zip(*g.items())
        for v in product(*values):
            yield dict(zip(keys, v))
    
    P = namedtuple('Params', params.keys())
    
    results = {}
    print('Initializing...', end='\r')
    for i, param in enumerate(grid(params)):
        p = P(**param)
        param['size'] = size_funcs[param['size']]
        result = be.run(**param)
        results[p] = result
        wr, sr, md = result.T.Strategy
        print(f'[{(i + 1):>3d}] {p}:\n      Win%={wr:.2%}, Sharpe={sr:.2f}, MDD={md:.2%}')
    
    df_results = pd.concat(results).reset_index()
    df_results.columns = ['lookback_window', 'minimum_occurrence', 'minimum_length',
                          'alpha', 'size', 'strategy', 'win_rate', 'sharpe', 'mdd']
    df_results.to_csv('misc/results.csv')

else:
    df_results = pd.read_csv('misc/results.csv', index_col=0)


display(HTML(df_results.to_html(max_rows=10)))

df_results_strategy = df_results[df_results.strategy=='Strategy'].drop('strategy', axis=1).melt(
    id_vars=['lookback_window', 'minimum_occurrence', 'minimum_length', 'alpha', 'size'],
    value_vars=['win_rate', 'sharpe', 'mdd'], var_name='statistic'
)
display(HTML(df_results_strategy.to_html(max_rows=10)))

sns.set({
    'font.size': 12, 'axes.labelsize': 12, 'legend.fontsize': 12, 
    'axes.titlesize': 12, 'xtick.labelsize': 12, 'ytick.labelsize': 12
})
sns.set_style({'axes.facecolor': 'white', 'axes.edgecolor': 'black', 'axes.linewidth': 2})

for hue in ['lookback_window', 'minimum_occurrence', 'minimum_length', 'alpha']:
    sns.catplot(
        x='size', y='value', hue=hue, col='statistic', kind='bar', capsize=.1, errwidth=2,
        sharey=False, palette=['red', 'orange', 'gold'], data=df_results_strategy
    )
    plt.savefig(f'misc/{hue}.png')