import argparse
import os
import time

import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.backends.cudnn as cudnn
import torchvision

from model import Net

parser = argparse.ArgumentParser(description="Train on market1501")
parser.add_argument("--data-dir",default='data',type=str)
parser.add_argument("--no-cuda",action="store_true")
parser.add_argument("--gpu-id",default=0,type=int)
parser.add_argument("--lr",default=0.1, type=float)
parser.add_argument("--interval",'-i',default=20,type=int)
parser.add_argument('--resume', '-r',action='store_true')
args = parser.parse_args()

# device
device = "cuda:{}".format(args.gpu_id) if torch.cuda.is_available() and not args.no_cuda else "cpu"
if torch.cuda.is_available() and not args.no_cuda:
    cudnn.benchmark = True

# data loading
root = args.data_dir
train_dir = os.path.join(root,"train")
test_dir = os.path.join(root,"test")
transform_train = torchvision.transforms.Compose([
    torchvision.transforms.RandomCrop((128,64),padding=4),
    torchvision.transforms.RandomHorizontalFlip(),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
transform_test = torchvision.transforms.Compose([
    torchvision.transforms.Resize((128,64)),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
trainloader = torch.utils.data.DataLoader(
    torchvision.datasets.ImageFolder(train_dir, transform=transform_train),
    batch_size=64,shuffle=True
)
testloader = torch.utils.data.DataLoader(
    torchvision.datasets.ImageFolder(test_dir, transform=transform_test),
    batch_size=64,shuffle=True
)
num_classes = max(len(trainloader.dataset.classes), len(testloader.dataset.classes))

# net definition
start_epoch = 0
net = Net(num_classes=num_classes)
if args.resume:
    assert os.path.isfile("./checkpoint/ckpt.t7"), "Error: no checkpoint file found!"
    print('Loading from checkpoint/ckpt.t7')
    checkpoint = torch.load("./checkpoint/ckpt.t7")
    # import ipdb; ipdb.set_trace()
    net_dict = checkpoint['net_dict']
    net.load_state_dict(net_dict)
    best_acc = checkpoint['acc']
    start_epoch = checkpoint['epoch']
net.to(device)

# loss and optimizer
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(), args.lr, momentum=0.9, weight_decay=5e-4)
best_acc = 0.

# train function for each epoch
def train(epoch):
    print("\nEpoch : %d"%(epoch+1))
    net.train()
    training_loss = 0.
    train_loss = 0.
    correct = 0
    total = 0
    interval = args.interval
    start = time.time()
    for idx, (inputs, labels) in enumerate(trainloader):
        # forward
        inputs,labels = inputs.to(device),labels.to(device)
        outputs = net(inputs)
        loss = criterion(outputs, labels)

        # backward
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # accumurating
        training_loss += loss.item()
        train_loss += loss.item()
        correct += outputs.max(dim=1)[1].eq(labels).sum().item()
        total += labels.size(0)

        # print
        if (idx+1)%interval == 0:
            end = time.time()
            print("[progress:{:.1f}%]time:{:.2f}s Loss:{:.5f} Correct:{}/{} Acc:{:.3f}%".format(
                100.*(idx+1)/len(trainloader), end-start, training_loss/interval, correct, total, 100.*correct/total
            ))
            training_loss = 0.
            start = time.time()

    return train_loss/len(trainloader), 1.- correct/total

def test(epoch):
    global best_acc
    net.eval()
    test_loss = 0.
    correct = 0
    total = 0
    start = time.time()
    with torch.no_grad():
        for idx, (inputs, labels) in enumerate(testloader):
            inputs, labels = inputs.to(device), labels.to(device)
            outputs = net(inputs)
            loss = criterion(outputs, labels)

            test_loss += loss.item()
            correct += outputs.max(dim=1)[1].eq(labels).sum().item()
            total += labels.size(0)

        print("Testing ...")
        end = time.time()
        print("[progress:{:.1f}%]time:{:.2f}s Loss:{:.5f} Correct:{}/{} Acc:{:.3f}%".format(
                100.*(idx+1)/len(testloader), end-start, test_loss/len(testloader), correct, total, 100.*correct/total
            ))

    # saving checkpoint
    acc = 100.*correct/total
    if acc > best_acc:
        best_acc = acc
        print("Saving parameters to checkpoint/ckpt.t7")
        checkpoint = {
            'net_dict':net.state_dict(),
            'acc':acc,
            'epoch':epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(checkpoint, './checkpoint/ckpt.t7')

    return test_loss/len(testloader), 1.- correct/total

# plot figure
x_epoch = []
record = {'train_loss':[], 'train_err':[], 'test_loss':[], 'test_err':[]}
fig = plt.figure()
ax0 = fig.add_subplot(121, title="loss")
ax1 = fig.add_subplot(122, title="top1err")
def draw_curve(epoch, train_loss, train_err, test_loss, test_err):
    global record
    record['train_loss'].append(train_loss)
    record['train_err'].append(train_err)
    record['test_loss'].append(test_loss)
    record['test_err'].append(test_err)

    x_epoch.append(epoch)
    ax0.plot(x_epoch, record['train_loss'], 'bo-', label='train')
    ax0.plot(x_epoch, record['test_loss'], 'ro-', label='val')
    ax1.plot(x_epoch, record['train_err'], 'bo-', label='train')
    ax1.plot(x_epoch, record['test_err'], 'ro-', label='val')
    if epoch == 0:
        ax0.legend()
        ax1.legend()
    fig.savefig("train.jpg")

# lr decay
def lr_decay():
    global optimizer
    for params in optimizer.param_groups:
        params['lr'] *= 0.1
        lr = params['lr']
        print("Learning rate adjusted to {}".format(lr))

def main():
    for epoch in range(start_epoch, start_epoch+40):
        train_loss, train_err = train(epoch)
        test_loss, test_err = test(epoch)
        draw_curve(epoch, train_loss, train_err, test_loss, test_err)
        if (epoch+1)%20==0:
            lr_decay()

if __name__ == '__main__':
    main()


import pandas as pd
import cloudscraper
import re
import os
from pathlib import Path
import json
import string
import statistics
import operator
from bs4 import BeautifulSoup
from pprint import pprint
from datetime import datetime
import unidecode
try:
    # ASSUME THIS IS A SUBMODULE IN A PACKAGE
    from . import showdown_constants as sc
except ImportError:
    # USE LOCAL IMPORT
    import showdown_constants as sc

class BaseballReferenceScraper:

# ------------------------------------------------------------------------
# INIT

    def __init__(self, name, year):
        is_full_career = year.upper() == 'CAREER'
        if is_full_career:
            year = 'CAREER'
        elif '(' in year:
            year = int(year[:4])
        elif '-' in year:
            # RANGE OF YEARS
            years = year.split('-')
            year_start = int(years[0].strip())
            year_end = int(years[1].strip())
            year = list(range(year_start,year_end+1))
        elif '+' in year:
            years = year.split('+')
            year = [int(x.strip()) for x in years]

        self.name = name
        self.error = None

        # PARSE MULTI YEARS
        if isinstance(year, list):
            is_multi_value_list = len(year) > 1
            self.is_multi_year = is_multi_value_list
            self.years = year
        else:
            self.is_multi_year = False
            self.years = [year]

        # CHECK FOR TEAM OVERRIDE
        self.team_override = None
        for team_id in sc.TEAM_COLOR_PRIMARY.keys():
            team_match = f'({team_id})' in self.name.upper()
            if team_match and not self.is_multi_year and not is_full_career:
                self.team_override = team_id

        # CHECK FOR TYPE OVERRIDE
        self.pitcher_override = '(PITCHER)' if '(PITCHER)' in self.name.upper() else None
        self.hitter_override = '(HITTER)' if '(HITTER)' in self.name.upper() else None

        # CHECK FOR BASEBALL REFERENCE ID
        self.is_name_a_bref_id = any(char.isdigit() for char in name)
        if self.is_name_a_bref_id:
            self.baseball_ref_id = name.split(' ')[0].lower()
        else:
            # CHECK DEFAULT CSV
            default_bref_id = self.__check_for_default_bref_id(name)
            if default_bref_id:
                self.baseball_ref_id = default_bref_id
            else:
                # SEARCH GOOGLE
                self.baseball_ref_id = self.search_google_for_b_ref_id(name, year)

        self.first_initial = self.baseball_ref_id[:1]

        # STORE BASEBALL REFERENCE ID USED FOR TRENDS DATA QUERY
        if "ohtansh01" in self.baseball_ref_id:
            self.baseball_ref_id_used_for_trends = f"ohtansh01 {'(Pitcher)' if self.pitcher_override else '(Hitter)'}"
        else:
            self.baseball_ref_id_used_for_trends = self.baseball_ref_id

# ------------------------------------------------------------------------
# SCRAPE WEBSITES

    def __check_for_default_bref_id(self, name):
        """Check for player name string in default csv with names and bref ids

        Purpose is to see if bot can avoid needing to scrape Google.

        Args:
          name: Full name of Player

        Returns:
          BrefId (If found), otherwise None
        """
        name_cleaned = name.replace("'", "").replace(".", "").lower().strip()
        player_id_filepath = os.path.join(Path(os.path.dirname(__file__)),'player_ids.csv')
        player_ids_pd = pd.read_csv(player_id_filepath)
        player_ids_pd_filtered = player_ids_pd.loc[player_ids_pd['name'] == name_cleaned]
        results_count = len(player_ids_pd_filtered)

        if results_count == 1:
            bref_id = player_ids_pd_filtered['brefid'].max()
            return bref_id if len(bref_id) > 0 else None
        else:
            return None

    def search_google_for_b_ref_id(self, name, year):
        """Convert name to a baseball reference Player ID.

        Goes by rank on google search for search: 'baseball reference "name" "year"'.
        This is needed to find the player_id used to scrape baseball reference.

        Args:
          name: Full name of Player
          year: Year for Player stats

        Raises:
          AttributeError: Cannot find Baseball Ref Page for name/year combo.

        Returns:
          Baseball Reference Id String (ex: 'Mike Piazza' -> 'piazzmi01').
        """

        name_for_url = name.replace(' ','+')

        # SET RESULT LIMIT TO INCREASE SPEED AND MINIMIZE ERRORS
        result_limit = 4
        url_search_name_and_year = 'https://www.google.com/search?q=baseball+reference+{}+{}+stats&num={}'.format(name_for_url, year, result_limit)
        soup_search_name_and_year = self.__soup_for_url(url_search_name_and_year)

        last_initial = self.__name_last_initial(name)
        search_results = soup_search_name_and_year.find_all("a", href=re.compile("https://www.baseball-reference.com/players/{}/".format(last_initial)))

        # GOOGLE WILL BLOCK REQUESTS IF IT DETECTS THE BOT.
        if len(soup_search_name_and_year.find_all(text="Why did this happen?")) > 0:
            self.error = 'Google has an overload of requests coming from your IP Address. Wait a few minutes and try again.'
            raise RuntimeError(self.error)

        # NO BASEBALL REFERENCE RESULTS FOR THAT NAME AND YEAR
        if search_results == []:
            self.error = f'Cannot Find BRef Page for {name} in {year}'
            raise AttributeError(self.error)

        top_result_url = search_results[0]["href"]
        player_b_ref_id = top_result_url.split('.shtml')[0].split('/')[-1]
        return player_b_ref_id

    def __soup_for_url(self, url, is_baseball_ref_page=False):
        """Converts html to BeautifulSoup object.

        Args:
          url: URL for the request.
          is_baseball_ref_page: Bool to denote if the URL is a link to baseball-reference.com.

        Returns:
          BeautifulSoup object with website data.
        """

        html = self.html_for_url(url)

        if is_baseball_ref_page:
            # A LOT OF BREF STATS ARE LOADED THROUGH JAVASCRIPT, BUT CAN BE FOUND IN COMMENTED HTML
            html = html.replace("<!--","")

        soup = BeautifulSoup(html, "lxml")

        return soup

    def html_for_url(self, url):
        """Make request for URL to get HTML

        Args:
          url: URL for the request.

        Raises:
          TimeoutError: 502 - BAD GATEWAY
          TimeoutError: 429 - TOO MANY REQUESTS TO BASEBALL REFERENCE

        Returns:
          HTML string for URL request.
        """

        scraper = cloudscraper.create_scraper()
        html = scraper.get(url)

        if html.status_code == 502:
          self.error = "502 - BAD GATEWAY"
          raise TimeoutError(self.error)
        if html.status_code == 429:
          self.error = "429 - TOO MANY REQUESTS TO BASEBALL REFERENCE. PLEASE TRY AGAIN IN A FEW MINUTES."
          raise TimeoutError(self.error)

        return html.text

# ------------------------------------------------------------------------
# PARSING HTML

    def player_statline(self):
        """Scrape baseball-reference.com to get all relevant stats

        Args:
          None

        Returns:
          Dict with all categories and stats.
        """

        # STANDARD STATS PAGE HAS MOST RELEVANT STATS NEEDED
        url_for_homepage_stats = 'https://www.baseball-reference.com/players/{}/{}.shtml'.format(self.first_initial,self.baseball_ref_id)
        soup_for_homepage_stats = self.__soup_for_url(url_for_homepage_stats, is_baseball_ref_page=True)

        master_stats_dict = {}
        is_full_career = self.years == ['CAREER']
        for year in self.years:
            # DEFENSE
            stats_dict = {'bref_id': self.baseball_ref_id, 'bref_url': url_for_homepage_stats}
            positional_fielding = self.positional_fielding(soup_for_homepage_stats=soup_for_homepage_stats,year=year)
            stats_dict.update(positional_fielding)

            # HAND / TYPE
            type = self.type(positional_fielding,year=year)

            # NATIONALITY
            nationality = self.nationality(soup_for_homepage_stats=soup_for_homepage_stats)

            # ROOKIE STATUS YEAR
            rookie_status_year = self.rookie_status_year(soup_for_homepage_stats=soup_for_homepage_stats)

            # WAR
            stats_dict.update({'bWAR': self.__bWar(soup_for_homepage_stats, year, type)})

            # COVER THE CASE OF NOT HAVING DATA FOR A PARTICULAR YEAR (EX: JOHN SMOLTZ 2000)
            if not type:
                continue

            name = self.player_name(soup_for_homepage_stats)
            years_played = self.__years_played_list(type=type, homepage_soup=soup_for_homepage_stats)
            stats_dict['type'] = type
            stats_dict['hand'] = self.hand(soup_for_homepage_stats, type)
            stats_dict['hand_throw'] = self.hand(soup_for_homepage_stats, type='Pitcher') # ALWAYS PASS PITCHER TO STORE THROWING HAND
            stats_dict['name'] = name
            stats_dict['years_played'] = years_played
            stats_dict['nationality'] = nationality
            stats_dict['rookie_status_year'] = rookie_status_year
            stats_dict['is_rookie'] = False if is_full_career or self.is_multi_year or rookie_status_year is None else ( int(year) <= rookie_status_year )

            # HITTING / HITTING AGAINST
            advanced_stats = self.advanced_stats(type,year=year)
            stats_dict.update(advanced_stats)

            # ICON THRESHOLDS
            icon_threshold_bools = self.__add_icon_threshold_bools(type=type, year=year, stats_dict=stats_dict, homepage_soup=soup_for_homepage_stats)
            stats_dict.update(icon_threshold_bools)

            # FILL BLANKS IF NEEDED
            keys_to_fill = ['SH','GIDP','IBB','HBP','SF']
            for key in keys_to_fill:
                if key in stats_dict.keys():
                    raw_value = stats_dict[key]
                    if len(str(raw_value)) == 0:
                        stats_dict[key] = 0

            if self.__is_pitcher_from_1901_to_1918(year=year,type=type):
                team_id = self.__team_w_most_games_played(type, soup_for_homepage_stats, years_filter_list=[int(year)])
                if team_id == 'TOT':
                    team_id = self.__parse_team_after_trade(soup_for_homepage_stats=soup_for_homepage_stats,year=year,type=type)
                stats_dict['team_ID'] = team_id

            # FULL CAREER
            if is_full_career:
                stats_dict['team_ID'] = self.__team_w_most_games_played(type, soup_for_homepage_stats)
                sprint_speed_list = []
                for year in years_played:
                    sprint_speed = self.statcast_sprint_speed(name=name, year=year)
                    if sprint_speed:
                        sprint_speed_list.append(sprint_speed)
                if len(sprint_speed_list) > 0:
                    stats_dict['sprint_speed'] = sum(sprint_speed_list) / len(sprint_speed_list)
                else:
                    stats_dict['sprint_speed'] = None
            else:
                stats_dict['sprint_speed'] = self.statcast_sprint_speed(name=name, year=year)

            # OUTS ABOVE AVERAGE (2016+)
            if 'outs_above_avg' not in stats_dict.keys(): # ONLY NEEDS TO RUN ONCE FOR MULTI-YEAR
                years_list = years_played if is_full_career else self.years
                years_as_ints = [int(y) for y in years_list]
                stats_dict['outs_above_avg'] = self.statcast_outs_above_average_dict(name=name, years=years_as_ints)

            # DERIVE 1B
            triples = 0 if '3B' not in stats_dict.keys() else int(stats_dict['3B'])
            doubles = 0 if '2B' not in stats_dict.keys() else int(stats_dict['2B'])
            stats_dict['1B'] = int(stats_dict['H']) - int(stats_dict['HR']) - triples - doubles
            master_stats_dict[year] = stats_dict

        if self.is_multi_year:
            # COMBINE INDIVIDUAL YEAR DATA
            stats_dict.update(self.__combine_multi_year_dict(master_stats_dict))
            stats_dict['team_ID'] = self.__team_w_most_games_played(type, soup_for_homepage_stats, years_filter_list=self.years)
            return stats_dict
        else:
            return stats_dict

    def positional_fielding(self, soup_for_homepage_stats, year):
        """Parse standard fielding metrics (tzr, games_played).

        Args:
          soup_for_homepage_stats: BeautifulSoup object with all stats on homepage.
          year: Year for Player stats

        Returns:
          Dict with name, tzr, and games played per position.
        """
        is_full_career = year == 'CAREER'
        if is_full_career:
            fielding_table = soup_for_homepage_stats.find('div', attrs = {'id': 'div_standard_fielding'})
            fielding_table_footer = fielding_table.find('tfoot')
            fielding_metrics_by_position = fielding_table_footer.find_all('tr')
        else:
            fielding_metrics_by_position = soup_for_homepage_stats.find_all('tr', attrs = {'id': '{}:standard_fielding'.format(year)})

        # POSITIONAL TZR
        all_positions = {}
        for index, position_info in enumerate(fielding_metrics_by_position, 1):
            # PARSE POSITION ATTRIBUTES
            is_position_found = position_info.find('td', attrs={'data-stat':'pos'})
            if is_position_found:
                position_name = position_info.find('td', attrs={'data-stat':'pos'}).get_text()
                games_played = position_info.find('td',attrs={'class':'right','data-stat':'G'}).get_text()

                if position_name != 'TOT':
                    # DRS (2003+)
                    drs_object = position_info.find('td',attrs={'class':'right','data-stat':'bis_runs_total'})
                    drs_rating = drs_object.get_text() if drs_object else None
                    drs_rating = None if (drs_rating or '') == '' else drs_rating

                    # ACCOUNT FOR SHORTENED OR ONGOING SEASONS
                    use_stat_per_yr = False
                    if is_full_career:
                        use_stat_per_yr = True
                    else:
                        today = datetime.today()
                        card_year_end_date = datetime(int(year), 10, 1)
                        is_year_end_date_before_today = today < card_year_end_date
                        drs_is_above_0 = int(drs_rating) > 0 if drs_rating else False
                        use_stat_per_yr = (str(year) == '2020' or is_year_end_date_before_today) and drs_is_above_0

                    if use_stat_per_yr:
                        drs_object = position_info.find('td',attrs={'class':'right','data-stat':'bis_runs_total_per_season'})
                        drs_rating = drs_object.get_text() if drs_object != None else 0
                        drs_rating = 0 if drs_rating == '' else int(drs_rating)

                    # TOTAL ZONE (1953-2003)
                    suffix = '_per_season' if use_stat_per_yr else ''
                    total_zone_object = position_info.find('td',attrs={'class':'right','data-stat':f'tz_runs_total{suffix}'})
                    total_zone_rating = total_zone_object.get_text() if total_zone_object else None
                    total_zone_rating = None if (total_zone_rating or '') == '' else total_zone_rating

                    # UPDATE POSITION DICTIONARY
                    position_dict = {
                        'Position{}'.format(index): position_name,
                        'gPosition{}'.format(index): games_played if games_played != '' else 0,
                        'tzPosition{}'.format(index): total_zone_rating,
                        'drsPosition{}'.format(index): drs_rating
                    }
                    all_positions.update(position_dict)

        # GET DEFENSIVE WAR IN CASE OF LACK OF TZR AVAILABILITY FOR SEASONS < 1952
        # UPDATE: SCRAPE TOTAL WAR AS WELL
        try:
            if is_full_career:
                summarized_header = self.__get_career_totals_row(div_id='div_batting_value',soup_object=soup_for_homepage_stats)
                num_seasons = float(summarized_header.find('th').get_text().split(" ")[0])
                dwar_object = summarized_header.find('td',attrs={'data-stat':'WAR_def'})
                dwar_rating = float(dwar_object.get_text()) if dwar_object != None else 0
                # USE AVG FOR CAREER
                dwar_rating = round(dwar_rating / num_seasons,2)
            else:
                player_value = soup_for_homepage_stats.find('tr', attrs = {'id': 'batting_value.{}'.format(year)})
                dwar_object = player_value.find('td',attrs={'class':'right','data-stat':'WAR_def'})
                dwar_rating = dwar_object.get_text() if dwar_object != None else 0
            all_positions.update({'dWAR': dwar_rating})
        except:
            all_positions.update({'dWAR': 0})

        return all_positions

    def __bWar(self, soup_for_homepage_stats, year, type):
        """Parse bWAR (baseball reference WAR)

        Args:
          soup_for_homepage_stats: BeautifulSoup object with all stats on homepage.
          year: Year for Player stats
          type: Pitcher or Hitter

        Returns:
          Float bWAR value
        """
        is_full_career = year == 'CAREER'
        is_pitcher = type == 'Pitcher'
        type_string = 'pitching' if is_pitcher else 'batting'
        war_type_suffix = '_pitch' if is_pitcher else ''
        try:
            if is_full_career:
                summarized_header = self.__get_career_totals_row(div_id=f"div_{type_string}_value",soup_object=soup_for_homepage_stats)
                # TOTAL WAR
                war_object = summarized_header.find('td',attrs={'data-stat':f"WAR{war_type_suffix}"})
                war_rating = float(war_object.get_text()) if war_object != None else 0
            else:
                player_value = soup_for_homepage_stats.find('tr', attrs = {'id': f"{type_string}_value.{year}"})
                # TOTAL WAR
                war_object = player_value.find('td',attrs={'class':'right','data-stat':f"WAR{war_type_suffix}"})
                war_rating = war_object.get_text() if war_object != None else 0
            return war_rating
        except:
            return 0.0

    def hand(self, soup_for_homepage_stats, type):
        """Parse hand of player.

        Batting Hand for hitter.
        Throwing Hand for pitcher.

        Args:
          soup_for_homepage_stats: BeautifulSoup object with all stats from homepage.

        Raises:
          ValueError: No metadata found.

        Returns:
          'Right', 'Left', or 'Both' string.
        """

        metadata = soup_for_homepage_stats.find('div', attrs = {'id': 'meta'})

        if metadata is None:
            self.error = 'No player metadata found'
            raise ValueError(self.error)
        # FIND THE HAND TAG IN THE METADATA LIST
        for strong_tag in metadata.find_all('strong'):
            hand_tag = 'Bats: ' if type == 'Hitter' else 'Throws: '
            if strong_tag.text == hand_tag:
                hand = strong_tag.next_sibling.replace('•','').rstrip()
                hand = 'Both' if hand == 'Unknown' else hand
                return hand

    def player_name(self, soup_for_homepage_stats):
        """Parse player name from baseball reference

        Args:
          soup_for_homepage_stats: BeautifulSoup object with all stats from homepage.

        Raises:
          ValueError: No player name found.

        Returns:
          'Right', 'Left', or 'Both' string.
        """

        metadata = soup_for_homepage_stats.find('div', attrs = {'id': 'meta'})

        if metadata is None:
            self.error = 'No player name found'
            raise ValueError(self.error)
        # FIND THE ITEMPROP TAG
        name_object = metadata.find('h1', attrs = {'itemprop': 'name'})
        if name_object is None:
            name_string = metadata.find('span').get_text()
        else:
            name_string = name_object.find('span').get_text()
        return name_string

    def nationality(self, soup_for_homepage_stats) -> str:
        """Parse player's nationality

        Args:
          soup_for_homepage_stats: BeautifulSoup object with all stats from homepage.

        Raises:
          ValueError: No player nationality found.

        Returns:
          'Right', 'Left', or 'Both' string.
        """

        # FIND METADATA DIV
        metadata = soup_for_homepage_stats.find('div', attrs = {'id': 'meta'})
        if metadata is None:
            self.error = "No player nationality found"
            raise ValueError(self.error)

        # FIND ALL CLASSES WITH THE KEYWORD "f-i" IN THE CLASS NAME.
        #   EX: <span class="f-i f-pr" style="">pr</span>
        # WHEN A PLAYER HAS 2 NATIONALITIES, CHOSE THE BIRTH COUNTRY
        for strong_tag in metadata.find_all('span', { "class": re.compile('.*f-i.*') }):
            country_abbr = strong_tag.text
            if country_abbr:
                return country_abbr.upper()

        # NO NATIONALITY WAS FOUND, RETURN NONE
        return None

    def rookie_status_year(self, soup_for_homepage_stats) -> int:
        """Parse the player's rookie status end year. Used for the "R" icon.

        Args:
          soup_for_homepage_stats: BeautifulSoup object with all stats from homepage.

        Returns:
          Int for the last year of rookie eligibility for the player.
        """

        # FIND METADATA DIV
        metadata = soup_for_homepage_stats.find('div', attrs = {'id': 'meta'})
        if metadata is None:
            return None

        # FIND THE HAND TAG IN THE METADATA LIST
        for strong_tag in metadata.find_all('strong'):
            if strong_tag.text == 'Rookie Status:':
                rookie_status_full_sentence = strong_tag.next_sibling.strip()
                rookie_status_only_ints = re.sub("[^0-9]", "", rookie_status_full_sentence)
                if len(rookie_status_only_ints) != 4:
                    continue
                try:
                    rookie_status_year = int(rookie_status_only_ints)
                    return rookie_status_year
                except:
                    continue

        # NO ROOKIE STATUS WAS FOUND, RETURN NONE
        return None

    def type(self, positional_fielding, year):
        """Guess Player Type (Pitcher or Hitter) based on games played at each position.

        Args:
          positional_fielding: Dict with positions, games_played, and tzr
          year: Year for Player stats

        Raises:
          AttributeError: This Player Played 0 Games. Check Player Name and Year.

        Returns:
          Either 'Hitter' or 'Pitcher' string.
        """

        games_as_pitcher = 0
        games_as_hitter = 0
        positions = int((len(positional_fielding)-1) / 4)
        # SPLIT GAMES BETWEEN TYPES
        for position_index in range(1, positions + 1):
            games = int(positional_fielding['gPosition{}'.format(position_index)])
            position = positional_fielding['Position{}'.format(position_index)]
            if position == 'P':
                games_as_pitcher += games
            else:
                games_as_hitter += games

        # CHECK FOR TYPE OVERRIDE
        is_pitcher_override = '(PITCHER)' in self.name.upper() and games_as_pitcher > 0
        is_hitter_override = '(HITTER)' in self.name.upper()

        # COMPARE GAMES PLAYED IN BOTH TYPES
        total_games = games_as_hitter + games_as_pitcher
        if total_games == 0:
            if self.is_multi_year:
                return None
            else:
                self.error = f'This Player Played 0 Games in {year}. Check Player Name and Year'
                raise AttributeError(self.error)
        elif is_pitcher_override:
            return "Pitcher"
        elif is_hitter_override:
            return "Hitter"
        elif games_as_pitcher < games_as_hitter:
            return "Hitter"
        else:
            return "Pitcher"

    def statcast_sprint_speed(self, name, year):
        """Sprint Speed for player from Baseball Savant (Only applicable to 2015+).

        Args:
          name: Full name of Player
          year: Year for Player stats

        Raises:
          AttributeError: This Player Played 0 Games. Check Player Name and Year.

        Returns:
          Average sprint speed for Player as Float.
        """

        # DATA ONLY AVAILABLE 2015+
        if int(year) < 2015:
            return None

        sprint_speed_url = 'https://baseballsavant.mlb.com/sprint_speed_leaderboard?year={}&position=&team=&min=0'.format(year)
        speed_data_html = self.html_for_url(sprint_speed_url)

        speed_data_extracted = re.search('var data = (.*);',speed_data_html).group(1)
        speed_list_all_players = json.loads(speed_data_extracted)

        for player_dict in speed_list_all_players:
            # FIND PLAYER IN LIST
            is_player_match = self.__statcast_name_match(bref_name=name, statcast_name=player_dict['name_display_last_first'])
            if is_player_match:
                speed = float(player_dict['r_sprint_speed_top50percent_pretty'])
                return speed
        # IF NOT FOUND, RETURN LEAGUE AVG
        default_speed = 26.25
        return default_speed

    def statcast_outs_above_average_dict(self, name, years):
        """Outs above average metric for players from Baseball Savant (Only applicable to 2016+).

        Args:
          name: Full name of Player
          year: List of years to include.

        Returns:
          Dictionary with position and outs above avg value
        """
        # DEFINE YEAR RANGE
        min_year = min(years)
        max_year = max(years)

        year_start_for_query = max(min_year, 2016)
        year_end_for_query = max_year
        # DATA ONLY AVAILABLE 2016+
        if max_year < 2016:
            return {}

        url = f'https://baseballsavant.mlb.com/leaderboard/outs_above_average?type=Fielder&startYear={year_start_for_query}&endYear={year_end_for_query}&position=&team=&min=0'
        oaa_html = self.html_for_url(url)

        data_extracted = re.search('var data = (.*);',oaa_html).group(1)
        oaa_list_all_players = json.loads(data_extracted)

        for player_dict in oaa_list_all_players:
            # FIND PLAYER IN LIST

            is_player_match = self.__statcast_name_match(bref_name=name, statcast_name=player_dict['entity_name'])
            if is_player_match:
                # MAKE REQUEST FOR DETAILED PAGE WITH EXACT NUMBERS
                statcast_player_id = player_dict['entity_id']
                player_detail_url = f'https://baseballsavant.mlb.com/savant-player/{statcast_player_id}?stats=statcast-r-fielding-mlb'
                player_detail_html = self.html_for_url(url=player_detail_url)
                fielding_data_extracted = re.search('infieldDefense: (.*),',player_detail_html)
                if fielding_data_extracted:
                    fielding_data_grouped = fielding_data_extracted.group(1)
                    fielding_data_jsons = json.loads(fielding_data_grouped)
                    fielding_data = {}
                    for fielding_row in fielding_data_jsons:
                        team_abbr = fielding_row['fld_abbreviation']
                        is_year_match = int(fielding_row['year']) in years
                        is_team_row = self.team_override == team_abbr if self.team_override else team_abbr != 'NA'
                        if is_year_match and is_team_row:
                            position = fielding_row['pos_name_short']
                            ooa = fielding_row['outs_above_average']
                            if ooa:
                                ooa_rounded = round(ooa, 3)
                                if position in fielding_data.keys():
                                    # POSITION IS ALREADY IN JSON, ADD TO IT
                                    fielding_data[position] += ooa_rounded
                                else:
                                    fielding_data[position] = ooa_rounded
                                # IF OF POSITION, ADD TO TOTAL OF DEFENSE
                                if position in ['LF','CF','RF']:
                                    if 'OF' in fielding_data.keys():
                                        # POSITION IS ALREADY IN JSON, ADD TO IT
                                        fielding_data['OF'] += ooa_rounded
                                    else:
                                        fielding_data['OF'] = ooa_rounded
                    return fielding_data

        # IF NOT FOUND, RETURN NONE
        return {}

    def __statcast_name_match(self, bref_name, statcast_name):
        """Compare first name and last name from bref vs statcast to match to a player.
        Returns true if both the first name and last name are a match.

        Args:
          bref_name: Full name of Player from baseball reference.
          statcast_name: String from a statcast leaderboard.

        Returns:
          True or False for whether the name matches.
        """

        # REMOVE SPECIAL CHARACTERS FROM THE NAME
        first_name_cleaned = unidecode.unidecode(bref_name.split(' ')[0].replace(".", ""))
        last_name = unidecode.unidecode(bref_name.split(' ')[1])
        full_name_baseball_savant = unidecode.unidecode(statcast_name)

        # REPLACE DECIMAL POINTS WITH EMPTY STRING
        try:
            full_name_baseball_savant = full_name_baseball_savant.replace(".",'')
        except:
            return False

        return first_name_cleaned in full_name_baseball_savant \
                and last_name in full_name_baseball_savant

    def advanced_stats(self, type, year):
        """Parse advanced stats page from baseball reference.

        Standard and ratio stats. For Pitchers, uses batting against table.

        Args:
          type: Player is Pitcher or Hitter.
          year: Year for Player stats

        Returns:
          Dict with standard and ratio statistics.
        """

        is_full_career = year == 'CAREER'

        # SCRAPE ADVANCED STATS PAGE
        page_suffix = '-bat' if type == 'Hitter' else '-pitch'
        url_advanced_stats = 'https://www.baseball-reference.com/players/{}/{}{}.shtml'.format(self.first_initial, self.baseball_ref_id, page_suffix)
        soup_for_advanced_stats = self.__soup_for_url(url_advanced_stats, is_baseball_ref_page=True)

        table_prefix = 'batting' if type == 'Hitter' else 'pitching'

        if is_full_career:
            standard_row = self.__get_career_totals_row(div_id=f'all_{table_prefix}_standard',soup_object=soup_for_advanced_stats)

        else:
            standard_row_key = f'{table_prefix}_standard.{year}'
            standard_row = soup_for_advanced_stats.find('tr',attrs={'class':'full','id': standard_row_key})

        advanced_stats = {}

        # BATTING AGAINST (PITCHERS ONLY)
        if type == 'Pitcher':
            if is_full_career:
                try:
                    batting_against_table = self.__get_career_totals_row(div_id='div_pitching_batting',soup_object=soup_for_advanced_stats)
                    advanced_stats.update(self.__parse_generic_bref_row(batting_against_table))
                except:
                    advanced_stats.update(self.__parse_splits_stats(year=year))
            else:
                # LOOK AT "SPLITS" PAGE FOR PRE-1916 PITCHERS
                if self.__is_pitcher_from_1901_to_1918(year=year,type=type):
                    advanced_stats.update(self.__parse_splits_stats(year=year))
                else:
                    batting_against_soup = soup_for_advanced_stats.find('table', attrs={'id': 'pitching_batting'})
                    partial_stats = self.__find_partial_team_stats_row(soup_object=batting_against_soup, team=self.team_override, year=year)
                    if self.team_override and partial_stats:
                        advanced_stats.update(partial_stats)
                    else:
                        batting_against_row = soup_for_advanced_stats.find('tr',attrs={'class':'full','id': f'pitching_batting.{year}'})
                        advanced_stats.update(self.__parse_generic_bref_row(batting_against_row))

        # STANDARD STATS
        standard_table_soup = soup_for_advanced_stats.find('div', attrs={'class': 'table_container', 'id': f'div_{table_prefix}_standard'})
        partial_stats_soup = self.__find_partial_team_stats_row(soup_object=standard_table_soup, team=self.team_override, year=year, return_soup_object=True)
        if self.team_override and partial_stats_soup:
            advanced_stats.update(self.__parse_standard_stats(type, partial_stats_soup, included_g_for_pitcher=is_full_career))
        else:
            standard_stats_dict = self.__parse_standard_stats(type, standard_row, included_g_for_pitcher=is_full_career)
            advanced_stats.update(standard_stats_dict)

        # PARSE AWARDS IF FULL CAREER
        if is_full_career or self.team_override:
            year_filter = '' if is_full_career else str(year)
            all_year_standard_stats = soup_for_advanced_stats.find_all('tr',attrs={'class':'full','id': re.compile(f"{table_prefix}_standard.{year_filter}")})
            awards_total = ""
            for year_stats in all_year_standard_stats:
                stats_dict = self.__parse_standard_stats(type, year_stats)
                if 'is_sv_leader' in stats_dict:
                    advanced_stats['is_sv_leader'] = stats_dict['is_sv_leader']
                year_award_summary = stats_dict['award_summary']
                if len(year_award_summary) > 0:
                    awards_total = f'{awards_total},{year_award_summary}'
            advanced_stats['award_summary'] = awards_total

        # IF A PLAYER PLAYED FOR MULTIPLE TEAMS IN ONE SEASON, SELECT THE LAST TEAM THEY PLAYED FOR.
        team_id_key = 'team_ID'
        if team_id_key in advanced_stats.keys():
            if advanced_stats[team_id_key] == 'TOT':
                advanced_stats[team_id_key] = self.__parse_team_after_trade(soup_for_homepage_stats=soup_for_advanced_stats,year=year,type=type)

        # FILL IN EMPTY STATS
        current_categories = advanced_stats.keys()
        if 'PA' not in current_categories:
            advanced_stats['is_stats_estimate'] = True
            # CHECK FOR BATTERS FACED
            if advanced_stats['batters_faced'] > 0:
                advanced_stats['PA'] = advanced_stats['batters_faced']
            # ESTIMATE PA AGAINST
            else:
                advanced_stats['PA'] = advanced_stats['IP'] * 4.25 # NEED TO ESTIMATE PA BASED ON AVG PA PER INNING

        keys_to_fill = ['SH','HBP','IBB','SF']
        for key in keys_to_fill:
            if key not in current_categories:
                advanced_stats[key] = 0

        if '2B' not in current_categories:
            maxDoubles = 0.25
            eraPercentile = self.__percentile(minValue=1.0, maxValue=5.0, value=advanced_stats['earned_run_avg'])
            advanced_stats['2B'] = int(advanced_stats['H'] * eraPercentile * maxDoubles)

        if '3B' not in current_categories:
            maxTriples = 0.025
            eraPercentile = self.__percentile(minValue=1.0, maxValue=5.0, value=advanced_stats['earned_run_avg'])
            advanced_stats['3B'] = int(advanced_stats['H'] * eraPercentile * maxTriples)

        if 'slugging_perc' not in current_categories:
            ab = advanced_stats['PA'] - advanced_stats['BB'] - advanced_stats['HBP']
            singles = advanced_stats['H'] - advanced_stats['2B'] - advanced_stats['3B'] - advanced_stats['HR']
            total_bases = (singles + (2 * advanced_stats['2B']) + (3 * advanced_stats['3B']) + (4 * advanced_stats['HR']))
            advanced_stats['AB'] = ab
            advanced_stats['TB'] = total_bases
            advanced_stats['slugging_perc'] = total_bases / ab

        if 'onbase_perc' not in current_categories:
            advanced_stats['onbase_perc'] = (advanced_stats['H'] + advanced_stats['BB']) / advanced_stats['PA']

        if 'batting_avg' not in current_categories:
            advanced_stats['batting_avg'] = advanced_stats['H'] / advanced_stats['AB']

        if 'SB' not in current_categories:
            advanced_stats['SB'] = 0

        # RATIO STATS
        ratio_table_key = f'div_{table_prefix}_ratio'
        if is_full_career:
            ratio_row = self.__get_career_totals_row(div_id=ratio_table_key,soup_object=soup_for_advanced_stats)
        else:
            ratio_table = soup_for_advanced_stats.find('div', attrs = {'id': ratio_table_key})
            partial_stats_soup = self.__find_partial_team_stats_row(soup_object=ratio_table, team=self.team_override, year=year, return_soup_object=True)
            if self.team_override and partial_stats_soup:
                ratio_row = partial_stats_soup
            else:
                ratio_row_key = f'{table_prefix}_ratio.{year}'
                ratio_row = soup_for_advanced_stats.find('tr', attrs = {'class': 'full', 'id': ratio_row_key})
        advanced_stats.update(self.__parse_ratio_stats(ratio_row=ratio_row, slg=advanced_stats['slugging_perc']))

        # IP/GS (STARTING PITCHER)
        if type == 'Pitcher':
            ip_per_gs = 0.0
            starter_stats_soup = soup_for_advanced_stats.find('div', attrs={'id': 'div_pitching_starter'})
            if starter_stats_soup:
                partial_stats_soup = self.__find_partial_team_stats_row(soup_object=starter_stats_soup, team=self.team_override, year=year, return_soup_object=True)
                row_accounting_for_team_override = partial_stats_soup if self.team_override and partial_stats_soup else starter_stats_soup.find('tr', attrs={'id': f'pitching_starter.{year}'})
                if row_accounting_for_team_override:
                    ip_per_start_object = row_accounting_for_team_override.find('td', attrs={'data-stat': 'innings_per_start'})
                    if ip_per_start_object:
                        ip_per_start_text = ip_per_start_object.get_text()
                        ip_per_gs = self.__convert_to_numeric(ip_per_start_text)
            advanced_stats['IP/GS'] = ip_per_gs

        return advanced_stats

    def __parse_standard_stats(self, type, soup_row, included_g_for_pitcher=False):
        """Parse standard batting table.

        Args:
          type: Player is Pitcher or Hitter.
          soup_row: BeautifulSoup row object with season stats.
          included_g_for_pitcher: Boolean for whether to parse 'G' for pitchers.

        Returns:
          Dict with standard hitting statistics.
        """

        standard_stats_dict = {}

        for category in soup_row:
            stat_category = category['data-stat']
            # LEAGUE LEADER?
            is_league_leader = '<strong>' in str(category) # LEAGUE LEADERS ON BASEBALL REF ARE DENOTED BY BOLD TEXT

            if stat_category in ('SV','HR','SB'):
                # SAVE THIS INFO FOR RP, HR, OR SB ICONS
                standard_stats_dict[f'is_{stat_category.lower()}_leader'] = is_league_leader

            stat = category.get_text()
            if stat_category in ['SF','IBB','CS'] and stat == '':
                stat = '0'
            stat = self.__convert_to_numeric(stat)

            if type == 'Pitcher':
                pitching_categories = ['earned_run_avg','team_ID','G','GS','W','SV','IP','H','2B','3B','HR','BB','SO','HBP','batters_faced','award_summary']
                fill_zeros = ['GS','W','SV','H','2B','3B','HR','BB','SO','HBP','batters_faced']
                if included_g_for_pitcher:
                    pitching_categories.append('G')
                if stat_category in pitching_categories:
                    if len(str(stat)) == 0 and stat_category in fill_zeros:
                        stat = 0
                    standard_stats_dict[stat_category] = stat
            else:
                if stat_category == 'SO' and len(str(stat)) == 0:
                    standard_stats_dict[stat_category] = 0
                else:
                    standard_stats_dict[stat_category] = stat

        return standard_stats_dict

    def __parse_splits_stats(self,year):
        """Parse standard statline from the "splits" page on Baseball Reference

        Args:
          year: Year for data stats (ex: 2021, CAREER)

        Returns:
          Dict with standard statistics.
        """
        url_splits = f'https://www.baseball-reference.com/players/split.fcgi?id={self.baseball_ref_id}&year={year}&t=p'
        soup_for_split = self.__soup_for_url(url_splits, is_baseball_ref_page=True)
        tables_w_incomplete_split = [a for a in soup_for_split.select('th[data-stat="incomplete_split"]')]
        stats = tables_w_incomplete_split[1]
        header_values = [self.__convert_to_numeric(sib['data-stat']) for sib in stats.next_siblings]
        stats_values = [self.__convert_to_numeric(sib.string) for sib in stats.next_siblings]
        batting_against_dict = dict(zip(header_values, stats_values))
        return batting_against_dict

    def __parse_generic_bref_row(self, row):
        """Parse hitting stats a pitcher allowed.

        Args:
          row: BeautifulSoup tr row object with stats

        Returns:
          Dict with statistics
        """

        final_stats_dict = {}
        if row is not None:
            for category in row:
                stat_category = category['data-stat']
                stat = category.get_text()
                stat = self.__convert_to_numeric(stat)
                if stat_category in ['IBB'] and stat == '':
                    stat = 0
                final_stats_dict[stat_category]= stat

        return final_stats_dict

    def __parse_ratio_stats(self, ratio_row, slg):
        """Parse out ratios (GB/AO, PU)

        Args:
          ratio_row: BeautifulSoup table row for ratios.
          slg: Slugging Pct, used if ratio's dont exist

        Returns:
          Dict with ratio statistics.
        """

        if ratio_row is None:
            # DEFAULT TO 50 / 50 SPLIT
            slg_percentile = self.__percentile(minValue=0.3, maxValue=0.5, value=slg)
            multiplier = 1.0 if slg_percentile < 0 else 1.0 - slg_percentile
            gb_ao_ratio = 1.5 * max(multiplier, 0.5)
            pu_ratio = 0.16 * max(multiplier, 0.5)
        else:
            gb_ao_ratio_raw = ratio_row.find('td',attrs={'class':'right','data-stat': 'go_ao_ratio'}).get_text()
            try:
                gb_ao_ratio = float(gb_ao_ratio_raw)
            except:
                gb_ao_ratio = 1.0
            pu_ratio_raw = ratio_row.find('td',attrs={'class':'right','data-stat': 'infield_fb_perc'})
            if pu_ratio_raw:
                # PU RATIO DATA AVAILABLE AFTER 1988
                pu_ratio_text = pu_ratio_raw.get_text()
                pu_ratio_text_cleaned = '20' if pu_ratio_text == '' else pu_ratio_text
                pu_ratio = int(pu_ratio_text_cleaned.replace('%','')) / 100.0
            else:
                pu_ratio = 0.20

        return {
            'GO/AO': gb_ao_ratio,
            'IF/FB': pu_ratio
        }

    def __parse_team_after_trade(self, soup_for_homepage_stats, year, type):
        """Parse the last team a player playe dfor in the given season.

        Args:
          soup_for_homepage_stats: BeautifulSoup object for advanced stats table.
          year: Year for Player stats
          type: Player is Pitcher or Hitter.

        Returns:
          Team Id for last team the player played on in year
        """

        header_prefix = 'batting' if type == 'Hitter' else 'pitching'
        standard_table = soup_for_homepage_stats.find('table',attrs={'id':f'{header_prefix}_standard'})
        if standard_table is None:
            return 'TOT'
        partial_objects_list = standard_table.find_all('tr',attrs={'class':'partial_table'})
        teams_list = []
        # ITERATE THROUGH PARTIAL SEASONS
        try:
            for partial_object in partial_objects_list:
                # SAVE TEAMS ONLY FOR year SEASON
                object_for_this_season = partial_object.find('th',attrs={'data-stat':'year_ID', 'csk': re.compile(str(year))})
                if object_for_this_season:
                    # PARSE TEAM ID
                    team_object = partial_object.find('td', attrs={'data-stat':'team_ID'})
                    if team_object is None:
                        continue
                    team_id = team_object.get_text()
                    if team_id != '' and team_id != 'TOT':
                        # ADD TO TEAMS LIST
                        teams_list.append(team_id)
            last_team = teams_list[-1]
            return last_team
        except:
            return 'TOT'

    def __add_icon_threshold_bools(self, type, year, stats_dict, homepage_soup):
        """Add 4 boolean flags for whether the player qualifies for Icons for a given stat

        Args:
          type: Player is Pitcher or Hitter.
          year: String representation for year of stats
          stats_dict: Real life stats in a dict (ex: H, BB, HR, etc)
          homepage_soup: Beautiful Soup object for the player's homepage on bRef

        Returns:
          Dictionary for threshold booleans
        """

        # ICON STATS
        icon_threshold_bools = {
            'is_above_hr_threshold': False,
            'is_above_so_threshold': False,
            'is_above_sb_threshold': False,
            'is_above_w_threshold': False,
        }
        stat_icon_mapping = {
            'HR': 17 if year == '2020' else 40,
            'SO': 96 if year == '2020' else 215,
            'SB': 15 if year == '2020' else 40,
            'W': 20,
        }
        for stat, threshold in stat_icon_mapping.items():
            bool_key_name = f"is_above_{stat.lower()}_threshold"

            if year == 'CAREER':
                list_of_values_for_stat = self.__get_stat_list_from_standard_table(type, homepage_soup, stat_key=stat)
                if list_of_values_for_stat:
                    list_cleaned = [0 if str(x) == '' else x for x in list_of_values_for_stat]
                    max_for_stat = max(list_cleaned)
                    is_qualified_for_icon = int(max_for_stat) >= threshold
                    icon_threshold_bools[bool_key_name] = is_qualified_for_icon
            else:
                if stat in stats_dict.keys():
                    if len(str(stats_dict[stat])) == 0:
                        icon_threshold_bools[bool_key_name] = False
                    else:
                        is_qualified_for_icon = int(stats_dict[stat]) >= threshold
                        icon_threshold_bools[bool_key_name] = is_qualified_for_icon

        return icon_threshold_bools

    def __career_year_range(self, table_prefix, advanced_stats_soup):
        """Parse the player's first and last seasons in the MLB

        Args:
          table_prefix: String for whether player is batter or pitcher
          advanced_stats_soup: BeautifulSoup object for advanced stats table.

        Returns:
          Tuple of start and end years of career (ex: 2001, 2009)
        """
        try:
            full_element_prefix = '{}_standard.'.format(table_prefix)
            all_seasons_list = advanced_stats_soup.find_all('tr',attrs={'class':'full','id': re.compile(full_element_prefix)})
            first_season = all_seasons_list[0].find('th',attrs={'class':'left','data-stat': 'year_ID'} )
            last_season = all_seasons_list[-1].find('th',attrs={'class':'left','data-stat': 'year_ID'} )
            year_of_first_season = str(first_season["csk"])
            year_of_last_season = str(last_season["csk"])
            return year_of_first_season, year_of_last_season
        except:
            return 2300, 2300

    def __combine_multi_year_dict(self, yearly_stats_dict):
        """Combine multiple years into one master final dataset

        Args:
          yearly_stats_dict: Dict of dicts per year.

        Returns:
          Flattened dictionary with combined stats
        """
        column_aggs = {
            '1B': 'sum',
            '2B': 'sum',
            '3B': 'sum',
            'AB': 'sum',
            'BB': 'sum',
            'CS': 'sum',
            'G': 'sum',
            'GIDP': 'sum',
            'H': 'sum',
            'HBP': 'sum',
            'HR': 'sum',
            'IBB': 'sum',
            'PA': 'sum',
            'R': 'sum',
            'RBI': 'sum',
            'SB': 'sum',
            'SF': 'sum',
            'SH': 'sum',
            'SO': 'sum',
            'TB': 'sum',
            'IP': 'sum',
            'SV': 'sum',
            'GS': 'sum',
            'GO/AO': 'mean',
            'IF/FB': 'mean',
            'sprint_speed': 'mean',
            'is_above_hr_threshold': 'max',
            'is_above_so_threshold': 'max',
            'is_above_sb_threshold': 'max',
            'is_above_w_threshold': 'max',
            'is_sv_leader': 'max',
            'is_hr_leader': 'max',
            'is_sb_leader': 'max',
            'award_summary': ','.join,
        }

        # FLATTEN MULTI YEAR STATS
        yearPd = pd.DataFrame.from_dict(yearly_stats_dict, orient='index')
        columns_to_remove = list(set(column_aggs.keys()) - set(yearPd.columns))
        if max(self.years) < 2015:
            columns_to_remove.append('sprint_speed')
        [column_aggs.pop(key) for key in columns_to_remove]

        avg_year = yearPd.groupby(by='name',as_index=False).agg(column_aggs)
        # CALCULATE RATES
        avg_year["batting_avg"] = round(avg_year['H'] / float(avg_year['AB']),3)
        avg_year["onbase_perc"] = round((avg_year['H'] + avg_year['BB'] + avg_year['HBP']) / float(avg_year['AB'] + avg_year['BB'] + avg_year['HBP'] + avg_year['SF']),3)
        avg_year["slugging_perc"] = round(avg_year['TB'] / avg_year['AB'],3)
        avg_year["onbase_plus_slugging"] = round(avg_year["onbase_perc"] + avg_year["slugging_perc"],3)

        avg_year_dict = avg_year.iloc[0].to_dict()

        # AGGREGATE DEFENSIVE METRICS
        avg_year_dict.update(self.__combine_multi_year_positions(yearly_stats_dict))

        return avg_year_dict

    def __combine_multi_year_positions(self, yearly_stats_dict):
        """Combine multiple year defensive positions and metrics
        into one master final dataset.

        Args:
          yearly_stats_dict: Dict of dicts per year.

        Returns:
          Flattened dictionary with combined stats
        """

        # OBJECT TO UPDATE
        defensive_fields_dict = {}
        positions_and_defense = {}
        positions_and_games_played = {}
        dWar_list = []

        # PARSE BY POSITION
        for year, stats in yearly_stats_dict.items():
            defensive_stats = {k:v for (k,v) in stats.items() if 'Position' in k or 'dWAR' in k}
            num_positions = int((len(defensive_stats.keys())-1) / 4)
            dWar_list.append(float(defensive_stats['dWAR']))
            for position_index in range(1, num_positions+1):
                position = defensive_stats['Position{}'.format(position_index)]
                # CHECK IF POSITION MATCHES PLAYER TYPE
                games_at_position = int(defensive_stats['gPosition{}'.format(position_index)])
                if position in positions_and_games_played.keys():
                    games_at_position += positions_and_games_played[position]
                positions_and_games_played[position] = games_at_position
                # IN-GAME RATING AT
                if position:
                    try:
                        drs_raw = defensive_stats['drsPosition{}'.format(position_index)]
                        tzr_raw = defensive_stats['tzPosition{}'.format(position_index)]
                        year_defense_metrics = {
                            'drs': int(drs_raw) if drs_raw else None,
                            'tzr': int(tzr_raw) if tzr_raw else None,
                        }
                    except:
                        year_defense_metrics = {'drs': 0, 'tzr': 0}
                    if position in positions_and_defense.keys():
                        for key in ['drs', 'tzr']:
                            current_stat_list = positions_and_defense[position][key]
                            current_stat_list.append(year_defense_metrics[key])
                            positions_and_defense[position][key] = current_stat_list
                    else:
                        positions_and_defense[position] = { k:[v] for (k,v) in year_defense_metrics.items() }

        # RE-INDEX POSITIONAL STATS
        positions_and_games_played_sorted_tuple = sorted(positions_and_games_played.items(), key=operator.itemgetter(1), reverse=True)
        for index, (position, games_played) in enumerate(positions_and_games_played_sorted_tuple, 1):
            drs_list = [df for df in positions_and_defense[position]['drs'] if df is not None]
            tzr_list = [tz for tz in positions_and_defense[position]['tzr'] if tz is not None]
            position_dict = {
                f"Position{index}": position,
                f"gPosition{index}": games_played,
                "dWar": statistics.median(dWar_list),
                f"drsPosition{index}": statistics.median(drs_list) if len(drs_list) > 0 else None,
                f"tzPosition{index}": statistics.median(tzr_list) if len(tzr_list) > 0 else None,
            }
            defensive_fields_dict.update(position_dict)

        return defensive_fields_dict

    def __get_career_totals_row(self, div_id, soup_object):
        """Grab first row of career totals data from any table

        Args:
          div_id: ID attribute of the div object
          soup_object: Beautiful Soup object to extract from

        Returns:
          Beautiful Soup object for career totals
        """

        div = soup_object.find('div', attrs = {'id': div_id})
        footer = div.find('tfoot')
        row = footer.find_all('tr')[0]
        return row

    def __years_played_list(self, type, homepage_soup):
        """Parse the standard batting/pitching table to get a list of the years
           a player played in.

        Args:
          homepage_soup: Beautiful Soup object for the player's homepage on bRef

        Returns:
          List of years as strings
        """
        table_prefix = 'batting' if type == 'Hitter' else 'pitching'
        standard_table = homepage_soup.find('div', attrs = {'id': f'all_{table_prefix}_standard'})
        year_soup_objects_list = standard_table.find_all('tr', attrs = {'id': re.compile(f'{table_prefix}_standard.')})
        years_parsed = [year['id'].split('.')[-1] for year in year_soup_objects_list]
        return years_parsed

    def __team_w_most_games_played(self, type, homepage_soup, years_filter_list=[]):
        """Parse the standard batting/pitching table to get a list of the years
           a player played in.

        Args:
          type: Player is Pitcher or Hitter.
          homepage_soup: Beautiful Soup object for the player's homepage on bRef
          years_filter_list: Optional list of years to include

        Returns:
          List of years as strings
        """
        table_prefix = 'batting' if type == 'Hitter' else 'pitching'
        standard_table = homepage_soup.find('div', attrs = {'id': f'all_{table_prefix}_standard'})
        year_soup_objects_list = standard_table.find_all('tr', attrs = {'id': re.compile(f'{table_prefix}_standard.')})
        teams_and_games_played = {}
        for year_object in year_soup_objects_list:
            year = int(year_object.find('th',attrs={'data-stat': 'year_ID'}).get_text())
            if len(years_filter_list) < 1 or (year in years_filter_list):
                team = year_object.find('td',attrs={'data-stat': 'team_ID'}).get_text()
                games = int(year_object.find('td',attrs={'data-stat': 'G'}).get_text())
                if team != 'TOT':
                    if team in teams_and_games_played.keys():
                        games += teams_and_games_played[team]
                    teams_and_games_played[team] = games
        if len(teams_and_games_played.keys()) > 0:
            team_w_most_games = sorted(teams_and_games_played.items(), key=operator.itemgetter(1), reverse=True)[0][0]
            return team_w_most_games
        else:
            return 'TOT'

    def __get_stat_list_from_standard_table(self, type, homepage_soup, stat_key):
        """Parse the standard batting/pitching table to get a list of values for a given stat.

        Args:
          type: Player is Pitcher or Hitter.
          homepage_soup: Beautiful Soup object for the player's homepage on bRef
          years_filter_list: Optional list of years to include

        Returns:
          List of years as strings
        """
        table_prefix = 'batting' if type == 'Hitter' else 'pitching'
        standard_table = homepage_soup.find('div', attrs = {'id': f'all_{table_prefix}_standard'})
        year_soup_objects_list = standard_table.find_all('tr', attrs = {'id': re.compile(f'{table_prefix}_standard.')})
        stat_list = []
        for year_object in year_soup_objects_list:
            stat_object = year_object.find('td',attrs={'data-stat': stat_key})
            if stat_object:
                stat = stat_object.get_text()
                stat = self.__convert_to_numeric(stat)
                stat_list.append(stat)

        if len(stat_list) < 1:
            return None

        return stat_list

    def __is_pitcher_from_1901_to_1918(self, year, type):
        """Checks to see if the player is a pitcher and the year is between 1901-1917.

        Args:
          year: Year for player stats
          type: String to designate player type ('Pitcher','Hitter')

        Returns:
          TRUE or FALSE bool
        """
        try:
            return int(year) > 1900 and int(year) < 1918 and type == 'Pitcher'
        except:
            return False

    def __find_partial_team_stats_row(self, soup_object, team, year, return_soup_object=False):
        """Iterates through each object to try to find specific team stats for a given year.

        Args:
          soup_object: Beautiful Soup object to search in
          team: Team ID to search for
          year: Year for player stats
          return_soup_object: Optionally return the soup object instead of the dictionary

        Returns:
          Soup object for the players statline for that specific team.
        """

        # TEAM OVERRIDE IS NONE, RETURN NONE
        if not team:
            return None

        # FIND ALL OPPORTUNITIES
        partial_seasons = soup_object.find_all('tr',attrs={'class':'partial_table'})

        # FILTER TO ROW WHERE TEAM AND YEAR MATCH
        for season in partial_seasons:
            data = self.__parse_generic_bref_row(season)
            if 'team_ID' in data.keys() and 'year_ID' in data.keys():
                if data['team_ID'] == team and str(data['year_ID']) == year:
                    return season if return_soup_object else data

        # NO MATCHES, RETURN NONE
        return None

# ------------------------------------------------------------------------
# HELPER METHODS

    def __name_last_initial(self, name):
        """Parse first initial of Player's last name.

        Need to account for >2 words and Suffixes (Jr, Sr, etc)

        Args:
          name: Full name of Player

        Returns:
          First Initial of Last Name (ex: 'Ken Griffey Jr' -> 'g')
        """

        remove_punctuation = str.maketrans('', '', string.punctuation)
        name_wo_punctuation = name.translate(remove_punctuation)

        name_no_suffix = name_wo_punctuation.lower()
        illegal_suffixes = [' jr', ' sr']
        for suffix in illegal_suffixes:
            if name_no_suffix.endswith(suffix):
                name_no_suffix = name_no_suffix.replace(suffix, '')

        last_initial = name_no_suffix.split(' ')[1][:1]
        return last_initial

    def __percentile(self,minValue,maxValue,value):
        """Get percentile of value given a range.

        Args:
          minValue: minimum of range
          maxValue: maximum of range
          value: value to compare

        Returns:
          float for percentile within range
        """

        percentile_raw = (value-minValue) / (maxValue-minValue)

        return min(percentile_raw,1.0) if percentile_raw > 0 else 0

    def __convert_to_numeric(self, string_value):
        """Will convert a string to either int or float if able, otherwise return as string

        Args:
          string_value: String for attribute

        Returns:
          Converted attribute
        """
        # CONVERT TYPE IF INT OR FLOAT

        if string_value is None:
            return 0

        if string_value.isdigit():
            return int(string_value)
        elif string_value.replace('.','',1).isdigit() and string_value.count('.') < 2:
            return float(string_value)
        else:
            # RETURN ORIGINAL STRING
            return string_value


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import logging

from mozsvc.config import get_configurator
from mozsvc.plugin import load_and_register

logger = logging.getLogger('tokenserver')

def includeme(config):
    config.include("cornice")
    config.include("mozsvc")
    config.scan("stokenserver.views")
    load_and_register("metadatadb", config)

def main(global_config, **settings):
    config = get_configurator(global_config, **settings)
    config.include(includeme)
    return config.make_wsgi_app()


#!/usr/bin/env python3
"""
Copy PDB models from pipeline to frontend
"""
import argparse
import os
import shutil
from pathlib import Path

import pandas as pd
from ruamel.yaml import YAML

def process_swissmodel(output_dir):
    """
    Identify swissmodel models in a directory and copy them to output_dir
    """
    proteins = [Path(i).stem for i in os.listdir('data/swissmodel') if i.endswith('.models')]
    for protein in proteins:
        config = pd.read_csv(f'data/swissmodel/{protein}.models', sep='\t', dtype={'model': str})

        if not len(config.index) > 0:
            continue

        Path(f'{output_dir}/{protein}').mkdir(exist_ok=True)
        for model, template in zip(config.model, config.template):
            shutil.copyfile(f'data/foldx/{protein}_{model}/model.pdb',
                            f'{output_dir}/{protein}/{template}.pdb')

def process_complex(output_dir):
    """
    Identify complex models in a directory and copy them to output_dir
    """
    yaml_reader = YAML(typ='safe')
    for model_dir in os.listdir('data/complex'):
        if not os.path.isdir(f'data/complex/{model_dir}'):
            continue

        config = yaml_reader.load(Path(f'data/complex/{model_dir}/model.yaml'))
        model = config['model'].split('.')[0]

        shutil.copyfile(f'data/complex/{model_dir}/model.pdb', f'{output_dir}/{model}.pdb')

def main(args):
    """Main"""
    if args.type == 'swissmodel':
        process_swissmodel(args.output)
    elif args.type == 'complex':
        process_complex(args.output)
    else:
        # Should never be able to get here, assuming argparse is working
        raise ValueError('Unrecognised "type" argument')

def parse_args():
    """Process input arguments"""
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('type', metavar='T', choices=('swissmodel', 'complex'),
                        help="Input directory format")
    parser.add_argument('output', metavar='O', help="Output directory")

    return parser.parse_args()

if __name__ == "__main__":
    main(parse_args())


import datetime
from typing import List, Optional

from passlib.context import CryptContext
import sqlalchemy
import sqlmodel

import reflex as rx

pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")

class Post(rx.Model, table=True):
    title: str
    body: str
    user_id: int = sqlmodel.Field(foreign_key="user.id")
    update_ts: datetime.datetime = sqlmodel.Field(
        default=None,
        sa_column=sqlalchemy.Column(
            "update_ts",
            sqlalchemy.DateTime(timezone=True),
            server_default=sqlalchemy.func.now(),
        ),
    )

    user: Optional["User"] = sqlmodel.Relationship(back_populates="posts", sa_relationship_kwargs={"lazy": "selectin"})
    flags: Optional[List["Flag"]] = sqlmodel.Relationship(back_populates="post")

    @classmethod
    def from_form_data(cls, form_data: dict) -> "Post":
        return cls(
            id=form_data.pop("post_id", None),
            **form_data,
        )

    def create_or_edit(self, session: sqlmodel.Session, user: "User") -> "Post":
        if not self.id:
            # Create post
            self.user = user
            session.add(self)
            return self

        # Editing post
        post = session.get(Post, self.id)
        if post.user_id != user.id and not user.is_mod:
            raise ValueError("Only moderators can edit other users' posts.")
        post.set(
            title=self.title,
            body=self.body,
            update_ts=datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0),
        )
        session.add(post)
        return post

    @staticmethod
    def cond_is_flagged(post: "Post", c1: rx.Component, c2: rx.Component | None = None) -> rx.Component:
        return rx.cond(post.flags & post.flags.length() > 0, c1, c2)

    @staticmethod
    def render(post: "Post") -> rx.Component:
        return rx.box(
            rx.hstack(rx.heading(post.title)),
            rx.hstack(
                rx.cond(
                    post.user,
                    rx.text(f"by {post.user.username}", font_size="0.7em"),
                ),
                rx.spacer(),
                rx.text(post.update_ts, font_size="0.7em"),
            ),
            rx.text(post.body),
            mod_tools(post),
        )

class User(
    rx.Model,
    table=True,  # type: ignore
):
    """A local User model with bcrypt password hashing."""

    username: str = sqlmodel.Field(unique=True, nullable=False, index=True)
    password_hash: str = sqlmodel.Field(default="", nullable=False)
    enabled: bool = True
    is_mod: bool = False
    is_admin: bool = False

    posts: List[Post] = sqlmodel.Relationship(back_populates="user")

    @staticmethod
    def hash_password(secret: str) -> str:
        """Hash the secret using bcrypt.

        Args:
            secret: The password to hash.

        Returns:
            The hashed password.
        """
        return pwd_context.hash(secret)

    def verify(self, secret: str) -> bool:
        """Validate the user's password.

        Args:
            secret: The password to check.

        Returns:
            True if the hashed secret matches this user's password_hash.
        """
        return pwd_context.verify(
            secret,
            self.password_hash,
        )

class Flag(rx.Model, table=True):
    post_id: int = sqlmodel.Field(foreign_key="post.id")
    user_id: int = sqlmodel.Field(foreign_key="user.id")
    message: str

    post: Optional[Post] = sqlmodel.Relationship(back_populates="flags")
    user: Optional[User] = sqlmodel.Relationship()

    @classmethod
    def from_form_data(cls, form_data: dict) -> "Flag":
        return cls(
            post_id=form_data.pop("flag_post_id", None),
            message=form_data.pop("message", None),
        )

    def create(self, session: sqlmodel.Session, user: User):
        post = session.get(Post, self.post_id)
        if not post:
            raise ValueError("Post not found.")
        self.user_id = user.id
        session.add(self)

class State(rx.State):
    active_user: Optional[User] = None
    error_message: str = ""

    def logout(self):
        self.active_user = None
        self.reset()

    def _is_mod(self, session: sqlmodel.Session) -> bool:
        if self.active_user is not None:
            session.add(self.active_user)
            return self.active_user.is_mod
        return False

class LoginForm(rx.Base):
    username: str
    password: str
    is_mod: bool = False
    is_admin: bool = False

    def login_or_create(self, session: sqlmodel.Session) -> User:
        user = session.exec(User.select.where(User.username == self.username)).first()
        if user and not user.verify(self.password):
            raise ValueError("Incorrect username / password.")
        if user and self.is_mod and not user.is_mod:
            raise ValueError(f"User {user.username} is not a moderator.")
        if not user:
            user = User(username=self.username, password_hash=User.hash_password(self.password), is_mod=self.is_mod)
            session.add(user)
            session.commit()
            session.refresh(user)
        return user

class LoginState(State):
    login_modal_is_shown: bool = False
    login_modal_error_message: str = ""

    def login(self, form_data: dict):
        try:
            form = LoginForm(**form_data)
        except ValueError as e:
            self.login_modal_error_message = str(e)
            return
        with rx.session() as session:
            try:
                self.active_user = form.login_or_create(session)
            except ValueError as e:
                self.login_modal_error_message = str(e)
                return
            self.login_modal_error_message = ""
            self.login_modal_is_shown = False
            return PostState.load_posts

    @classmethod
    def modal(cls) -> rx.Component:
        return rx.modal(
            header="Login",
            close_button="Close",
            body=rx.form(
                rx.vstack(
                    rx.text(cls.login_modal_error_message),
                    rx.input(placeholder="Username", id="username"),
                    rx.password(placeholder="Password", id="password"),
                    rx.checkbox("User is Moderator?", type_="checkbox", id="is_mod"),
                    rx.button("Submit", type_="submit"),
                ),
                on_submit=cls.login,
            ),
            is_open=cls.login_modal_is_shown,
            on_close=cls.set_login_modal_is_shown(False),
        )

    @classmethod
    def login_logout_button(cls) -> rx.Component:
        return rx.cond(
            cls.active_user,
            rx.button(f"Logout {cls.active_user.username}", on_click=State.logout),
            rx.button("Login", on_click=cls.set_login_modal_is_shown(True)),
        )

def hidden_input_with_ref(id_: str) -> rx.Component:
    hidden_input = rx.input(id=id_, type_="hidden")
    # Accessing the hidden field value on the frontend via refs is possible
    hidden_input_value = rx.Var.create(
        f"refs['{hidden_input.get_ref()}']?.current?.value",
        _var_is_local=False,
    )
    return hidden_input, hidden_input_value

class PostState(State):
    posts: List[Post] = []
    post_modal_is_shown: bool = False
    post_modal_error_message: str = ""

    def load_posts(self):
        if not self.active_user:
            return
        load_options = []
        with rx.session() as session:
            if self._is_mod(session):
                # only moderators can see the flags
                load_options.append(
                    sqlalchemy.orm.selectinload(Post.flags).options(
                        sqlalchemy.orm.selectinload(Flag.user),
                    ),
                )
            if self._is_mod(session):
                print("I'm a mod!")
            self.posts = session.exec(
                Post.select
                .options(*load_options)
                .limit(15)
                .order_by(Post.update_ts.desc())
            ).all()
        self.error_message = ""

    def new_post(self):
        self.post_modal_is_shown = True

    def edit_post(self, post_id: int):
        with rx.session() as session:
            post = session.get(Post, post_id)
        if not post:
            self.error_message = "Post not found."
            self.post_modal_is_shown = False
            yield
            return
        self.post_modal_is_shown = True
        yield
        return [
            rx.set_value("post_id", post.id),
            rx.set_value("title", post.title),
            rx.set_value("body", post.body),
        ]

    def delete_post(self, post_id: int):
        with rx.session() as session:
            if not self._is_mod(session):
                self.error_message = "Only moderators can delete posts."
                return

            post = session.get(Post, post_id)
            session.delete(post)
            session.commit()
        return PostState.load_posts

    def submit_post(self, form_data: dict):
        if not self.active_user:
            self.post_modal_error_message = "Please log in to post."
            return
        try:
            post = Post.from_form_data(form_data)
        except ValueError as e:
            self.post_modal_error_message = str(e)
            return
        with rx.session() as session:
            try:
                post.create_or_edit(session, self.active_user)
                session.commit()
            except ValueError as e:
                self.post_modal_error_message = str(e)
                return
        self.post_modal_error_message = ""
        self.post_modal_is_shown = False
        return PostState.load_posts

    @classmethod
    def edit_post_modal(cls) -> rx.Component:
        hidden_post_id_input, post_id_value = hidden_input_with_ref("post_id")
        return rx.modal(
            header=rx.cond(
                post_id_value,
                rx.text(f"Edit Post ({post_id_value})"),
                rx.text("New Post"),
            ),
            close_button="Close",
            body=rx.form(
                hidden_post_id_input,
                rx.vstack(
                    rx.text(cls.post_modal_error_message),
                    rx.input(placeholder="Title", id="title"),
                    rx.input(placeholder="Body", id="body"),
                    rx.button("Submit", type_="submit"),
                ),
                on_submit=cls.submit_post,
            ),
            is_open=cls.post_modal_is_shown,
            on_close=cls.set_post_modal_is_shown(False),
        )

class FlagState(State):
    flag_modal_is_shown: bool = False
    flag_modal_error_message: str = ""

    def submit_flag(self, form_data: dict):
        if not self.active_user:
            self.flag_modal_error_message = "Please log in to flag."
            return
        try:
            flag = Flag.from_form_data(form_data)
        except ValueError as e:
            self.flag_modal_error_message = str(e)
            return
        with rx.session() as session:
            try:
                flag.create(session, user=self.active_user)
                session.commit()
            except ValueError as e:
                self.flag_modal_error_message = str(e)
                return
        self.flag_modal_error_message = ""
        self.flag_modal_is_shown = False
        return PostState.load_posts

    def delete_flag(self, flag_id: int):
        with rx.session() as session:
            if not self._is_mod(session):
                self.error_message = "Only moderators can delete flags."
                return

            flag = session.get(Flag, flag_id)
            session.delete(flag)
            session.commit()
        return PostState.load_posts

    def show_flag_modal(self, post_id: int | None):
        self.flag_modal_is_shown = True
        yield
        if post_id is not None:
            return rx.set_value("flag_post_id", post_id)

    @classmethod
    def flag_post_button(cls, post: Post) -> rx.Component:
        hidden_post_id_input, post_id_value = hidden_input_with_ref("flag_post_id")
        return rx.fragment(
            rx.button("Flag", on_click=cls.show_flag_modal(post.id)),
            rx.modal(
                header=f"Flag Post ({post_id_value})",
                close_button="Close",
                body=rx.form(
                    hidden_post_id_input,
                    rx.vstack(
                        rx.text(cls.flag_modal_error_message),
                        rx.input(placeholder="Why is this content inappropriate?", id="message"),
                        rx.button("Submit", type_="submit"),
                    ),
                    on_submit=cls.submit_flag,
                ),
                is_open=cls.flag_modal_is_shown,
                on_close=cls.set_flag_modal_is_shown(False),
            ),
        )

    @classmethod
    def flags_accordion(cls, post: Post) -> rx.Component:
        def flag_row(flag: cls):
            return rx.hstack(
                rx.text(flag.message),
                rx.text(flag.user.username),
                rx.button("X", on_click=cls.delete_flag(flag.id), size="sm"),
            )

        return Post.cond_is_flagged(
            post,
            rx.accordion(
                rx.accordion_item(
                    rx.accordion_button("Flags 🏴☠", rx.accordion_icon()),
                    rx.accordion_panel(rx.foreach(post.flags, flag_row)),
                ),
                allow_multiple=True,
            ),
        )

def mod_tools(post: Post):
    return rx.cond(
        State.active_user.is_mod,
        rx.fragment(
            rx.hstack(
                rx.button("Edit", on_click=PostState.edit_post(post.id)),
                rx.button("Delete", on_click=PostState.delete_post(post.id)),
            ),
            FlagState.flags_accordion(post),
        ),
        FlagState.flag_post_button(post),
    )

def index():
    return rx.fragment(
        rx.vstack(
            rx.heading("Welcome to the Mini Twitter!"),
            rx.text(State.error_message),
            LoginState.modal(),
            LoginState.login_logout_button(),
            rx.cond(
                State.active_user,
                rx.button("New Post", on_click=PostState.new_post),
            ),
            PostState.edit_post_modal(),
            rx.foreach(PostState.posts, Post.render),
        ),
    )

app = rx.App()
app.add_page(index, on_load=PostState.load_posts)
app.compile()


from collections import namedtuple

from bottle_utils.i18n import lazy_gettext as _

__all__ = ('LBAND', 'KUBAND', 'L_PRESETS', 'KU_PRESETS', 'PRESETS')

Preset = namedtuple('Preset', ('label', 'index', 'values'))

LBAND = 'l'
KUBAND = 'ku'

L_PRESETS = [
    # Translators, name of the L-band tuner preset covering AF-EU-ME
    Preset(_('Africa-Europe-Middle East (25E)'), 1, {
        'frequency': '1545.94',
        'uncertainty': '4000',
        'symbolrate': '4200',
        'sample_rate': '1',
        'rf_filter': '20',
        'descrambler': True,
        # Translators, used as coverage area of a transponder
        'coverage': _('Africa, Europe, Middle East'),
    }),
    # Translators, name of the L-band tuner preset covering the Americas
    Preset(_('North and South America (98W)'), 2, {
        'frequency': '1539.8725',
        'uncertainty': '4000',
        'symbolrate': '4200',
        'sample_rate': '1',
        'rf_filter': '20',
        'descrambler': True,
        # Translators, used as coverage area of a transponder
        'coverage': _('North and South America'),
    }),
    # Translators, name of the L-band tuner preset covering Asia-Pacific
    Preset(_('Asia-Pacific (144E)'), 3, {
        'frequency': '1545.9525',
        'uncertainty': '4000',
        'symbolrate': '4200',
        'sample_rate': '1',
        'rf_filter': '20',
        'descrambler': True,
        # Translators, used as coverage area of a transponder
        'coverage': _('Asia, Oceania'),
    }),
]

KU_PRESETS = [
    Preset('Galaxy 19 (97.0W)', 1, {
        'frequency': '11929',
        'symbolrate': '22000',
        'polarization': 'v',
        'delivery': 'DVB-S',
        'modulation': 'QPSK',
        # Translators, used as coverage area of a transponder
        'coverage': _('North America'),
    }),
    Preset('Hotbird 13 (13.0E)', 2, {
        'frequency': '11471',
        'symbolrate': '27500',
        'polarization': 'v',
        'delivery': 'DVB-S',
        'modulation': 'QPSK',
        # Translators, used as coverage area of a transponder
        'coverage': _('Europe, North Africa'),
    }),
    Preset('Intelsat 20 (68.5E)', 3, {
        'frequency': '12522',
        'symbolrate': '27500',
        'polarization': 'v',
        'delivery': 'DVB-S',
        'modulation': 'QPSK',
        # Translators, used as coverage area of a transponder
        'coverage': _('North and West Europe, Subsaharan Africa'),
    }),
    Preset('AsiaSat 5 C-band (100.5E)', 4, {
        'frequency': '3960',
        'symbolrate': '30000',
        'polarization': 'h',
        'delivery': 'DVB-S',
        'modulation': 'QPSK',
        # Translators, used as coverage area of a transponder
        'coverage': _('Middle East, Asia, Australia'),
    }),
    Preset('Eutelsat (113.0W)', 5, {
        'frequency': '12089',
        'symbolrate': '11719',
        'polarization': 'h',
        'delivery': 'DVB-S',
        'modulation': 'QPSK',
        # Translators, used as coverage area of a transponder
        'coverage': _('North, Middle, and South America'),
    }),
    Preset('ABS-2 (74.9E)', 6, {
        'frequency': '11734',
        'symbolrate': '44000',
        'polarization': 'h',
        'delivery': 'DVB-S',
        'modulation': 'QPSK',
        # Translators, used as coverage area of a transponder
        'coverage': _('India'),
    }),
]

PRESETS = {
    LBAND: L_PRESETS,
    KUBAND: KU_PRESETS,
}


# Notes:-
# This code uses CART instead of c4.5. They both are pretty close to each other. Refer documentation for difference.
# This code uses SAMME.R for Adaboost instead of ADaboost.MI used for boosting in the paper.
#
# Links to documentations
# http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.BaggingClassifier.html#sklearn.ensemble.BaggingClassifier
# http://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html
# http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html#sklearn.ensemble.ExtraTreesClassifier
# http://scikit-learn.org/stable/modules/classes.html#module-sklearn.ensemble
# http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier.fit

# CLEANING
# Lots of cleaning needed. Convert everything to a constant format to parse. Tab separate, comma separated, colon separated, etc
# Add missing headers
# Prep missing values using imputing.
# Categorical data needs to be mapped to dummy numerical values for trees to work properly.

# Import stuff
import numpy as np
import pandas as pd
import scipy.stats as stats
import matplotlib.pyplot as plt
import statsmodels.api as sm
import sklearn
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.cross_validation import cross_val_score
from sklearn import cross_validation
from sklearn.preprocessing import Imputer
import timeit
import time

CSV_SEP = ','

# get list of datasets of UCI rep to use
def get_dataset_list():
    datasets = [
                'data/anneal.data.txt',
                'data/segment.dat.txt',
                'data/krkopt.data.txt',
                'data/house-votes-84.data.txt',
                'data/soybean-large.data.txt',
                'data/sat.trn.txt',
                'data/sonar.all-data.txt',
                'data/letter-recognition.data.txt',
                'data/winequality-white.csv',
                'data/winequality-red.csv'

                # 'splice.data.txt' - Need to enter comma after every char ,rest cleaning is done.
                #'data/glass.data.txt', - This is continuous
                #'data/heart.dat.txt',- This is continuous
                ]

    return datasets

# Do bagging.
# X : {array-like, sparse matrix} of shape = [n_samples, n_features]
# Y : array-like, shape = [n_samples]
def do_bagging(X, y):
    bagging = BaggingClassifier(DecisionTreeClassifier(), 200, 0.67, 1.0, True, True)
    return cross_val_score(bagging, X, y, cv=10)

# Do boosting
# X : {array-like, sparse matrix} of shape = [n_samples, n_features]
# Y : array-like, shape = [n_samples]
def do_boosting(X, y):
    boosting = AdaBoostClassifier(DecisionTreeClassifier(), n_estimators=100)
    return cross_val_score(boosting, X, y, cv=10)

# Do Randomization
# X : {array-like, sparse matrix} of shape = [n_samples, n_features]
# Y : array-like, shape = [n_samples]
def do_randomization(X, y):
    random = ExtraTreesClassifier(200)
    return cross_val_score(random, X, y, cv=10)

# Do plain vanilla CART
# X : {array-like, sparse matrix} of shape = [n_samples, n_features]
# Y : array-like, shape = [n_samples]
def do_cart(X, y):
    cart = DecisionTreeClassifier()
    return cross_val_score(cart, X, y, cv=10)

# Convert accuracy scores to error rates.
def convert_to_error_rate(score):
    error = []
    for foldscore in score:
        errorscore = 1 - foldscore
        error.append(errorscore)
    return error

# Remove missing values from data and replace then with NaN for imputer to work.
def remove_missing_values(arr):
    X = arr
    for row_ind, row in enumerate(arr):
        for col_ind, element in enumerate(row):
            if (element == '?'):
                X[row_ind][col_ind] = float('NaN') #Fixed number to identify missing values.
    return X

# Convert categorical feature data to dummy values to CART can work.
def convert_categorical_data(arr):
    X = []

    for row_ind, row in enumerate(arr):
        # create a map from your variable names to unique integers:
        intmap = dict([(val, i) for i, val in enumerate(set(row))])
        # make the new array hold corresponding integers instead of strings:
        new_vals = [intmap[val] for val in row]
        X.append(new_vals)
    return X

def time_fn( fn, *args, **kwargs ):
    start = time.clock()
    results = fn( *args, **kwargs )
    end = time.clock()
    fn_name = fn.__module__ + "." + fn.__name__
    print(fn_name + ": " + str(end-start) + "s")
    return [results, end-start]

# Main
scores = []
error_rates = []
mean_error_rates = []
mean_scores = []
meanTimes = []
for dataset_url in get_dataset_list():
    print("Testing: "+dataset_url)
    datascore = []
    dataerrorrate = []

    # Read data.
    df = pd.read_csv(dataset_url, CSV_SEP)

    # Remove non attribute columns.
    Y = df['class'].values
    if ('instance' in df):
        df.drop('instance', 1)
    df = df.drop('class', 1)

    # Prep data.
    df.fillna(df.mode().iloc[0])
    X = df.as_matrix()
    X = remove_missing_values(X)
    X = convert_categorical_data(X)
    imp = Imputer(strategy='most_frequent', axis=0)
    imp.fit(X, Y)
    X = imp.transform(X)

    # Get cross validation scores.
    cart_score, cartTime = time_fn(do_cart, X, Y)
    bagging_score, bagTime = time_fn(do_bagging, X, Y)
    boosting_score, boostTime = time_fn(do_boosting, X, Y)
    random_score, randomTime = time_fn(do_randomization, X, Y)

    print("Cart: {0} bag: {1} boost: {2} random: {3}".format(cartTime, bagTime, boostTime, randomTime))
    meanTimes.append([cartTime, bagTime, boostTime, randomTime])

    # Store scores.
    datascore.append(cart_score)
    datascore.append(bagging_score)
    datascore.append(boosting_score)
    datascore.append(random_score)
    scores.append(datascore)

    # Get error rates.
    cart_error = convert_to_error_rate(cart_score)
    bagging_error = convert_to_error_rate(bagging_score)
    boosting_error = convert_to_error_rate(boosting_score)
    random_error = convert_to_error_rate(random_score)

    # Store error rates.
    dataerrorrate.append(cart_error)
    dataerrorrate.append(bagging_error)
    dataerrorrate.append(boosting_error)
    dataerrorrate.append(random_error)
    error_rates.append(dataerrorrate)

    # Calculate mean error rates.

    mean_error_rates.append([np.mean(cart_error), np.mean(bagging_error), np.mean(boosting_error), np.mean(random_error)])
    mean_scores.append([np.mean(cart_score), np.mean(bagging_score), np.mean(boosting_score), np.mean(random_score)])

print("Average Times: ")
cartTime = np.mean([x[0] for x in meanTimes])
print("Cart: {0}".format(cartTime))
bagTime = np.mean([x[1] for x in meanTimes])
print("Bag: {0}".format(bagTime))
boostTime = np.mean([x[2] for x in meanTimes])
print("Boost: {0}".format(boostTime))
randomTime = np.mean([x[3] for x in meanTimes])
print("Random: {0}".format(randomTime))

names = dict()
names[0] = "Annealing (9)"
names[1] = "Voting (15)"
names[2] = "King Rook King Pawn (31)"
names[3] = "Soybean (6)"
names[4] = "Segment (4)"
names[5] = "Satellite (8)"
names[6] = "Sonar (1)"
names[7] = "Letter Recognition (2)"
names[8] = "Wine quality - white"
names[9] = "Wine quality - red"

# Plot box plot - scores.
for idx, score in enumerate(scores):
    plt.boxplot(score, labels=['CART', 'Bagging', 'Boosting', 'Randomization'])
    plt.ylabel('Score')
    plt.xlabel('Classification Method')
    plt.title('Classification Score - ' + names[idx])
    plt.show()

# Plot box plot - Error rates.
for idx, errorrate in enumerate(error_rates):
    plt.boxplot(errorrate, labels=['CART', 'Bagging', 'Boosting', 'Randomization'])
    plt.ylabel('Error rates')
    plt.xlabel('Classification Method')
    plt.title('Error Rates -' + names[idx])
    plt.show()

# Plot error rates of various algorithms
mean_error_rates = np.array(mean_error_rates)
plt.plot(mean_error_rates.T[0], label='CART')
plt.plot(mean_error_rates.T[1], label='Bagging')
plt.plot(mean_error_rates.T[2], label='Boosting')
plt.plot(mean_error_rates.T[3], label='Randomization')
plt.legend(loc='best')
plt.ylabel('Error rates')
plt.xlabel('Data Set')
plt.title('Line plot of error rates for various datasets using various algorithms')
plt.show()

mean_scores = np.array(mean_scores)
plt.plot(mean_scores.T[0], label='CART')
plt.plot(mean_scores.T[1], label='Bagging')
plt.plot(mean_scores.T[2], label='Boosting')
plt.plot(mean_scores.T[3], label='Randomization')
plt.legend(loc='best')
plt.ylabel('Classification scores')
plt.xlabel('Data Set')
plt.title('Line plot of classication scores for various datasets using various algorithms')
plt.show()

mean_times = np.array(meanTimes)
plt.plot(mean_times.T[0], label='CART')
plt.plot(mean_times.T[1], label='Bagging')
plt.plot(mean_times.T[2], label='Boosting')
plt.plot(mean_times.T[3], label='Randomization')
plt.legend(loc='best')
plt.ylabel('Run times')
plt.xlabel('Data Set')
plt.title('Line plot of run times for various datasets using various algorithms')
plt.show()