import requests
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Reads in the PPG for each team in 2022
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2022-04-05"

html = requests.get(url).content
ppg2022_list = pd.read_html(html)
ppg2022 = ppg2022_list[0]

# Reads in the Opposing team's PPG for each team in 2022
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2022-04-05"

html = requests.get(url).content
oppPpg2022_list = pd.read_html(html)
oppPpg2022 = oppPpg2022_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2022 = oppPpg2022.drop(oppPpg2022.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2022 = oppPpg2022.rename(columns={'2021': 'OppPPG'})

ppg2022 = ppg2022.drop(ppg2022.columns[[0,3,4,5,6,7]], axis=1)
ppg2022 = ppg2022.rename(columns={'2021': 'PPG'})

# Reads in the PPG for each team in 2021
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2021-04-06"

html = requests.get(url).content
ppg2021_list = pd.read_html(html)
ppg2021 = ppg2021_list[0]

# Reads in the Opposing team's PPG for each team in 2021
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2021-04-06"

html = requests.get(url).content
oppPpg2021_list = pd.read_html(html)
oppPpg2021 = oppPpg2021_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2021 = oppPpg2021.drop(oppPpg2021.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2021 = oppPpg2021.rename(columns={'2020': 'OppPPG'})

ppg2021 = ppg2021.drop(ppg2021.columns[[0,3,4,5,6,7]], axis=1)
ppg2021 = ppg2021.rename(columns={'2020': 'PPG'})

# Reads in the PPG for each team in 2019
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2019-04-09"

html = requests.get(url).content
ppg2019_list = pd.read_html(html)
ppg2019 = ppg2019_list[0]

# Reads in the Opposing team's PPG for each team in 2019
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2019-04-09"

html = requests.get(url).content
oppPpg2019_list = pd.read_html(html)
oppPpg2019 = oppPpg2019_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2019 = oppPpg2019.drop(oppPpg2019.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2019 = oppPpg2019.rename(columns={'2018': 'OppPPG'})

ppg2019 = ppg2019.drop(ppg2019.columns[[0,3,4,5,6,7]], axis=1)
ppg2019 = ppg2019.rename(columns={'2018': 'PPG'})

# Reads in the PPG for each team in 2018
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2018-04-03"

html = requests.get(url).content
ppg2018_list = pd.read_html(html)
ppg2018 = ppg2018_list[0]

# Reads in the Opposing team's PPG for each team in 2018
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2018-04-03"

html = requests.get(url).content
oppPpg2018_list = pd.read_html(html)
oppPpg2018 = oppPpg2018_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2018 = oppPpg2018.drop(oppPpg2018.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2018 = oppPpg2018.rename(columns={'2017': 'OppPPG'})

ppg2018 = ppg2018.drop(ppg2018.columns[[0,3,4,5,6,7]], axis=1)
ppg2018 = ppg2018.rename(columns={'2017': 'PPG'})

# Reads in the PPG for each team in 2017
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2017-04-04"

html = requests.get(url).content
ppg2017_list = pd.read_html(html)
ppg2017 = ppg2017_list[0]

# Reads in the Opposing team's PPG for each team in 2017
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2017-04-04"

html = requests.get(url).content
oppPpg2017_list = pd.read_html(html)
oppPpg2017 = oppPpg2017_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2017 = oppPpg2017.drop(oppPpg2017.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2017 = oppPpg2017.rename(columns={'2016': 'OppPPG'})

ppg2017 = ppg2017.drop(ppg2017.columns[[0,3,4,5,6,7]], axis=1)
ppg2017 = ppg2017.rename(columns={'2016': 'PPG'})

# Reads in the PPG for each team in 2016
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2016-04-05"

html = requests.get(url).content
ppg2016_list = pd.read_html(html)
ppg2016 = ppg2016_list[0]

# Reads in the Opposing team's PPG for each team in 2016
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2016-04-05"

html = requests.get(url).content
oppPpg2016_list = pd.read_html(html)
oppPpg2016 = oppPpg2016_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2016 = oppPpg2016.drop(oppPpg2016.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2016 = oppPpg2016.rename(columns={'2015': 'OppPPG'})

ppg2016 = ppg2016.drop(ppg2016.columns[[0,3,4,5,6,7]], axis=1)
ppg2016 = ppg2016.rename(columns={'2015': 'PPG'})

# Reads in the PPG for each team in 2015
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2015-04-06"

html = requests.get(url).content
ppg2015_list = pd.read_html(html)
ppg2015 = ppg2015_list[0]

# Reads in the Opposing team's PPG for each team in 2015
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2015-04-06"

html = requests.get(url).content
oppPpg2015_list = pd.read_html(html)
oppPpg2015 = oppPpg2015_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2015 = oppPpg2015.drop(oppPpg2015.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2015 = oppPpg2015.rename(columns={'2014': 'OppPPG'})

ppg2015 = ppg2015.drop(ppg2015.columns[[0,3,4,5,6,7]], axis=1)
ppg2015 = ppg2015.rename(columns={'2014': 'PPG'})

# Reads in the PPG for each team in 2014
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2014-04-07"

html = requests.get(url).content
ppg2014_list = pd.read_html(html)
ppg2014 = ppg2014_list[0]

# Reads in the Opposing team's PPG for each team in 2014
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2014-04-07"

html = requests.get(url).content
oppPpg2014_list = pd.read_html(html)
oppPpg2014 = oppPpg2014_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2014 = oppPpg2014.drop(oppPpg2014.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2014 = oppPpg2014.rename(columns={'2013': 'OppPPG'})

ppg2014 = ppg2014.drop(ppg2014.columns[[0,3,4,5,6,7]], axis=1)
ppg2014 = ppg2014.rename(columns={'2013': 'PPG'})

# Reads in the PPG for each team in 2013
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2013-04-08"

html = requests.get(url).content
ppg2013_list = pd.read_html(html)
ppg2013 = ppg2013_list[0]

# Reads in the Opposing team's PPG for each team in 2013
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2013-04-08"

html = requests.get(url).content
oppPpg2013_list = pd.read_html(html)
oppPpg2013 = oppPpg2013_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2013 = oppPpg2013.drop(oppPpg2013.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2013 = oppPpg2013.rename(columns={'2012': 'OppPPG'})

ppg2013 = ppg2013.drop(ppg2013.columns[[0,3,4,5,6,7]], axis=1)
ppg2013 = ppg2013.rename(columns={'2012': 'PPG'})

# Reads in the PPG for each team in 2012
url = "https://www.teamrankings.com/ncaa-basketball/stat/points-per-game?date=2012-04-02"

html = requests.get(url).content
ppg2012_list = pd.read_html(html)
ppg2012 = ppg2012_list[0]

# Reads in the Opposing team's PPG for each team in 2012
url = "https://www.teamrankings.com/ncaa-basketball/stat/opponent-points-per-game?date=2012-04-02"

html = requests.get(url).content
oppPpg2012_list = pd.read_html(html)
oppPpg2012 = oppPpg2012_list[0]

# Drops the repetitive columns and Renames the meaningful columns
oppPpg2012 = oppPpg2012.drop(oppPpg2012.columns[[0,3,4,5,6,7]], axis=1)
oppPpg2012 = oppPpg2012.rename(columns={'2011': 'OppPPG'})

ppg2012 = ppg2012.drop(ppg2012.columns[[0,3,4,5,6,7]], axis=1)
ppg2012 = ppg2012.rename(columns={'2011': 'PPG'})

# Reads in team names and conferences of teams in NCAA Tournament for 2022
url = "https://en.wikipedia.org/wiki/2022_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2022 = pd.DataFrame()
MMteams2022 = pd.concat([MMteams_list[5], MMteams_list[6], MMteams_list[7], MMteams_list[8]])
MMteams2022 = MMteams2022.reset_index(drop=True)

#Merges ppg2022 and OppPPG2022 as well as standardizing the School Column
points2022 = ppg2022.merge(oppPpg2022)
points2022 = points2022.rename(columns={'Team': 'School'})
points2022['School'] = points2022['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2022
points2022.loc[154, 'School'] = 'New Mexico State'
points2022.loc[217, 'School'] = 'Cal State Fullerton'
points2022.loc[192, 'School'] = "Saint Mary's"
points2022.loc[18, 'School'] = "North Carolina"
points2022.loc[148, 'School'] = "Virginia Tech"
points2022.loc[274, 'School'] = "Saint Peter's"
points2022.loc[223, 'School'] = "TCU"
points2022.loc[110, 'School'] = "Loyola Chicago"
points2022.loc[1, 'School'] = "South Dakota State"
points2022.loc[158, 'School'] = "Jacksonville State"
points2022.loc[205, 'School'] = "Texas Southern"
points2022.loc[78, 'School'] = "Texas A&M–Corpus Christi"

# Creates a dataframe for all 2022 data
MMteams2022 = pd.merge(MMteams2022, points2022)
MMteams2022['year'] = 2022

# Reads in team names and conferences of teams in NCAA Tournament for 2021
url = "https://en.wikipedia.org/wiki/2021_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2021 = pd.DataFrame()
MMteams2021 = pd.concat([MMteams_list[4], MMteams_list[5], MMteams_list[6], MMteams_list[7]])
MMteams2021 = MMteams2021.reset_index(drop=True)

#Merges ppg2021 and OppPPG2021 as well as standardizing the School Column
points2021 = ppg2021.merge(oppPpg2021)
points2021 = points2021.rename(columns={'Team': 'School'})
points2021['School'] = points2021['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2021
points2021.loc[90, 'School'] = 'UC Santa Barbara'
points2021.loc[23, 'School'] = 'Eastern Washington'
points2021.loc[192, 'School'] = "Grand Canyon"
points2021.loc[249, 'School'] = "Appalachian State"
points2021.loc[137, 'School'] = "UConn"
points2021.loc[179, 'School'] = "St. Bonaventure"
points2021.loc[103, 'School'] = "UNC Greensboro"
points2021.loc[81, 'School'] = "Abilene Christian"
points2021.loc[323, 'School'] = "Mount St. Mary's"
points2021.loc[107, 'School'] = "Texas Southern"
points2021.loc[58, 'School'] = "North Carolina"
points2021.loc[130, 'School'] = "Virginia Tech"
points2021.loc[30, 'School'] = "West Virginia"
points2021.loc[176, 'School'] = "Loyola Chicago"
points2021.loc[64, 'School'] = "Georgia Tech"

# Creates a dataframe for all 2021 data
MMteams2021 = pd.merge(MMteams2021, points2021)
MMteams2021['year'] = 2021

# Reads in team names and conferences of teams in NCAA Tournament for 2019
url = "https://en.wikipedia.org/wiki/2019_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2019 = pd.DataFrame()
MMteams2019 = pd.concat([MMteams_list[4], MMteams_list[5], MMteams_list[6], MMteams_list[7]])
MMteams2019 = MMteams2019.reset_index(drop=True)

#Merges ppg2019 and OppPPG2019 as well as standardizing the School Column
points2019 = ppg2019.merge(oppPpg2019)
points2019 = points2019.rename(columns={'Team': 'School'})
points2019['School'] = points2019['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2019
points2019.loc[125, 'School'] = "Virginia Tech"
points2019.loc[50, 'School'] = 'Mississippi State'
points2019.loc[150, 'School'] = 'UCF'
points2019.loc[276, 'School'] = "North Carolina Central"
points2019.loc[154, 'School'] = "North Dakota State"
points2019.loc[53, 'School'] = "St. John's"
points2019.loc[52, 'School'] = "Northern Kentucky"
points2019.loc[102, 'School'] = "Fairleigh Dickinson"
points2019.loc[90, 'School'] = "Prairie View A&M"
points2019.loc[81, 'School'] = "Ole Miss"
points2019.loc[147, 'School'] = "Saint Mary's"
points2019.loc[86, 'School'] = "Gardner–Webb"
points2019.loc[2, 'School'] = "North Carolina"
points2019.loc[45, 'School'] = "New Mexico State"
points2019.loc[78, 'School'] = "Northeastern"
points2019.loc[207, 'School'] = "Abilene Christian"

# Creates a dataframe for all 2018 data
MMteams2019 = pd.merge(MMteams2019, points2019)
MMteams2019['year'] = 2019

# Reads in team names and conferences of teams in NCAA Tournament for 2018
url = "https://en.wikipedia.org/wiki/2018_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2018 = pd.DataFrame()
MMteams2018 = pd.concat([MMteams_list[3], MMteams_list[4], MMteams_list[5], MMteams_list[6]])
MMteams2018 = MMteams2018.reset_index(drop=True)

#Merges ppg2018 and OppPPG2018 as well as standardizing the School Column
points2018 = ppg2018.merge(oppPpg2018)
points2018 = points2018.rename(columns={'Team': 'School'})
points2018['School'] = points2018['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2018
points2018.loc[231, 'School'] = "Loyola Chicago"
points2018.loc[216, 'School'] = "UMBC"
points2018.loc[21, 'School'] = 'North Carolina'
points2018.loc[11, 'School'] = "South Dakota State"
points2018.loc[208, 'School'] = "UNC Greensboro"
points2018.loc[253, 'School'] = "North Carolina Central"
points2018.loc[64, 'School'] = "Texas Southern"
points2018.loc[36, 'School'] = "West Virginia"
points2018.loc[38, 'School'] = "Virginia Tech"
points2018.loc[70, 'School'] = "St. Bonaventure"
points2018.loc[62, 'School'] = "Stephen F. Austin"
points2018.loc[211, 'School'] = "Cal State Fullerton"
points2018.loc[83, 'School'] = "LIU Brooklyn"
points2018.loc[17, 'School'] = "TCU"
points2018.loc[105, 'School'] = "New Mexico State"
points2018.loc[122, 'School'] = "College of Charleston"
points2018.loc[109, 'School'] = "Penn"

# Creates a dataframe for all 2018 data
MMteams2018 = pd.merge(MMteams2018, points2018)
MMteams2018['year'] = 2018

# Reads in team names and conferences of teams in NCAA Tournament for 2017
url = "https://en.wikipedia.org/wiki/2017_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2017 = pd.DataFrame()
MMteams2017 = pd.concat([MMteams_list[3], MMteams_list[4], MMteams_list[5], MMteams_list[6]])
MMteams2017 = MMteams2017.reset_index(drop=True)

#Merges ppg2017 and OppPPG2017 as well as standardizing the School Column
points2017 = ppg2017.merge(oppPpg2017)
points2017 = points2017.rename(columns={'Team': 'School'})
points2017['School'] = points2017['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2017
points2017.loc[130, 'School'] = "SMU"
points2017.loc[170, 'School'] = "South Carolina"
points2017.loc[39, 'School'] = 'Virginia Tech'
points2017.loc[13, 'School'] = "UNC Wilmington"
points2017.loc[37, 'School'] = "East Tennessee State"
points2017.loc[62, 'School'] = "New Mexico State"
points2017.loc[284, 'School'] = "Mount St. Mary's"
points2017.loc[329, 'School'] = "Jacksonville State"
points2017.loc[165, 'School'] = "North Carolina Central"
points2017.loc[18, 'School'] = "West Virginia"
points2017.loc[187, 'School'] = "Saint Mary's"
points2017.loc[77, 'School'] = "Florida Gulf Coast"
points2017.loc[85, 'School'] = "South Dakota State"
points2017.loc[8, 'School'] = "North Carolina"
points2017.loc[135, 'School'] = "Middle Tennessee"
points2017.loc[120, 'School'] = "Northern Kentucky"
points2017.loc[136, 'School'] = "Texas Southern"

# Creates a dataframe for all 2017 data
MMteams2017 = pd.merge(MMteams2017, points2017)
MMteams2017['year'] = 2017

# Reads in team names and conferences of teams in NCAA Tournament for 2016
url = "https://en.wikipedia.org/wiki/2016_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2016 = pd.DataFrame()
MMteams2016 = pd.concat([MMteams_list[3], MMteams_list[4], MMteams_list[5], MMteams_list[6]])
MMteams2016 = MMteams2016.reset_index(drop=True)

#Merges ppg2016 and OppPPG2016 as well as standardizing the School Column
points2016 = ppg2016.merge(oppPpg2016)
points2016 = points2016.rename(columns={'Team': 'School'})
points2016['School'] = points2016['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2016
points2016.loc[102, 'School'] = "Miami"
points2016.loc[157, 'School'] = "UConn"
points2016.loc[96, 'School'] = 'South Dakota State'
points2016.loc[137, 'School'] = "UNC Asheville"
points2016.loc[61, 'School'] = "Saint Joseph's"
points2016.loc[266, 'School'] = "Northern Iowa"
points2016.loc[41, 'School'] = "UNC Wilmington"
points2016.loc[8, 'School'] = "Green Bay"
points2016.loc[187, 'School'] = "Cal State Bakersfield"
points2016.loc[7, 'School'] = "North Carolina"
points2016.loc[35, 'School'] = "West Virginia"
points2016.loc[28, 'School'] = "Stephen F. Austin"
points2016.loc[86, 'School'] = "Florida Gulf Coast"
points2016.loc[74, 'School'] = "Fairleigh Dickinson"
points2016.loc[232, 'School'] = "Little Rock"
points2016.loc[177, 'School'] = "Middle Tennessee"

# Creates a dataframe for all 2016 data
MMteams2016 = pd.merge(MMteams2016, points2016)
MMteams2016['year'] = 2016

# Reads in team names and conferences of teams in NCAA Tournament for 2015
url = "https://en.wikipedia.org/wiki/2015_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2015 = pd.DataFrame()
MMteams2015 = pd.concat([MMteams_list[3], MMteams_list[4], MMteams_list[5], MMteams_list[6]])
MMteams2015 = MMteams2015.reset_index(drop=True)

#Merges ppg2015 and OppPPG2015 as well as standardizing the School Column
points2015 = ppg2015.merge(oppPpg2015)
points2015 = points2015.rename(columns={'Team': 'School'})
points2015['School'] = points2015['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2015
points2015.loc[42, 'School'] = "West Virginia"
points2015.loc[125, 'School'] = "Northeastern"
points2015.loc[156, 'School'] = 'New Mexico State'
points2015.loc[12, 'School'] = "North Carolina"
points2015.loc[133, 'School'] = "Texas Southern"
points2015.loc[71, 'School'] = "Coastal Carolina"
points2015.loc[218, 'School'] = "Northern Iowa"
points2015.loc[78, 'School'] = "North Carolina State"
points2015.loc[105, 'School'] = "SMU"
points2015.loc[74, 'School'] = "St. John's"
points2015.loc[7, 'School'] = "Stephen F. Austin"
points2015.loc[4, 'School'] = "Eastern Washington"
points2015.loc[235, 'School'] = "North Dakota State"
points2015.loc[28, 'School'] = "North Florida"
points2015.loc[111, 'School'] = "Robert Morris"

# Creates a dataframe for all 2015 data
MMteams2015 = pd.merge(MMteams2015, points2015)
MMteams2015['year'] = 2015

# Reads in team names and conferences of teams in NCAA Tournament for 2014
url = "https://en.wikipedia.org/wiki/2014_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

MMteams2014 = pd.DataFrame()
MMteams2014 = pd.concat([MMteams_list[4], MMteams_list[5], MMteams_list[6], MMteams_list[7]])
MMteams2014 = MMteams2014.reset_index(drop=True)

#Merges ppg2014 and OppPPG2014 as well as standardizing the School Column
points2014 = ppg2014.merge(oppPpg2014)
points2014 = points2014.rename(columns={'Team': 'School'})
points2014['School'] = points2014['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2014
points2014.loc[50, 'School'] = "Stephen F. Austin"
points2014.loc[128, 'School'] = "Western Michigan"
points2014.loc[156, 'School'] = 'Eastern Kentucky'
points2014.loc[42, 'School'] = "Mount St. Mary's"
points2014.loc[35, 'School'] = "North Carolina"
points2014.loc[113, 'School'] = "UConn"
points2014.loc[85, 'School'] = "George Washington"
points2014.loc[129, 'School'] = "Saint Joseph's"
points2014.loc[106, 'School'] = "North Carolina Central"
points2014.loc[178, 'School'] = "Milwaukee"
points2014.loc[170, 'School'] = "Coastal Carolina"
points2014.loc[72, 'School'] = "North Dakota State"
points2014.loc[36, 'School'] = "New Mexico State"
points2014.loc[15, 'School'] = "Louisiana-Lafayette"
points2014.loc[60, 'School'] = "Texas Southern"
points2014.loc[44, 'School'] = "Massachusetts"

# Creates a dataframe for all 2014 data
MMteams2014 = pd.merge(MMteams2014, points2014)
MMteams2014['year'] = 2014

# Reads in team names and conferences of teams in NCAA Tournament for 2013
url = "https://en.wikipedia.org/wiki/2013_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html, header=1)

MMteams2013 = pd.DataFrame()
MMteams2013 = pd.concat([MMteams_list[3], MMteams_list[4],MMteams_list[5], MMteams_list[6]])
MMteams2013 = MMteams2013.reset_index(drop=True)

#Merges ppg2013 and OppPPG2013 as well as standardizing the School Column
points2013 = ppg2013.merge(oppPpg2013)
points2013 = points2013.rename(columns={'Team': 'School'})
points2013['School'] = points2013['School'].apply(lambda x: x[:-2] + 'State' if x[-3:] == 'St' else x)

# Renames the teams that conflict with MMteams2013
points2013.loc[15, 'School'] = "North Carolina"
points2013.loc[49, 'School'] = "South Dakota State"
points2013.loc[9, 'School'] = 'Northwestern State'
points2013.loc[51, 'School'] = "Florida Gulf Coast"
points2013.loc[177, 'School'] = "Western Kentucky"
points2013.loc[6, 'School'] = "Ole Miss"
points2013.loc[106, 'School'] = "Miami"
points2013.loc[215, 'School'] = "James Madison"
points2013.loc[2, 'School'] = "Long Island"
points2013.loc[78, 'School'] = "Middle Tennessee"
points2013.loc[29, 'School'] = "Saint Mary's (CA)"
points2013.loc[163, 'School'] = "New Mexico State"
points2013.loc[288, 'School'] = "North Carolina A&T"

# Creates a dataframe for all 2013 data
MMteams2013 = pd.merge(MMteams2013, points2013)
MMteams2013['year'] = 2013

# Reads in team names and conferences of teams in NCAA Tournament for 2012
url = "https://en.wikipedia.org/wiki/2012_NCAA_Division_I_men%27s_basketball_tournament"

html = requests.get(url).content
MMteams_list = pd.read_html(html)

# Resets Column Names since the 2012 webpage gave the tables a header
MMteams2012 = pd.DataFrame()
MMteams_list[4].columns = ['Seed', 'School', 'Conference', 'Record', 'Berth type', 'Overall rank']
MMteams_list[5].columns = ['Seed', 'School', 'Conference', 'Record', 'Berth type', 'Overall rank']
MMteams_list[6].columns = ['Seed', 'School', 'Conference', 'Record', 'Berth type', 'Overall rank']
MMteams_list[7].columns = ['Seed', 'School', 'Conference', 'Record', 'Berth type', 'Overall rank']
MMteams2012 = pd.concat([MMteams_list[4],MMteams_list[5], MMteams_list[6], MMteams_list[7]])
MMteams2012 = MMteams2012.reset_index(drop=True)

#Merges ppg2012 and OppPPG2012 as well as standardizing the School Column
points2012 = ppg2012.merge(oppPpg2012)
points2012 = points2012.rename(columns={'Team': 'School'})
points2012['School'] = points2012['School'].apply(lambda x: x[:-2] + 'State' if x[-2:] == 'St' else x)

# Renames the teams that conflict with MMteams2012
points2012.loc[94, 'School'] = "Southern Miss"
points2012.loc[85, 'School'] = "West Virginia"
points2012.loc[96, 'School'] = 'St. Bonaventure'
points2012.loc[171, 'School'] = "Loyola"
points2012.loc[8, 'School'] = "UNC Asheville"
points2012.loc[2, 'School'] = "North Carolina"
points2012.loc[42, 'School'] = "Saint Mary's"
points2012.loc[324, 'School'] = "South Florida"
points2012.loc[100, 'School'] = "Detroit Mercy"
points2012.loc[13, 'School'] = "New Mexico State"
points2012.loc[141, 'School'] = "Mississippi Valley State"
points2012.loc[233, 'School'] = "Western Kentucky"
points2012.loc[43, 'School'] = "Long Beach State"
points2012.loc[22, 'School'] = "South Dakota State"

# Creates a dataframe for all 2012 data
MMteams2012 = pd.merge(MMteams2012, points2012)
MMteams2012['year'] = 2012

# Combines all data from 2012-2022
MMteams = pd.concat([MMteams2022, MMteams2021, MMteams2019, MMteams2018, MMteams2017, MMteams2016, MMteams2015, MMteams2014, MMteams2013, MMteams2012], ignore_index=True)

# Resets Column names as some did match, and deletes the extra columns
MMteams['Overall Seed'] = MMteams['Overall rank'].combine_first(MMteams['Overall Seed'])
MMteams['Overall Seed'] = MMteams['Overall rank[4]'].combine_first(MMteams['Overall Seed'])
MMteams = MMteams.drop(['Overall rank', 'Coach', 'Overall rank[4]'], axis=1)

# Renames some conferences as different years used different names for the same conference
MMteams['Conference'] = MMteams['Conference'].str.replace('Pac-12', 'Pac–12')
MMteams['Conference'] = MMteams['Conference'].str.replace('ASUN', 'Atlantic Sun')
MMteams['Conference'] = MMteams['Conference'].str.replace('CAA', 'Colonial')
MMteams['Conference'] = MMteams['Conference'].str.replace('Ivy League', 'Ivy')
MMteams['Conference'] = MMteams['Conference'].str.replace('WCC', 'West Coast')
MMteams['Conference'] = MMteams['Conference'].str.replace('Summit League', 'Summit')
MMteams['Conference'] = MMteams['Conference'].str.replace('Conference USA', 'C-USA')
MMteams['Conference'] = MMteams['Conference'].str.replace('NEC', 'Northeast')
MMteams['Conference'] = MMteams['Conference'].str.replace('Mid American', 'MAC')
MMteams['Conference'] = MMteams['Conference'].str.replace('MVC', 'Missouri Valley')
MMteams['Berth type'] = MMteams['Berth type'].str.replace('Auto', 'Automatic')
MMteams['Berth type'] = MMteams['Berth type'].str.replace('Automaticmatic', 'Automatic')
MMteams['Berth type'] = MMteams['Berth type'].str.replace('At–large', 'At-Large')
MMteams['Berth type'] = MMteams['Berth type'].str.replace('At-large', 'At-Large')

# Resets PPG and OppPPG to floats to allow averaging
MMteams['PPG'] = MMteams['PPG'].astype(float)
MMteams['OppPPG'] = MMteams['OppPPG'].astype(float)

# Group by year and calculate the mean of PPG and OppPPG
grouped = MMteams.groupby('year')[['PPG', 'OppPPG']].mean()

# Plot the dual bar chart
grouped.plot(kind='bar', width=0.8, figsize=(8, 6))
plt.xlabel('Year')
plt.ylabel('Points per game')
plt.title('Average PPG and OppPPG by year')
plt.legend(['PPG', 'OppPPG'])
plt.show()

# There is not too significant of a trend on this chart, both averages went up a little in 2017 and 2018, but not too
# significantly, or long enough to consider it a trend.

# Group by year and calculate the mean of PPG and OppPPG
MMteams['Seed'] = MMteams['Seed'].astype(str)

# Removes all *'s and #'s from the Seed Column, and makes Seed a Float
MMteams['Seed'] = MMteams['Seed'].str.replace('*', '')
MMteams['Seed'] = MMteams['Seed'].str.replace('#', '')
MMteams['Seed'] = MMteams['Seed'].astype(float)

# Plot the dual bar chart
avg_seed = MMteams.groupby('Conference')['Seed'].mean()
avg_seed_sorted = avg_seed.sort_values(ascending=False)

# Create a bar chart of the average seed by conference
fig, ax = plt.subplots(figsize=(10, 10))

# plot the bar chart
ax.barh(avg_seed_sorted.index, avg_seed_sorted.values)

# customize the chart
ax.set_title('Average Seed by Conference')
ax.set_xlabel('Conference')
ax.set_ylabel('Average Seed')
ax.tick_params(axis='x', rotation=10)
plt.show()

# There is an obvious jump between the Atlantic 10 and Conference USA on this chart. The reason is likely because the
# conferences below the A10 usually get one sometimes two teams into the tournament, and since their teams are not as good
# as the other conferences their teams struggle to gain a high enough Strength of Schedule to both make the tournament,
# as well as earning top seeds

# Counts how many occurences of each conference are found in the dataset
n = 15
conference_counts = MMteams['Conference'].value_counts()
top_conferences = conference_counts[:n].index.tolist()
MMteams.loc[~MMteams['Conference'].isin(top_conferences), 'Conference'] = '1 Bid Conferences'
conference_counts = MMteams['Conference'].value_counts()

# Plots the resulting Pie Chart
plt.figure(figsize=(8,8))
plt.pie(conference_counts, labels=conference_counts.index, autopct='%1.1f%%')
plt.title('Conference Distribution')
plt.show()

# The 1-bid Conferences were the conferences in which between 2012 and 2022 only had one team each year. An interesting note
# is that the "Power 5" were not the top 5 conferences on the list as the SEC and the Pac-12 both fell underneath the Big
# East in this chart


#!/usr/bin/env python3
import pandas as pd
import matplotlib.pyplot as plt

import gc_analysis

PARAMS = {'font.size': 20}

def make_flareplot(datafile, output):
    hbond = pd.read_csv(datafile, sep=r'\t', index_col=False)

    sorting = ["D1", "A1", "D2", "A2", "D3", "A3", "D4"]
    test = {val: number for number, val in enumerate(sorting)}
    hbond = hbond.replace(test)

    hbond.nhb=hbond.nhb/500000.0
    hbond = hbond.set_index(['acceptor', 'donor'])['nhb']

    flareplot = gc_analysis.flareplot(hbond)
    flareplot.replace_labels(test.keys())
    flareplot.fig.savefig(output)

if __name__ == '__main__':
    with plt.rc_context(PARAMS):
        make_flareplot('example.dat', 'example.png')


from __future__ import division
import numpy as np
import funcs as fun
from plots import plot3d, plot_conf_int, fig_bias_var, seaborn_heatmap
from sklearn.exceptions import ConvergenceWarning
from sklearn.utils.testing import ignore_warnings
np.random.seed(777)

#------------------------------------------------
def ex_a(N, n):
    """OLS on Franke function"""
    print("a) Ordinary Least Square:")
    x = np.sort(np.random.uniform(0, 1, N))
    y = np.sort(np.random.uniform(0, 1, N))

    X, Y = np.meshgrid(x, y)
    noise = 0.2*np.random.normal(0, 1, size=X.shape)
    Z = fun.FrankeFunction(X, Y)
    z = np.ravel(Z)
    Zn = fun.FrankeFunction(X, Y) + noise
    M = fun.DesignMatrix(X, Y, n)

    beta_OLS = fun.OLS(M, z)
    y_tilde = M @ beta_OLS
    mse_OLS = fun.MSE(Z, y_tilde)
    r2score_OLS = fun.R2score(Z, y_tilde)
    var_OLS = fun.VAR(y_tilde)
    Bias_OLS = fun.BIAS(Z, y_tilde)

    print("MSE = %.6f" %mse_OLS)
    print("R2-score = %.6f" %r2score_OLS)
    print("Variance = %.6f" %var_OLS)
    print("Bias = %.6f" %Bias_OLS)

    plot_conf_int(N, hyperparam=1, method="OLS")
    plot3d(X, Y, Z, Zn, method=None)

#------------------------------------------------

def ex_b(N):
    """Resampling with k-fold OLS"""
    print("b) Resampling of test data with k-fold:")
    x = np.sort(np.random.uniform(0, 1, N))
    y = np.sort(np.random.uniform(0, 1, N))

    X, Y = np.meshgrid(x, y)
    noise = 0.2*np.random.normal(0, 1, size=X.shape)
    Z = fun.FrankeFunction(X, Y)
    Zn = fun.FrankeFunction(X, Y) + noise

    dim_params = [4, 5, 6, 7, 8, 9, 10]
    A = np.zeros((9, len(dim_params)))

    for i, dims in enumerate(dim_params):
        Mse, R2, Var, Bias = fun.k_fold_CV_franke(X, Y, Z, folds=5, dim=dims,
                                        hyperparam=1, method="OLS", train=False)
        Mse_n, R2_n, Var_n, Bias_n = fun.k_fold_CV_franke(X, Y, Zn, folds=5,
                                        dim=dims, hyperparam=1, method="OLS", train=False)
        A[:,i] = dims, Mse, Mse_n, R2, R2_n, Var, Var_n, Bias, Bias_n
        print("Franke function without noise:")
        print("MSE k-fold = %.10f" %Mse)
        print("R2-score k-fold = %.10f" %R2)
        print("Variance k-fold = %.10f" %Var)
        print("Bias k-fold = %.10f" %Bias)
        print("Franke function with noise:")
        print("MSE k-fold = %.10f" %Mse_n)
        print("R2-score k-fold = %.10f" %R2_n)
        print("Variance k-fold = %.10f" %Var_n)
        print("Bias k-fold = %.10f" %Bias_n)
    fun.make_tab(A, task="ex_b_OLS", string="6f")

#------------------------------------------------

def ex_c(N):
    """Bias-variance tradeoff OLS"""
    print("c) Plotting bias-variance tradeoff:")
    x = np.sort(np.random.uniform(0, 1, N))
    y = np.sort(np.random.uniform(0, 1, N))

    X, Y = np.meshgrid(x, y)
    fig_bias_var(X, Y, p=11, method="OLS")

#------------------------------------------------

def ex_d(N, n):
    """Ridge Regression"""
    print("d) Ridge analysis:")
    x = np.sort(np.random.uniform(0, 1, N))
    y = np.sort(np.random.uniform(0, 1, N))

    X, Y = np.meshgrid(x, y)
    noise = 0.2*np.random.normal(0, 1, size=X.shape)
    Z = fun.FrankeFunction(X, Y)
    z = np.ravel(Z)
    Zn = fun.FrankeFunction(X, Y) + noise
    zn = np.ravel(Zn)
    M = fun.DesignMatrix(X, Y, n)
    lambda_Ridge = np.logspace(-7, 1, 9)

    A = np.zeros((5, len(lambda_Ridge)))

    # Without resampling and with noise
    for j, lambd in enumerate(lambda_Ridge):
        beta_Ridge = fun.Ridge(M, zn, hyperparam=lambd)
        y_tilde = M @ beta_Ridge
        mse_Ridge = fun.MSE(Zn, y_tilde)
        r2score_Ridge = fun.R2score(Zn, y_tilde)
        var_Ridge = fun.VAR(y_tilde)
        bias_Ridge = fun.BIAS(Zn, y_tilde)
        A[:,j] = lambd, mse_Ridge, r2score_Ridge, var_Ridge, bias_Ridge

        print("Lambda=", lambd)
        print("MSE =", mse_Ridge)
        print("R2-score =", r2score_Ridge)
        print("Variance =", var_Ridge)
        print("Bias = ", bias_Ridge)
    fun.make_tab(A, task="ex_d_Ridge", string="5e")

    # With resampling
    lambda_params = [1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-10]
    dim_params = [5, 6, 7, 8, 9, 10, 11]
    R2 = np.zeros((len(dim_params), (len(lambda_params))))
    R2_n = np.zeros_like(R2)
    #for i, dims in enumerate(dim_params):
        #for j, lambd in enumerate(lambda_params):
            #Mse, R2[i][j], Var, Bias = fun.k_fold_CV_franke(X, Y, Z, folds=5,
                                        #dim=dims, hyperparam=lambd,
                                        #method="Ridge", train=False)
            #Mse_n, R2_n[i][j], Var_n, Bias_n = fun.k_fold_CV_franke(X, Y, Zn,
                                                #folds=5, dim=dims, hyperparam=lambd,
                                                #method="Ridge", train=False)

    # Test best dimension (n) and model complexity (lambda)
    #seaborn_heatmap(R2, lambda_params, dim_params, noise="No", method="Ridge")
    #seaborn_heatmap(R2_n, lambda_params, dim_params, noise="Yes", method="Ridge")

    # Run through different hyperparameters
    B = np.zeros((9, len(lambda_params)))
    #for j, lambd in enumerate(lambda_params):
        #plot_conf_int(N, hyperparam=lambd, method="Ridge")

        #Mse, R2, Var, Bias = fun.k_fold_CV_franke(X, Y, Z, folds=5, dim=n,
                                #hyperparam=lambd, method="Ridge", train=False)
        #Mse_n, R2_n, Var_n, Bias_n = fun.k_fold_CV_franke(X, Y, Zn, folds=5,
                                #dim=n, hyperparam=lambd, method="Ridge", train=False)
        #B[:,j] = lambd, Mse, Mse_n, R2, R2_n, Var, Var_n, Bias, Bias_n

        #print("Franke function without noise:")
        #print("Lambda:", lambd)
        #print("MSE k-fold = %.10f" %Mse)
        #print("R2-score k-fold = %.10f" %R2)
        #print("Variance k-fold = %.10f" %Var)
        #print("Bias k-fold = %.10f" %Bias)
        #print("Franke function with noise:")
        #print("Lambda:", lambd)
        #print("MSE k-fold = %.10f" %Mse_n)
        #print("R2-score k-fold = %.10f" %R2_n)
        #print("Variance k-fold = %.10f" %Var_n)
        #print("Bias k-fold = %.10f" %Bias_n)
    #fun.make_tab(B, task="ex_d_kfold", string="5e")

    #fig_bias_var(X, Y, p=12, method="Ridge")

#-------------------------------------------------------

@ignore_warnings(category=ConvergenceWarning)
def ex_e(N, n):
    """Lasso Regression"""
    print("e) Lasso analysis:")
    x = np.sort(np.random.uniform(0, 1, N))
    y = np.sort(np.random.uniform(0, 1, N))

    X, Y = np.meshgrid(x, y)
    noise = 0.2*np.random.normal(0, 1, size=X.shape)
    Z = fun.FrankeFunction(X, Y)
    z = np.ravel(Z)
    Zn = fun.FrankeFunction(X, Y) + noise
    zn = np.ravel(Zn)
    M = fun.DesignMatrix(X, Y, n)

    lambda_Lasso = np.logspace(-10, 0, 9)
    A = np.zeros((5, len(lambda_Lasso)))
    # Without resampling and noise
    for j, i in enumerate(lambda_Lasso):
        beta_Lasso = fun.Lasso(M, zn, hyperparam=i)  # lambda=0 should give ~OLS
        y_tilde = M @ beta_Lasso
        mse_Lasso = fun.MSE(Zn, y_tilde)
        r2score_Lasso = fun.R2score(Zn, y_tilde)
        var_Lasso = fun.VAR(y_tilde)
        bias_Lasso = fun.BIAS(Zn, y_tilde)
        A[:,j] = i, mse_Lasso, r2score_Lasso, var_Lasso, bias_Lasso

        print("Lambda:", i)
        print("MSE =", mse_Lasso)
        print("R2-score =", r2score_Lasso)
        print("Variance =", var_Lasso)
        print("Bias = ", bias_Lasso)

    fun.make_tab(A, task="ex_e_Lasso", string="5e")

    # With resampling
    lambda_params = [1e-5, 1e-6, 1e-7, 1e-8, 1e-10, 1e-12]
    dim_params = [5, 6, 7, 8, 9, 10, 12]
    R2 = np.zeros((len(dim_params), (len(lambda_params))))
    R2_n = np.zeros_like(R2)
    #for i, dims in enumerate(dim_params):
        #for j, lambd in enumerate(lambda_params):
            #Mse, R2[i][j], Var, Bias = fun.k_fold_CV_franke(X, Y, Z, folds=5, dim=dims, hyperparam=lambd, method="Lasso", train=False)
            #Mse_n, R2_n[i][j], Var_n, Bias_n = fun.k_fold_CV_franke(X, Y, Zn, folds=5, dim=dims, hyperparam=lambd, method="Lasso", train=False)

    # Test best dimension (n) and model complexity (lambda)
    #seaborn_heatmap(R2, lambda_params, dim_params, noise="No", method="Lasso")
    #seaborn_heatmap(R2_n, lambda_params, dim_params, noise="Yes", method="Lasso")

    B = np.zeros((9, len(lambda_params)))
    #for j, lambd in enumerate(lambda_params):
        #plot_conf_int(N, hyperparam=lambd, method="Lasso")

        #Mse, R2, Var, Bias = fun.k_fold_CV_franke(X, Y, Z, folds=5, dim=n, hyperparam=lambd, method="Lasso", train=False)
        #Mse_n, R2_n, Var_n, Bias_n = fun.k_fold_CV_franke(X, Y, Zn, folds=5, dim=n, hyperparam=lambd, method="Lasso", train=False)
        #B[:,j] = lambd, Mse, Mse_n, R2, R2_n, Var, Var_n, Bias, Bias_n

        #print("Franke function without noise:")
        #print("Lambda:", lambd)
        #print("MSE k-fold = %.10f" %Mse)
        #print("R2-score k-fold = %.10f" %R2)
        #print("Variance k-fold = %.10f" %Var)
        #print("Bias k-fold = %.10f" %Bias)
        #print("Franke function with noise:")
        #print("Lambda:", lambd)
        #print("MSE k-fold = %.10f" %Mse_n)
        #print("R2-score k-fold = %.10f" %R2_n)
        #print("Variance k-fold = %.10f" %Var_n)
        #print("Bias k-fold = %.10f" %Bias_n)
    #fun.make_tab(B, task="ex_e_kfold", string="5e")

    #fig_bias_var(X, Y, p=12, method="Lasso")

#-------------------------------------------------------

@ignore_warnings(category=ConvergenceWarning)
def best(N, n, hyperparam, method=""):
    x = np.sort(np.random.uniform(0, 1, N))
    y = np.sort(np.random.uniform(0, 1, N))

    X, Y = np.meshgrid(x, y)
    noise = 0.2*np.random.normal(0, 1, size=X.shape)

    Zn = fun.FrankeFunction(X, Y) + noise
    z = np.ravel(Zn)

    M = fun.DesignMatrix(X, Y, n)
    if method == "OLS":
        beta = fun.OLS(M, z)
    elif method == "Ridge":
        beta = fun.Ridge(M, z, hyperparam)
    elif method == "Lasso":
        beta = fun.Lasso(M, z, hyperparam)

    y_tilde = M @ beta
    mse = fun.MSE(Zn, y_tilde)
    r2score = fun.R2score(Zn, y_tilde)
    var = fun.VAR(y_tilde)
    bias = fun.BIAS(Zn, y_tilde)
    print("MSE = %.6f" %mse)
    print("R2-score = %.6f" %r2score)
    print("Variance = %.6f" %var)
    print("Bias = %.6f" %bias)
    if method == "OLS":
        A = [n, mse, r2score, var, bias]
    else:
        A = [hyperparam, n, mse, r2score, var, bias]

    fun.make_tab(A, task="best_"+str(method), string="6f")
    plot3d(X, Y, Zn, y_tilde, method)

if __name__ == "__main__":
    """
    NB! Be careful if running several tasks at once. Running several tasks at
    the same time may affect the results you get, for some reason.
    Comment out the desired task to run (both below and above in the functions)
    """
    #ex_a(N=50, n=5)
    #ex_b(N=50)
    #ex_c(N=50)
    #ex_d(N=50, n=5)
    #ex_e(N=50, n=5)
    #best(N=50, n=9, hyperparam=1, method="OLS")
    #best(N=50, n=9, hyperparam=1e-7, method="Ridge")
    #best(N=50, n=9, hyperparam=1e-7, method="Lasso")
    pass


from rest_framework import serializers
from .models import Store, Product, PriceEntry, UserPoints
from django.contrib.auth.models import User

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('id', 'username')

class StoreSerializer(serializers.ModelSerializer):
    class Meta:
        model = Store
        fields = ('id', 'name', 'address', 'created_at', 'updated_at')

class ProductSerializer(serializers.ModelSerializer):
    class Meta:
        model = Product
        fields = ('id', 'name', 'description', 'barcode', 'created_at', 'updated_at')

class PriceEntrySerializer(serializers.ModelSerializer):
    reported_by = UserSerializer(read_only=True)

    class Meta:
        model = PriceEntry
        fields = ('id', 'product', 'store', 'price', 'date_reported', 'reported_by', 'created_at', 'updated_at')

    def create(self, validated_data):
        # Get the user from the context
        user = self.context['request'].user

        # Check if the user is authenticated
        if user and user.is_authenticated:
            validated_data['reported_by'] = user
            price_entry = super().create(validated_data)

            # Award points for price report
            points_awarded = 10  # This could also be a setting or a calculated value
            UserPoints.objects.get_or_create(user=user)[0].add_points(points_awarded)

            return price_entry
        else:
            raise serializers.ValidationError("Users must be authenticated to create a price entry.")

class UserPointsSerializer(serializers.ModelSerializer):
    user = UserSerializer(read_only=True)

    class Meta:
        model = UserPoints
        fields = ('id', 'user', 'total_points', 'created_at', 'updated_at')


from gensim.corpora.dictionary import Dictionary
from gensim.models import TfidfModel, LdaMulticore
import pandas as pd
import csvfile, modeling
import time

print("---------------------테슬라 주식 데이터 Dataframe ------------------------")

# 테슬라 주식 데이터 Dataframe
# 전체 날짜
#테슬라 주식 병합 (2020 ~ 20203.09)
stockfilepath = './yumi/data/'
stockSaveFilepath = './yumi/data/merge/'
stockFileName = 'merge_stock_data'
#csvfile.stock_merge_csv(stockfilepath, stockSaveFilepath, stockFileName)

mergeFilePath = './yumi/data/merge/'

#테슬라 주식 dataFrame
stocks_df = pd.DataFrame()
stocks_df = csvfile.read_csv(mergeFilePath, stockFileName)

#datetime으로 타입 변환
stocks_df['날짜'] = pd.to_datetime(stocks_df['날짜'])
#print('주식 데이터 shape : ', stocks_df.shape)

#거래량 M를 제외한 숫자로만 표기습을 진행함
stocks_df['거래량'] = pd.to_numeric(stocks_df['거래량'].str.replace('M', ''))

#날짜와 거래량 추출 : stock_df
stock_df = stocks_df[['날짜','거래량']]
#print(stock_df[:10])

print("---------------------테슬라 기사 데이터 Dataframe ------------------------")

newsFilePath = './yumi/data/merge/'
newsFileName = 'tesla_news_cleaninngWords'

#테슬라 뉴스 dataFrame
words_df = pd.DataFrame()
words_df = csvfile.read_csv(newsFilePath, newsFileName)

#주식 데이터와 열 이름 통일
words_df = words_df.rename(columns={'date' : '날짜'})

#datetime으로 타입 변환
words_df['날짜'] = pd.to_datetime(words_df['날짜'])

#기사 데이터 날짜별 기사 수 추출
news_cnt_daily = words_df.groupby(words_df['날짜']).count()['nouns_content']
#print(news_cnt_daily)

daily_news_count = news_cnt_daily.reset_index()
daily_news_count.columns = ['날짜', '기사갯수']
print(daily_news_count.info())

print("---------------------기사와 주식 병합 Dataframe ------------------------")
# 주식 데이터와 기사 데이터를 병합
data_df = pd.merge(stock_df, daily_news_count, how='outer', on='날짜')
#print(data_df.info())

#결측지 제거 : 주가가 없는 날짜 제거 (주말)
data_df = data_df.dropna()
print(data_df.info())

#날짜별 기사개수/주식거래량 수
stock_daily_volume = data_df['거래량']
news_daily_cnt = data_df['기사갯수']

#총 기사개수/거래량 수
#stock_volume = data_df['거래량'].sum()
#news_cnt = data_df['기사갯수'].sum()
#print('2023년 기사 개수 :', news_cnt)

print('날짜별 기사개수 : {}, 날짜별 주식 거래양 수 : {}'.format(news_daily_cnt, stock_daily_volume))

print("---------------------라쏘 회귀 ------------------------")
# 라쏘 회귀  :
# 선형 회귀으로 비용함수인 mse를 최소하하는 방향으로 머신러닝 모델이 학습을 진행
# TF-IDF는 단어의 빈도수(TF)와 단어가 들어있는 문서 수의 반비례 하는 수(IDF)를 곱한값
# 기사 수(news_cnt) : X, 주식거래량(stock_volume) : y
# 각 데이터를 StandardScaler/MinMaxScaler로 표준화하여 모델링

X = data_df['기사갯수'].values.reshape(-1,1)
y = data_df['거래량'].values.reshape(-1,1)
alpha = 0.001
n_splits = 5
alphas = [0.001, 0.01, 0.1, 1, 10, 100]

#라쏘회귀 실행
print(modeling.LASSO_KFold(X, y, alpha, n_splits))

# 최적의 alpha 찾기
print(modeling.optimize_alpha(X, y, alphas, n_splits))

print("---------------------LDA 모델링------------------------")

def save_topics_csv(lda, num_topics, save_result_to: str = './yumi/data/lda.csv'):
  # LDA 모델의 토픽 리스트를 csv파일로 저장
  topics = pd.Series(lda.print_topics(num_topics=num_topics, num_words=10))
  topics.to_csv(save_result_to, mode='w', encoding='utf-8', header=['list'], index_label='topic')

#딕셔너리 생성 : 다시하기
dic = Dictionary()
#clean_words = words_df['nouns_content']
clean_words = words_df['nouns_content'].apply(lambda x: str(x).split())
id2word = Dictionary(clean_words)
#print(id2word)

corpus_TDM = []
for doc in clean_words:
  #print(doc)
  result = id2word.doc2bow(doc)
  corpus_TDM.append(result)

#tfidf로 벡터화 적용
tfidf = TfidfModel(corpus_TDM)
corpus_TFIDF = tfidf[corpus_TDM]

#LDA 모델링
if __name__ == '__main__':
  start_time = time.time()
  n = 30 #토픽의 개수
  #worker(프로세스 수),토픽수.passes(매개변수) 수를 조정하여
  #속도를 높일 수 있음
  lda = LdaMulticore(corpus=corpus_TFIDF,
                    id2word=id2word,
                    num_topics=n,
                    random_state=100,
                    passes=15,
                    workers=6)

  for t in lda.print_topics():
    print(t[0],":",t[1])

  #로그 혼란도(0에 가까울수록 성능이 높음)
  lda.log_perplexity(corpus_TFIDF)

  #다양도(1에 가까울 수록 성능이 높음)
  topn = 25
  top_words = set()

  for topic in range(lda.num_topics):
      for word, prob in lda.show_topic(topic, topn=topn):
          top_words.add(word)
  len(top_words) / (lda.num_topics * topn)

  #csv 파일 저장
  save_topics_csv(lda, n)
  end_time = time.time()

  execution_time = end_time - start_time
  print(f"실행 시간: {execution_time} 초")


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains

options = Options()
options.add_argument('--disable-notifications')

chrome = webdriver.Chrome('./chromedriver', options=options)
chrome.get("https://www.binance.com/zh-TW/markets")

my_list = list()

time.sleep(5)
# click button
python_button = chrome.find_elements_by_xpath('//*[@id="__APP"]/div[1]/main/div/div[2]/div/div/div[2]/div[1]/div[1]/div/button[4]')[0]
python_button.click()
# click button
python_button = chrome.find_elements_by_xpath('//*[@id="__APP"]/div[1]/main/div/div[2]/div/div/div[2]/div[2]/div/div/div[3]')[0]
python_button.click()
crypts = chrome.find_elements_by_class_name('css-1c1ahuy')
for crypt in crypts:
    my_list.append(crypt)
    print(crypt.text)
ele = crypts[19]
while True:
    chrome.execute_script("arguments[0].scrollIntoView();", ele)
    time.sleep(1)
    crypts1 = chrome.find_elements_by_class_name('css-1c1ahuy')
    temp = set(crypts1).difference(my_list)
    if(len(temp)<=0):
        break
    for crypt in set(crypts1).difference(crypts):
        my_list.append(crypt)
        print(crypt.text)
    ele = crypts1[len(crypts1) - 1]


import os
import time
import argparse

import numpy as np

import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP

from ConTNet import build_model
from optimizer import build_optimizer
from lr_scheduler import build_lr_scheduler
from criterion import build_criterion
from data import build_loader

from utils import accuracy, reduce_tensor, resume_model, save_model
from timm.utils import AverageMeter

import warnings
warnings.filterwarnings("ignore")

def parse_args():
    parser = argparse.ArgumentParser(description='ConTNet')

    # data and model
    parser.add_argument('--data_path', type=str, help='path to dataset')
    parser.add_argument('--arch', type=str, default='ConT-M',
                        choices=['ConT-M', 'ConT-B', 'ConT-S', 'ConT-Ti'],
                        help='the architecture of ConTNet')

    # model hypeparameters
    parser.add_argument('--use_avgdown', type=bool, default=False,
                        help='If True, using avgdown downsampling shortcut')
    parser.add_argument('--relative', type=bool, default=False,
                        help='If True, using relative position embedding')
    parser.add_argument('--qkv_bias', type=bool, default=True)
    parser.add_argument('--pre_norm', type=bool, default=False)

    # base setting
    parser.add_argument('--eval', default=None, type=str,
                        help='only validation')
    parser.add_argument('--batch_size', default=512, type=int,
                        help='batch size')
    parser.add_argument('--workers', default=8, type=int,
                        help='number of data loading workers')
    parser.add_argument('--epoch', default=200, type=int,
                        help='number of total epochs to run')
    parser.add_argument('--warmup_epoch', default=10, type=int,
                        help='the num of warmup epochs')
    parser.add_argument('--resume', default=None, type=str,
                        help='resume file path')
    parser.add_argument('--init_lr', default=5e-4, type=float,
                        help='a low initial learning rata for adamw optimizer')
    parser.add_argument('--wd', default=0.5, type=float,
                        help='a high weight decay setting for adamw optimizer')
    parser.add_argument('--momentum', default=0.9, type=float,
                        help='momentum for sgd')
    parser.add_argument('--optim', default='AdamW', type=str, choices=['AdamW', 'SGD'],
                        help='optimizer supported by PyTorch')
    parser.add_argument('--print_freq', default=100, type=int,
                        help='frequency of printing train info')
    parser.add_argument('--save_path', default='weights', type=str,
                        help='the path to saving the checkpoints')
    parser.add_argument('--save_best', default=True, type=bool,
                        help='saveing the checkpoint has the best acc')

    # aug&reg
    parser.add_argument('--mixup', default=0.8, type=float,
                        help='using mixup and set alpha value')
    parser.add_argument('--autoaug', default='rand-m9-mstd0.5-inc1', type=str,
                        help='using auto-augmentation')
    parser.add_argument('-ls','--label-smoothing', default=0.1, type=float,
                        help='if > 0, using label-smothing')

    # distributed parallel triaining
    parser.add_argument("--local_rank", type=int, required=True, help='local rank for DDP')

    return parser.parse_args()

def launch_worker(local_rank):
    # print(local_rank)
    if not torch.cuda.is_available():
        raise ValueError(f'CPU-only training is not supported')
    torch.backends.cudnn.benchmark = True
    torch.cuda.set_device(local_rank)
    dist.init_process_group(backend='nccl', init_method='env://')
    dist.barrier()

def train(loader, model, criterion, optimizer, mixup_fn, scheduler, print_freq, epoch):
    model.train()
    if dist.get_rank() == 0:
        print(f'\n=> Training epoch{epoch}')

    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    end = time.time()
    for i, (images, targets) in enumerate(loader):
        images = images.cuda(non_blocking=True)
        targets = targets.cuda(non_blocking=True)

        if mixup_fn:
            images, targets_ = mixup_fn(images, targets)

        # forward
        outputs = model(images)

        # update acc1, acc5
        acc1, acc5 = accuracy(outputs, targets, topk=(1, 5))
        acc1 = reduce_tensor(acc1)
        acc5 = reduce_tensor(acc5)
        top1.update(acc1.item(), targets.size(0))
        top5.update(acc5.item(), targets.size(0))

        # compute loss and backward
        loss = criterion(outputs, targets_)
        loss = reduce_tensor(loss)
        losses.update(loss.item(), targets_.size(0))
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        scheduler.step_update(epoch * len(loader) + i)

        # update using time
        interval = torch.tensor([time.time() - end])
        interval = reduce_tensor(interval.cuda())
        batch_time.update(interval.item())
        end = time.time()

        if i % print_freq == 0 and dist.get_rank() == 0:
            lr = optimizer.param_groups[0]['lr']
            sep = '| '
            print(f'Epoch: [{epoch}] | [{i}/{len(loader)}] lr: {lr:.8f} '+ sep +
                f'loss {losses.val:.4f} ({losses.avg:.4f}) '+ sep +
                f'Top1.acc {top1.val:6.2f} ' + sep +
                f'Top5.acc {top5.val:6.2f} ' + sep +
                f'time {batch_time.val:.4f} ({batch_time.avg:.4f}) ' + sep
                )

@torch.no_grad()
def validate(val_loader, model, criterion, epoch=None):
    model.eval()

    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    end = time.time()
    for i, (images, targets) in enumerate(val_loader):
        images = images.cuda(non_blocking=True)
        targets = targets.cuda(non_blocking=True)

        # forward
        outputs = model(images)

        loss = criterion(outputs, targets)
        loss = reduce_tensor(loss)
        losses.update(loss.item(), images.size(0))

        # update acc1, acc5
        acc1, acc5 = accuracy(outputs, targets, topk=(1, 5))
        acc1 = reduce_tensor(acc1)
        acc5 = reduce_tensor(acc5)
        top1.update(acc1.item(), targets.size(0))
        top5.update(acc5.item(), targets.size(0))

        # update using time
        interval = torch.tensor([time.time() - end])
        interval = reduce_tensor(interval.cuda())
        batch_time.update(interval.item())
        end = time.time()

    if dist.get_rank() == 0:
        stat = f"epoch {epoch}" if epoch is not None else "Only"
        print(f'=> Validation {stat}')
        sep = '| '
        print(f'loss {losses.avg:.4f} '+ sep +
            f'Top1.acc {top1.avg:6.2f} ' + sep +
            f'Top5.acc {top5.avg:6.2f} ' + sep +
            f'time {batch_time.avg:.4f} ' + sep
            )

    return top1.avg, top5.avg, losses.avg

def main(config):
    # set up ddp
    launch_worker(config.local_rank)
    # build loader
    train_loader, val_loader = build_loader(config.data_path, config.autoaug, config.batch_size, config.workers)
    # build model
    model=build_model(config.arch, config.use_avgdown, config.relative, config.qkv_bias, config.pre_norm)
    model = DDP(model.cuda(), device_ids=[config.local_rank])
    # build optimizer
    optimizer=build_optimizer(model, config.optim, config.init_lr, config.wd, config.momentum)
    # build learning scheduler
    scheduler=build_lr_scheduler(config.epoch, config.warmup_epoch, optimizer, len(train_loader))
    # build criterion and mixup
    train_criterion, mixup_fn =build_criterion(config.mixup, config.label_smoothing)
    val_criterion = torch.nn.CrossEntropyLoss()
    # init acc1 and start epoch
    best_acc1 = 0.0
    start_epoch = 0

    # only validation
    if config.eval:
        if os.path.isfile(config.eval):
            model.load_state_dict(torch.load(config.eval)['model'])
            validate(val_loader, model, val_criterion)
            return
        else:
            print(f"=> !!!!!!! no checkpoint found at '{config.eval}'\n")
            print(f"=> !!!!!!! validation is stopped")
            return

    # resume training
    if not config.resume:
        print(f"=>Training is from scratch")
    else:
        if os.path.isfile(config.resume):
            model, optimizer, scheduler, start_epoch, best_acc1 = resume_model(config.resume, model, optimizer, scheduler)
        else:
            print(f"=> !!!!!!! no checkpoint found at '{config.resume}'\n")

    # training
    for epoch in range(start_epoch, args.epoch):
        train_loader.sampler.set_epoch(epoch)

        train(train_loader, model, train_criterion, optimizer, mixup_fn, scheduler, config.print_freq, epoch)

        acc1, acc5, loss = validate(val_loader, model, val_criterion, epoch)

        best_acc1 = max(best_acc1, acc1)
        is_best = (best_acc1 == acc1)

        if dist.get_rank() == 0:
            print('\n******************\t',
                   f'\nBest Top1.acc {best_acc1:6.2f}\t',
                    '\n******************\t')

            # save model
            if not config.save_best or is_best:
                save_model(config.save_path, model, optimizer, scheduler, best_acc1, epoch, is_best)

if __name__ == '__main__':
    # build configs
    args = parse_args()
    # launch
    main(config=args)
    print('=> Finished!')