text:
#!/usr/bin/env python
# coding: utf-8

# In[24]:

from jupyterthemes import get_themes
from jupyterthemes.stylefx import set_nb_theme
set_nb_theme('onedork')

# In[25]:

get_ipython().system('pip install matplotlib')

# In[26]:

import matplotlib.pyplot as plt

# In[27]:

squarea=[1,4,8,17,25,30,49,64,81,99]
plt.plot(squarea)
plt.show()

# In[28]:

import matplotlib.pyplot as plt
squares=[1,4,8,17,25,30,49,64,81,99]
plt.plot(squares)

plt.axis([0,8,0,90])
plt.show()

# In[29]:

import matplotlib.pyplot as plt
squares=[1,4,8,17,25,30,49,64,81,99]

plt.plot(squares,linewidth=7)
plt.title("Demo Chart",fontsize=32)
plt.xlabel("value",fontsize=23)
plt.ylabel("Square",fontsize=20)
plt.tick_params(axis='both',labelsize=7,color='red')
plt.axis([0,10,0,90])
plt.show()

# In[30]:

import matplotlib.pyplot as plt
data1=[1,4,8,17,25,30,49,64,81]
data2=[1,2,4,6,8,10,12,14,16]
seq=[1,2,3,4,5,6,7,8,9]
plt.plot(seq,data1,seq,data2,linewidth=3)
plt.title("Demo Chart",fontsize=32)
plt.xlabel("value",fontsize=23)
plt.ylabel("Square",fontsize=20)
plt.tick_params(axis='both',labelsize=7,color='red')
plt.show()

# In[31]:

import matplotlib.pyplot as plt
data1=[1,4,8,17,25,30,49,64,81]
data2=[1,2,4,6,8,10,12,14,16]
data3=[1,6,16,26,36,46,56,66,99]
data4=[1,1,56,31,44,85,6,77,9]
seq=[1,2,3,4,5,6,7,8,9]
plt.plot(seq,data1,'g--',seq,data2,'r-.',seq,data3,'b-',seq,data4,'y-',linewidth=7)
plt.title("Demo Chart",fontsize=32)
plt.xlabel("value",fontsize=23)
plt.ylabel("Square",fontsize=20)
plt.tick_params(axis='both',labelsize=7,color='red')
plt.show()
plt.savefig('demo.png')

# In[ ]:

# In[ ]:

# In[32]:

import matplotlib.pyplot as plt
Benz = [3080, 4620, 6549] # Benz線條
BMW = [3800, 2590, 4388] # BMW線條
Lexus = [6500, 5430, 7450] # Lexus線條
seq = [2018, 2019, 2020] # 年度
plt.xticks(seq)
plt.plot(seq, Benz,'--o',label='benz')
plt.plot(seq, BMW,'-s',label='BMW')
plt.plot(seq, Lexus,'-^',label='Lexus')
plt.title("Sales Report", fontsize=24)
plt.xlabel("Year", fontsize=3)
plt.ylabel("Number of Sales", fontsize=14)
plt.legend(loc='best')
plt.tick_params(axis='both', labelsize=12, color='red')
plt.show()
plt.savefig('legend.png')

# In[ ]:

# In[33]:

import matplotlib.pyplot as plt
plt.scatter(5,5,1000,'red')
plt.show()

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[34]:

import matplotlib.pyplot as plt
xpt=[1,2,3,4,5]
ypt=[1,4,9,16,25]
plt.scatter(xpt,ypt)
plt.show()

# In[35]:

xlst=[]
for  n in range(6):
    xlst.append(n)
print(xlst)

# In[36]:

xlst=[n for n in range(6)]
print(xlst)

# In[37]:

import matplotlib.pyplot as plt
xpt = list(range(1,101))
ypt = [x**2 for x in xpt ]# 取x裡 變平方 給y
plt.scatter(xpt, ypt,color='y')
plt.show()

# In[38]:

import matplotlib.pyplot as plt
data01=[1,4,9,16,25,36,49,64,81]
data02=[1,2,4,6,8,10,12,14,16]
seq=[1,2,3,4,5,6,7,8,9]
plt.figure(1)
plt.plot(seq,data01,'-o')
plt.figure(2)
plt.plot(seq,data02,'--s')
plt.title("Demo chart 2",fontsize=32)
plt.xlabel("x-value",fontsize=24)
plt.ylabel("y-value",fontsize=20)
plt.tick_params(axis='both',labelsize=12,color='red')# axis 是軸；axis 可选 x、y、both
plt.show()

# subplot(x1,x2,x3)
#  x1 垂直幾張  x2 水平幾張  x3第幾張

# ![image.png](attachment:image.png)
#
#
#

# In[ ]:

# In[ ]:

# In[39]:

import matplotlib.pyplot as plt
data01=[1,4,9,16,25,36,49,64,81]
data02=[1,2,4,6,8,10,12,14,16]
seq=[1,2,3,4,5,6,7,8,9]
plt.subplot(2,1,1)
plt.subplots_adjust(hspace = 0.9  )
plt.plot(seq,data01,'-or')
plt.subplot(2,1,2)
plt.plot(seq,data02,'--sb')
plt.title("Demo chart 2",fontsize=22,loc='center')
plt.xlabel("x-value",fontsize=14)
plt.ylabel("y-value",fontsize=10)
plt.tick_params(axis='both',labelsize=8,color='red')# axis 是軸；axis 可选 x、y、both
plt.show()

# left  = 0.125  # the left side of the subplots of the figure
# right = 0.9    # the right side of the subplots of the figure
# bottom = 0.1   # the bottom of the subplots of the figure
# top = 0.9      # the top of the subplots of the figure
# wspace = 0.2   # the amount of width reserved for blank space between subplots,
#                # expressed as a fraction of the average axis width
# hspace = 0.2   # the amount of height reserved for white space between subplots,
#                # expressed as a fraction of the average axis height
# ————————————————
# 版权声明：本文为CSDN博主「GuokLiu」的原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接及本声明。
# 原文链接：https://blog.csdn.net/qq_33039859/article/details/79424858

# In[40]:

get_ipython().system('pip install twstock')
get_ipython().system('pip install lxml')

# In[41]:

import twstock
stock2330= twstock.Stock("2330")
print(stock2330)

# sid 股票代號字串
# open 近31天的開盤價(元)串列
# high 近31天的最高價(元)串列
# low 近31天的最低價(元)串列
# close or price 近31天的收盤價(元)串列
# capacity 近31天的成交量(股)串列
# transaction 近31天的成交筆數(筆)串列
# turnover 近31天的成交金額(元)串列
# change 近31天的漲跌幅(元)串列
# date 近31天的交易日期datatime串列
# data 近31天的全部資料內容串列
# raw_data 近31天的原始資料串列

# In[ ]:

# In[42]:

import twstock
stock2330 = twstock.Stock("2330")
print("股票代號 : ", stock2330.sid)
print("股票收盤價 : ", stock2330.price)

# In[43]:

import matplotlib.pyplot as plt
import twstock
stock2330= twstock.Stock("2330")#先抓，再丟給它

plt.rcParams['font.family'] = 'Adobe Gothic Std'

plt.title(u"台積電",fontsize=24)
plt.plot(stock2330.price)#再丟給它，畫圖
plt.savefig('台積電.png')
plt.show()

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[44]:

#查詢系統已安裝的中文字體
from matplotlib import font_manager
font_set = {f.name for f in
font_manager.fontManager.ttflist}
for f in font_set:
    print(f)

# In[45]:

#檢驗是否可以顯示中文
from matplotlib import pyplot as plt
plt.rcParams['font.family'] = 'Adobe Gothic Std'
plt.text(.5, .5, '台積電')
plt.show()

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# fetch_31() 最近31天的交易資料(Data 物件)串列
# fetch(year, month) 指定年月的交易資料(Data 物件)串列
# fetch_from(year, month) 指定年月至今的交易資料(Data 物件)串列
# moving_average(data,days) 串列數據data的days日平均值串列
# continuous(data) 串列data持續上漲天數

# In[46]:

stock2330=twstock.Stock("2330")
stock2330.fetch_from(2020,6)

plt.rcParams['font.family'] = 'Adobe Gothic Std'

plt.rcParams['axes.unicode_minus']=False

plt.title(u'台積電',fontsize=23)#橫軸名稱 u"中文字串"
plt.xlabel(u"2020年3月以來的交易天數",fontsize=13)
plt.ylabel(u"價格",fontsize=15)
plt.plot(stock2330.price)
plt.show()

# day2__CSV
#
#
#
#
#

# In[47]:

import csv

# with open(檔案名稱) as csvFile     #csvFile自己命名的檔案名
# 或
# csvFile =open(檔案名案)

# In[48]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/csvReport.csv'
with open(fn) as csvFile:
    csvReader=csv.reader(csvFile)#讀檔案建立Reader物件
    listReader=list (csvReader)#將資料轉成串列
print(listReader)

# In[49]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/csvReport.csv'
with open(fn) as csvFile:
    csvReader=csv.reader(csvFile)#讀檔案建立Reader物件
    for row in csvReader:#用迴圈列出csvReader物件內容
        print("Row %s = " % csvReader.line_num,row)

# In[50]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/csvReport.csv'
with open(fn) as csvFile:
    csvReader=csv.reader(csvFile)#讀檔案建立Reader物件
    listReport= list(csvReader)#將資料轉成串列
    for row in listReport:#使用迴圈列出串列內容
        print(row)

# In[51]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/csvReport.csv'
with open(fn) as csvFile:
    csvReader=csv.reader(csvFile)#讀檔案建立Reader物件
    listReport= list(csvReader)#將資料轉成串列
print(listReport[0][1],listReport[0][5])#第？列第？個
print(listReport[1][5],listReport[1][3])
print(listReport[2][5],listReport[2][6])

# In[52]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/csvPeople.csv'
with open(fn) as csvFile:
    csvDictReader=csv.DictReader(csvFile)#讀檔案建立DictReader物件
    for row in csvDictReader:
        print(row)

# In[ ]:

# In[53]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/csvPeople.csv'
with open(fn) as csvFile:
    csvDictReader=csv.DictReader(csvFile)#讀檔案建立DictReader物件
    for row in csvDictReader:
        print(row['first_name'],row['last_name'])

# 寫入CSV檔案
#
# 使用writer物件
# with open(‘檔案名稱’,’w’) as csvFile:
# outWriter = csv.writer(csvFile)

# In[54]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/writerow.csv'
with open(fn,'w') as csvFile:
    csvWriter=csv.writer(csvFile,delimiter='\t')#建立Writer物件
    #資料寫入CSV檔時,預設的分隔符號是逗號,
    #用delimiter更改分隔符號
    csvWriter.writerow(['Name','Age','City'])
    csvWriter.writerow(['Tang','35','taipei'])
    csvWriter.writerow(['Aggie','40','San Francisco'])

# In[55]:

import csv
fn='py_six/day2-20210724T050933Z-001/day2/writedict.csv'
with open(fn,'w') as csvFile:
    fields=['Name','Age','City']
    dictWriter = csv.DictWriter(csvFile,fieldnames=fields)
    dictWriter.writeheader()#寫入標題
    dictWriter.writerow({'Name':'Tanf','Age':'35','City':'Taipei'})
    dictWriter.writerow({'Name':'Aggie','Age':'40','City':'San Francisco'})

# In[56]:

import csv
dictList=[{'Name':'Tanf','Age':'35','City':'Taipei'},{'Name':'Aggie','Age':'40','City':'San Francisco'}]
fn='py_six/day2-20210724T050933Z-001/day2/write_dictlist.csv'
with open(fn,'w') as csvFile:
    fields=['Name','Age','City']
    dictWriter = csv.DictWriter(csvFile,fieldnames=fields)
    dictWriter.writeheader()#寫入標題
    for row in dictList:
        dictWriter.writerow(row)

# 瀏覽器跟網站之間的交換資料,只能文字交換
# JSON全名JavaScript Object Notation,可知最初
# 是由JavaScript開發的
# 流程是將JavaScript物件轉換成JSON,將JSON
# 傳送到伺服器,並從伺服器接收JSON,將JSON轉
# 換成JavaScript物件
#
# JSON的兩種資料格式
# ■ 物件(object):一般用大括號{ }表示
# ■ 陣列(array):一般用中括號[]表示
#
# json格式中字串需用雙引號,文件裡不可以有註解
#
# -物件(object)
# 例如:
# {“Name”:“Aggie”, “Age”:25}
# # {鍵:值} 鍵要是文字
#
# -陣列(array)
# 陣列是由一系列的值(value)所組成,由左中括號“[”
# 開始,右中括號“]”鍵結束
#
#

# 使用json.dump()模組
#
#
#
# python資料__JSON資料

# dict _____________object

# list, tuple ______array

# str, unicode______string

# int, float, long__number

# True______________true

# False_____________false

# None______________null

# In[57]:

import json
listNumbers=[1,10,20,30,40,50]
tupleNumbers=(1,5,10,15,20,25)
jsonData1=json.dumps(listNumbers)#python 轉 json
jsonData2=json.dumps(tupleNumbers)
print("串列轉換json的陣列",jsonData1)
print("元組轉換json的陣列",jsonData2)
print("json陣列在Python的資料類型",type(jsonData1))

# 將json格式資料轉成Python資料
# 使用json模組內的loads()

# In[58]:

import json
jsonObj='{"bob":88,"allen":66,"chris":77}'
dictObj=json.loads(jsonObj)
print(dictObj)
print(type(dictObj))

# In[59]:

{"Asia":[{"Japan":"Toyko"},{"Taiwn":"Taipei"}]}  #注意

# In[60]:

import json
obj='{"Asia":[{"Japan":"Toyko"},{"Taiwan":"Tapei"}]}'
json_obj=json.loads(obj)#json 轉 python
print(json_obj)
print(json_obj["Asia"])
print(json_obj["Asia"][0])
print(json_obj["Asia"][1])
print(json_obj["Asia"][0]["Japan"])

# In[61]:

import json
dictObj={'bob':88,'allen':66,'chris':77}#字典
fn='py_six/day2-20210724T050933Z-001/day2/save_1.json'
with open(fn,'w') as fnObj:#打開 給 寫入
    json.dump(dictObj,fnObj)#字典 轉 json
with open(fn,'r') as fnObj:#打開 給 讀出
    data=json.load(fnObj)#json 轉 python
print(data)
print(data)

# In[62]:

import json
objlist=[{"日本":"Japan","首都":"Tykyo"},{'美州':"USA",'首都':'Washington'}]
fn='py_six/day2-20210724T050933Z-001/day2/save_1.json'
with open(fn,'w') as fnObj:
    json.dump(objlist,fnObj)
with open(fn,'r') as fnObj:
    data1=json.load(fnObj)
print(data1)
print(type(data1))

# In[63]:

import json
objlist=[{"日本":"Japan","首都":"Tykyo"},{'美州':"USA",'首都':'Washington'}]
fn='py_six/day2-20210724T050933Z-001/day2/save_1.json'
with open(fn,'w',encoding='utf-8') as fnObj:#注意  很像
    json.dump(objlist,fnObj,indent=6,ensure_ascii=False)#是  dump 不是dumps
with open(fn,'r',encoding='utf-8') as fnObj:#注意  很像
        kk=json.load(fnObj)#是 load 不是loads
print(kk)

# In[ ]:

# Pandas
#
# Pandas來自panel、dataframe、series,這三個單
# 字,也是Pandas三個資料結構
# ■ Panel
# ■ DataFrame
# ■ Series

# In[64]:

get_ipython().system('pip install pandas')

# In[ ]:

# Series是一種一維的陣列資料結構
# 陣列可以是整數、浮點數、字串
# 也可以是Python(例如:字串list、字典dict...)
# Numpy的ndarray,純量
# 語法如下:
# pandas.Series(data=None,index=None, dtype=None, options, ...)

# In[65]:

import pandas as pd
s=pd.Series([11,22,33,44,55])
print(s)

# In[66]:

import pandas as pd
myindex =[3,8,9]
value=[20,55,67]
s=pd.Series(value,index=myindex)
print(s)

# In[67]:

import pandas as pd
drink=['tea','coca','coffee']
price=[33,12,59]
s=pd.Series(price,index=drink)
print(s)

# DataFrame
#
# DataFrame是一種二維的陣列資料結構
# 陣列可以是整數、浮點數、字串
# 也可以是Python(例如:字串list、字典dict...)
# Numpy的ndarray,純量
# 語法如下:
# pandas.DataFrame(data=None,index=None, dtype=None, options, ...)
#
# _______
# 組合一維陣列Series物件成為二維陣列的
# DataFrame
#
# 55
# pandas.concat([Series1,Series2,... ],axis=1)
#
#

# In[68]:

import pandas as pd
years=range(2018,2021)
tokyo=pd.Series([13,15,19],index=years)
hongkong=pd.Series([25,26,27],index=years)
singapore=pd.Series([30,29,31],index=years)
print("\n")
citydf0=pd.concat([tokyo,hongkong,singapore])#預設axis=0
print(type(citydf0))
print(citydf0)
print("\n")
citydf=pd.concat([tokyo,hongkong,singapore],axis=1)
print(type(citydf))
print(citydf)

# In[69]:

import pandas as pd
years=range(2018,2021)
tokyo=pd.Series([13,15,19],index=years)
hongkong=pd.Series([25,26,27],index=years)
singapore=pd.Series([30,29,31],index=years)
citydf=pd.concat([tokyo,hongkong,singapore],axis=1)
cities=['beijing','hongkong','singapore']
citydf.columns=cities#欄位屬性
print(type(citydf))
print(citydf)

# In[70]:

import pandas as pd
data =[{'apple':50,'Orange':28,'Cherry':80},{'apple':50,'Cherry':80}]
fruits = pd.DataFrame(data)
print(fruits)

# In[71]:

import pandas as pd
cities = {'country':['Taiwan', 'Japan', 'Singapore'],'town':['Taipei','Tokyo','Singapore'],'population':[200, 1600, 600]}
citydf = pd.DataFrame(cities)
print(citydf)

# In[72]:

#:前面是橫軸   水果是3種 各一 共三組 放第一筆資料
# _____________城市是 一個字典裡  一個橫軸 有三組  共三筆  ； 第二個橫軸 有三組  共三筆.....

# In[73]:

import pandas as pd
cities = {'country':['Taiwan', 'Japan', 'Singapore'],'town':['Taipei','Tokyo','Singapore'],'population':[200, 1600, 600]}
rowindex = ['first', 'second', 'third']
citydf = pd.DataFrame(cities, index=rowindex)
print(citydf)

# In[ ]:

# In[74]:

#columns欄名  index列名

# In[75]:

import pandas as pd
cities = {'country':['Taiwan', 'Japan', 'Singapore'],'town':['Taipei','Tokyo','Singapore'],'population':[200, 1600, 600]}
citydf = pd.DataFrame(cities,columns=["town","population"],index=cities["country"])
#citydf.columns=cities (想ˋ一 下\比較 \查資料
print(citydf)

# Series和DataFrame物件完成後,接著可以開始執行資料分析跟處理
#
#
# 屬性 說明
# at 使用index和columns內容取得或設定單一元素內容與陣列內容
# iat 用index和columns編號取得或設定單一元素內容
# loc 使用index和columns內容取得或設定整個row或columns資料或陣列內容
# iloc 用index和columns編號取得或設定整個row或columns資料

# Pandas用to_csv()將DataFrame物件寫入CSV檔案
# to_csv(path= None, sep=’,’, header=True,index=True,encoding=None,...)
# path : 檔案路徑
# sep:分隔字元,預設是‘,’
# header:是否保留columns,預設True
# index:是否保留index,預設True
# encoding:檔案編碼方式

# In[76]:

import pandas as pd
course=['Chinese','English','Math','Natural','Society']
chinese=[14,12,10,13,12]
eng=[13,14,11,10,15]
math=[15,8,13,9,15]
nature=[15,10,13,10,15]
social=[12,11,14,9,14]
df=pd.DataFrame([chinese,eng,math,nature,social],columns=course,index=range(1,6))
df.to_csv("py_six/day2-20210724T050933Z-001/day2/university_a.csv")
df.to_csv("py_six/day2-20210724T050933Z-001/day2/university_b.csv",header=False,index=False)

# Pandas用read_csv()將DataFrame物件讀取CSV檔案
# read_csv(path= None, sep=’,’, index_col=None,names=None, encoding=None,userows=None,usecols=None, ...)
#
# path : 檔案路徑
# sep:分隔字元,預設是‘,’
# header:設定那一row為欄位標籤,預設是0。當參數有names時,此為None。
# 如果所讀取的檔案有欄位標籤時,需設定此header值
# index_col:指出第幾個欄位column是索引,預設是None
# encoding:檔案編碼方式
# nrows:設定讀取前幾個row
# usecols:設定讀取前幾個欄位

# In[77]:

ttr=pd.read_csv("py_six/day2-20210724T050933Z-001/day2/university_b.csv",names=course)
print(ttr)

# 最常使用plot(),基本語法如下:
# plot(x=None, kind = “XX”, title = None, legend=True,rot=None,...)
#
# kind是選擇繪圖模式,預設是line,常見的選項是bar、hist、box、pie、scatter、...等。
# rot是旋轉刻度

# In[78]:

import pandas as pd
import matplotlib.pyplot as plt

population=[860,1100,1450,1800,2020,2200,2260]
tw=pd.Series(population,index=range(1950,2011,10))
tw.plot(title='Population in Taiwan')
plt.xlabel("Year")
plt.ylabel("Population")
plt.show()

# In[79]:

import pandas as pd
import matplotlib.pyplot as plt
cities={'population':[1000,850,800,1500,600,800],'town':['New York','Chicago','Bangkok','Tokyo','Singpore','HongKong']}
tw1=pd.DataFrame(cities,columns=['population'],index=cities['town'])
tw1.plot(title='Population in the world')
plt.xlabel('City')
plt.ylabel('Population')
plt.show( )

# In[80]:

import pandas as pd
import matplotlib.pyplot as plt
cities={'population':[1000,850,800,1500,600,800],'town':['New York','Chicago','Bangkok','Tokyo','Singpore','HongKong']}
tw2=pd.DataFrame(cities,columns=['population'],index=cities['town'])
tw2.plot(title='Population in the world', kind='bar')#kind參數可更改圖表
plt.xlabel('City')
plt.ylabel('Population')
plt.show( )

# In[ ]:

# pie()常用的語法:
# pie(x,options,...)
# x是一個串列,options表示可以選擇的參數:
# ■ labels:標籤
# ■ colors:顏色
# ■ explode :是否從圓餅圖分離,0表示不分離,數值越大越分離,可
# 從0.1開始
# ■ autopct:表示項目百分比格式,基本語法是“%格式%%”,例如:
# “%1.2f%%”表示整數1位數,小數是2位數

# In[81]:

import  pandas as pd
import matplotlib.pyplot as plt

fruits=['Apple','Bananas','Grapes','Pear','Oranges']
s=pd.Series([2300,5000,1200,2500,2900],index=fruits,name='Fruits Shop')
explode1=[0.4,0,0,0.2,0]# 圓餅圖 要分離嗎? 數字越大 分越開
s.plot.pie(explode=explode1,autopct='%1.2f%%')# 預設格式:%____%%
plt.show()

# In[ ]:

# In[82]:

get_ipython().system('pip install pandas-datareader#安裝pandas-datareader股票模組')

# In[83]:

import pandas_datareader as pdr
df_2330=pdr.DataReader('2330.TW','yahoo')
print(df_2330)

# In[84]:

import pandas_datareader as pdr
startTime='2020-03-01'
endTime='2020-06-30'
df_2330=pdr.DataReader('2330.TW','yahoo',startTime,endTime)
print(df_2330)

# In[ ]:

# In[ ]:

# Open 開盤價
# High 最高價
# Low 最低價
# Close 收盤價
# Volume 交易量
# Adj Close 經過調整的收盤價

# day3
#
#
#

# 雖然串列(list),元組(tuple)可以執行一維陣列或
# 多為陣列的運算,但真正在運算時:
# ■ 執行速度慢
# ■ 需要較多的系統資源
#
# Numpy模組主要是支援多維度空間的陣列與矩陣
# 運算,可以產生矩陣,並擴充到圖表
# import numpy as np

# 陣列(array)類似Python串列(list),不過array資料
# 型態必須是"相同",不同於list可以不同。
#
# 陣列是程式語言的一種基本資料結構,
# 屬於循序性的資料結構。
#
# 信箱依序編號,1、2、3、4、5,號碼是存取資料的
# 索引,稱為「一維陣列」。

# 使用linspace()方法,可以產生相同等距的陣列
# ■ linspace(start, end, num) #最常使用簡化的語法
# start是起始值,end是結束值,num產生多少個等距點的的陣列
# 值,預設值是5
#
# ■ arange(start, stop, step)
# start是起始值如果省略預設值是0,stop是結束值但所產生的陣
# 列不包含此值, step是陣列相鄰元素的間距,預設值是1

# In[85]:

import numpy as np
x1= np.linspace(0,10,num=11)
print(type(x1),x1)# 浮點數ㄛ
x2= np.arange(0,11,1)
print(type(x2),x2)
x3= np.arange(11)
print(type(x3),x3)

# In[ ]:

# In[86]:

import matplotlib.pyplot as plt
import numpy as np

xpt=np.linspace(0,10,500)#建立含500個元素
ypt1=np.sin(xpt)#y陣列的變化
ypt2=np.cos(xpt)
plt.scatter(xpt, ypt1,color=(0,1,0))#綠色
plt.scatter(xpt,ypt2)#預設顏色
plt.show()

# In[87]:

import matplotlib.pyplot as plt
import numpy as np

xpt=np.linspace(0,5,500)
ypt=1-0.5*np.abs(xpt-2)
lwidths=(1+xpt)**2
plt.scatter(xpt,ypt,s=lwidths,color=(0,1,0))
plt.show()
rty=plt.scatter(xpt,ypt,s=50,c=ypt,cmap='hsv')
plt.show(rty)

# In[ ]:

# 在scatter()中,增加color的設定參數是c,此時color
# 的值是陣列(或是串列),增加cmap(color map),指
# 定色彩映射值
# https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html

# Numpy模組所建立的陣列資料型態稱
# ndarray(n-dimension array)。ndarray陣列特色如
# 下:
# ■ 陣列大小是固定
# ■ 陣列元素內容的資料型態是相同
# 認識ndarray
#
# 19
#
# 屬性 說明
# ndarray.dtype 陣列元素型態
# ndarray.itemsize 陣列元素資料型態大小,單位是位元組
# ndarray.ndim 陣列的維度
# ndarray.shape 陣列維度元素個數的元祖
# ndarray.size 陣列元素個數

# In[88]:

import numpy as np
x=np.array([1,2,3])
print(x.dtype)#列印x陣列元素型態,例如int64,指64位元
print(x.itemsize)#列印x陣列元素大小,一個位元組是8個位元,會回傳8
print(x.ndim)#列印x陣列維度
print(x.shape)#列印陣列外形,3是第1維元素個數
print(x.size)#列印x陣列元素個數

# In[89]:

import numpy as np
x=np.array([1,2,3])
y=x+8
print(y)
print('='*8)
y=np.array([10,20,30])
z=x+y
print(z)
print('='*8)
z=x/y
print(z)
print('='*8)

# In[90]:

import numpy as np
x=np.array([[1,2,3],[4,5,6]])
print(x.dtype)
print(x.itemsize)
print(x.ndim)
print(x.shape)
print(x.size)
print(x)

# x=np.zeros((2,3))

# x=np.ones((2,3))

# In[95]:

import numpy as np
x=np.array([[1,2,3],[4,5,6]])
print(x[0,2])
print(x[1,1])
print("========")
x[1,2]=20
print(x)

# In[104]:

import numpy as np
x=np.array([[1,2,3],[4,5,6]]) #多維矩陣
print(x[0])#row=0
print(x[0,])#row=0
print(x[0,:])#row=0
print('======')
print(x[:,1])#不管 列  取第一欄
print(list(x))
xx11=np.square(x)
print(xx11)

# In[108]:

u=np.array([[7,2,3,4,5],[2,3,4,5,6],[3,4,5,6,7]])
#逗號前面 是 列  ，後面是 欄   ，清清楚楚
print(u[:3,0])
print(u[:2,3:5])
print(u[1:])
print('='*10)
print(u)

# In[109]:

e=np.array([1,2,3,4,5,6])
g=e.reshape(2,3)#想轉幾維  都可以
print(g)
o=g.reshape(3,2)
print(o)

# In[111]:

a=np.array([[1,2,3],[4,5,6]])
y=x.ravel()
print(y)

b=np.array([[2,3,4],[5,6,7]])
z=b.flatten()
print(z)
'''
首先声明两者所要实现的功能是一致的（将多维数组降位一维），两者的区别在于返回拷贝（copy）还是返回视图（view），
numpy.flatten()返回一份拷贝，对拷贝所做的修改不会影响（reflects）原始矩阵，
而numpy.ravel()返回的是视图（view，也颇有几分C/C++引用reference的意味），会影响（reflects）原始矩阵。
————————————————
'''

# In[112]:

import numpy as np
x=np.array([1,2,3])
y=np.array([5,6,7])
print(np.concatenate([x,y]))
print('='*35)
z=[6,7,8]
print(np.concatenate([x,y,z]))
#使用np.concatanate串接

# In[114]:

import numpy as np
grid=np.array([[1,2,4],[4,7,8]])
print(np.concatenate([grid,grid]))
print('='*90)
print(np.concatenate([grid,grid],axis=1))

# ![00123.png](attachment:00123.png)
#

# In[119]:

import numpy as np
x=np.array([1,2,6])
grid=np.array([[9,8,7],[6,5,4]])
print(np.vstack([x,grid]))#垂直疊加   放下面
print('='*70)
y=np.array([[33],[33]])
print(np.hstack([grid,y]))#水平疊加   橫橫放後面

# hsplit()將陣列依垂直方向切割
# vsplit()將陣列依水平方向切割
# 1. ...
# 2. y1,y2 = np.hsplit(x,2)
# 3. ...
# 4. y1,y2 = np.vsplit(x,2)

# 向量內積
#
# 如 A[a1,a2,a3],B[b1,b2,b3]
# A。B=a1*b1+a2*b2+a3*b3
# np.inner(A,B) or  np.dot(A,B)
#

# In[121]:

import numpy as np
x=np.array([1,2,5])
y=np.array([4,6,3])
z1=np.inner(x,y)
print(z1)
z2=np.dot(x,y)
print(z2)

# 向量外積
#
# ![71105.png](attachment:71105.png)

# In[122]:

import  numpy as np
x=np.array([1,2,3])
y=np.array([5,6,7])
z=np.outer(x,y)
print(z)

# <實作>轉置矩陣

# In[123]:

import numpy as np
x=np.arange(10).reshape(5,2)
print(x)
print('='*45)
y1=x.transpose()
print(y1)
print('='*46)
y2=x.T

# In[125]:

import numpy as np
A=0
X=np.arange(1,101).reshape(10,10)
print(X)
for x in X:
    A+=x
print(type(A))
print('A=',A)
sum=0
for a in A:
    sum+=a
print(type(sum))

# 使用plt.imshow(img, cmap=‘xx’)
# ■ 參數img可以是圖片,也可以是矩陣數據。這個函數常用在機器學
# 習檢測神經網路的輸出

# In[126]:

import numpy as np
import pandas as pd
s=pd.Series(np.arange(0,9,2))
print(s)

# In[127]:

import numpy as np
import pandas as pd
s=pd.Series([1,2,4])
ss=np.square(s)#平方
print(ss)

# In[152]:

import numpy as np
import pandas as pd
df1=pd.DataFrame([[1,2,3],[4,np.nan,6],[7,8,np.nan]])#np.nan  空值
x=df1.isna()
y=df1.notna()
print(df1)
print('='*80)
print(x)
print('='*80)
print(y)
print('='*80+"\n")
print(df1.isnull())
print("\n")
print(df1.isnull().sum())
print('='*80)

# %%

# In[155]:

aa=df1.fillna(0)# nan ;補0
print(aa)
print("\n")
bb=df1.dropna() #刪 nan 的 rows
print(bb)
print("\n")
cc=df1.dropna(axis='columns')# 刪 nan的 columns
print(cc)

# In[156]:

get_ipython().system('pip install scipy')

# 4x+3y = 15
# x-z=1
# 7y+2z=11

# In[158]:

import numpy as np
from scipy import linalg
coeff=np.array([[4,3,0],[1,0,-1],[0,7,2]])
deps=np.array([15,1,11])
ans=linalg.solve(coeff,deps)
print(ans)

# 使用scipy.stats 內的randint()建立指定區間均勻分
# 布的隨機整數,語法如下:
# stats.randint(low, high, size, options) #options其他參數

# In[160]:

import scipy.stats as st
rv=st.randint(low=1,high=11)
xx=rv.rvs(size=5)
print(rv)
print(xx)

# scipy.stats模組,提供統計最常見的函數
# ■ mean(low,high,loc=0):算術平均數 #loc設定mean的算法
# ■ var(low,high,loc=0):變異數
# ■ std(low,high,loc=0):標準差
# ■ median(low,high,loc=0):中位數
#
# norm()建立常態分佈模型:
# norm(loc=0,scale=1)
# loc是mean預設是0,scale是標準差std預設是1

# In[162]:

import matplotlib.pyplot as plt
import scipy.stats as st
x=st.norm.rvs(size=1200)
plt.hist(x)
plt.ylabel('Times')
plt.show()

# optimization模組
#
# 內有許多功能,處理最佳化、找最
# 小值、曲線擬合、解方程式的根
# 解一元二次方程式的根
# ■ root(fun, x0, options,...) #options是較少用的參數
# ■ fun是要解的函數名稱,x0是初始迭代值(可以用不同的
# 參數值,會有不同的結果)

# In[188]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scipy.stats as st

from scipy.optimize import fsolve,root
from scipy.integrate import odeint
def f(lalala):
    x,y=lalala[0],lalala[1]

    return [
        x**2-6*x+8-y,
        2*x+3-y
    ]
solved=fsolve(f,[0,1])
print(solved)

print("Program done!")

'''
plt.plot(f,color='red',label='f1')
plt.plot(f,color='blue',label='f2')
plt.legend()
#plt.title('%.5fx%.5f=y' % (A, B))
plt.show()
'''
print("="*50)

rangex1 = np.linspace(-2,8)
rangey1_1,rangey1_2 = 2*np.sin(rangex1),rangex1-1
plt.figure(1)
plt.plot(rangex1,rangey1_1,'r',rangex1,rangey1_2,'b--')
plt.title('$2sin(x)$ and $x-1$')

def f1(x):
    return np.sin(x)*2-x+1

sol1_root = root(f1,[2])
sol1_fsolve = fsolve(f1,[2])
plt.scatter(sol1_fsolve,2*np.sin(sol1_fsolve),linewidths=9)
plt.show()

## 2、求解線性方程組{3X1+2X2=3;X1-2X2=5}
def f2(x):
    return np.array([3*x[0]+2*x[1]-3,x[0]-2*x[1]-5])

sol2_root = root(f2,[0,0])
sol2_fsolve = fsolve(f2,[0,0])
print(sol2_fsolve) # [2. -1.5]

a = np.array([[3,2],[1,-2]])
b = np.array([3,5])
x = np.linalg.solve(a,b)
print(x) # [2. -1.5]
## 3、求解非線性方程組
def f3(x):
    return np.array([2*x[0]**2+3*x[1]-3*x[2]**3-7,
                    x[0]+4*x[1]**2+8*x[2]-10,
                    x[0]-2*x[1]**3-2*x[2]**2+1])

sol3_root = root(f3,[0,0,0])
sol3_fsolve = fsolve(f3,[0,0,0])
print(sol3_fsolve)

## 4、非線性方程
def f4(x):
    return np.array(np.sin(2*x-np.pi)*np.exp(-x/5)-np.sin(x))
init_guess =np.array([[0],[3],[6],[9]])
sol4_root = root(f4,init_guess)
sol4_fsolve = fsolve(f4,init_guess)
print(sol4_fsolve)
t = np.linspace(-2,12,2000)
y1 = np.sin(2*t-np.pi)*np.exp(-t/5)
y2 = np.sin(t)
plt.figure(2)
a , = plt.plot(t,y1,label='$sin(2x-\pi)e^{-x/5}$')
b , = plt.plot(t,y2,label='$sin(x)$')
plt.scatter(sol4_fsolve,np.sin(sol4_fsolve),linewidths=8)
plt.title('$sin(2x-\pi)e^{-x/5}$ and $sin(x)$')
plt.legend()

# In[194]:

import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
def func(x, a, b, c):
    return a * np.exp(-b * x) + c
#定義數據以使其適合某些噪聲：

xdata = np.linspace(0, 4, 50)
y = func(xdata, 2.5, 1.3, 0.5)
np.random.seed(1729)
y_noise = 0.2 * np.random.normal(size=xdata.size)
ydata = y + y_noise
plt.plot(xdata, ydata, 'b-', label='data')
#適合函數func的參數a，b，c：

popt, pcov = curve_fit(func, xdata, ydata)
popt
plt.plot(xdata, func(xdata, *popt), 'r-',
label='fit:a=%5.3f, b=%5.3f, c=%5.3f' % tuple(popt))

#將優化限製在以下區域0 <= a <= 3，0 <= b <= 1和0 <= c <= 0.5：

popt, pcov = curve_fit(func, xdata, ydata, bounds=(0, [3., 1., 0.5]))
popt

plt.plot(xdata, func(xdata, *popt), 'g--',
label='fit:a=%5.3f, b=%5.3f, c=%5.3f' % tuple(popt))
plt.xlabel('x')
plt.ylabel('y')
plt.legend()
plt.show()

# In[195]:

get_ipython().system('pip install opencv-python')

# In[ ]:

#%%

import numpy as np
import cv2
img=cv2.imread("D:\桌面/ttt.png")

cv2.imshow('Lena',img)
cv2.waitKey(0)#??????????????????????????????????

# %%

#%%
# In[24]:

import cv2 as cv
import matplotlib.pyplot as plt
import numpy as np
#讀取圖片 默認為BGR 需要轉成RGB並plt印出來
img = cv.imread("00123.png")
img = cv.cvtColor(img,cv.COLOR_BGR2RGB)
plt.imshow(img)
plt.show()

# In[ ]:

# In[ ]:

'''
import cv2 as cv
import matplotlib.pyplot as plt
import numpy as np
#讀取圖片 默認為BGR 需要轉成RGB並plt印出來
img = cv.imread("00123.png")
img = cv.cvtColor(img,cv.COLOR_BGR2RGB)

height, width, channels =img.shape #print height, width,channels
img_updown = np.zeros([height,width, 3], dtype=np.uint8)
for i in range(height):
    for j in range(width):
        img_updown[height-i-1,j] =img[i,j]
cv.imwrite("00123.png",img_updown);

plt.imshow()
plt.show()

# In[ ]:

# In[ ]:

'''

height, width, channels =img.shape #print height,width, channels
img_RL = np.zeros([height,width, 3], dtype=np.uint8)
for i in range(height):
    for j in range(width):
        img_RL[i, width-j-1] =img[i,j]

cv2.imwrite("RL.bmp",img_RL);
'''

# In[ ]:

# In[ ]:

'''
height, width, channels =img.shape #print height,width, channels

img_DIG = np.zeros([height,width, 3], dtype=np.uint8)
for i in range(height):
    for j in range(width):
        img_DIG[j,i]=img[i,j]
cv2.imwrite("DIG.bmp",img_DIG);

'''

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# #分別代表(影像,頂點座標,對向頂點座標,顏色,線條寬度（-1為填滿）)
# cv.rectangle(img,(x,y),(x+w,y+h),(0,255,0),2)
# #畫圓型 (影像,圓心座標,半徑,顏色,線條寬度)
# cv.circle(img,(x,y),30,(0,255,0),3)
# #畫多邊形 (設定座標點)
# arr = np.array([[20,20],[30,30],[50,30]],np.int32)
# #多邊形（影像,頂點座標,封閉形,顏色,線條寬度）
# cv.polylines(img, [arr], True, (255, 255, 0), 4)
# #分別代表(影像,頂點座標,對向頂點座標,顏色,線條寬度（-1為填滿）)
# cv.rectangle(img,(x,y),(x+w,y+h),(0,255,0),2)
# #畫圓型 (影像,圓心座標,半徑,顏色,線條寬度)
# cv.circle(img,(x,y),30,(0,255,0),3)
# #畫多邊形 (設定座標點)
# arr = np.array([[20,20],[30,30],[50,30]],np.int32)
# #多邊形（影像,頂點座標,封閉形,顏色,線條寬度）
# cv.polylines(img, [arr], True, (255, 255, 0), 4)

# In[ ]:

max_stars_repo_name: windllsnow/python-test

max_issues_repo_name: windllsnow/python-test

max_forks_repo_name: windllsnow/python-test

reason: import numpy,import scipy,from scipy


text:
# Copyright (c) Yuta Saito, Yusuke Narita, and ZOZO Technologies, Inc. All rights reserved.
# Licensed under the Apache 2.0 License.

"""Regression Model Class for Estimating the Q functions in Cascade-DR."""
from dataclasses import dataclass

import numpy as np
from sklearn.base import BaseEstimator
from sklearn.base import clone
from sklearn.base import is_classifier
from sklearn.utils import check_scalar

from obp.utils import check_array

@dataclass
class SlateRegressionModel(BaseEstimator):
    """Machine learning model to estimate the Q functions for OPE of ranking policies.

    Note
    -------
    Q function at position :math:`l` is defined as
    :math:`\\hat{Q}_l := \\hat{Q}_l(x, a(1), \\ldots, a(k)) \\approx \\mathbb{E}[ \\sum_{l'=l}^L \\alpha_{l'} r(l') | x, a(1), \\ldots, a(l)]`).

    Q function is estimated recursively, and then used in Cascade-DR.
    Please refer to Section 3.1 of Kiyohara et al.(2022) for more details.

    Parameters
    ------------
    base_model: BaseEstimator
        A machine learning model used to estimate the Q function.

    len_list: int
        Length of a list of actions in a recommendation/ranking interface, slate size.
        When Open Bandit Dataset is used, 3 should be set.

    n_unique_action: int
        Number of unique actions.

    fitting_method: str, default='normal'
        Method to fit the regression model.
        Must be either of ['normal', 'iw'] where 'iw' stands for importance weighting.

    Reference
    ------------
    Haruka Kiyohara, Yuta Saito, Tatsuya Matsuhiro, Yusuke Narita, Nobuyuki Shimizu, and Yasuo Yamamoto.
    "Doubly Robust Off-Policy Evaluation for Ranking Policies under the Cascade Behavior Model.", 2022.

    """

    base_model: BaseEstimator
    len_list: int
    n_unique_action: int
    fitting_method: str = "normal"

    def __post_init__(self):
        """Initialize Class."""
        check_scalar(self.n_unique_action, "n_unique_action", int, min_val=2)
        check_scalar(self.len_list, "len_list", int, min_val=1)
        if not (
            isinstance(self.fitting_method, str)
            and self.fitting_method in ["normal", "iw"]
        ):
            raise ValueError(
                f"`fitting_method` must be either 'normal' or 'iw', but {self.fitting_method} is given"
            )
        if not isinstance(self.base_model, BaseEstimator):
            raise ValueError(
                "`base_model` must be BaseEstimator or a child class of BaseEstimator"
            )
        if is_classifier(self.base_model):
            raise ValueError("`base_model` must be a regressor, not a classifier")
        self.base_model_list = [clone(self.base_model) for _ in range(self.len_list)]
        self.action_context = np.eye(self.n_unique_action)

    def fit(
        self,
        context: np.ndarray,
        action: np.ndarray,
        reward: np.ndarray,
        pscore_cascade: np.ndarray,
        evaluation_policy_pscore_cascade: np.ndarray,
        evaluation_policy_action_dist: np.ndarray,
    ):
        """Fit the regression model on given logged bandit data.

        Parameters
        ----------
        context: array-like, shape (n_rounds, dim_context)
            Context vectors observed for each data, i.e., :math:`x_i`.

        action: array-like, (n_rounds * len_list,)
            Actions observed at each slot in a ranking/slate in logged bandit data, i.e., :math:`a_{i}(l)`,
            which is chosen by the behavior policy :math:`\\pi_b`.

        reward: array-like, shape (n_rounds * len_list,)
            Slot-level rewards observed for each data in logged bandit data, i.e., :math:`r_{i}(l)`.

        pscore_cascade: array-like, shape (n_rounds * len_list,)
            Joint probabilities of behavior policy choosing a particular sequence of actions from the top position to the :math:`l`-th position (:math:`a_{1:l}`).

        evaluation_policy_pscore_cascade: array-like, shape (n_rounds * len_list,)
            Joint probabilities of evaluation policy choosing a particular sequence of actions from the top position to the :math:`l`-th position (:math:`a_{1:l}`). This type of action choice probabilities corresponds to the cascade model.

        evaluation_policy_action_dist: array-like (n_rounds * len_list * n_unique_actions, )
            Plackett-luce style action distribution induced by evaluation policy
            (action choice probabilities at each slot given previous action choices)
            , i.e., :math:`\\pi_e({a'}_t(k) | x_i, a_i(1), \\ldots, a_i(l-1)) \\forall {a'}_t(k) \\in \\mathcal{A}`.

        """
        check_array(array=context, name="context", expected_dim=2)
        check_array(array=action, name="action", expected_dim=1)
        check_array(array=reward, name="reward", expected_dim=1)
        check_array(array=pscore_cascade, name="pscore_cascade", expected_dim=1)
        check_array(
            array=evaluation_policy_pscore_cascade,
            name="evaluation_policy_pscore_cascade",
            expected_dim=1,
        )
        check_array(
            array=evaluation_policy_action_dist,
            name="evaluation_policy_action_dist",
            expected_dim=1,
        )
        if not (
            action.shape
            == reward.shape
            == pscore_cascade.shape
            == evaluation_policy_pscore_cascade.shape
            == (context.shape[0] * self.len_list,)
        ):
            raise ValueError(
                "Expected `action.shape == reward.shape == pscore_cascade.shape == evaluation_policy_pscore_cascade.shape"
                " == (context.shape[0] * len_list, )`"
                ", but found it False"
            )
        if evaluation_policy_action_dist.shape != (
            context.shape[0] * self.len_list * self.n_unique_action,
        ):
            raise ValueError(
                "Expected `evaluation_policy_action_dist.shape == (context.shape[0] * len_list * n_unique_action, )`"
                ", but found it False"
            )
        if not (
            np.issubdtype(action.dtype, np.integer)
            and action.min() >= 0
            and action.max() < self.n_unique_action
        ):
            raise ValueError(
                "`action` elements must be integers in the range of [0, n_unique_action)"
            )
        if np.any(pscore_cascade <= 0) or np.any(pscore_cascade > 1):
            raise ValueError("`pscore_cascade` must be in the range of (0, 1]")
        if np.any(evaluation_policy_pscore_cascade <= 0) or np.any(
            evaluation_policy_pscore_cascade > 1
        ):
            raise ValueError(
                "`evaluation_policy_pscore_cascade` must be in the range of (0, 1]"
            )
        if not np.allclose(
            np.ones(
                evaluation_policy_action_dist.reshape((-1, self.n_unique_action)).shape[
                    0
                ]
            ),
            evaluation_policy_action_dist.reshape((-1, self.n_unique_action)).sum(
                axis=1
            ),
        ):
            raise ValueError(
                "`evaluation_policy_action_dist[i * n_unique_action : (i+1) * n_unique_action]` "
                "must sum up to one for all i."
            )
        # (n_rounds_ * len_list, ) -> (n_rounds_, len_list)
        action = action.reshape((-1, self.len_list))
        reward = reward.reshape((-1, self.len_list))
        iw = (evaluation_policy_pscore_cascade / pscore_cascade).reshape(
            (-1, self.len_list)
        )

        # (n_rounds_, )
        n_rounds_ = len(action)
        sample_weight = np.ones(n_rounds_)

        for pos_ in range(self.len_list)[::-1]:
            X, y = self._preprocess_for_reg_model(
                context=context,
                action=action,
                reward=reward,
                evaluation_policy_action_dist=evaluation_policy_action_dist,
                position_=pos_,
            )

            if self.fitting_method == "iw":
                sample_weight = iw[:, pos_]

            self.base_model_list[pos_].fit(X, y, sample_weight=sample_weight)

    def predict(
        self,
        context: np.ndarray,
        action: np.ndarray,
    ):
        """Predict the Q function values.

        Parameters
        -----------
        context: array-like, shape (n_rounds_of_new_data, dim_context)
            Context vectors for new data.

        action: array-like, shape (n_rounds_of_new_data * len_list, )
            Action vectors for new data.

        Returns
        -----------
        q_hat: array-like, shape (n_rounds_of_new_data * len_list * n_unique_action, )
            Estimated Q function values of new data.
            :math:`\\hat{Q}_{i,l}(x_i, a_i(1), \\ldots, a_i(l-1), a_i(l)) \\forall a_i(l) \\in \\mathcal{A}`.

        """
        check_array(array=context, name="context", expected_dim=2)
        check_array(array=action, name="action", expected_dim=1)
        if action.shape != (context.shape[0] * self.len_list,):
            raise ValueError(
                "Expected `action.shape == (context.shape[0] * len_list, )`"
                ", but found it False"
            )
        n_rounds_of_new_data = len(context)
        # (n_rounds_of_new_data * len_list, ) -> (n_rounds_of_new_data, len_list)
        action = action.reshape((-1, self.len_list))
        # (n_rounds_, len_list, n_unique_action, )
        q_hat = np.zeros((n_rounds_of_new_data, self.len_list, self.n_unique_action))
        for pos_ in range(self.len_list)[::-1]:
            # the action vector shrinks every time as the position_ decreases
            # (n_rounds_of_new_data, position_ - 1)
            action = action[:, :pos_]
            # (n_rounds_of_new_data, dim_context) -> (n_rounds_of_new_data * n_unique_action, dim_context)
            context_ = []
            # (n_rounds_of_new_data, position_) -> (n_rounds_of_new_data * n_unique_action, position_)
            action_ = []
            for i in range(n_rounds_of_new_data):
                for a_ in range(self.n_unique_action):
                    context_.append(context[i])
                    action_.append(np.append(action[i], a_))
            # (n_rounds_of_new_data * n_unique_action, dim_context + position_)
            X = np.concatenate([context_, action_], axis=1)
            # (n_rounds_of_new_data * n_unique_action, ) -> (n_rounds_of_new_data, n_unique_action)
            q_hat[:, pos_, :] = (
                self.base_model_list[pos_]
                .predict(X)
                .reshape((-1, self.n_unique_action))
            )
        # (n_rounds_of_new_data * len_list * n_unique_action, )
        return q_hat.flatten()

    def fit_predict(
        self,
        context: np.ndarray,
        action: np.ndarray,
        reward: np.ndarray,
        pscore_cascade: np.ndarray,
        evaluation_policy_pscore_cascade: np.ndarray,
        evaluation_policy_action_dist: np.ndarray,
    ):
        """Fit the regression model on given logged bandit data and predict the Q function values on the same data.

        Parameters
        ----------
        context: array-like, shape (n_rounds, dim_context)
            Context vectors observed for each data, i.e., :math:`x_i`.

        action: array-like, (n_rounds * len_list,)
            Actions observed at each slot in a ranking/slate in logged bandit data, i.e., :math:`a_{i}(l)`,
            which is chosen by the behavior policy :math:`\\pi_b`.

        reward: array-like, shape (n_rounds * len_list,)
            Slot-level rewards observed for each data in logged bandit data, i.e., :math:`r_{i}(l)`.

        pscore_cascade: array-like, shape (n_rounds * len_list,)
            Joint probabilities of behavior policy choosing a particular sequence of actions from the top position to the :math:`l`-th position (:math:`a_{1:l}`).

        evaluation_policy_pscore_cascade: array-like, shape (n_rounds * len_list,)
            Joint probabilities of evaluation policy choosing a particular sequence of actions from the top position to the :math:`l`-th position (:math:`a_{1:l}`). This type of action choice probabilities corresponds to the cascade model.

        evaluation_policy_action_dist: array-like (n_rounds * len_list * n_unique_actions, )
            Plackett-luce style action distribution induced by evaluation policy
            (action choice probabilities at each slot given previous action choices)
            , i.e., :math:`\\pi_e(a_i(l) | x_i, a_i(1), \\ldots, a_i(l-1)) \\forall a_i(l) \\in \\mathcal{A}`.

        Returns
        -----------
        q_hat: array-like, shape (n_rounds_of_new_data * len_list * n_unique_action, )
            Estimated Q functions for new data by the regression model.

        """
        self.fit(
            context=context,
            action=action,
            reward=reward,
            pscore_cascade=pscore_cascade,
            evaluation_policy_pscore_cascade=evaluation_policy_pscore_cascade,
            evaluation_policy_action_dist=evaluation_policy_action_dist,
        )
        # (n_rounds_test, len_list, n_unique_action, )
        return self.predict(context=context, action=action)

    def _preprocess_for_reg_model(
        self,
        context: np.ndarray,
        action: np.ndarray,
        reward: np.ndarray,
        evaluation_policy_action_dist: np.ndarray,
        position_: int,
    ):
        """Preprocess feature vectors and target variables for training a regression model.

        Note
        -----
        Please override this method if you want to use another feature enginnering
        for training the regression model.

        Parameters
        -----------
        context: array-like, shape (n_rounds_, dim_context)
            Context vectors in the training set of logged bandit data.

        action: array-like, (n_rounds_ * len_list, )
            Actions observed at each slot in a ranking/slate in logged bandit data, i.e., :math:`a_{i}(l)`,
            which is chosen by the behavior policy :math:`\\pi_b`.

        reward: array-like, shape (n_rounds_ * len_list, )
            Slot-level rewards observed for each data in logged bandit data, i.e., :math:`r_{i}(l)`.

        evaluation_policy_action_dist: array-like (n_rounds_ * len_list * n_unique_actions, )
            Plackett-luce style action distribution induced by evaluation policy
            (action choice probabilities at each slot given previous action choices)
            , i.e., :math:`\\pi_e(a_i(l) | x_i, a_i(1), \\ldots, a_i(l-1)) \\forall a_i(l) \\in \\mathcal{A}`.

        position_: int
            Position id (slot) in a slate.

        Returns
        -----------
        X, y: array-like, shape(n_rounds, )
            Input and target vectors in prediction.

        """
        n_rounds_ = len(context)
        # (n_rounds_, len_list) -> (n_rounds_, position_)
        action = action[:, : position_ + 1]
        # (n_rounds_, len_list) -> (n_rounds_, )
        reward = reward[:, position_]
        # estimator input
        X = np.concatenate([context, action], axis=1)
        # estimate the Q function at the next position
        # (n_rounds_, )
        if position_ + 1 == self.len_list:
            q_hat_at_next_position = np.zeros(n_rounds_)
        else:
            # (n_rounds_ * len_list * n_unique_action, ) -> (n_rounds_, len_list, n_unique_action) -> (n_rounds_, len_list) -> (n_rounds_ * n_unique_action, )
            evaluation_policy_action_dist_at_next_position = (
                evaluation_policy_action_dist.reshape(
                    (-1, self.len_list, self.n_unique_action)
                )[:, position_ + 1, :]
            ).flatten()
            # (n_rounds_, dim_context) -> (n_rounds_ * n_unique_action, dim_context)
            context_ = []
            # (n_rounds_, position_ + 1) -> (n_rounds_ * n_unique_action, position_ + 1)
            action_ = []
            for i in range(n_rounds_):
                for a_ in range(self.n_unique_action):
                    context_.append(context[i])
                    action_.append(np.append(action[i], a_))
            X_ = np.concatenate([context_, action_], axis=1)
            # (n_rounds_ * n_unique_action, ) -> (n_rounds_, )
            q_hat_at_next_position = self.base_model_list[position_ + 1].predict(X_)
            # the expected Q function under the evaluation policy
            # (n_rounds_ * n_unique_action, ) -> (n_rounds_, n_unique_action) -> (n_rounds_, )
            q_hat_at_next_position = (
                (
                    evaluation_policy_action_dist_at_next_position
                    * q_hat_at_next_position
                )
                .reshape((-1, self.n_unique_action))
                .sum(axis=1)
            )
        # (n_rounds_, )
        y = reward + q_hat_at_next_position
        # (n_rounds_, dim_context + position_), (n_rounds_, )
        return X, y

ext: py

reason: import numpy


text:
import logging
from baseline.embeddings import create_embeddings_reduction
from baseline.model import ClassifierModel, register_model
from baseline.pytorch.torchy import *
from baseline.utils import listify, write_json
from eight_mile.pytorch.layers import *
import torch.backends.cudnn as cudnn
import os
cudnn.benchmark = True

logger = logging.getLogger('baseline')

class ClassifierModelBase(nn.Module, ClassifierModel):
    """Base for all baseline implementations of token-based classifiers

    This class provides a loose skeleton around which the baseline models
    are built.  It is built on the PyTorch `nn.Module` base, and fulfills the `ClassifierModel` interface.
    To override this class, the use would typically override the `create_layers` function which will
    create and attach all sub-layers of this model into the class, and the `forward` function which will
    give the model some implementation to call on forward.
    """

    def __init__(self):
        super().__init__()
        self.gpu = False

    @classmethod
    def load(cls, filename: str, **kwargs) -> 'ClassifierModelBase':
        device = kwargs.get('device')
        if not os.path.exists(filename):
            filename += '.pyt'
        model = torch.load(filename, map_location=device)
        model.gpu = False if device == 'cpu' else model.gpu
        return model

    def save(self, outname: str):
        logger.info('saving %s' % outname)
        torch.save(self, outname)
        basename, _ = os.path.splitext(outname)
        write_json(self.labels, basename + ".labels")

    @classmethod
    def create(cls, embeddings, labels, **kwargs) -> 'ClassifierModelBase':

        model = cls()
        model.pdrop = kwargs.get('pdrop', 0.5)
        model.lengths_key = kwargs.get('lengths_key')
        model.gpu = not bool(kwargs.get('nogpu', False))
        model.labels = labels
        model.create_layers(embeddings, **kwargs)
        logger.info(model)
        return model

    def cuda(self, device=None):
        self.gpu = True
        return super().cuda(device=device)

    def create_loss(self):
        return nn.NLLLoss()

    def make_input(self, batch_dict, perm=False, numpy_to_tensor=False):

        """Transform a `batch_dict` into something usable in this model

        :param batch_dict: (``dict``) A dictionary containing all inputs to the embeddings for this model
        :return:
        """
        example_dict = dict({})
        perm_idx = None

        # Allow us to track a length, which is needed for BLSTMs
        if self.lengths_key is not None:
            lengths = batch_dict.get(self.lengths_key)
            if lengths is not None:
                if numpy_to_tensor:
                    lengths = torch.from_numpy(lengths)
                lengths, perm_idx = lengths.sort(0, descending=True)
                if self.gpu:
                    lengths = lengths.cuda()
                example_dict['lengths'] = lengths

        for key in self.embeddings.keys():
            tensor = batch_dict[key]
            if numpy_to_tensor:
                tensor = torch.from_numpy(tensor)
            if perm_idx is not None:
                tensor = tensor[perm_idx]
            if self.gpu:

                if isinstance(tensor, torch.Tensor):
                    tensor = tensor.cuda()
                elif is_sequence(tensor):
                    t = list(tensor)
                    for i in range(len(t)):
                        t[i] = t[i].cuda()
                        tensor = tuple(t)
            example_dict[key] = tensor

        y = batch_dict.get('y')
        if y is not None:
            if numpy_to_tensor:
                y = torch.from_numpy(y)
            if perm_idx is not None:
                y = y[perm_idx]
            if self.gpu:
                y = y.cuda()
            example_dict['y'] = y

        if perm:
            return example_dict, perm_idx

        return example_dict

    def predict_batch(self, batch_dict: Dict[str, TensorDef], **kwargs) -> TensorDef:
        numpy_to_tensor = bool(kwargs.get('numpy_to_tensor', True))
        examples, perm_idx = self.make_input(batch_dict, perm=True, numpy_to_tensor=numpy_to_tensor)
        with torch.no_grad():
            probs = self(examples).exp()
            probs = unsort_batch(probs, perm_idx)
        return probs

    def predict(self, batch_dict: Dict[str, TensorDef], raw: bool = False, dense: bool = False, **kwargs):
        self.eval()
        probs = self.predict_batch(batch_dict, **kwargs)
        if raw and not dense:
            logger.warning(
                "Warning: `raw` parameter is deprecated pass `dense=True` to get back values as a single tensor")

            dense = True
        if dense:
            return probs
        results = []
        batchsz = probs.size(0)
        for b in range(batchsz):
            outcomes = [(self.labels[id_i], prob_i) for id_i, prob_i in enumerate(probs[b])]
            results.append(outcomes)
        return results

    def get_labels(self) -> List[str]:
        return self.labels

    def create_layers(self, embeddings: Dict[str, TensorDef], **kwargs):
        """This method defines the model itself, and must be overloaded by derived classes

        This function will update `self` with the layers required to execute the `call()` method

        :param embeddings: The input feature indices
        :param kwargs:
        :return:
        """

class EmbedPoolStackClassifier(ClassifierModelBase):

    """Provides a simple but effective base for most `ClassifierModel`s

   This class provides a common base for classifiers by identifying and codifying
   and idiomatic pattern where a typical classifier may be though of as a composition
   between a stack of embeddings, followed by a pooling operation yielding a fixed length
   tensor, followed by one or more dense layers, and ultimately, a projection to the output space.

   To provide an useful interface to sub-classes, we override the `create_layers` to provide a hook
   for each layer identified in this idiom, and leave the implementations up to the sub-class.

   We also fully implement the `forward` method.

   """

    def create_layers(self, embeddings: Dict[str, TensorDef], **kwargs):
        self.embeddings = self.init_embed(embeddings, **kwargs)
        self.pool_model = self.init_pool(self.embeddings.output_dim, **kwargs)
        self.stack_model = self.init_stacked(self.pool_model.output_dim, **kwargs)
        self.output_layer = self.init_output(self.stack_model.output_dim, **kwargs)

    def init_embed(self, embeddings: Dict[str, TensorDef], **kwargs) -> BaseLayer:
        """This method creates the "embedding" layer of the inputs, with an optional reduction

        :param embeddings: A dictionary of embeddings

        :Keyword Arguments: See below
        * *embeddings_reduction* (defaults to `concat`) An operator to perform on a stack of embeddings
        * *embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))

        :return: The output of the embedding stack followed by its reduction.  This will typically be an output
          with an additional dimension which is the hidden representation of the input
        """
        reduction = kwargs.get('embeddings_reduction', kwargs.get('embed_reduction_type', 'concat'))
        reduction = create_embeddings_reduction(embed_reduction_type=reduction, **kwargs)
        embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))
        return EmbeddingsStack(embeddings, embeddings_dropout, reduction=reduction)

    def init_pool(self, input_dim: int, **kwargs) -> BaseLayer:
        """Produce a pooling operation that will be used in the model

        :param input_dim: The input dimension size
        :param kwargs:
        :return: A pooling operation
        """

    def init_stacked(self, input_dim: int, **kwargs) -> BaseLayer:
        """Produce a stacking operation that will be used in the model

        :param input_dim: The input dimension size
        :param kwargs:
        :return: A stacking operation (or None)
        """
        hszs = listify(kwargs.get('hsz', []))
        if not hszs:
            return PassThru(input_dim)
        return DenseStack(input_dim, hszs, pdrop_value=self.pdrop)

    def init_output(self, input_dim: int, **kwargs) -> BaseLayer:
        """Produce the final output layer in the model

        :param input_dim: The input hidden size
        :param kwargs:
        :return:
        """
        return Dense(input_dim, len(self.labels), activation=kwargs.get('output_activation', 'log_softmax'))

    @property
    def embed_output_dim(self):
        return self.stack_model.output_dim

    def embed(self, inputs):
        lengths = inputs.get("lengths")
        embedded = self.embeddings(inputs)
        embedded = (embedded, lengths)
        pooled = self.pool_model(embedded)
        stacked = self.stack_model(pooled)
        return stacked

    def forward(self, inputs: Dict[str, TensorDef]) -> TensorDef:
        """Forward execution of the model.  Sub-classes typically shouldnt need to override

        :param inputs: An input dictionary containing the features and the primary key length
        :return: A tensor
        """
        stacked = self.embed(inputs)
        return self.output_layer(stacked)

@register_model(task='classify', name='default')
class ConvModel(EmbedPoolStackClassifier):
    """Current default model for `baseline` classification.  Parallel convolutions of varying receptive field width
    """

    def init_pool(self, input_dim: int, **kwargs) -> BaseLayer:
        """Do parallel convolutional filtering with varied receptive field widths, followed by max-over-time pooling

        :param input_dim: Embedding output size
        :param kwargs: See below

        :Keyword Arguments:
        * *cmotsz* -- (``int``) The number of convolutional feature maps for each filter
            These are MOT-filtered, leaving this # of units per parallel filter
        * *filtsz* -- (``list``) This is a list of filter widths to use

        :return: A pooling layer
        """
        cmotsz = kwargs['cmotsz']
        filtsz = kwargs['filtsz']
        return WithoutLength(WithDropout(ParallelConv(input_dim, cmotsz, filtsz, "relu", input_fmt="bth"), self.pdrop))

@register_model(task='classify', name='lstm')
class LSTMModel(EmbedPoolStackClassifier):
    """A simple single-directional single-layer LSTM. No layer-stacking.
    """

    def init_pool(self, input_dim: int, **kwargs) -> BaseLayer:
        """LSTM with dropout yielding a final-state as output

        :param input_dim: The input word embedding depth
        :param kwargs: See below

        :Keyword Arguments:
        * *rnnsz* -- (``int``) The number of hidden units (defaults to `hsz`)
        * *rnntype/rnn_type* -- (``str``) The RNN type, defaults to `lstm`, other valid values: `blstm`
        * *hsz* -- (``int``) backoff for `rnnsz`, typically a result of stacking params.  This keeps things simple so
          its easy to do things like residual connections between LSTM and post-LSTM stacking layers

        :return: A pooling layer
        """
        unif = kwargs.get('unif')
        hsz = kwargs.get('rnnsz', kwargs.get('hsz', 100))
        if type(hsz) is list:
            hsz = hsz[0]
        weight_init = kwargs.get('weight_init', 'uniform')
        rnntype = kwargs.get('rnn_type', kwargs.get('rnntype', 'lstm'))
        if rnntype == 'blstm':
            return BiLSTMEncoderHidden(input_dim, hsz, 1, self.pdrop, unif=unif, batch_first=True, initializer=weight_init)
        return LSTMEncoderHidden(input_dim, hsz, 1, self.pdrop, unif=unif, batch_first=True, initializer=weight_init)

class NBowModelBase(EmbedPoolStackClassifier):
    """Neural Bag-of-Words Model base class.  Defines stacking of fully-connected layers, but leaves pooling to derived
    """

    def init_stacked(self, input_dim: int, **kwargs) -> BaseLayer:
        """Produce a stacking operation that will be used in the model, defaulting to a single layer

        :param input_dim: The input dimension size
        :param kwargs: See below

        :Keyword Arguments:
        * *hsz* -- (``List[int]``) The number of hidden units (defaults to 100)
        """
        kwargs.setdefault('hsz', [100])
        return super().init_stacked(input_dim, **kwargs)

@register_model(task='classify', name='nbow')
class NBowModel(NBowModelBase):
    """Neural Bag-of-Words average pooling (standard) model"""

    def init_pool(self, input_dim: int, **kwargs) -> BaseLayer:
        """Do average pooling on input embeddings, yielding a `dsz` output layer

        :param input_dim: The word embedding depth
        :param kwargs: None
        :return: The average pooling representation
        """
        return MeanPool1D(input_dim)

@register_model(task='classify', name='nbowmax')
class NBowMaxModel(NBowModelBase):
    """Max-pooling model for Neural Bag-of-Words.  Sometimes does better than avg pooling
    """

    def init_pool(self, input_dim: int, **kwargs) -> BaseLayer:
        """Do max pooling on input embeddings, yielding a `dsz` output layer

        :param input_dim: The word embedding depth
        :param kwargs: None
        :return: The max pooling representation
        """
        return MaxPool1D(input_dim)

@register_model(task='classify', name='fine-tune')
class FineTuneModelClassifier(ClassifierModelBase):
    """Fine-tune based on pre-pooled representations"""

    def init_embed(self, embeddings: Dict[str, TensorDef], **kwargs) -> BaseLayer:
        """This method creates the "embedding" layer of the inputs, with an optional reduction

        :param embeddings: A dictionary of embeddings

        :Keyword Arguments: See below
        * *embeddings_reduction* (defaults to `concat`) An operator to perform on a stack of embeddings
        * *embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))

        :return: The output of the embedding stack followed by its reduction.  This will typically be an output
          with an additional dimension which is the hidden representation of the input
        """
        reduction = kwargs.get('embeddings_reduction', kwargs.get('embed_reduction_type', 'concat'))
        reduction = create_embeddings_reduction(embed_reduction_type=reduction, **kwargs)
        embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))
        return EmbeddingsStack(embeddings, embeddings_dropout, reduction=reduction)

    def init_stacked(self, input_dim: int, **kwargs) -> BaseLayer:
        """Produce a stacking operation that will be used in the model

        :param input_dim: The input dimension size
        :param kwargs:
        :return: A stacking operation (or None)
        """
        hszs = listify(kwargs.get('hsz', []))
        if not hszs:
            return PassThru(input_dim)
        return DenseStack(input_dim, hszs, pdrop_value=self.pdrop)

    def init_output(self, input_dim: int, **kwargs) -> BaseLayer:
        """Produce the final output layer in the model

        :param input_dim: The input hidden size
        :param kwargs:
        :return:
        """
        return WithDropout(Dense(input_dim, len(self.labels), activation=kwargs.get('output_activation', 'log_softmax'),
                                 unif=kwargs.get('output_unif', 0.0)), pdrop=kwargs.get('output_dropout', 0.0))

    def create_layers(self, embeddings: Dict[str, TensorDef], **kwargs):
        self.embeddings = self.init_embed(embeddings, **kwargs)
        self.stack_model = self.init_stacked(self.embeddings.output_dim, **kwargs)
        self.output_layer = self.init_output(self.stack_model.output_dim, **kwargs)

    @property
    def embed_output_dim(self):
        return self.stack_model.output_dim

    def embed(self, inputs):
        base_layers = self.embeddings(inputs)
        stacked = self.stack_model(base_layers)
        return stacked

    def forward(self, inputs):
        stacked = self.embed(inputs)
        return self.output_layer(stacked)

@register_model(task='classify', name='fine-tune-dual')
class FineTuneDualModelClassifier(FineTuneModelClassifier):
    """Fine-tune based on pre-pooled representations"""

    def init_embed(self, embeddings: Dict[str, TensorDef], **kwargs) -> BaseLayer:
        """This method creates the "embedding" layer of the inputs, with an optional reduction

        :param embeddings: A dictionary of embeddings

        :Keyword Arguments: See below
        * *embeddings_reduction* (defaults to `concat`) An operator to perform on a stack of embeddings
        * *embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))

        :return: The output of the embedding stack followed by its reduction.  This will typically be an output
          with an additional dimension which is the hidden representation of the input
        """
        reduction = kwargs.get('embeddings_reduction', kwargs.get('embed_reduction_type', 'concat-subtract'))
        reduction = create_embeddings_reduction(embed_reduction_type=reduction, **kwargs)
        embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))
        if len(embeddings) != 1:
            raise Exception("Currently we only support a single embedding")

        key_name = list(embeddings.keys())[0]
        key1 = f"{key_name}[0]"
        key2 = f"{key_name}[1]"
        embeddings_dual = {key1: embeddings[key_name], key2: embeddings[key_name]}
        return EmbeddingsStack(embeddings_dual, embeddings_dropout, reduction=reduction)

@register_model(task='classify', name='fine-tune-dual-cosine')
class FineTuneDualCosineSimModelClassifier(FineTuneModelClassifier):
    """Fine-tune based on pre-pooled representations"""
    def init_embed(self, embeddings: Dict[str, TensorDef], **kwargs) -> BaseLayer:
        """This method creates the "embedding" layer of the inputs, with an optional reduction

        :param embeddings: A dictionary of embeddings

        :Keyword Arguments: See below
        * *embeddings_reduction* (defaults to `concat`) An operator to perform on a stack of embeddings
        * *embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))

        :return: The output of the embedding stack followed by its reduction.  This will typically be an output
          with an additional dimension which is the hidden representation of the input
        """
        reduction = kwargs.get('embeddings_reduction', 'concat')
        embeddings_dropout = float(kwargs.get('embeddings_dropout', 0.0))
        if len(embeddings) != 1:
            raise Exception("Currently we only support a single embedding")

        key_name = list(embeddings.keys())[0]
        key1 = f"{key_name}[0]"
        key2 = f"{key_name}[1]"
        embeddings_dual = {key1: embeddings[key_name], key2: embeddings[key_name]}
        return EmbeddingsStack(embeddings_dual, embeddings_dropout, reduction=reduction)

    def init_output(self, input_dim: int, **kwargs) -> BaseLayer:
        assert(len(self.labels) == 2)
        return nn.Identity()

    def create_loss(self):
        return CosineSimilarityLoss()

@register_model(task='classify', name='fine-tune-dual-ocl')
class FineTuneDualOCLModelClassifier(FineTuneDualCosineSimModelClassifier):

    def create_loss(self):
        return OnlineContrastiveLoss()

@register_model(task='classify', name='fine-tune-paired')
class FineTunePairedClassifierModel(FineTuneModelClassifier):

    """Fine-tuning model for pairs

    This model encodes a pair as a single utterance using some encoding scheme defined in
    ``_convert_pair`` which is fed directly into the fine-tuning model.

    For BERT, this simply encodes the input key pair as a single utterance while building
    a token-type vector.

    For the input, we will assume that the vectorizer will be producing a start token and an end token.
    We will simply remove the start token from the second sentence and concatenate
    [CLS] this is sentence one [SEP]

    [CLS] this is sentence two [SEP]

    """
    def _convert_pair(self, key, batch_dict, numpy_to_tensor, example_dict):

        toks = batch_dict[f"{key}"]
        token_type_key = f"{key}_tt"
        tt = batch_dict.get(token_type_key)
        if numpy_to_tensor:
            toks = torch.from_numpy(toks)
            if tt is not None:
                tt = torch.from_numpy(tt)
        if self.gpu:
            toks = toks.cuda()
            if tt is not None:
                tt = tt.cuda()
        if tt is not None:
            example_dict[key] = (toks, tt)
        else:
            example_dict[key] = toks

    def make_input(self, batch_dict: Dict, perm: bool = False, numpy_to_tensor: bool = False):

        """Transform a `batch_dict` into something usable in this model

        :param batch_dict: A dictionary containing all inputs to the embeddings for this model
        :param perm: Should we permute the batch, this is not supported here
        :param numpy_to_tensor: Do we need to convert from numpy to a ``torch.Tensor``
        :return: An example dictionary for processing
        """
        example_dict = dict({})
        # Allow us to track a length, which is needed for BLSTMs
        for key in self.embeddings.keys():
            self._convert_pair(key, batch_dict, numpy_to_tensor, example_dict)

        y = batch_dict.get('y')
        if y is not None:
            if numpy_to_tensor:
                y = torch.from_numpy(y)
            if self.gpu:
                y = y.cuda()
            example_dict['y'] = y

        return example_dict

@register_model(task='classify', name='composite')
class CompositePoolingModel(ClassifierModelBase):
    """Fulfills pooling contract by aggregating pooling from a set of sub-models and concatenates each"""

    def init_pool(self, dsz, **kwargs):
        SubModels = [eval(model) for model in kwargs.get('sub')]
        sub_models = [SM.init_pool(self, dsz, **kwargs) for SM in SubModels]
        return CompositePooling(sub_models)

    def make_input(self, batch_dict):
        """Because the sub-model could contain an LSTM, make sure to sort lengths descending

        :param batch_dict:
        :return:
        """
        inputs = super().make_input(batch_dict)
        lengths = inputs['lengths']
        lengths, perm_idx = lengths.sort(0, descending=True)
        for k, value in inputs.items():
            inputs[k] = value[perm_idx]
        return inputs

max_stars_repo_name: minsukchang/baseline

max_issues_repo_name: minsukchang/baseline

max_forks_repo_name: minsukchang/baseline

reason: from numpy


text:
import re
from PIL import ImageFilter
import random

def pre_question(question,max_ques_words):
    question = re.sub(
        r"([,.'!?\"()*#:;~])",
        '',
        question.lower(),
    ).replace('-', ' ').replace('/', ' ')
    question = question.rstrip(' ')

    #truncate question
    question_words = question.split(' ')
    if len(question_words)>max_ques_words:
        question = ' '.join(question_words[:max_ques_words])

    return question

def pre_caption(caption,max_words):
    caption = re.sub(
        r"([,.'!?\"()*#:;~])",
        '',
        caption.lower(),
    ).replace('-', ' ').replace('/', ' ').replace('<person>', 'person')

    caption = re.sub(
        r"\s{2,}",
        ' ',
        caption,
    )
    caption = caption.rstrip('\n')
    caption = caption.strip(' ')

    #truncate caption
    caption_words = caption.split(' ')
    if len(caption_words)>max_words:
        caption = ' '.join(caption_words[:max_words])

    return caption

from vqaTools.vqaEval import VQAEval
from refTools.evaluation.refEvaluation import RefEvaluation

import json
import os
import numpy as np
import torch
import torch.distributed as dist
import torch.nn.functional as F

import utils
from tqdm import tqdm

def vqa_eval(vqa, result_file, test_ques_path):
    vqaRes = vqa.loadRes(result_file, test_ques_path)
    # create vqaEval object by taking vqa and vqaRes
    vqaEval = VQAEval(vqa, vqaRes, n=2)  # n is precision of accuracy (number of places after decimal), default is 2
    # evaluate results
    vqaEval.evaluate()

    # print accuracies
    print("\n")
    print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall']))
    print("Per Answer Type Accuracy is the following:")
    for ansType in vqaEval.accuracy['perAnswerType']:
        print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType]))
    print("\n")

    return vqaEval

def collect_result(result, result_dir, filename, is_json=True, is_list=True):
    if is_json:
        result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,utils.get_rank()))
        final_result_file = os.path.join(result_dir, '%s.json'%filename)
        json.dump(result,open(result_file,'w'))
    else:
        result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,utils.get_rank()))
        final_result_file = os.path.join(result_dir, '%s.pth'%filename)
        torch.save(result,result_file)

    dist.barrier()

    result = None
    if utils.is_main_process():
        # combine results from all processes
        if is_list:
            result = []
        else:
            result = {}
        for rank in range(utils.get_world_size()):
            if is_json:
                result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,rank))
                res = json.load(open(result_file,'r'))
            else:
                result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,rank))
                res = torch.load(result_file)
            if is_list:
                result += res
            else:
                result.update(res)

    return result

def save_result(result, result_dir, filename, is_json=True, is_list=True):
    if is_json:
        result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,utils.get_rank()))
        final_result_file = os.path.join(result_dir, '%s.json'%filename)
        json.dump(result,open(result_file,'w'))
    else:
        result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,utils.get_rank()))
        final_result_file = os.path.join(result_dir, '%s.pth'%filename)
        torch.save(result,result_file)

    dist.barrier()

    if utils.is_main_process():
        # combine results from all processes
        if is_list:
            result = []
        else:
            result = {}
        for rank in range(utils.get_world_size()):
            if is_json:
                result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,rank))
                res = json.load(open(result_file,'r'))
            else:
                result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,rank))
                res = torch.load(result_file)
            if is_list:
                result += res
            else:
                result.update(res)
        if is_json:
            json.dump(result,open(final_result_file,'w'))
        else:
            torch.save(result,final_result_file)

        print('result file saved to %s'%final_result_file)
    dist.barrier()
    return final_result_file

def grounding_eval(results,dets,cocos,refer,alpha,mask_size=24):

    correct_A_d, correct_B_d, correct_val_d = 0, 0, 0
    correct_A, correct_B, correct_val = 0, 0, 0
    num_A,num_B,num_val = 0,0,0

    for res in tqdm(results):

        ref_id = res['ref_id']
        ref = refer.Refs[ref_id]
        ref_box = refer.refToAnn[ref_id]['bbox']
        image = refer.Imgs[ref['image_id']]

        mask = res['pred'].cuda().view(1,1,mask_size,mask_size)
        mask = F.interpolate(mask,size = (image['height'],image['width']), mode='bicubic').squeeze()

        # rank detection boxes
        max_score = 0
        for det in dets[str(ref['image_id'])]:
            score = mask[int(det[1]):int(det[1]+det[3]),int(det[0]):int(det[0]+det[2])]
            area = det[2]*det[3]
            score = score.sum() / area**alpha
            if score>max_score:
                pred_box = det[:4]
                max_score = score

        IoU_det = computeIoU(ref_box, pred_box)

        if ref['split']=='testA':
            num_A += 1
            if IoU_det >= 0.5:
                correct_A_d += 1
        elif ref['split']=='testB':
            num_B += 1
            if IoU_det >= 0.5:
                correct_B_d += 1
        elif ref['split']=='val':
            num_val += 1
            if IoU_det >= 0.5:
                correct_val_d += 1

    eval_result = {'val_d':correct_val_d/num_val,'testA_d':correct_A_d/num_A,'testB_d':correct_B_d/num_B}

    for metric, acc in eval_result.items():
        print(f'{metric}: {acc:.3f}')

    return eval_result

# IoU function
def computeIoU(box1, box2):
    # each box is of [x1, y1, w, h]
    inter_x1 = max(box1[0], box2[0])
    inter_y1 = max(box1[1], box2[1])
    inter_x2 = min(box1[0]+box1[2]-1, box2[0]+box2[2]-1)
    inter_y2 = min(box1[1]+box1[3]-1, box2[1]+box2[3]-1)

    if inter_x1 < inter_x2 and inter_y1 < inter_y2:
        inter = (inter_x2-inter_x1+1)*(inter_y2-inter_y1+1)
    else:
        inter = 0
    union = box1[2]*box1[3] + box2[2]*box2[3] - inter
    return float(inter)/union

class GaussianBlur(object):
    """Gaussian blur augmentation in SimCLR https://arxiv.org/abs/2002.05709"""

    def __init__(self, sigma=[.1, 2.]):
        self.sigma = sigma

    def __call__(self, x):
        sigma = random.uniform(self.sigma[0], self.sigma[1])
        x = x.filter(ImageFilter.GaussianBlur(radius=sigma))
        return x

max_stars_repo_name: dashstander/TCL

max_issues_repo_name: dashstander/TCL

max_forks_repo_name: dashstander/TCL


text:
import pandas as pd
import numpy as np
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from defaults_3 import default_config
import os

def get_data():
	# 划分数据集
	data = np.load('./data/DatawithrawL.npy')
	train = data[:int(data.shape[0]*0.8), :]
	test = data[int(data.shape[0]*0.8):, :]
	train_x = train[:, :-1]
	train_label = train[:, -1]
	test_x = test[:, :-1]
	test_label = test[:, -1]

	# 转换为Dataset数据格式
	lgb_train = lgb.Dataset(train_x, train_label)
	lgb_eval = lgb.Dataset(test_x, test_label, reference=lgb_train)
	return test_x, test_label, lgb_train, lgb_eval

def get_kfold_data(k):
	# 划分数据集
	data = np.load('./data/DatawithrawL.npy')
	np.random.shuffle(data)
	data_part = data.reshape(k, -1, data.shape[1])
	trains = []
	tests = []
	for i in range(k):
		test = data_part[i, :, :]
		tests.append(test)
		train = np.delete(data_part, i, axis=0)
		train = np.concatenate(train, axis=0)
		trains.append(train)
	trains = np.array(trains)
	tests = np.array(tests)
	train_X = trains[:, :, :-1]
	train_label = trains[:, :, -1]
	test_X = tests[:, :, :-1]
	test_label = tests[:, :, -1]
	return train_X, train_label, test_X, test_label

def get_config(config_file):
	FIXED_params = {
    'task': 'train',
    'boosting_type': 'gbdt',  # 设置提升类型
    'objective': 'regression',  # 目标函数
    'metric': 'l2',  # 评估函数
	'learning_rate': 0.01,
	'num_leaves': 31,  # 叶子节点数
    'verbose': -1,  # <0 显示致命的, =0 显示错误 (警告), >0 显示信息
	}
	SEARCH_params = default_config()
	SEARCH_params.merge_from_file(os.path.join('./config_3', config_file))
	params = {**FIXED_params, **SEARCH_params}
	return params

def get_single_data():
	# 划分数据集
	data = np.load('./data/DatawithrawL.npy')
	test = data[:int(data.shape[0]*0.2), :]
	train = data[int(data.shape[0]*0.2):, :]
	train_x = train[:, :-1]
	train_label = train[:, -1]
	test_x = test[:, :-1]
	test_label = test[:, -1]

	# 转换为Dataset数据格式
	lgb_train = lgb.Dataset(train_x, train_label)
	lgb_eval = lgb.Dataset(test_x, test_label, reference=lgb_train)
	return test_x, test_label, lgb_train, lgb_eval

def get_all_data():
	data = np.load('./data/DatawithrawL.npy')
	train_x = data[:, :-1]
	train_label = data[:, -1]
	lgb_train = lgb.Dataset(train_x, train_label)
	return train_x, train_label, lgb_train


text:
import pytest
import mxnet as mx
import subprocess
from os.path import abspath

from mxnet import gluon
from mxnet.gluon import nn
from numpy.testing import assert_almost_equal

def dummy_data(ctx, batch_size=10):
    return [mx.nd.random.uniform(shape=shape, ctx=ctx) for shape in ([batch_size, 1, 28, 28], [batch_size])]

def binary_lenet(**qconv_kwargs):
    net = nn.HybridSequential(prefix="")
    with net.name_scope():
        net.add(gluon.nn.Conv2D(channels=64, kernel_size=5))
        net.add(gluon.nn.Activation(activation='tanh'))
        net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
        net.add(gluon.nn.BatchNorm(axis=1, center=True, scale=True))

        net.add(gluon.nn.QActivation())
        net.add(gluon.nn.QConv2D(**qconv_kwargs))
        net.add(gluon.nn.BatchNorm(axis=1, center=True, scale=True))
        net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))

        # The Flatten layer collapses all axis, except the first one, into one axis.
        net.add(gluon.nn.Flatten())

        net.add(gluon.nn.QActivation())
        net.add(gluon.nn.QDense(1000))
        net.add(gluon.nn.BatchNorm(axis=1, center=True, scale=True))
        net.add(gluon.nn.Activation(activation='tanh'))

        net.add(gluon.nn.Dense(10))
    return net

@pytest.mark.parametrize("ctx", [mx.cpu()])
@pytest.mark.parametrize("qconv_kwargs", [{"channels": 64, "kernel_size": 5},
                                          {"channels": 64, "kernel_size": 3, "padding": 1}])
def test_save(ctx, qconv_kwargs):
    prefix, symbol_file, param_file = "binarized_", "test-symbol.json", "test-0000.params"
    orig_net = binary_lenet(**qconv_kwargs)
    orig_net.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)

    # This dummy pass is needed to initialize binary layers correctly!
    data, _ = dummy_data(ctx)
    expected = orig_net(data)

    # Training loop would be here

    # Export as symbol, so it can be used with C API
    orig_net.hybridize()

    # This dummy pass is needed to make correct symbol export possible, but does not replace the first one
    data, label = dummy_data(ctx)
    _ = orig_net(data)
    orig_net.export("test", epoch=0)

    # Intermediate symbolic model, non-compressed
    net1 = mx.gluon.nn.SymbolBlock.imports(symbol_file, ['data'], param_file=param_file, ctx=ctx)
    out1 = net1(data)

    output = subprocess.check_output(["build/tools/binary_converter/model-converter", param_file])

    # Compressed symbolic model
    net2 = mx.gluon.nn.SymbolBlock.imports(prefix + symbol_file, ['data'], param_file=prefix + param_file, ctx=ctx)
    out2 = net2(data)

    assert_almost_equal(expected.asnumpy(), out1.asnumpy())
    assert_almost_equal(expected.asnumpy(), out2.asnumpy())


text:
import numpy

"""
Naive recursive implementation
"""
def fib_recursive(n):
    if n < 2:
        return 1
    return fib_recursive(n - 1) + fib_recursive(n - 2)

"""
Iterative implementation of O(n)
"""
def fib_iter(n):
    a, b = 1, 1
    for _ in xrange(n):
        a, b = a + b, a
    return b

"""
Matrix multiplication implementation of O(log N)
"""
def fib_matpow(n):
    m = numpy.matrix('1 1; 1 0') ** n
    return m.item(0)

"""
Closed form solution of the recurrence relation
"""
def fib_phi(n):
    phi = (1 + math.sqrt(5)) / 2.0
    psi - (1 - math.sqrt(5)) / 2.0
    return int((phi ** (n+1) - psi ** (n+1)) / math.sqrt(5))

"""
Generating functions
"""
def fib(n):
    return (4 << n*(3+n)) // ((4 << 2*n) - 1) & ((2 << n) - 1)

max_stars_repo_name: RyanZHe/fibonacci

max_issues_repo_name: RyanZHe/fibonacci

max_forks_repo_name: RyanZHe/fibonacci


text:
#!/usr/bin/python3

import os, sys, time
import glob
import copy
import re
import datetime
import calendar
import pprint
import functools
import pprint as pprint2
from pathlib import Path

from typing import Callable

import pyLARDA.NcReader as NcReader
import pyLARDA.ParameterInfo as ParameterInfo
#import pyLARDA.DataBuffer as DataBuffer
#import pyLARDA.MeteoReader as MeteoReader
#import pyLARDA.Spec as Spec
import pyLARDA.peakTree as peakTree
import pyLARDA.trace_reader as trace_reader
import pyLARDA.helpers as h
import pyLARDA.Transformations as Transf

import numpy as np
from operator import itemgetter
import collections
import json
import requests, msgpack
from tqdm import tqdm
#import cbor2

import logging
logger = logging.getLogger(__name__)

DATEstrfmt = "%Y%m%d-%H%M%S"

def convert_regex_date_to_dt(re_date):
    """convert a re_date dict to datetime

    .. warning::

        When using 2 digit years (i.e. RPG) a 20 will
        be added in front

    Args:
        re_date (dict): result of the regex search with keys
    Returns:
        datetime
    """
    l = []
    if len(re_date['year']) == 2:
        re_date['year'] = '20' + re_date['year']
    for k in ['year', 'month', 'day', 'hour', 'minute', 'second']:
        if k in re_date.keys():
            l.append(int(re_date[k]))
    return datetime.datetime(*l)

def convert_to_datestring(datepattern, f):
    """convert the date in a (file-)string to dt

    Args:
        datepatttern: a python regex definition with named groups
        f: the string
    Returns:
        datetime
    """
    try:
        dt = convert_regex_date_to_dt(
            re.search(datepattern, f).groupdict())
    except AttributeError:
        logger.warning(f'No matching data pattern "{datepattern}" in file: "{f}"')
        return -1

    return dt.strftime(DATEstrfmt)

def setupreader(paraminfo) -> Callable:
    """obtain the reader from the paraminfo

    """

    if paraminfo["ncreader"] == 'timeheight_limrad94':
        reader = NcReader.timeheightreader_rpgfmcw(paraminfo)
    elif paraminfo["ncreader"] == 'spec_limrad94':
        reader = NcReader.specreader_rpgfmcw(paraminfo)
    elif paraminfo["ncreader"] == 'spec_rpgpy':
        reader = NcReader.specreader_rpgpy(paraminfo)
    elif paraminfo["ncreader"] == 'spec_kazr':
        reader = NcReader.specreader_kazr(paraminfo)
    elif paraminfo["ncreader"] in ['aux', 'aux_all_ts']:
        reader = NcReader.auxreader(paraminfo)
    elif paraminfo["ncreader"] in ['scan_timeheight', 'scan_time']:
        reader = NcReader.scanreader_mira(paraminfo)
    elif paraminfo['ncreader'] == 'peakTree':
        reader = peakTree.peakTree_reader(paraminfo)
    elif paraminfo['ncreader'] == 'trace':
        reader = trace_reader.trace_reader(paraminfo)
    elif paraminfo['ncreader'] == 'trace2':
        reader = trace_reader.trace_reader2(paraminfo)
    elif paraminfo["ncreader"] == 'pollyraw':
        reader = NcReader.reader_pollyraw(paraminfo)
    elif paraminfo["ncreader"] == 'mrrpro_spec':
        paraminfo.update({"ncreader": "spec", "compute_velbins":"mrrpro"})
        reader = NcReader.reader(paraminfo)
    elif paraminfo["ncreader"] == "wyoming_sounding_txt":
        reader = NcReader.reader_wyoming_sounding(paraminfo)
    else:
        reader = NcReader.reader(paraminfo)

    return reader

def setup_valid_date_filter(valid_dates) -> Callable:
    """validator function for chunks of valid dates

    Args:
        valid_dates: list of [begin, end] in 'YYYYMMDD'

    Returns:
        a single argument ('YYYYMMDD-HHMMSS') validator function
    """
    def date_filter(e):
        datepair, f = e
        f_b, f_e = datepair
        #print(valid_dates, datepair, f_b, f_e)
        #print([(f_b >= valid[0] and f_e <= valid[1]) for valid in valid_dates])
        return any([(f_b[:-7] >= valid[0] and f_e[:-7] <= valid[1]) for valid in valid_dates])

    return date_filter

def path_walk(top, topdown = False, followlinks = False):
    """pendant for os.walk
    """

    names = list(top.iterdir())

    dirs = [node for node in names if node.is_dir() is True]
    nondirs = [node for node in names if node.is_dir() is False]

    if topdown:
        yield top, dirs, nondirs

    for name in dirs:
        if followlinks or name.is_symlink() is False:
            for x in path_walk(name, topdown, followlinks):
                yield x

    if topdown is not True:
        yield top, dirs, nondirs

def end_1sec_earlier(date):
    dt = datetime.datetime.strptime(date, DATEstrfmt)
    return (dt-datetime.timedelta(seconds=1)).strftime(DATEstrfmt)

def guess_end(dates):
    """estimate the end of a file

    Returns:
        list of pairs [begin, end]
    """
    if len(dates) > 1:
        guessed_duration = (datetime.datetime.strptime(dates[-1], DATEstrfmt) -
            datetime.datetime.strptime(dates[-2], DATEstrfmt))
    else:
        guessed_duration = datetime.timedelta(seconds=(24*60*60)-1)
    # quick fix guessed duration not longer than 24 h
    if guessed_duration >= datetime.timedelta(days=1):
        guessed_duration = datetime.timedelta(seconds=(24*60*60)-1)
    last_d = (
        datetime.datetime.strptime(dates[-1], DATEstrfmt) + guessed_duration
    ).strftime(DATEstrfmt)
    ends = [end_1sec_earlier(d) for d in dates[1:]] + [last_d]
    return list(zip(dates, ends))

class Connector_remote:
    """connect the data (from the a remote source) to larda

    Args:
        camp_name (str): campaign name
        system (str): system identifier
        plain_dict (dict): connector meta info
        uri (str): address of the remote source
    """
    def __init__(self, camp_name, system, plain_dict, uri):
        self.camp_name = camp_name
        self.system = system
        self.params_list = list(plain_dict['params'].keys())
        print(self.system, self.params_list)
        self.plain_dict = plain_dict
        self.uri = uri

    def collect(self, param, time_interval, *further_intervals, **kwargs) -> dict:
        """collect the data from a parameter for the given intervals

        Args:
            param (str) identifying the parameter
            time_interval: list of begin and end datetime
            *further_intervals: range, velocity, ...
            **interp_rg_join: interpolate range during join

        Returns:
            data_container
        """
        resp_format = 'msgpack'
        interval = ["-".join([str(h.dt_to_ts(dt)) for dt in time_interval])]
        interval += ["-".join([str(i) for i in pair]) for pair in further_intervals]
        stream = True if resp_format == "msgpack" else False
        params = {"interval": ','.join(interval), 'rformat': resp_format}
        params.update(kwargs)
        resp = requests.get(self.uri + '/api/{}/{}/{}'.format(self.camp_name, self.system, param),
                            params=params, stream=stream)
        logger.debug("fetching data from: {}".format(resp.url))
        if resp_format == "msgpack":
            block_size = 1024
            pbar = tqdm(unit="B", total=(int(resp.headers.get('content-length', 0))//block_size)*block_size, unit_divisor=1024, unit_scale=True)
            content = bytearray()
            for data in resp.iter_content(block_size):
                content.extend(data)
                pbar.update(len(data))

        if resp.status_code != 200:
            if resp_format == "msgpack":
                print("Error at Backend")
                print(content.decode("unicode_escape"))
            else:
                print(resp.json())
            raise ConnectionError("bad status code of response {}".format(resp.status_code))

        starttime = time.time()
        # if resp_format == 'bin':
        #     data_container = cbor2.loads(resp.content)
        if resp_format == 'msgpack':
            logger.info("msgpack version {}".format(msgpack.version))
            if msgpack.version[0] < 1:
                data_container = msgpack.loads(content, encoding='utf-8')
            else:
                data_container = msgpack.loads(content, strict_map_key=False)
        elif resp_format == 'json':
            data_container = resp.json()

        #print("{:5.3f}s decode data".format(time.time() - starttime))
        starttime = time.time()
        for k in ['ts', 'rg', 'vel', 'var', 'mask', 'vel_ch2', 'vel_ch3']:
            if k in data_container and type(data_container[k]) == list:
                data_container[k] = np.array(data_container[k])
        logger.info("loaded data container from remote: {}".format(data_container.keys()))
        #print("{:5.3f}s converted to np arrays".format(time.time() - starttime))
        return data_container

    def description(self, param):
        """get the description str"""
        resp = requests.get(self.uri + '/description/{}/{}/{}'.format(self.camp_name, self.system, param))
        if resp.status_code != 200:
            raise ConnectionError("bad status code of response {}".format(resp.status_code))

        logger.warning(resp.text)
        return resp.text

    def get_as_plain_dict(self) -> dict:
        """put the most important information of the connector into a plain dict (for http tranfer)"""

        return self.plain_dict

class Connector:
    """connect the data (from the ncfiles/local sources) to larda

    Args:
        system (str): system identifier
        system_info (dict): dict info loaded from toml
        valid_dates (list of lists): list of begin and end datetime
        description_dir (optional): dir with the description rst
    """
    def __init__(self, system, system_info, valid_dates, description_dir=None):
        self.system = system
        self.system_info = system_info
        self.valid_dates = valid_dates
        self.params_list = list(system_info["params"].keys())
        self.description_dir = description_dir
        logger.info("params in this connector {} {}".format(self.system, self.params_list))
        logger.debug('connector.system_info {}'.format(system_info))

    def __str__(self):
        s = "connector for system {} \ncontains parameters: ".format(self.system)
        s += " ".join(self.params_list)
        return s

    def build_filehandler(self):
        """scrape the directories and build the filehandler
        """
        pathdict = self.system_info['path']

        filehandler = {}
        for key, pathinfo in pathdict.items():
            all_files = []
            current_regex = pathinfo['matching_subdirs'] if 'matching_subdirs' in pathinfo else ''

            # 1. match the names and subdirs with regex
            for root, _, files in path_walk(Path(pathinfo['base_dir'])):
                #print('walk ', root, dirs, len(list(files)))
                abs_filepaths = [f for f in files if re.search(current_regex, str(f))]
                logger.debug("valid_files {} {}".format(root, [f for f in files if re.search(current_regex, str(f))]))
                #print("skipped_files {} {}".format(root, [f for f in files if not re.search(current_regex, str(f))]))
                all_files += abs_filepaths
                #files = [f for f in os.listdir('.') if re.match(r'[0-9]+.*\.jpg', f)]

            # remove basedir (not sure if that is a good idea)
            all_files = [str(p).replace(pathinfo['base_dir'], "./") for p in all_files]
            #logger.debug('filelist {} {}'.format(len(all_files), all_files[:10]))

            # 2. extract the dates with another regex
            dates = [convert_to_datestring(pathinfo["date_in_filename"], str(f))\
                     for f in all_files]
            all_files = [f for _, f in sorted(zip(dates, all_files), key=lambda pair: pair[0])]
            dates = sorted(dates)

            # 3. estimate the duration a file covers
            date_pairs = guess_end(dates) if dates else []

            # 4. validate with the durations
            valid_date_filter = setup_valid_date_filter(self.valid_dates)
            singlehandler = list(filter(
                valid_date_filter,
                list(zip(date_pairs, all_files))))

            filehandler[key] = singlehandler
        #pprint.pprint(filehandler)
        self.filehandler = filehandler

    def save_filehandler(self, path, camp_name):
        """save the filehandler to json file"""
        savename = 'connector_{}.json'.format(self.system)
        pretty = {'indent': 2, 'sort_keys':True}
        #pretty = {}

        if not os.path.isdir(path+'/'+camp_name):
            os.makedirs(path+'/'+camp_name)

        with open(path+'/'+camp_name+'/'+savename, 'w') as outfile:
                json.dump(self.filehandler, outfile, **pretty)
                logger.info('saved connector to {}/{}/{}'.format(path,camp_name,savename))

    def load_filehandler(self, path, camp_name):
        """load the filehandler from the json file"""
        filename = "connector_{}.json".format(self.system)
        with open(path+'/'+camp_name+'/'+filename) as json_data:
                self.filehandler = json.load(json_data)

    def collect(self, param, time_interval, *further_intervals, **kwargs) -> dict:
        """collect the data from a parameter for the given intervals

        Args:
            param (str) identifying the parameter
            time_interval: list of begin and end datetime
            *further_intervals: range, velocity, ...
            **interp_rg_join: interpolate range during join

        Returns:
            data_container
        """

        paraminfo = self.system_info["params"][param]
        if 'interp_rg_join' not in paraminfo:
            # default value
            paraminfo['interp_rg_join'] = False
        if 'interp_rg_join' in kwargs:
            paraminfo['interp_rg_join'] = kwargs['interp_rg_join']
        base_dir = self.system_info['path'][paraminfo['which_path']]["base_dir"]
        logger.debug("paraminfo at collect {}".format(paraminfo))
        if len(time_interval) == 2:
            begin, end = [dt.strftime(DATEstrfmt) for dt in time_interval]
            # cover all three cases: 1. file only covers first part
            # 2. file covers middle part 3. file covers end
            #print(begin, end)
            flist = [e for e in self.filehandler[paraminfo['which_path']] \
                     if (e[0][0] <= begin < e[0][1])
                     or (e[0][0] > begin and e[0][1] < end)
                     or (e[0][0] <= end <= e[0][1])]
            assert len(flist) > 0, "no files available"
        elif len(time_interval) == 1:
            begin = time_interval[0].strftime(DATEstrfmt)
            flist = [e for e in self.filehandler[paraminfo['which_path']] if e[0][0] <= begin < e[0][1]]
            assert len(flist) == 1, "flist too long or too short: {}".format(len(flist))

        #[print(e, (e[0][0] <= begin and e[0][1] > begin), (e[0][0] > begin and e[0][1] < end), (e[0][0] <= end and e[0][1] >= end)) for e in flist]

        load_data = setupreader(paraminfo)
        datalist = [load_data(base_dir + e[1], time_interval, *further_intervals) for e in flist]
        # [print(e.keys) if e != None else print("NONE!") for e in datalist]
        # reader returns none, if it detects no data prior to begin
        # now these none values are filtered from the list
        assert len(datalist) > 0, 'No data found for parameter: {}'.format(param)
        datalist = list(filter(lambda x: x != None, datalist))
        #Transf.join(datalist[0], datalist[1])
        data = functools.reduce(Transf.join, datalist)

        return data

    def description(self, param) -> str:
        paraminfo = self.system_info["params"][param]
        #print('connector local paraminfo: ' + paraminfo['variable_name'])

        # Prints the nicely formatted dictionary
        # this is the python pprint function, not the larda.helpers function
        pp = pprint2.PrettyPrinter(indent=4)
        logger.info(pp.pformat(paraminfo))

        if 'description_file' not in paraminfo:
            return 'no description file defined in config'
        if self.description_dir == None:
            return 'description dir not set'

        description_file = self.description_dir / paraminfo['description_file']
        logger.info('load description file {}'.format(description_file))

        with open(description_file, 'r', encoding="utf-8") as f:
            descr = f.read()
        descr = "\n"+descr+"\n"
        logger.warning(descr)
        return descr

    def get_as_plain_dict(self) -> dict:
        """put the most important information of the connector into a plain dict (for http tranfer)

        Returns:
            connector information

            .. code::

                {params: {param_name: fileidentifier, ...},
                avail: {fileidentifier: {"YYYYMMDD": no_files, ...}, ...}
        """
        return {
            'params': {e: self.system_info['params'][e]['which_path'] for e in self.params_list},
            'avail': {k: self.files_per_day(k) for k in self.filehandler.keys()}
        }

    def files_per_day(self, which_path) -> dict:
        """replaces ``days_available`` and ``day_available``

        Returns:
            dict with days and no of files

            .. code::

                {'YYYYMMDD': no of files, ...}
        """
        fh = self.filehandler[which_path]
        groupedby_day = collections.defaultdict(list)
        for d, f in fh:
            groupedby_day[d[0][:8]] += [f]
        no_files_per_day = {k: len(v) for k, v in groupedby_day.items()}
        return no_files_per_day

max_stars_repo_name: jroettenbacher/larda

max_issues_repo_name: jroettenbacher/larda

max_forks_repo_name: jroettenbacher/larda