import utils.get_url as get_url
from utils.dbmenu import conn_util
import requests
import json
import time
from utils.req import req_util
import utils.normal as normal
from bs4 import BeautifulSoup
import re
import threading
from utils.proxy import refresh_db_ip
from utils.proxy import search_new_ip

# 定时获取首页推荐的视频信息 入口函数
def main_get_recommend_video():
    url = get_url.get_recomend_list_url ()
    try:
        response = requests.get (url)
        response.raise_for_status ()
        # response.encoding = response.apparent_encoding
        result = json.loads (response.text).get ('data').get ('archives')
    except Exception as e:
        print (e)
        pass
    total = len(result)
    for (i,video) in enumerate(result):
        aid = video.get ('aid')
        # 尝试更新vinfo
        # 检测aid是否存在于数据库中 存在则跳过
        try:
            count_of_aid = conn_ob.count_aid (aid)
            if count_of_aid == 0:
                bvid = video.get ('bvid')
                area = video.get ('tname')
                vname = video.get ('title')
                uid = video.get ('owner').get ('mid')
                ctime = video.get ('ctime')
                cre_time = normal.timestamp2str (ctime)
                # 尝试更新userinfo
                # 如果uid存在于数据库中，则跳过
                count_of_uid = conn_ob.count_uid (uid)
                conn_ob.insert_vinfo (aid, bvid, uid, vname, area, cre_time)
                if count_of_uid == 0:
                    uname = video.get ('owner').get ('name')
                    try:
                        conn_ob.insert_userinfo (uid, uname)
                    except:
                        pass
                # 获取该视频的tag
                download_video_tag (aid)
        except:
            print (f"\rdownloading failed recommend {(i + 1) / total : 3.2%}...... \n", end="")
            continue
        print(f"\rdownloading recommend {(i+1)/total : 3.2%}......",end="")
    return 0

def download_vresult_row(aid):
    req_ob = req_util ()
    url = get_url.get_vinfo_from_aid (aid)
    result = req_ob.request_content (url)
    timestamp = normal.timestamp2str (int (time.time ()))
    info = json.loads (result)["data"]
    view = info["view"]
    danmaku = info["danmaku"]
    reply = info["reply"]
    favorite = info["favorite"]
    coin = info["coin"]
    like = info["like"]
    dislike = info["dislike"]
    conn_ob.insert_vresult (timestamp, aid, view, reply, coin, like, dislike, favorite, danmaku)

# 下载 目前 所有列表里面的视频的成绩信息 入口函数
def main_download_all_vresult_now():
    aid_list = conn_ob.select_all_aid ()
    for i, row in enumerate (aid_list):
        try:
            download_vresult_row (row["aid"])
            time.sleep(1)
        except:
            pass
        print (f"\r Downloading vresult {(i + 1) / len (aid_list):3.2%} ...", end="")
    print ("\rDownload vresult finished!")

# 获取热词及热词相关的20个最新视频并储存 入口函数
def main_get_hot_word_video():
    req_ob = req_util ()
    url = get_url.get_hotword_url ()
    content = req_ob.request_content (url)
    con_list = json.loads (content)
    timestamp = con_list["timestamp"]
    formatime = normal.timestamp2str (timestamp)
    hotlist = con_list["list"]
    total = len (hotlist)
    for i, row in enumerate (hotlist):
        hotword = row["keyword"]
        rank = row["pos"]
        conn_ob.insert_hotword (formatime, rank, hotword)
        try:
            download_hot_word_aid (hotword)
        except:
            pass
        print (f"\rDownloading hotword {i / total:3.2%}", end="")
    print ("\rDownloaded!")

def download_hot_word_aid(hotword):
    req_ob = req_util (PROXY=True)
    url = get_url.get_newest_search_url (hotword)
    content = req_ob.request_content (url)
    soup = BeautifulSoup (content, 'lxml')
    video = soup.find_all ("li", attrs={"class": "video-item matrix"})
    for v in video:
        bvid = re.findall (r"/(BV[0-9a-zA-Z]{10})", v.next.attrs["href"])[0]
        aid = normal.get_aid_from_BV (bvid)
        conn_ob.insert_aid_hotword (aid, hotword)
        try:
            # 获取并储存热词对应该视频的详细信息
            download_detail_vinfo (bvid)
        except:
            pass
        try:
            # 获取并储存热词对应该视频的tag
            download_video_tag (aid)
        except:
            pass
    return

def download_video_tag(aid):
    req_ob = req_util ()
    url = get_url.get_tag_url (aid)
    content = req_ob.request_content (url)
    info = json.loads (content)
    for row in info["data"]:
        tag_name = row["tag_name"]
        count = conn_ob.count_aid_tag (aid, tag_name)
        if count == 0:
            conn_ob.insert_aid_tag (aid, tag_name)
    return

def download_detail_vinfo(bvid):
    req_ob = req_util ()
    url = get_url.get_vinfo_by_bvid (bvid)
    content = req_ob.request_content (url)
    info = json.loads (content)["data"]
    aid = info["aid"]
    count_aid = conn_ob.count_aid (aid)
    if count_aid == 0:
        uid = info["owner"]["mid"]
        vname = info["title"]
        area = info["tname"]
        ctime = info["ctime"]
        cre_time = normal.timestamp2str (ctime)
        conn_ob.insert_vinfo (aid, bvid, uid, vname, area, cre_time)
        count_uid = conn_ob.count_uid (uid)
        if count_uid == 0:
            uname = info["owner"]["name"]
            conn_ob.insert_userinfo (uid, uname)
    return

def download_follower_by_uid(uid):
    req_ob = req_util ()
    url = get_url.get_follower_info (uid)
    content = req_ob.request_content (url)
    timestamp = normal.timestamp2str (int (time.time ()))
    info = json.loads (content)["data"]
    follower_num = info["follower"]
    conn_ob.insert_follower (timestamp, uid, follower_num)
    return

def main_download_follower_now():
    uid_list = conn_ob.select_all_uid ()
    for i, row in enumerate (uid_list):
        try:
            download_follower_by_uid (row["uid"])
        except:
            pass
        print (f"\rDownloading follower {(i + 1) / len (uid_list):3.2%} ...", end="")
    print ("\rDownload follower finished!")

def run_download_recommend(counter=1, dual=5):
    print (f"\ndownload recommend start，间隔{dual}小时下载一次")
    # 下载20次推荐首页的视频相关信息，每次下载间隔12小时
    for i in range (1, counter + 1):
        print (f"第{i}次 推荐视频 下载开始")
        try:
            main_get_recommend_video ()
        except Exception as e:
            print (e)
            pass
        time.sleep (dual * 60 * 60)
        print (f"第{i}次 推荐视频 下载完成")

def run_download_hotword(counter=20, dual=6, asleep=0.2):
    time.sleep (asleep * 60 * 60)
    print (f"\ndownload hotword start，间隔{dual}小时下载一次")
    # 下载10次热词相关视频及热词，每次间隔24小时
    for i in range (1, counter + 1):
        print (f"第{i}次 热词视频 下载开始")
        try:
            main_get_hot_word_video ()
        except Exception as e:
            print (e)
            pass
        time.sleep (dual * 60 * 60)
        print (f"第{i}次 热词视频 下载完成")

def run_download_list_vresult(asleep=1, dual=6):
    time.sleep (asleep * 60 * 60)
    print (f"\ndownload vresult start , 间隔{dual}小时下载一次该时段内的视频详细成绩信息")
    # 在程序运行期间，每隔六小时运行一次
    i = 1
    while True:
        print (f"第{i}次 详细信息 下载开始")
        try:
            main_download_all_vresult_now ()
        except Exception as e:
            print (e)
            pass
        time.sleep (dual * 60 * 60)
        print (f"第{i}次 详细信息 下载完成")
        i += 1

def run_download_follower(asleep=1.5, dual=12):
    time.sleep (asleep * 60 * 60)
    print (f"\ndownload vresult start , 间隔{dual}小时下载一次该时段内的视频详细成绩信息")
    # 在程序运行期间，每隔六小时运行一次
    i = 1
    while True:
        print (f"第{i}次 follwer 下载开始")
        try:
            main_download_follower_now()
        except Exception as e:
            print (e)
            pass
        time.sleep (dual * 60 * 60)
        print (f"第{i}次 follower 下载完成")
        i += 1

def run_refresh_ips(dual=12):
    while True:
        time.sleep(dual * 60 * 60)
        print("刷新ip池可用ip")
        refresh_db_ip()
        ip_count = conn_ob.count_all_ip()
        if ip_count < 15:
            search_new_ip()

def main():
    print("正在等待mysql启动，请等待")
    time.sleep(20)
    refresh_db_ip ()
    t1 = threading.Thread (target=run_download_recommend)
    t2 = threading.Thread (target=run_download_hotword)
    t3 = threading.Thread (target=run_download_list_vresult)
    t4 = threading.Thread (target=run_download_follower)
    t5 = threading.Thread (target=run_refresh_ips)
    t1.start ()
    t2.start ()
    t3.start ()
    t4.start ()
    t5.start ()

conn_ob = conn_util ()
if __name__ == '__main__':
    main()


import requests
import csv
import os
from bs4 import BeautifulSoup
from selenium import webdriver
from get_keyword import getKeywordsForMulti, getKeywords
import time

def getLinks():
    options = webdriver.ChromeOptions()
    options.add_argument('headless')

    driver = webdriver.Chrome(os.path.dirname(os.path.realpath(__file__)) + "/chromedriver", chrome_options=options)
    driver.get("https://medium.com/netflix-techblog")
    driver.implicitly_wait(7)

    SCROLL_PAUSE_TIME = 3

    # Get scroll height
    last_height = driver.execute_script("return document.body.scrollHeight")

    while True:
        # Scroll down to bottom
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

        # Wait to load page
        time.sleep(SCROLL_PAUSE_TIME)

        # Calculate new scroll height and compare with last scroll height
        new_height = driver.execute_script("return document.body.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height

    link_list = []
    already_list = []

    try:
        f = open(os.path.dirname(os.path.realpath(__file__)) + "/data/netflix.csv", "r", encoding='utf-8')
        rdr = csv.reader(f)
        for line in rdr:
            already_list.append(line[2])
        f.close()
    except FileNotFoundError:
        f = open(os.path.dirname(os.path.realpath(__file__)) + "/data/netflix.csv", "w", encoding='utf-8', newline='')
        writefile = csv.writer(f)
        writefile.writerow(["title", "content", "url", "cnt", "source", "keyword", "image", "createdAt", "priority"])
        f.close()

    source = driver.page_source
    soup = BeautifulSoup(source, "lxml")
    driver.close()

    links_1 = soup.find_all("div", {"class": "col u-xs-marginBottom10 u-paddingLeft9 u-paddingRight12 u-paddingTop0 u-sm-paddingTop20 u-paddingBottom25 u-size4of12 u-xs-size12of12 u-marginBottom30"})
    links_2 = soup.find_all("div", {"class": "u-lineHeightBase postItem"})
    links_3 = soup.find_all("div", {"class": "col u-xs-marginBottom10 u-paddingLeft0 u-paddingRight0 u-marginBottom60"})

    for l in links_1:
        link = l.find("a")
        # print(link['href'])
        if link['href'] not in already_list:
            link_list.append(link['href'])

    for l in links_2:
        link = l.find("a")
        # print(link['href'])
        if link['href'] not in already_list:
            link_list.append(link['href'])

    for l in links_3:
        link = l.find("a")
        # print(link['href'])
        if link['href'] not in already_list:
            link_list.append(link['href'])

    return link_list

def getData(file, link, multi):
    r = requests.get(link)
    content_source = r.text
    content_soup = BeautifulSoup(content_source, "lxml")

    title = content_soup.select_one('div.section-content > div > h1').text.replace(u'\xa0',' ').replace('\t',' ').lstrip().rstrip().replace("\"", "").replace("\'", "")
    created_at = content_soup.find("time")['datetime'].split("T")[0]

    content = ""
    section_content = content_soup.find_all("div", {"class": "section-content"})
    for s in section_content:

        image = ""
        img = s.find("img")
        if img is not None:
            image = img['src']
        print(image)

        contents = s.find_all("p")
        for c in contents:
            content += c.text
    content = content.replace(u'\xa0',' ').replace('\t',' ').replace('<br>', ' ').replace("\n", ' ').lstrip().rstrip().replace("\"", "").replace("\'", "")
    source = "netflix"

    if multi:
        keyword_list = getKeywordsForMulti(title.lower(), content.lower(), source, False)
    else:
        keyword_list = getKeywords(title.lower(), content.lower(), source, False)

    if keyword_list:
        _keyword = ",".join(keyword_list)
    else:
        _keyword = ""

    file.writerow([title, content[:200] + "...", link, 0, source, _keyword, image, created_at, 0])

    updater = open(os.path.dirname(os.path.realpath(__file__)) + "/data/update.csv", "a", encoding='utf-8', newline='')
    update_writer = csv.writer(updater)
    update_writer.writerow([title, content[:200] + "...", link, 0, source, _keyword, image, created_at, 0])
    updater.close()

def main(multi):
    print("netflix")
    start = time.time()

    link_list = getLinks()

    file = open(os.path.dirname(os.path.realpath(__file__)) + "/data/netflix.csv", "a", encoding='utf-8', newline='')
    writefile = csv.writer(file)
    for i in range(len(link_list)):
        getData(writefile, link_list[i], multi)
    file.close()

    print(time.time() - start)

if __name__ == "__main__":
    main()


import collections
import auxiliary

def breadth_first_search(initial_state, goal_state):
    open_set = collections.deque()
    closed_set = set()
    come_from = dict()

    open_set.append(initial_state)
    come_from[initial_state] = None

    while len(open_set) > 0:
        state = open_set.popleft()

        if state == goal_state:
            return auxiliary.construct_path(come_from, state)

        for child_state in auxiliary.get_children(state):
            if child_state in closed_set:
                continue
            if child_state not in open_set:
                come_from[child_state] = state
                open_set.append(child_state)

        closed_set.add(state)

if __name__ == "__main__":
    initial = ((2, 8, 1), (4, 6, 3), (0, 7, 5))
    goal = ((1, 2, 3), (8, 0, 4), (7, 6, 5))
    # initial = ((8, 6, 7), (2, 5, 4), (3, 0, 1))
    # goal = ((1, 2, 3), (4, 5, 6), (7, 8, 0))
    path = breadth_first_search(initial, goal)
    auxiliary.print_path(path)


import os
import sys

import theano
import theano.tensor as T
from theano import printing
from theano.ifelse import ifelse

import numpy as np
from numpy import random as rng

import config
from Conv1d import Conv1DLayer
from NN4LogReg import NN4LogReg
from NN4Normal import NN4Normal
from ResNet4Distance import ResNet
from DilatedResNet4Distance import DilatedResNet
from utils import OuterConcatenate, MidpointFeature
import DistanceUtils

from config import Response2LabelName, Response2LabelType

##this class calculates the convolution of 1D sequence and then convert it to a 2D matrix using OuterConcatenate
class Conv1D2Matrix:

	def __init__(self, rng, input, n_in, n_hiddens=[], halfWinSize=0, mask=None):
	## input has shape (batchSize, seqLen, n_in) where n_in is the number of features at each position
	## all the sequences aligned at the right hand side, mask is used for the left hand side positions
	## mask has shape (batchSize, #positions_to_be_masked )

	## n_hiddens specify the number of hidden neurons at all the hidden layers
	## halfWinSize specify the half window size of the convolution operation

		self.input = input
		self.mask = mask
		self.n_in = n_in
		self.halfWinSize = halfWinSize

		self.layers = []
		self.params = []
		self.paramL1 = 0
		self.paramL2 = 0

		n_out_at_last_layer = n_in
		output_at_last_layer = input

		for i in range(len(n_hiddens)):
			layer = Conv1DLayer(rng, input=output_at_last_layer, numOfInFeatures=n_out_at_last_layer, numOfOutFeatures=n_hiddens[i], halfWinSize=halfWinSize, mask=mask)
			self.layers.append(layer)
			n_out_at_last_layer = n_hiddens[i]
			output_at_last_layer = layer.output

			self.params += layer.params
			self.paramL1 += layer.paramL1
			self.paramL2 += layer.paramL2

		## conv_out has shape (batchSize, seqLen, n_out_at_last_layer ), it is the final convolution result
		conv_out = output_at_last_layer

		## now we take the OuterConcatenate operation, the result has shape (batchSize, seqLen, seqLen, 2 * n_out_at_last_layer)
		self.output = OuterConcatenate (input = conv_out)
		self.n_out = 2 * n_out_at_last_layer

		##add the below two lines for convenience
		self.conv_out = conv_out
		self.n_conv_out = n_out_at_last_layer

def TestConv1D2Matrix():
	x = T.tensor3('x')
	conv1dlayer = Conv1D2Matrix( rng=np.random.RandomState(), input=x, n_in=2, n_hiddens=[3], halfWinSize=1)
	f = theano.function([x], conv1dlayer.output)
	a = np.random.uniform(0, 1, (2, 7, 2)).astype(np.float32)
	print(a)
	print(f(a))

## 2D convolution on a matrix for distance matrix prediction
## no pooling is allowed here
class Conv2D4DistMatrix:
	def __init__(self, rng, input, n_in=1, n_hiddens=None, halfWinSize=0, mask=None, activation=T.nnet.relu):
		## input has shape (batchSize, nRows, nCols, n_in) where n_in is the number of input features at each element
		## here we assume that input[0], input[1], ...., input[batchSize -1 ] are aligned at the right bottom
		## nrows and ncols are unknown
		## mask has shape (batchSize, #rows_to_be_masked, nCols), mask is a binary matrix with 0 indicates padding positions and 1 real data position

		##hwsz is the half window size of the filter shape
		##mask is used to reduce noise introduced by padding. all the matrices in a batch shall be aligned at the right bottom corner

		self.input = input
		self.n_in = n_in
		self.n_hiddens = n_hiddens
		self.halfWinSize = halfWinSize
		self.mask = mask

		##the window size of our filters is always odd
		wSize = 2*halfWinSize + 1

		self.params = []
		self.paramL1 = 0
		self.paramL2 = 0

		## reshape input so that it has shape (batchSize, n_in, nRows, nCols)
		output_in_last_layer = input.dimshuffle(0, 3, 1, 2)

		n_out_in_last_layer = n_in

		for i in range(len(n_hiddens)):
			W_shape = (n_hiddens[i], n_out_in_last_layer, wSize, wSize)

			W_values = np.asarray(
			rng.uniform( low = - np.sqrt(6. / (n_out_in_last_layer * wSize * wSize + n_hiddens[i])),
			 high =  np.sqrt(6. / (n_out_in_last_layer * wSize * wSize + n_hiddens[i])),
			size = W_shape
			),
			dtype = theano.config.floatX
			)
			if activation == T.nnet.relu:
				W_values = np.asarray(
				rng.normal( scale = np.sqrt(2. / (n_out_in_last_layer * wSize * wSize + n_hiddens[i])), size = W_shape),
				dtype = theano.config.floatX
				)

			b_shape = (n_hiddens[i], )
			b_values = np.asarray (rng.uniform(low = -.0, high =.0, size = b_shape), dtype=theano.config.floatX )

			W = theano.shared (value = W_values, name = 'conv2d_W_' + str(i), borrow = True)
			b = theano.shared (value = b_values, name = 'conv2d_b_' + str(i), borrow = True)

			shape_in_last_layer = (None, n_out_in_last_layer, None, None)

			## Please make sure that conv2d_out has shape (batchSize, n_hiddens[i], nRows, nCols)
			conv2d_out = T.nnet.conv2d(output_in_last_layer, W, input_shape = shape_in_last_layer, filter_shape=W_shape, border_mode='half')
			if activation is not None:
				out2 = activation(conv2d_out + b.dimshuffle('x', 0, 'x', 'x'))
			else:
				out2 = (conv2d_out + b.dimshuffle('x', 0, 'x', 'x'))

			if mask is not None:
				## mask has shape (batchSize, #rows_to_be_masked, nCols)

				## a subtensor of out2 along the horiz direction
				out2_sub_horiz = out2[:, :, :mask.shape[1], :]
				mask_horiz = mask.dimshuffle(0, 'x', 1, 2)
				out3 = T.set_subtensor(out2[:, :, :mask.shape[1], :], T.mul(out2_sub_horiz, mask_horiz) )

				## a subtensor of out3 along the vertical direction
				out3_sub_vertical = out3[:, :, :, :mask.shape[1] ]
				mask_vertical = mask.dimshuffle(0, 'x', 2, 1)
				out4 = T.set_subtensor(out3[:, :, :, :mask.shape[1] ], T.mul(out3_sub_vertical, mask_vertical) )

				output_in_last_layer = out4
			else:
				output_in_last_layer = out2

			n_out_in_last_layer = n_hiddens[i]

			self.params += [W, b]
			self.paramL1 += (abs(W).sum() +  abs(b).sum())
			self.paramL2 += ((W**2).sum() + (b**2).sum())

		##change the shape of the final output to (batchSize, nRows, nCols, n_out)
		self.output = output_in_last_layer.dimshuffle(0, 2, 3, 1)
		self.n_out = n_out_in_last_layer

def TestConv2D4DistMatrix():
	x = T.tensor4('x')
	conv2dlayer = Conv2D4DistMatrix( rng=np.random.RandomState(), input=x, n_in=2, n_hiddens=[3], halfWinSize=1)
	f = theano.function([x], conv2dlayer.output)
	a = np.random.uniform(0, 1, (2, 7, 7, 2)).astype(np.float32)
	print(a)
	print(f(a))

"""
def SeparateTrainByRange(rng, conv_out, n_in, numLabels, n_hiddens_logreg=None):

    batchSize, dataLen, dataLen, tmp_n = conv_out.shape

    n_in4logreg = n_in

    ##theano currently does not have triu for tensor3, so we have to do some work here
    M1s = T.ones((dataLen, dataLen), dtype=np.int8)
    M1s_3d = T.ones((batchSize, dataLen, dataLen), dtype=np.int8)
    Sep24Mat = T.triu(M1s, 24) + T.tril(M1s, -24)
    Sep12Mat = T.triu(M1s, 12) + T.tril(M1s, -12)
    Sep6Mat = T.triu(M1s, 6) + T.tril(M1s, -6)

    LRselection = Sep24Mat.dimshuffle('x',0,1)
    MRselection = (Sep12Mat - Sep24Mat).dimshuffle('x', 0, 1)
    SRselection = (Sep6Mat - Sep12Mat ).dimshuffle('x', 0, 1)
    LRselection = T.mul( LRselection, M1s_3d )
    MRselection = T.mul( MRselection, M1s_3d )
    SRselection = T.mul( SRselection, M1s_3d )

    selections = [ LRselection, MRselection, SRselection ]

    regressors = []
    output_2d = T.ones((batchSize, dataLen, dataLen), dtype=np.int8)*(-1)
    output_2d_prob = T.zeros((batchSize, dataLen, dataLen, numLabels), dtype=theano.config.floatX)

    for sel in selections:
        selected = conv_out[sel.nonzero()]
        regressor = MLLogReg(rng=rng, input=selected, n_in=n_in4logreg, n_out=numLabels, n_hiddens=n_hiddens_logreg)
        regressors.append(regressor)
        output_2d = T.set_subtensor(self.output_2d[ sel.nonzero() ], T.cast(regressor.y_pred, 'int8') )
        output_2d_prob = T.set_subtensor(self.output_2d_prob[ sel.nonzero() ], regressor.p_y_given_x )

    return selections, regressors, output_2d, output_2d_prob
"""

## this is the main class for the prediction of inter-residue distance, hydrogen bonding and beta-pairing
class ResNet4DistMatrix:
	def __init__(self, rng, seqInput, matrixInput, mask_seq=None, mask_matrix=None,
				 embedInput=None, modelSpecs = None):
		"""
		seqInput has shape (batchSize, seqLen, n_in_seq)
		matrixInput has shape (batchSize, seqLen, seqLen, n_in_matrix)
		mask_seq has shape (batchSize, #cols_to_be_masked)
		mask_matrix has shape (batchSize, #rows_to_be_masked, seqLen)
		embedInput has shape (batchSize, seqLen, n_in2)
		"""

		assert (modelSpecs is not None)

		self.modelSpecs = modelSpecs
		self.responses = modelSpecs['responses']

		## set the number of hidden neurons and number of layers
		n_in_seq = modelSpecs['n_in_seq']
		n_in_matrix = modelSpecs['n_in_matrix']
		n_hiddens_seq = modelSpecs['conv1d_hiddens']
		n_hiddens_matrix = modelSpecs['conv2d_hiddens']
		n_hiddens_logreg = modelSpecs['logreg_hiddens']
		seq_repeats = modelSpecs['conv1d_repeats']
		matrix_repeats = modelSpecs['conv2d_repeats']

		## half win size for convolutional operation
		if modelSpecs['network'].startswith('DilatedResNet'):
			hwsz_matrix = modelSpecs['conv2d_hwszs']
			hwsz_seq = [modelSpecs['conv1d_hwsz'] ]* len(n_hiddens_seq)
			dilation_seq = [1] * len(n_hiddens_seq)
			dilation_matrix = modelSpecs['conv2d_dilations']
		else:
			hwsz_matrix=modelSpecs['halfWinSize_matrix']
			hwsz_seq=modelSpecs['halfWinSize_seq']

		## masks to reduce impact of padding zeros
		self.mask_1d = mask_seq
		self.mask_2d = mask_matrix

		self.layers = []

		# sequence convolution
		if modelSpecs['network'].startswith('DilatedResNet'):
			seqConv = DilatedResNet(rng, input=seqInput, n_in=n_in_seq, n_hiddens=n_hiddens_seq, n_repeats=seq_repeats, halfWinSize=hwsz_seq, dilation=dilation_seq, mask=mask_seq, activation=modelSpecs['activation'], batchNorm=modelSpecs['batchNorm'], version=modelSpecs['network'])
		else:
			seqConv = ResNet(rng, input=seqInput, n_in=n_in_seq, n_hiddens=n_hiddens_seq, n_repeats=seq_repeats, halfWinSize=hwsz_seq, mask=mask_seq, activation=modelSpecs['activation'], batchNorm=modelSpecs['batchNorm'], version=modelSpecs['network'])
		self.layers.append(seqConv)

		## transform 1d sequence to 2d matrix
		seq2matrixMode = modelSpecs['seq2matrixMode']
		seq2matrixLayers = []
		embedLayers = []

		## determine if we shall use the sequential features or not. The sequential features include sequence profile (PSSM), predicted secondary structure and predicted solvent accessibility
		## useSequentialFeatures is True by default
		useSequentialFeatures = ('UseSequentialFeatures' in modelSpecs \
								 and (modelSpecs['UseSequentialFeatures'] is True))

		## use OuterConcatenation operation to convert sequence features into pairwise features
		if 'OuterCat' in seq2matrixMode and useSequentialFeatures:
			##midpointfeature has shape (batchSize, seqLen, seqLen, n_midpoint_out)
			midpointfeature, n_midpoint_out = MidpointFeature(seqConv.output, seqConv.n_out)

			##remove noise in midpointfeature
			## mask_matrix is used to reduce noise introduced by padding positions
			mid_subtensor = midpointfeature[:, :mask_matrix.shape[1], :, :]
			midpointfeature = T.set_subtensor(mid_subtensor, T.mul(mask_matrix.dimshuffle(0,1,2,'x'), mid_subtensor) )
			mid_subtensor2 = midpointfeature[:, :, :mask_matrix.shape[1], : ]
			midpointfeature = T.set_subtensor(mid_subtensor2, T.mul(mask_matrix.dimshuffle(0,2,1,'x'), mid_subtensor2) )

			## here we use convolution with halfWinSize=0 to reduce model complexity
			compressLayer = Conv2D4DistMatrix(rng, input=midpointfeature, n_in=n_midpoint_out, n_hiddens=seq2matrixMode['OuterCat'], halfWinSize=0, mask=mask_matrix )
			seq2matrixLayers.append(compressLayer)

		## embedding primary sequence and/or predicted secondary structure
		if embedInput is not None:
			from EmbeddingLayer import MetaEmbeddingLayer

			if 'Seq+SS' in seq2matrixMode:
				n_out_embed = seq2matrixMode['Seq+SS']
			elif 'SeqOnly' in seq2matrixMode:
				n_out_embed = seq2matrixMode['SeqOnly']
			else:
				print('At least one of two embedding modes Seq+SS or SeqOnly shall be specified.')
				sys.exit(-1)

			embedLayer = MetaEmbeddingLayer(embedInput, modelSpecs['n_in_embed'], n_out_embed)
			seq2matrixLayers.append(embedLayer)
			embedLayers.append(embedLayer)

		self.layers += seq2matrixLayers

		input_2d = T.concatenate( [ matrixInput ] + [ layer.output for layer in seq2matrixLayers], axis=3)
		n_input2d = n_in_matrix + sum([ layer.n_out for layer in seq2matrixLayers ])

		if modelSpecs['network'].startswith('ResNet'):
			matrixConv=ResNet(rng, input=input_2d, n_in=n_input2d, n_hiddens=n_hiddens_matrix, n_repeats=matrix_repeats, halfWinSize=hwsz_matrix, mask=mask_matrix, activation=modelSpecs['activation'], batchNorm=modelSpecs['batchNorm'], version=modelSpecs['network'])
		elif modelSpecs['network'].startswith('DilatedResNet'):
			matrixConv=DilatedResNet(rng, input=input_2d, n_in=n_input2d, n_hiddens=n_hiddens_matrix, n_repeats=matrix_repeats, halfWinSize=hwsz_matrix, dilation=dilation_matrix, mask=mask_matrix, activation=modelSpecs['activation'], batchNorm=modelSpecs['batchNorm'], version=modelSpecs['network'])
		else:
			print('Unimplemented deep network type: ', modelSpecs['network'])
			exit(-1)

		self.layers.append(matrixConv)

		conv_out = matrixConv.output

		selected = conv_out.dimshuffle(3, 0, 1, 2).flatten(2).dimshuffle(1, 0)
		n_in4logreg = matrixConv.n_out

		self.outputList = []
		self.output_probList = []
		self.predictors = []

		self.params4var = []
		self.paramL14var = 0
		self.paramL24var = 0

		for res in modelSpecs['responses']:
			labelType = Response2LabelType(res)
			predictor = None

			if labelType.startswith('Discrete'):
				assert (config.responseValueDims[labelType] == 1)
				predictor = NN4LogReg(rng=rng, input=selected, n_in=n_in4logreg, n_out=config.responseProbDims[labelType], n_hiddens=n_hiddens_logreg)
			elif labelType.startswith('LogNormal') or labelType.startswith('Normal'):
				predictor = NN4Normal(rng=rng, input=selected, n_in=n_in4logreg, n_variables=config.responseValueDims[labelType], n_out=config.responseProbDims[labelType], n_hiddens=n_hiddens_logreg)
				## recording parameters specific for variance prediction
				self.params4var += predictor.params4var
				self.paramL14var += predictor.paramL14var
				self.paramL24var += predictor.paramL24var
			else:
				print('incorrect response name or label type: ', res)
				exit(-1)

			self.layers.append(predictor)
			self.predictors.append(predictor)

			## output in 2d matrix
			output_2d = predictor.y_pred.reshape( (conv_out.shape[0], conv_out.shape[1], conv_out.shape[2], config.responseValueDims[labelType]) )
			output_2d_prob = predictor.output.reshape( (conv_out.shape[0], conv_out.shape[1], conv_out.shape[2], config.responseProbDims[labelType] ) )

			self.outputList.append( output_2d )
			self.output_probList.append( output_2d_prob )

		self.output = T.concatenate( self.outputList, axis=3 )
		self.output_prob = T.concatenate( self.output_probList, axis=3 )

		## collect all the model parameters and their norms
		self.params = []
		self.paramL2 = 0
		self.paramL1 = 0

		for layer in self.layers:
			self.params  += layer.params
			self.paramL2 += layer.paramL2
			self.paramL1 += layer.paramL1

	def loss(self, zList, weightList=None, useMeanOnly=False):
		losses = []
		if weightList is not None and len(weightList)>0:
			for predictor, z, w in zip(self.predictors, zList, weightList):
				if (z.ndim == 3 ):
					zflat = z.flatten().dimshuffle(0, 'x')
				elif (z.ndim == 4):
					zflat = z.dimshuffle(3, 0, 1, 2).flatten(2).dimshuffle(1, 0)
				else:
					print('unsupported ndim for z in errors():', z.ndim)
					exit(-1)
				assert (w.ndim == 3)
				wflat = w.flatten().dimshuffle(0, 'x')

				## zflag shall have shape (batchSize, valueDims) and wflag shall have shape (batchSize, 1)
				losses.append(  predictor.loss(zflat, useMeanOnly=useMeanOnly, sampleWeight=wflat)   )
		else:
			for predictor, z in zip(self.predictors, zList):
				if (z.ndim == 3):
					zflat = z.flatten().dimshuffle(0, 'x')
				elif (z.ndim == 4):
					zflat = z.dimshuffle(3, 0, 1, 2).flatten(2).dimshuffle(1, 0)
				else:
					print('unsupported ndim for z in errors():', z.ndim)
					exit(-1)

			## zflag shall have shape (batchSize, valueDims)
			losses.append(  predictor.loss(zflat, useMeanOnly=useMeanOnly )  )

		## losses is a list of scalar
		return T.stack(losses)

	## calculate errors for one response with label Type Discrete12C, Discrete25C, Discrete52C, return a tensor with ndim=1
	def errors4one(self, z, out, weight=None, distLabelType='12C'):
		distBins = config.distCutoffs[distLabelType]
		label8 = DistanceUtils.LabelsOfOneDistance(config.ContactDefinition, distBins)
		label15 = DistanceUtils.LabelsOfOneDistance(config.InteractionLimit, distBins)

		z3C = T.cast( T.ge(z, label8), 'int32') + T.cast( T.ge(z, label15), 'int32')
		o3C = T.cast( T.ge(out, label8), 'int32') + T.cast( T.ge(out, label15), 'int32')

		if weight is not None:
			err = T.sum( T.mul(weight, T.neq(o3C, z3C) ) )*1./T.sum(weight)
		else:
			err = T.mean( T.neq(o3C , z3C) )

		## err is s scalar, convert it to a tensor with ndim=1
		return T.stack([err] )

	## this function returns a vector of errors, the size of this vector is equal to the sum of ValueDims for all the responses
	def errors(self, zList, weightList=None):
		errs = []
		if weightList is not None and len(weightList)>0:
			for res, predictor, z, w, o in zip(self.responses, self.predictors, zList, weightList, self.outputList):
				labelType = Response2LabelType(res)
				numLabels = config.responseProbDims[labelType]

				## if the label type is Discrete25C, Discrete52C, Discrete12C
				if labelType.startswith('Discrete') and numLabels > 3:
					assert (z.ndim == 3 and config.responseValueDims[labelType] == 1 )
					o2 = o.flatten(3)
					## here we convert 12C, 25C, and 52C to 3C for error calculation, which makes the result easier to interpret
					errs.append( self.errors4one(z, o2, weight=w, distLabelType=labelType[len('Discrete'): ] ) )
				else:
					## call the error function of each predictor
					if (z.ndim == 3 ):
						zflat = z.flatten().dimshuffle(0, 'x')
					elif (z.ndim == 4 ):
						zflat = z.dimshuffle(3, 0, 1, 2).flatten(2).dimshuffle(1, 0)
					else:
						print('unsupported ndim for z in errors():', z.ndim)
						exit(-1)

					assert (w.ndim == 3)
					wflat = w.flatten().dimshuffle(0, 'x')
					e = predictor.errors(zflat, sampleWeight=wflat)
					## e is a tensor with ndim=1
					errs.append(e)

		else:
			for res, predictor, z, o in zip(self.responses, self.predictors, zList, self.outputList):
				labelType = Response2LabelType(res)
				numLabels = config.responseProbDims[labelType]
				if labelType.startswith('Discrete') and numLabels > 3 :
					assert (z.ndim == 3 and config.responseValueDims[labelType] == 1 )
					o2 = o.flatten(3)
					errs.append( self.errors4one(z, o, distLabelType=labelType[len('Discrete'): ] ) )
				else:
					## call the error function of each predictor
					if (z.ndim == 3):
						zflat = z.flatten().dimshuffle(0, 'x')
					elif (z.ndim == 4):
						zflat = z.dimshuffle(3, 0, 1, 2).flatten(2).dimshuffle(1, 0)
					else:
						print('unsupported ndim for z in errors():', z.ndim)
						exit(-1)
					e = predictor.errors(zflat)
					## e is a tensor with ndim=1
					errs.append(e)

		return T.concatenate(errs)

	## this function returns the long-, medium- and short-range prediction accuracy when proteinLen*topRatio predicted contacts are evaluated
	## note that the average accuracy is only for one batch. To calculate the avg accuracy of multiple batches, u may have to take into consideration
	## the fact that two batches may have different numbers of proteins
	## this function returns a 3*2 matrix where rows 0, 1, and 2 correspond to long-, medium- and short-range accuracy.
	## column 0 is the accuracy and column 1 is the average number of predicted contacts among all the top proteinLen*topRatio predicted contacts
	## the average number of predicted contacts may not be equal to avg(proteinLen)* topRatio when topRatio is big and proteinLen is small
	##def TopAccuracyByRange(self, zList, topRatio=0.5):
	def TopAccuracyByRange(self, zList):
		currentResponse = None
		topRatio = 0.5

	## in this function, we assume that pred is a tensor3 of floatX and truth is a matrix
	## pred has shape (dataLen, dataLen, 2) and truth has shape (dataLen, dataLen)
	## we also assume that label 0 is positive and label 1 is negative

	## the result is not 100% accurate for non-symmetric response, e.g., hydrogen-bonding matrix
	def TopAccuracy2C(pred=None, truth=None, symmetric=False):
		M1s = T.ones_like(truth, dtype=np.int8)
		LRsel = T.triu(M1s, 24)
		MLRsel = T.triu(M1s, 12)
		SMLRsel = T.triu(M1s, 6)
		MRsel = MLRsel - LRsel
		SRsel = SMLRsel - MLRsel

		dataLen = truth.shape[0]

		pred0 = pred[:,:,0]

		if symmetric:
			avg_pred = (pred0 + pred0.dimshuffle(1, 0) )/2.0
		else:
			avg_pred = pred0

		#pred_truth = T.concatenate( (avg_pred, truth.dimshuffle(0, 1, 'x') ), axis=2)
		pred_truth = T.stack( [avg_pred, T.cast(truth, 'int32')], axis=2 )

		accuracyList = []
		for Rsel in [ LRsel, MRsel, MLRsel, SRsel]:
			selected_pred_truth = pred_truth[Rsel.nonzero()]

			## sort by the predicted value for label 0 from the largest to the smallest
			selected_pred_truth_sorted = selected_pred_truth[ (selected_pred_truth[:,0]).argsort()[::-1] ]

			#print 'topRatio =', topRatio
			numTops = T.minimum( T.iround(dataLen * topRatio), selected_pred_truth_sorted.shape[0])

			selected_sorted_truth = T.cast(selected_pred_truth_sorted[:, -1], 'int32')
			numTruths = T.bincount(selected_sorted_truth, minlength=2 )
			numCorrects = T.bincount( selected_sorted_truth[0:numTops], minlength=2 )
			#numTops = T.minimum(numTops, numTruths[0])
			accuracyList.append( T.stack( [numCorrects[0] *1./(numTops + 0.001), numTops, numTruths[0] ], axis=0)  )

		return T.stacklists( accuracyList )

	def TopAccuracyNormal(pred=None, truth=None, symmetric=True):
		truth_new = T.ge(truth, config.ContactDefinition )

		if pred.ndim == 2:
			pred_new = -pred.dimshuffle(0, 1, 'x')
		else:
			pred_new = -pred

		return TopAccuracy2C(pred = pred_new, truth = truth_new, symmetric=symmetric)

	def TopAccuracyLogNormal(pred=None, truth=None, symmetric=True):
		truth_new = T.ge(truth, T.log(config.ContactDefinition) )

		if pred.ndim == 2:
			pred_new = -pred.dimshuffle(0, 1, 'x')
		else:
			pred_new = -pred

		return TopAccuracy2C(pred = pred_new, truth = truth_new, symmetric=symmetric)

	## in this function, we assume that pred is tensor3 of float and truth is a matrix of int8 or int
	## pred has shape (dataLen, dataLen, numLabels), having the predicted probability of each label
	## truth has shape (dataLen, dataLen)
	def TopAccuracyMultiC(pred=None, truth=None, subType=None, symmetric=True):
		## convert pred and truth to 2C
		distBins = config.distCutoffs[subType]
		label8 = DistanceUtils.LabelsOfOneDistance(config.ContactDefinition, distBins)
		label15 = DistanceUtils.LabelsOfOneDistance(config.InteractionLimit, distBins)
		truth1 = T.cast( T.ge(truth, label8), 'int32')
		truth_new = truth1
		pred1 = T.sum(pred[:,:,:label8], axis=2, keepdims=True)
		pred2 = T.sum(pred[:,:,label8:], axis=2, keepdims=True)
		pred_new = T.concatenate( (pred1, pred2), axis=2)
		return TopAccuracy2C(pred=pred_new, truth=truth_new, symmetric=symmetric)

	##def EvaluateAccuracy(pred_prob, truth, pad_len, response):
	def EvaluateAccuracy(pred_prob, truth, pad_len):
		pred_in_correct_shape = T.cast( pred_prob[pad_len:, pad_len: ], dtype=theano.config.floatX)
		truth_in_correct_shape = truth[pad_len:, pad_len: ]

		labelType = Response2LabelType(currentResponse)
		atomType = Response2LabelName(currentResponse)
		symmetric = ( atomType in ['CaCa', 'CbCb', 'CgCg', 'Beta'] )

		if labelType.startswith('LogNormal'):
			return TopAccuracyLogNormal(pred=pred_in_correct_shape, truth=truth_in_correct_shape, symmetric=symmetric)
		elif labelType.startswith('Normal'):
			return TopAccuracyNormal(pred=pred_in_correct_shape, truth=truth_in_correct_shape, symmetric=symmetric)
		elif labelType.startswith('Discrete'):
			subType = labelType[len('Discrete'): ]
			if subType.startswith('2C'):
				return TopAccuracy2C(pred=pred_in_correct_shape, truth=truth_in_correct_shape, symmetric=symmetric)
			else:
				return TopAccuracyMultiC(pred=pred_in_correct_shape, truth=truth_in_correct_shape, subType=subType, symmetric=symmetric)
		else:
			print('unsupported label type in EvaluateAccuracy: ', labelType)
			exit(-1)

		accuracyList = []
		for res, out_prob, z, ratio in zip(self.responses, self.output_probList, zList, self.modelSpecs['topRatios'] ):
			## currently TopAccuracy only works when the dimension of each z is 3
			assert z.ndim == 3
			if self.mask_1d is not None:
				paddingLens = self.mask_1d.shape[1] - T.sum(self.mask_1d, axis=1)
			else:
				paddingLens = T.zeros_like(z[:,0,0], dtype=np.int32)
			currentResponse = res
			topRatio = ratio
			##here we use scan to calculate accuracy for each protein
			result, updates = theano.scan(fn=EvaluateAccuracy, outputs_info=None, sequences=[out_prob, z, paddingLens] )
			accuracy = T.mean(result, axis=0)
			accuracyList.append(accuracy)

		return T.stacklists(accuracyList)

def BuildModel(modelSpecs, forTrain=True):
	rng = np.random.RandomState()

	## x is for sequential features and y for matrix (or pairwise) features
	x = T.tensor3('x')
	y = T.tensor4('y')

	## mask for x and y, respectively
	xmask = T.bmatrix('xmask')
	ymask = T.btensor3('ymask')

	xem = None
	##if any( k in modelSpecs['seq2matrixMode'] for k in ('SeqOnly', 'Seq+SS') ):
	if config.EmbeddingUsed(modelSpecs):
		xem = T.tensor3('xem')
		distancePredictor = ResNet4DistMatrix( rng, seqInput=x,
											   matrixInput=y, mask_seq=xmask, mask_matrix=ymask,
											   embedInput=xem, modelSpecs=modelSpecs )
	else:
		distancePredictor = ResNet4DistMatrix( rng, seqInput=x,
											   matrixInput=y, mask_seq=xmask, mask_matrix=ymask,
											   modelSpecs=modelSpecs )

	## labelList is a list of label tensors, each having shape (batchSize, seqLen, seqLen) or (batchSize, seqLen, seqLen, valueDims[response] )
	labelList = []
	if forTrain:
		## when this model is used for training. We need to define the label variable
		for response in modelSpecs['responses']:
			labelType = Response2LabelType(response)
			rValDims = config.responseValueDims[labelType]

			if labelType.startswith('Discrete'):
				if rValDims > 1:
				## if one response is a vector, then we use a 4-d tensor
				## wtensor is for 16bit integer
					labelList.append( T.wtensor4('Tlabel4' + response ) )
				else:
					labelList.append( T.wtensor3('Tlabel4' + response ) )
			else:
				if rValDims > 1:
					labelList.append( T.tensor4('Tlabel4' + response ) )
				else:
					labelList.append( T.tensor3('Tlabel4' + response ) )

	## weightList is a list of label weight tensors, each having shape (batchSize, seqLen, seqLen)
	weightList = []
	if len(labelList)>0 and modelSpecs['UseSampleWeight']:
		weightList = [ T.tensor3('Tweight4'+response) for response in modelSpecs['responses'] ]

	## for prediction, both labelList and weightList are empty
	return distancePredictor, x, y, xmask, ymask, xem, labelList, weightList

def TestResNet4DistMatrix():
	x = T.tensor3('x')
	y = T.tensor4('y')
	xmask = T.bmatrix('xmask')
	ymask = T.btensor3('ymask')
	selection = T.wtensor3('selection')

	import cPickle
	fh = open('seqDataset4HF.pkl')
	data = cPickle.load(fh)
	fh.close()

	distancePredictor = ResNet4DistMatrix( rng=np.random.RandomState(), seqInput=x, matrixInput=y, n_in_seq=data[0][0].shape[2], n_in_matrix=data[1][0].shape[3], n_hiddens_seq=[3, 5], n_hiddens_matrix=[2], hwsz_seq=4, hwsz_matrix=4, mask_seq=xmask, mask_matrix=ymask)

	"""
	f = theano.function([x, y, xmask, ymask], distancePredictor.output_1d)
	g = theano.function([x, y, xmask, ymask], distancePredictor.output_2d)
	"""

	dataLen = 300
	batchSize = 60
	a = np.random.uniform(0, 1, (batchSize, dataLen, 20)).astype(np.float32)
	b = np.random.uniform(0, 1, (batchSize, dataLen, dataLen, 3)).astype(np.float32)
	amask = np.zeros((batchSize, 0)).astype(np.int8)
	bmask = np.zeros((batchSize, 0, dataLen)).astype(np.int8)
	sel = np.ones((batchSize, dataLen, dataLen)).astype(np.int8)
	#print a
	#print b
	c = np.random.uniform(0, 3, (batchSize, dataLen, dataLen)).round().astype(np.int8)
	np.putmask(c, c>=2, 2)
	"""
	c[0, 1, 13]=1
	c[0, 2, 15]=1
	c[0, 4, 16]=1

	c[0, 1, 27]=1
	c[0, 2, 28]=1
	c[0, 4, 29]=1

	c[1, 0, 13]=2
	c[1, 1, 15]=2
	c[1, 3, 16]=2

	c[2, 0, 23]=2
	c[2, 1, 25]=2
	c[2, 3, 26]=2
	"""
	#sel = c

	#out1d = f(a, b, amask, bmask)
	#out2d = g(a, b, amask, bmask)

	#print out1d
	#print out2d

	z = T.btensor3('z')
	loss = distancePredictor.loss(z, selection)
	errs = distancePredictor.ErrorsByRange(z)
	accs = distancePredictor.TopAccuracyByRange(z)
	confM = distancePredictor.confusionMatrix(z)

	h = theano.function([x, y, xmask, ymask, selection, z], confM, on_unused_input='ignore')
	#l, e, accu = h(a, b, amask, bmask, sel, c)

	cms = []
	for i in np.arange(5):
		cm = h(data[0][i], data[1][i], data[2][i], data[3][i], data[4][i], data[5][i])
		print(cm)
		cms.append(cm)

	sumofcms = np.sum(cms, axis=0)*1.

	for i in range(sumofcms.shape[0]):
		sumofcms[i] /= np.sum(sumofcms[i])

	confusions = sumofcms
	print(confusions)
	print(np.sum(confusions[0]))
	print(np.sum(confusions[1]))
	print(np.sum(confusions[2]))

	"""
	params = contactPredictor.params
	cost = loss + 0.1 * contactPredictor.paramL2
	grads = [ T.grad(cost, p) for p in params ]
	fg = theano.function([x, y, xmask, ymask, selection, z], grads)
	g0 = fg(a, b, amask, bmask, sel, c)

	param_shapes = [ p.get_value().shape for p in params ]
	print(g0)
	"""

if __name__ == "__main__":
	#TestOuterConcatenate()

	#TestConv1D2Matrix()

	#TestConv2D4DistMatrix()

	TestResNet4DistMatrix()

	#TestMidpointFeature()


from dbt.main import parse_args
from dbt.config.runtime import RuntimeConfig
from dbt.parser.manifest import ManifestLoader
from dbt.adapters.factory import register_adapter
from dbt.tracking import track_invocation_start, track_invocation_end, flush
from contextlib import contextmanager
from dbt.tracking import disable_tracking

class DBT:
    def __init__(self, dbt_args):
        disable_tracking()
        self.parsed = parse_args(args=dbt_args)
        self.config = RuntimeConfig.from_args(self.parsed)
        register_adapter(self.config)

    def get_manifest(self):
        with tracker(config=self.config, args=self.parsed):
            return ManifestLoader.get_full_manifest(config=self.config)

@contextmanager
def tracker(config, args):
    track_invocation_start(config, args)
    try:
        yield
        track_invocation_end(config, args, result_type="ok")
    finally:
        flush()


# -*- coding: utf-8 -*-
"""

"""
import json
import time
from collections import OrderedDict

TEDSLISTFILE = 'tedslist.txt'
SCRIPTLISTFILE = 'scriptlist.txt'
SENSORDATAFILE = 'sensordatafile.txt'
CONTROLLERDATAFILE = 'controllerdatafile.txt'
PLOTLISTFILE = 'plotlistfile.txt'
NEW_TEDS = "new_teds"

HEADERS = {"Content-type": "application/json",
           "Accept": "text/plain",
           "User-Agent": "ToRMConsole"}

CONSOLE_PAGE = "/tormconsole"
CONSOLE_TEDS_PAGE = "/tormconsole/teds"
NEW_TEDS_PAGE = "/new_teds"
REMOVE_TEDS_PAGE = "/tormconsole/remove"
CONSOLE_PLOT_DATA = "/tormconsole/plotdata"
CONSOLE_CONTROLLERS_DATA = "/tormconsole/controllers"

TEDS_FOLDER = "teds"
FILE_FOLDER = "files"
SCRIPT_FOLDER = "scripts"

RX_NAME_ID= 'ID'
## TEDS field related constant strings
# Strings for the TEDS field names
TEDS_NAME_NAME = 'Name'
TEDS_NAME_ID = 'ID'
TEDS_NAME_ACTIVE = 'Active'
TEDS_NAME_ADDRESS = 'SPI Address'
TEDS_NAME_CLKSPD = 'Clock Speed'
TEDS_NAME_SPI_SELECT = 'SPI Select'
TEDS_NAME_CSHIGH = 'Chip Select Idle High'
TEDS_NAME_CLOCKPOLPHASE = 'Clock Polarization/Phase Mode'
TEDS_NAME_SETUP = 'Data for Setup'

TEDS_NAME_REGISTER = 'Register'
TEDS_REGISTER_FRIENDNAME = 'Register Name'
TEDS_NAME_ACTIVEREGISTER = 'Active Register'
TEDS_NAME_ACCESSDATA = 'SPI Access Bytes'
TEDS_NAME_NEXT_INSTRUCTION = 'Next Instruction'
TEDS_NAME_SIUNIT = 'SI Unit'
TEDS_NAME_MINVALUE = 'Minimum Value'
TEDS_NAME_MAXVALUE = 'Maximum Value'
TEDS_NAME_CONVCONST = 'Value Constant Convertion'
TEDS_NAME_FIRSTDATABYTE = 'First Data Byte'
TEDS_NAME_LASTDATABYTE = 'Last Data Byte'
TEDS_NAME_FIRSTDATABIT = 'First Data Bit'
TEDS_NAME_LASTDATABIT = 'Last Data Bit'
TEDS_NAME_ISACTUATOR = 'Is Actuator'
TEDS_NAME_ACTUATORVALUE = 'Actuator Value'
TEDS_NAME_ISSENSOR = 'Is Sensor'
TEDS_NAME_ACTIVATEINTERRUPT = 'Activate Interruption'
TEDS_NAME_WARNINGTHRESHOLD = 'Warning Threshold'
TEDS_NAME_DANGERTHRESHOLD = 'Danger Threshold'

# Strings for the TEDS field tips
TEDS_TIP_NAME = 'Name for user identification.'
TEDS_TIP_ID = 'TEDS unique identification number.'
TEDS_TIP_ACTIVE = 'Set if the transducer will be used in the system.'
TEDS_TIP_ADDRESS = 'SPI physical address connected to the system. \
    It has a maximum value of 1024.'
TEDS_TIP_CLKSPD = 'Transducer SPI clock speed.'
TEDS_TIP_SPI_SELECT = 'Raspberry pin SPI selection GPIO(7/8)'
TEDS_TIP_CSHIGH = 'Set if chip select is idle high and active low.'
TEDS_TIP_CLOCKPOLPHASE = 'Clock polarization/phase mode: [pol|pha] 0b00=0... \
    0b11=3  '
TEDS_TIP_SETUP = 'Setup/Configuration bytes needed for the transducer initialization.'

TEDS_TIP_ACTIVEREGISTER = 'Set if the register will be used.'
TEDS_TIP_ACCESSDATA = 'Bytes used to access the register.'
TEDS_TIP_NEXT_INSTRUCTION = 'Interested Data came on the next instruction'
TEDS_TIP_SPI_SELECT = 'Raspberry pin SPI selection GPIO(7/8)'
TEDS_TIP_SIUNIT = 'The SI physical unit of the register.'
TEDS_TIP_MINVALUE = 'Minimum value it is able to measure.'
TEDS_TIP_MAXVALUE = 'Maximum value it is able to measure.'
TEDS_TIP_CONVCONST = 'Constant used to convert to digital input/output data'
TEDS_TIP_FIRSTDATABYTE = 'Start Byte of the data. Starts with B0 (first \
    sent/received byte), B1...'
TEDS_TIP_LASTDATABYTE = 'Last sent/recieved Data Byte'
TEDS_TIP_FIRSTDATABIT = 'The number of the position of the first bit (Most \
    Significant Bit) of the data in the First Data Byte. \
    Byte = (MSB)|b7|b6|b5|b4|b3|b2|b1|b0|(LSB)'
TEDS_TIP_LASTDATABIT = 'The number of the position of the last bit (Least \
    Significant Bit) of the data in the Last Data Byte. \
    Byte = (MSB)|b7|b6|b5|b4|b3|b2|b1|b0|(LSB)'
TEDS_TIP_ISACTUATOR = 'Checkbox confirming the transducer\'s actuation \
    function.'
TEDS_TIP_ACTUATORVALUE = 'Actuator target value'
TEDS_TIP_ISSENSOR = 'Checkbox confirming the transducer\'s sensing function.'
TEDS_TIP_ACTIVATEINTERRUPT = 'Activate Interruption'
TEDS_TIP_WARNINGTHRESHOLD = 'Warning  Threshold Value'
TEDS_TIP_DANGERTHRESHOLD = 'Danger  Threshold Value'

# TEDS dictionaries
TEDS_BUILD_DICT = [{'name': TEDS_NAME_NAME, 'type': 'str', 'value': '', 'tip': TEDS_TIP_NAME},
        {'name': TEDS_NAME_ID, 'type': 'str', 'value': '', 'tip': TEDS_TIP_ID, 'readonly': True},
        {'name': TEDS_NAME_ACTIVE, 'type': 'bool', 'value': True, 'tip': TEDS_TIP_ACTIVE},
        {'name': TEDS_NAME_ADDRESS, 'type': 'int', 'value': 1, 'limits': (1, 2 ** 10), 'tip': TEDS_TIP_ADDRESS},
        {'name': TEDS_NAME_CLKSPD, 'type': 'int', 'value': 0, 'siPrefix': True, 'suffix': 'kHz', 'tip': TEDS_TIP_CLKSPD},
        {'name': TEDS_NAME_SPI_SELECT, 'type': 'int', 'value': 1, 'tip':TEDS_TIP_SPI_SELECT},
        {'name': TEDS_NAME_CSHIGH, 'type': 'bool', 'value': True, 'tip': TEDS_TIP_CSHIGH},
        {'name': TEDS_NAME_CLOCKPOLPHASE, 'type': 'int', 'value': 0, 'limits': (0, 2 ** 2 - 1), 'tip': TEDS_TIP_CLOCKPOLPHASE},
        {'name': TEDS_NAME_SETUP, 'type': 'str', 'value': '', 'tip': TEDS_TIP_SETUP}
        ]

REGISTER_BUILD_DICT = {'name': TEDS_NAME_REGISTER, 'type': 'group', 'renamable': True, 'removable': True, 'children':[
                {'name':TEDS_REGISTER_FRIENDNAME, 'type': 'str', 'value': ''},
                {'name': TEDS_NAME_ACTIVEREGISTER, 'type': 'bool', 'value': False, 'tip': TEDS_TIP_ACTIVEREGISTER},
                {'name': TEDS_NAME_ACCESSDATA, 'type': 'str', 'value': '', 'tip': TEDS_TIP_ACCESSDATA},
                {'name': TEDS_NAME_NEXT_INSTRUCTION, 'type':'bool','value':False, 'tip': TEDS_TIP_NEXT_INSTRUCTION},
                {'name': TEDS_NAME_SIUNIT, 'type': 'str', 'value': '', 'tip': TEDS_TIP_SIUNIT},
                {'name': TEDS_NAME_MINVALUE, 'type': 'float', 'value': 0, 'tip': TEDS_TIP_MINVALUE},
                {'name': TEDS_NAME_MAXVALUE, 'type': 'float', 'value': 0, 'tip': TEDS_TIP_MAXVALUE},
                {'name': TEDS_NAME_CONVCONST, 'type': 'float', 'value': 0, 'tip': TEDS_TIP_CONVCONST},
                {'name': TEDS_NAME_FIRSTDATABYTE, 'type': 'int', 'value': 0, 'tip': TEDS_TIP_FIRSTDATABYTE},
                {'name': TEDS_NAME_LASTDATABYTE, 'type': 'int', 'value': 0, 'tip': TEDS_TIP_LASTDATABYTE},
                {'name': TEDS_NAME_FIRSTDATABIT, 'type': 'int', 'value': 0, 'limits': (0, 2 ** 3 - 1), 'tip': TEDS_TIP_FIRSTDATABIT},
                {'name': TEDS_NAME_LASTDATABIT, 'type': 'int', 'value': 0, 'limits': (0, 2 ** 3 - 1), 'tip': TEDS_TIP_LASTDATABIT},
                {'name': TEDS_NAME_ISACTUATOR, 'type': 'bool', 'value': False, 'tip': TEDS_TIP_ISACTUATOR},
                {'name': TEDS_NAME_ACTUATORVALUE, 'type': 'float', 'value': 0, 'tip': TEDS_TIP_ACTUATORVALUE},
                {'name': TEDS_NAME_ISSENSOR, 'type': 'bool', 'value': False, 'tip': TEDS_TIP_ISSENSOR},
                {'name': TEDS_NAME_ACTIVATEINTERRUPT, 'type': 'bool', 'value': False, 'tip': TEDS_TIP_ACTIVATEINTERRUPT},
                {'name': TEDS_NAME_WARNINGTHRESHOLD, 'type': 'float', 'value': 0, 'tip': TEDS_TIP_WARNINGTHRESHOLD},
                {'name': TEDS_NAME_DANGERTHRESHOLD, 'type': 'float', 'value': 0, 'tip': TEDS_TIP_DANGERTHRESHOLD}
                ]}

RX_SETUP_BUILD_DICT = [{'name': 'Motherboard', 'type': 'float', 'value': 0},
                {'name': 'Slot', 'type': 'float', 'value': 0},
                {'name': RX_NAME_ID, 'type': 'str', 'value': '', 'readonly': True},
                {'name': 'Rx_Gate_Pair', 'type': 'int', 'value': 0 },
                {'name': 'Gain(Low/High)', 'type': 'int'}
                ]

CONTROLLERS_PARAMETERS = (TEDS_NAME_MINVALUE, TEDS_NAME_MAXVALUE,
                          TEDS_NAME_SIUNIT, TEDS_NAME_ACTUATORVALUE)

def get_list_from_text(inputstring):
    """Convert string type to list type at the TEDS field's input"""
    outputlist = []

    auxlist = inputstring.split(';')
    if inputstring != '':

        for item in auxlist:
            auxlist2 = item.split(',')
            intlist = []

            for item in auxlist2:
                intlist.append(int(item))

            outputlist.append(intlist)

    return outputlist

def get_text_from_list(inputlist):
    """Convert list type to string type at the TEDS field's input"""
    outputstring = ''
    auxlist3 = []

    for sublist in inputlist:
        auxlist = []

        for item in sublist:
            auxlist.append(str(item))

        auxlist2 = ','.join(auxlist)
        auxlist3.append(auxlist2)

    outputstring = ';'.join(auxlist3)
    return outputstring

def id_generator():
    """Create an unique identifier"""
    return time.strftime('%Y%b%d%H%M%S',time.gmtime())

def save_teds(datateds, fileteds):
    """Save TEDS in a file .ted from a dictionary"""
    with open(fileteds + '.ted', 'w') as auxfile:
        json.dump(datateds, auxfile, indent=1)

def load_teds(fileteds):
    """Load TEDS from a file .ted to a dictionary"""
    with open(fileteds + '.ted', 'r') as auxfile:
        datateds = json.load(auxfile)
    return datateds

def load_teds_ordereddict(fileteds):
    """Load TEDS from a file .ted to a ordered dictionary"""
    with open(fileteds + '.ted', 'r') as auxfile:
        datateds = json.load(auxfile, object_pairs_hook=OrderedDict)
    return datateds

def build_dict_from_ordereddict(orddict):
    """Create a new dictionary with the meaningful values"""
    newdict = OrderedDict()
    for key in orddict:
        if orddict[key][0] != None:
            newdict[key] = orddict[key][0]
        else:
            newdict[key] = build_dict_from_ordereddict(orddict[key][1])
    return newdict

def search_entry_index(entry, builddict):
    """ """
    if type(builddict) == type([]):
        for item in builddict:
            if item['name'] == entry:
                return builddict.index(item)

    elif type(builddict) == type({}):
        for item in builddict['children']:
            if item['name'] == entry:
                return builddict['children'].index(item)

    else:
        return None

def set_teds_fields_from_dict(inputdict, teds, basetedsdic, baseregisterdict):
    """Uses the dictionary to set the TEDS fields"""
    for key in inputdict:
        if type(inputdict[key]) == type(OrderedDict()):
            teds.addChild(baseregisterdict, autoIncrementName=True)
            teds.child(key).setName(key)

            for subkey in inputdict[key]:
                if ((subkey == TEDS_NAME_ACCESSDATA)):
                    teds.child(key).child(subkey).setValue(
                        get_text_from_list(inputdict[key][subkey]))
                else:
                    teds.child(key).child(subkey).setValue(inputdict[key][subkey])
        else:
            if key == TEDS_NAME_SETUP:
                teds.child(key).setValue(get_text_from_list(inputdict[key]))
            else:
                teds.child(key).setValue(inputdict[key])
                # if teds does not have the inputdict key?

def set_teds_list_from_dict(inputdict, tedslist):
    """ """
    for key in inputdict:
        tedslist.addChild({'name': key, 'type': 'group', 'children': [
            {'name': inputdict[key], 'type': 'bool', 'value': True}
            ]})

def set_teds_fields_from_file(fileteds, teds, basetedsdic, baseregisterdict):
    """Load TEDS from file and set the fields"""
    auxdict = load_teds_ordereddict(fileteds)
    set_teds_fields_from_dict(auxdict, teds, basetedsdic, baseregisterdict)


from Interprete.OperacionesConExpresiones.Opera_Relacionales import Opera_Relacionales
from Interprete.Condicionantes.Condicion import Condicion
from Interprete.SELECT.select import select
from Interprete.Arbol import Arbol

from Parser.Reportes.Nodo import Nodo

reservadas = {

	# Boolean Type
	'boolean': 'BOOLEAN',
	'true': 'TRUE',
	'false': 'FALSE',
	'order': 'ORDER',

	'into': 'INTO',

	# operator Precedence
	'isnull': 'ISNULL',
	'notnull': 'NOTNULL',

	# Definition
	'replace': 'REPLACE',
	'owner': 'OWNER',
	'show': 'SHOW',
	'databases': 'DATABASES',
	'MAP' : 'MAP',
	'LIST' : 'LIST',

	# Inheritance
	'inherits': 'INHERITS',

	# ESTRUCTURAS DE CONSULTA Y DEFINICIONES
	'select'  : 'SELECT',
	'insert'  : 'INSERT',
	'update'  : 'UPDATE',
	'drop'  : 'DROP',
	'delete'  : 'DELETE',
	'alter'  : 'ALTER',
	'constraint'  : 'CONSTRAINT',
	'from' : 'FROM',
	'group' : 'GROUP',
	'by'  : 'BY',
	'where'  : 'WHERE',
	'having'   : 'HAVING',
	'create' : 'CREATE',
	'type' : 'TYPE',
	'primary' : 'PRIMARY',
	'foreign' : 'FOREIGN',
	'add' : 'ADD',
	'rename' : 'RENAME',
	'set' : 'SET',
	'key' : 'KEY',
	'if' : 'IF',
	'unique' : 'UNIQUE',
	'references' : 'REFERENCES',
	'check' : 'CHECK',
	'column' : 'COLUMN',
	'database' : 'DATABASE',
	'table' : 'TABLE',
	'text' : 'TEXT',
	'float' : 'FLOAT',
	'values' : 'VALUES',
	'int' : 'INT',
	'default' : 'DEFAULT',
	'null' : 'NULL',

	# TIPOS NUMERICOS
	'smallint' : 'SMALLINT',
	'integer' : 'INTEGER',
	'bigint' : 'BIGINT',
	'decimal' : 'DECIMAL',
	'numeric' : 'NUMERIC',
	'real' : 'REAL',
	'double' : 'DOUBLE',
	'precision' : 'PRECISION',
	'money' : 'MONEY',
	'varying' : 'VARYING',
	'varchar' : 'VARCHAR',
	'character' : 'CHARACTER',
	'char' : 'CHAR',
	'clear' : 'CLEAR',

	# TIPOS EN FECHAS
	'timestamp' : 'TIMESTAMP',
	'date' : 'DATE',
	'time' : 'TIME',
	'interval' : 'INTERVAL',
	'year' : 'YEAR',
	'day' : 'DAY',
	'hour' : 'HOUR',
	'minute' : 'MINUTE',
	'second' : 'SECOND',
	'to' : 'TO',

	# ENUM
	'enum'  : 'ENUM',

	# OPERADORES LOGICOS
	'and' : 'AND',
	'or'  : 'OR',
	'not'   : 'NOT',

	# PREDICADOS DE COMPARACION
	'between'   : 'BETWEEN',
	'unknown' : 'UNKNOWN',
	'is'    : 'IS',
	'distinct'  : 'DISTINCT',

	# FUNCIONES MATEMATICAS
	'abs' : 'ABS',
	'cbrt' : 'CBRT',
	'ceil' : 'CEIL',
	'ceiling' : 'CEILING',
	'degrees' : 'DEGREES',
	'div' : 'DIV',
	'exp' : 'EXP',
	'factorial' : 'FACTORIAL',
	'floor' : 'FLOOR',
	'gcd' : 'GCD',
	'lcm' : 'LCM',
	'ln' : 'LN',
	'log' : 'LOG',
	'log10' : 'LOG10',
	'min_scale' : 'MIN_SCALE',
	'mod' : 'MOD',
	'pi' : 'PI',
	'power' : 'POWER',
	'radians' : 'RADIANS',
	'round' : 'ROUND',
	'scale' : 'SCALE',
	'sign' : 'SIGN',
	'sqrt' : 'SQRT',
	'trim_scale' : 'TRIM_SCALE',
	'truc' : 'TRUC',
	'width_bucker' : 'WIDTH_BUCKET',
	'random' : 'RANDOM',
	'setseed' : 'SETSEED',
	'contains' : 'CONTAINS',
	'remove': 'REMOVE',

	# FUNCIONES DE AGREGACION
	'count' : 'COUNT',
	'sum' : 'SUM',
	'avg' : 'AVG',
	'max' : 'MAX',
	'min' : 'MIN',

	# FUNCIONES TRIGONOMETRICAS
	'acos' : 'ACOS',
	'acosd' : 'ACOSD',
	'asin' : 'ASIN',
	'asind' : 'ASIND',
	'atan' : 'ATAN',
	'atand' : 'ATAND',
	'atan2' : 'ATAN2',
	'atan2d' : 'ATAN2D',
	'cos' : 'COS',
	'cosd' : 'COSD',
	'cot' : 'COT',
	'cotd' : 'COTD',
	'sin' : 'SIN',
	'sind' : 'SIND',
	'tan' : 'TAN',
	'tand' : 'TAND',
	'sinh' : 'SINH',
	'cosh' : 'COSH',
	'tanh' : 'TANH',
	'asinh' : 'ASINH',
	'acosh   ' : 'ACOSH',
	'atanh' : 'ATANH',
	'SIZE' : 'SIZE',

	# FUNCIONES DE CADENA BINARIAS
	'length' : 'LENGTH',
	'substring' : 'SUBSTRING',
	'trim' : 'TRIM',
	'get_byte' : 'GET_BYTE',
	'md5' : 'MD5',
	'set_byte' : 'SET_BYTE',
	'sha256' : 'SHA256',
	'substr' : 'SUBSTR',
	'convert' : 'CONVERT',
	'encode' : 'ENCODE',
	'decode' : 'DECODE',

	# COINCidenCIA POR PATRON
	'like' : 'LIKE',
	'ilike' : 'ILIKE',
	'similar' : 'SIMILAR',
	'as' : 'AS',
	'couter' : 'COUTER',

	# SUBQUERYS
	'in' : 'IN',
	'exists' : 'EXISTS',
	'any' : 'ANY',
	'all' : 'ALL',
	'some' : 'SOME',

	# JOINS
	'join' : 'JOIN',
	'inner' : 'INNER',
	'left' : 'LEFT',
	'right' : 'RIGHT',
	'full' : 'FULL',
	'outer' : 'OUTER',
	'on' : 'ON',

	# ORDENAMIENTO DE FILAS
	'asc' : 'ASC',
	'desc' : 'DESC',
	'nulls' : 'NULLS',
	'first' : 'FIRST',
	'last' : 'LAST',

	# EXPRESIONES
	'case' : 'CASE',
	'when' : 'WHEN',
	'then' : 'THEN',
	'end' : 'END',
	'greatest' : 'GREATEST',
	'least' : 'LEAST',

	# LIMITE Y OFFSET
	'limit' : 'LIMIT',
	'offset' : 'OFFSET',

	# CONSULTAS DE COMBINACION
	'union' : 'UNION',
	'intersect' : 'INTERSECT',
	'except' : 'EXCEPT'

}

tokens  = [
	'PT',
	'DOBPTS',
	'MAS',
	'MENOS',
	'MULTI',
	'DIVISION',
	'MODULO',
	'IGUAL',
	'PARIZQ',
	'PARDER',
	'PTCOMA',
	'COMA',
	'TKNOT',
	'NOTBB',
	'ANDBB',
	'ORBB',
	'NUMERAL',
	'TKEXP',
	'SHIFTIZQ',
	'SHIFTDER',
	'IGUALQUE',
	'DISTINTO',
	'MAYORIG',
	'MENORIG',
	'MAYORQUE',
	'MENORQUE',
	'CORIZQ',
	'CORDER',
	'DOSPTS',
	'TKDECIMAL',
	'ENTERO',
	'CADENA',
	'CADENADOBLE',
	'ID'
] + list(reservadas.values())

# Tokens
t_PT        = r'\.'
t_DOBPTS    = r'::'
t_CORIZQ      = r'\['
t_CORDER      = r']'

t_MAS       = r'\+'
t_MENOS     = r'-'
t_TKEXP     = r'\^'
t_MULTI     = r'\*'
t_DIVISION  = r'/'
t_MODULO   = r'%'
t_IGUAL     = r'='
t_PARIZQ    = r'\('
t_PARDER    = r'\)'
t_PTCOMA    = r';'
t_COMA      = r','
t_TKNOT       = r'!'
t_NOTBB     = r'~'
t_ANDBB     = r'&'
t_ORBB      = r'\|'

t_NUMERAL   = r'\#'

t_SHIFTIZQ  = r'<<'
t_SHIFTDER  = r'>>'
t_IGUALQUE  = r'=='
t_DISTINTO  = r'!='
t_MAYORIG   = r'>='
t_MENORIG   = r'<='
t_MAYORQUE  = r'>'
t_MENORQUE  = r'<'
t_DOSPTS    = r':'

def t_ENTERO(t):
	r'\d+'
	try:
		t.value = int(t.value)
	except ValueError:
		print("Integer value too large %d", t.value)
		t.value = 0
	return t

def t_TKDECIMAL(t):
	r'\d+\.\d+'
	try:
		t.value = float(t.value)
	except ValueError:
		print("Float value too large %d", t.value)
		t.value = 0
	return t

def t_CADENA(t):
	r'\'.*?\''
	t.value = t.value[1:-1]
	return t

def t_CADENADOBLE(t):
	r'\".*?\"'
	t.value = t.value[1:-1]
	return t

def t_ID(t):
	 r'[a-zA-Z_][a-zA-Z_0-9]*'
	 t.type = reservadas.get(t.value.lower(),'ID')
	 return t

t_ignore = " \t"

def t_newline(t):
	r'\n+'
	t.lexer.lineno += len(t.value) # t.value.count("\n")

def t_COMENTARIO_SIMPLE(t):
	r'\--.*\n'
	t.lexer.lineno += 1

def t_COMENTARIO_MULTILINEA(t):
	r'/\*(.|\n)*?\*/'
	t.lexer.lineno += t.value.count('\n')

def t_error(t):
	print("Caracter NO Valido: '%s'" % t.value[0])
	t.lexer.skip(1)
	print(t.value[0])

# -----------------------------------------------------------------------------------
# ---------------------- SINTACTICO -------------------------------------------------
# -----------------------------------------------------------------------------------

#-----------------------
import ply.lex as lex
lexer2 = lex.lex()
#-----------------------
precedence = (
	#('left','CONCAT'),
	#('left','MENOR','MAYOR','IGUAL','MENORIGUAL','MAYORIGUAL','DIFERENTE'),
	('left','MENORQUE','MAYORQUE','IGUAL','MENORIG','MAYORIG','DISTINTO'),
	('left','MAS','MENOS'),
	('left','MULTI','DIVISION','MODULO'),
	('left','TKEXP'),
	#('right','UMENOS'),
)

def p_init(t):
	'init : sentences'
	t[0] = Nodo("init", "")
	t[0].add(t[1])

def p_sentences(t):
	'''
		sentences   : sentences setInstruccions
					| setInstruccions
	'''
	if len(t) == 3:  # listawhere atributoselect
		t[1].add(t[2])
		t[0] = t[1]
	else:  # atributoselect
		t[0] = Nodo("sentences", "")
		t[0].add(t[1])

def p_setInstruccions(t):
	'''
		setInstruccions   : sentence PTCOMA
	'''
	t[0] = Nodo("setInstruccions", "")
	t[0].add(t[1])

def p_sentence(t):
	'''
		sentence     : ddl
	'''
	t[0] = Nodo("sentence", "")
	t[0].add(t[1])

def p_ddl(t):
	'''
		ddl  : select
			 | table_create
			 | insert
			 | update
			 | deletetable
	'''
	t[0] = Nodo("ddl", "")
	t[0].add(t[1])

def p_select(t):
	'''
		select  : SELECT listavalores FROM exp listawhere
	'''
	t[0] = Nodo("select", "")
	t[1].add(Nodo(t[1], ''))
	t[0].add(t[2])
	t[1].add(Nodo(t[3], ''))
	t[0].add(t[4])
	t[0].add(t[5])

def p_listawhere(t):
	'''
		listawhere  : listawhere atributoselect
					| atributoselect
	'''
	if len(t) == 3:  # listawhere atributoselect
		t[1].add(t[2])
		t[0] = t[1]
	else:  # atributoselect
		t[0] = Nodo("listawhere", "")
		t[0].add(t[1])

def p_atributoselect(t):
	'''
		atributoselect  : WHERE exp
						| ORDER BY exp ordenamiento
						| GROUP BY exp
						| LIMIT exp
	'''
	t[0] = Nodo("atributoselect", "")
	if t[1].lower() == "where":
		t[1].add(Nodo(t[1], ''))
		t[1].add(t[2])
	elif t[1].lower() == "order":
		t[1].add(Nodo(t[1]+' '+t[2], ''))
		t[1].add(t[3])
		t[1].add(t[4])
	elif t[1].lower() == "group":
		t[1].add(Nodo(t[1] + ' ' + t[2], ''))
		t[1].add(t[3])
	elif t[1].lower() == "limit":
		t[1].add(Nodo(t[1], ''))
		t[1].add(t[2])

def p_ordenamiento(t):
	'''
		ordenamiento   : ASC
					   | DESC
	'''
	t[0] = Nodo("ordenamiento", "")
	t[1].add(Nodo(t[1], ''))

def p_listavalores(t):
	'''
		listavalores   : listavalores COMA exp
					   | exp
	'''
	if len(t) == 4: #listavalores COMA exp
		t[1].add(t[3])
		t[0] = t[1]
	else: # exp
		t[0] = Nodo("listavalores", "")
		t[0].add(t[1])

def p_exp(t):
	'''
		exp   : MAS exp
			  | MENOS exp
			  | exp TKEXP 	exp
			  | exp MULTI 	exp
			  | exp DIVISION exp
			  | exp MODULO exp
			  | exp MAS exp
			  | exp MENOS exp
			  | exp BETWEEN exp
			  | exp IN exp
			  | exp LIKE exp
			  | exp ILIKE exp
			  | exp SIMILAR exp
			  | exp IGUAL exp
			  | exp MAYORQUE exp
			  | exp MENORQUE exp
			  | exp MAYORIG exp
			  | exp MENORIG exp
			  | exp IS exp
			  | exp ISNULL exp
			  | exp NOTNULL exp
			  | NOT exp
			  | exp AND exp
			  | exp OR exp
	'''
	t[0] = Nodo("exp", "")
	if len(t) == 3:#LITERAL exp
		t[0].add(Nodo(t[1], ''))
		t[0].add(t[2])
	elif len(t) == 4:#exp LITERAL exp
		t[0].add(t[1])
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])

def p_expSimples(t):
	'''
		exp         :   ID
					| ENTERO
					| TKDECIMAL
					| TRUE
					| FALSE
					| CADENADOBLE
					| CADENA
	'''
	t[0] = Nodo("exp", "")
	t[0].add(Nodo(t[1], ''))

# ---------------CREATE TABLE---------------
def p_table_create(t):
	'''
		table_create : CREATE TABLE ID PARIZQ lista_table COMA listadolprimary PARDER
					 | CREATE TABLE ID PARIZQ lista_table PARDER
					 | CREATE TABLE IF NOT EXISTS ID PARIZQ lista_table COMA listadolprimary PARDER
					 | CREATE TABLE IF NOT EXISTS ID PARDER lista_table PARDER
	'''
	t[0] = Nodo("table_create", "")
	if len(t) == 9:
		#CREATE TABLE ID PARIZQ lista_table COMA listadolprimary PARDER
		t[1].add(Nodo(t[1]+' '+t[2]+' '+t[3],''))
		t[1].add(t[5])
		t[1].add(t[7])
	elif len(t) == 7:
		#CREATE TABLE ID PARIZQ lista_table PARDER
		t[1].add(Nodo(t[1]+' '+t[2]+' '+t[3],''))
		t[1].add(t[5])
	elif len(t) == 12:
		#CREATE TABLE IF NOT EXISTS ID PARIZQ lista_table COMA listadolprimary PARDER
		t[1].add(Nodo(t[1]+' '+t[2]+' '+t[3]+' '+t[4]+' '+t[5]+' '+t[6],''))
		t[1].add(t[8])
		t[1].add(t[10])
	elif len(t) == 10:
		#CREATE TABLE IF NOT EXISTS ID PARDER lista_table PARDER
		t[1].add(Nodo(t[1]+' '+t[2]+' '+t[3]+' '+t[4]+' '+t[5]+' '+t[6],''))
		t[1].add(t[8])

def p_lista_table(t):
	'''
		lista_table  : lista_table COMA atributo_table
					 | atributo_table
	'''
	if len(t) == 3: #lista_table COMA atributo_table
		t[1].add(t[3])
		t[0] = t[1]
	else: # atributo_table
		t[0] = Nodo("lista_table", "")
		t[0].add(t[1])

def p_listadolprimary(t):
	'''
		listadolprimary  : listadolprimary COMA lista_primary
						 | lista_primary
	'''
	if len(t) == 3: #listadolprimary COMA lista_primary
		t[1].add(t[3])
		t[0] = t[1]
	else: # lista_primary
		t[0] = Nodo("listadolprimary", "")
		t[0].add(t[1])

def p_lista_primary(t):
	'''
		lista_primary : PRIMARY KEY PARIZQ listaids PARDER
					  | FOREIGN KEY PARIZQ listaids PARDER REFERENCES ID PARIZQ listaids PARDER
					  | CONSTRAINT ID CHECK PARIZQ exp PARDER
					  | UNIQUE PARIZQ listaids PARDER
	'''
	t[0] = Nodo("lista_primary", "")
	if t[1].tolower() == 'primary':#PRIMARY KEY PARIZQ listaids PARDER
		t[0].add(Nodo(t[1]+''+t[2], ''))
		t[1].add(t[4])
	elif t[1].tolower() == 'foreign':#FOREIGN KEY PARIZQ listaids PARDER REFERENCES ID PARIZQ listaids PARDER
		t[0].add(Nodo(t[1]+''+t[2], ''))
		t[1].add(t[4])
		t[0].add(Nodo(t[6]+''+t[7], ''))
		t[1].add(t[9])
	elif t[1].tolower() == 'contraint': #CONSTRAINT ID CHECK PARIZQ exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))
		t[1].add(t[5])
	elif t[1].tolower() == 'unique':#UNIQUE PARIZQ listaids PARDER
		t[0].add(Nodo(t[1], ''))
		t[1].add(t[3])

def p_atributo_table(t):
	'''
		atributo_table : ID  tipocql listaespecificaciones
					   | ID  tipocql
	'''
	t[0] = Nodo("atributo_table", "")
	if len(t)==3:#ID  tipocql
		t[0].add(Nodo(t[1], ''))
		t[1].add(t[2])
	else:#ID  tipocql listaespecificaciones
		t[0].add(Nodo(t[1], ''))
		t[1].add(t[2])
		t[1].add(t[3])

def p_listaespecificaciones(t):
	'''
		listaespecificaciones  : listaespecificaciones especificaciones
							   | especificaciones
	'''
	if len(t) == 3: #listaespecificaciones especificaciones
		t[1].add(t[2])
		t[0] = t[1]
	else: # especificaciones
		t[0] = Nodo("listaespecificaciones", "")
		t[0].add(t[1])

def p_especificaciones(t):
	'''
		especificaciones : UNIQUE
						 | NULL
						 | PRIMARY KEY
						 | NOT NULL
						 | REFERENCES ID
						 | CONSTRAINT ID
						 | DEFAULT exp
						 | CHECK PARIZQ exp PARDER
	'''
	t[0] = Nodo("especificaciones", "")

	if len(t)==3:
		if t[1].tolower()=='default':# DEFAULT exp
			t[0].add(Nodo(t[1], ''))
			t[0].add(t[2])
		else:  #| PRIMARY KEY | NOT NULL | REFERENCES ID | CONSTRAINT ID
			t[0].add(Nodo(t[1], ''))
			t[0].add(Nodo(t[2], ''))

	elif len(t) == 5:  #CHECK PARIZQ exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))

	else: #UNIQUE | NULL | PRIMARY KEY | NOT NULL
		t[0].add(Nodo(t[1], ''))

def p_tipocql(t):
	'''
		tipocql : ID
				| tipo
	'''
	t[0] = Nodo("tipo", "")
	t[0].add(Nodo(t[1], ''))

def p_tipo(t):
	'''
		 tipo : SMALLINT
			  | INTEGER
			  | BIGINT
			  | DECIMAL
			  | NUMERIC
			  | REAL
			  | TIME
			  | MONEY
			  | TEXT
			  | DATE
			  | TIMESTAMP
			  | INTERVAL
			  | BOOLEAN
			  | DOUBLE 	  PRECISION
			  | VARCHAR   PARIZQ 		 exp PARDER
			  | CHARACTER PARIZQ 		 exp PARDER
			  | CHAR 	  PARIZQ 	     exp PARDER
			  | CHARACTER VARYING PARIZQ exp PARDER
	'''
	t[0] = Nodo("tipo", "")

	if len(t) ==2:
	#SMALLINT | INTEGER | BIGINT | DECIMAL |
	# NUMERIC | REAL | TIME | MONEY | TEXT | DATE | TIMESTAMP |
	# INTERVAL | BOOLEAN
		t[0].add(Nodo(t[1], ''))
	elif len(t) == 3:#DOUBLE 	  PRECISION
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
	elif len(t) ==5:# VARCHAR   PARIZQ exp PARDER | CHARACTER PARIZQ
		# exp PARDER | CHAR PARIZQ exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))
	elif len(t) ==6:
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))
		t[0].add(t[4])
		t[0].add(Nodo(t[5], ''))

# ---------------INSERT---------------
def p_insert(t):
	'''
		insert : INSERT INTO ID VALUES PARIZQ listavalores PARDER
			   | INSERT INTO ID PARIZQ listaids PARDER VALUES PARIZQ listavalores PARDER
	'''
	t[0] = Nodo("insert", "")
	if len(t) ==7: #INSERT INTO ID VALUES PARIZQ listavalores PARDER
		t[0].add(Nodo(t[1]+' '+t[2]+' '+t[3]+' '+t[4], ''))
		t[0].add(t[6])

	elif len(t) == 11: #INSERT INTO ID PARIZQ listaids PARDER VALUES PARIZQ listavalores PARDER
		t[0].add(Nodo(t[1]+' '+t[2]+' '+t[3], ''))
		t[0].add(t[5])
		t[0].add(Nodo(t[7], ''))
		t[0].add(t[9])

def p_listaids(t):
	'''
		listaids : listaids COMA ID
				 | ID
	'''
	if len(t) == 4: #listaids COMA ID
		t[1].add(Nodo(t[3],''))
		t[0] = t[1]
	else: # ID
		t[0] = Nodo("listaids", "")
		t[0].add(Nodo(t[1],''))

# ---------------UPDATE---------------
def p_update(t):
	'''
		update : UPDATE ID SET listaupdate WHERE exp
			   | UPDATE ID SET listaupdate
	'''
	t[0] = Nodo("update", "")

	if len(t)==7:#UPDATE ID SET listaupdate WHERE exp
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))
		t[0].add(t[4])
		t[0].add(Nodo(t[5], ''))
		t[0].add(t[6])
	elif len(t)==5:#UPDATE ID SET listaupdate
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))
		t[0].add(t[4])

def p_listaupdate(t):
	'''
		listaupdate : listaupdate COMA asignacionupdate
					| asignacionupdate
	'''
	if len(t) == 4: # listaupdate COMA asignacionupdate
		t[1].add(t[3])
		t[0] = t[1]
	else:           # asignacionupdate
		t[0] = Nodo("listaupdate", "")
		t[0].add(t[1])

def p_asignacionupdate(t):
	'''
		asignacionupdate : acceso IGUAL exp
	'''
	t[0] = Nodo("asignacionupdate", "")
	t[0].add(t[1])
	t[0].add(Nodo(t[2], ''))
	t[0].add(t[3])

# TODO: Segun la gramatica, hace falta la produccion 'Variable' cuya exp. reg. es: "@[A-Za-z][_A-Za-z0-9]*"
def p_acceso(t):
	'''
		acceso : acceso PT ID
			   | acceso  CORIZQ exp CORDER
			   | ID
	'''
	t[0] = Nodo("acceso", "")

	if len(t) == 4:#acceso PT ID
		t[0].add(t[1])
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))
		pass
	elif len(t) == 5: #acceso  CORIZQ exp CORDER
		t[0].add(t[1])
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))
	elif len(t) == 2: # ID
		t[0].add(Nodo(t[1], ''))

def p_acceso1(t):
	'''
		acceso : acceso PT funcioncollection
	'''
	t[0] = Nodo("acceso", "")
	t[0].add(t[1])
	t[0].add(Nodo(t[2], ''))
	t[0].add(t[3])

def p_funcioncollection(t):
	'''
		funcioncollection   : INSERT    PARIZQ exp COMA exp PARDER
							| INSERT    PARIZQ exp PARDER
							| SET       PARIZQ exp COMA exp PARDER
							| REMOVE    PARIZQ exp PARDER
							| SIZE      PARIZQ PARDER
							| CLEAR     PARIZQ PARDER
							| CONTAINS 	PARIZQ exp PARDER
							| LENGTH    PARIZQ PARDER
							| SUBSTRING PARIZQ exp COMA exp PARDER
	'''
	t[0] = Nodo("funcioncollection", "")

	if t[1].lower() == 'insert':

		if len(t) ==7: #INSERT PARIZQ exp COMA exp PARDER
			t[0].add(Nodo(t[1],''))
			t[0].add(Nodo(t[2],''))
			t[0].add(t[3])
			t[0].add(Nodo(t[4],''))
			t[0].add(t[5])
			t[0].add(Nodo(t[6],''))

		elif len(t) == 5: #INSERT PARIZQ exp PARDER
			t[0].add(Nodo(t[1],''))
			t[0].add(Nodo(t[2],''))
			t[0].add(t[3])
			t[0].add(Nodo(t[4],''))

	elif t[1].lower() == 'set': #SET PARIZQ exp COMA exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))
		t[0].add(t[5])
		t[0].add(Nodo(t[6], ''))

	elif t[1].lower() == 'remove': #REMOVE PARIZQ exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))

	elif t[1].lower() == 'size': #SIZE PARIZQ PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))

	elif t[1].lower() == 'clear': #CLEAR PARIZQ PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))

	elif t[1].lower() == 'contains': #CONTAINS PARIZQ exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))

	elif t[1].lower() == 'length': #LENGTH PARIZQ PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(Nodo(t[3], ''))

	elif t[1].lower() == 'substring': #SUBSTRING PARIZQ exp COMA exp PARDER
		t[0].add(Nodo(t[1], ''))
		t[0].add(Nodo(t[2], ''))
		t[0].add(t[3])
		t[0].add(Nodo(t[4], ''))
		t[0].add(t[5])
		t[0].add(Nodo(t[6], ''))

# ---------------DELETE---------------
def p_deletetable(t):
	'''
		deletetable : DELETE FROM ID WHERE exp
					| DELETE FROM ID
					| DELETE listaatributos FROM ID WHERE exp
					| DELETE listaatributos FROM ID
	'''
	t[0] = Nodo("deletetable", "")

	if len(t)==6: # DELETE FROM ID WHERE exp
		t[0].add(Nodo(t[1],''))
		t[0].add(Nodo(t[2],''))
		t[0].add(Nodo(t[3],''))
		t[0].add(Nodo(t[4],''))
		t[0].add(t[5])

	elif len(t) == 4:  # DELETE FROM ID
		t[0].add(Nodo(t[1],''))
		t[0].add(Nodo(t[2],''))
		t[0].add(Nodo(t[3],''))

	elif len(t) == 7:  # DELETE listaatributos FROM ID WHERE exp
		t[0].add(Nodo(t[1],''))
		t[0].add(t[2])
		t[0].add(Nodo(t[3],''))
		t[0].add(Nodo(t[4],''))
		t[0].add(Nodo(t[6],''))
		t[0].add(t[7])

	elif len(t) == 5:  # DELETE listaatributos FROM ID
		t[0].add(Nodo(t[1],''))
		t[0].add(t[2])
		t[0].add(Nodo(t[3],''))
		t[0].add(Nodo(t[4],''))

def p_listaatributos(t):
	'''
		listaatributos : listaatributos COMA acceso
					   | acceso
	'''
	if len(t) == 4:
		t[1].add(t[3])
		t[0] = t[1]
	else:
		t[0] = Nodo("listaatributos","")
		t[0].add(t[1])

# ---------------ERROR SINTACTICO---------------
def p_error(t):
	print(t)
	print("Error sintáctico en '%s'" % t.value)

import ply.yacc as yacc
parser = yacc.yacc()

def parse(input) :
	global cadena,lisErr, dot
	#parser = yacc.yacc()
	lexer2.lineno=1
	#par= parser.parse("ADD")
	#print(par)
	return parser.parse(input)


# wczytanie potrzebnych bibliotek:
import roboticstoolbox as rtb
import numpy as np
from spatialmath import *
from spatialmath.base import *
from spatialmath.base.symbolic import *
import math
import matplotlib.pyplot as plt
# ...

# definicje funkcji:

def zadanie_3():
    Tps = np.eye(4)
    Tbs = np.eye(4)
    Rps = np.array([[-1, 0, 0],
                    [0, 1, 0],
                    [0, 0, -1]])
    Rbs = np.array([[0, -1, 0],
                    [1, 0, 0],
                    [0, 0, 1]])
    tps = np.array([3, -6, 4])
    tbs = np.array([2, 5, 0])
    Tbs[0:3, 0:3] = Rbs
    Tbs[0:3, 3] = tbs.T
    Tbs = SE3(Tbs)
    Tps[0:3, 0:3] = Rps.T
    Tps[0:3, 3] = -(Rps.T)@tps
    Tps = SE3(Tps)
    dp = np.eye(4)
    db = np.eye(4)
    ds = np.eye(4)
    dp1 = np.array([1, -2, 2])
    dp[0:3, 3] = dp1.T
    dp = SE3(dp)
    ds = Tps@dp
    db = Tbs@ds
    db = SE3(db)
    print("Zadanie 3.1: ", db.t)
    #zadanie 3.2
    trplot(transl(0, 0, 0), frame='B', rviz=True, width=1)
    trplot(Tbs.A, color='red', frame='S')
    p = Tbs@Tps
    p = SE3(p)
    trplot(p.A, color='black', frame='P')
    plt.quiver(0, 0, 0, -2, 7, 2)
    plt.show()

# ...

# wykonywanie wybranej funkcji
if __name__ == '__main__':
    zadanie_3()


"""
Based on tutorial
https://medium.com/@ageitgey/machine-learning-is-fun-part-3-deep-learning-and-convolutional-neural-networks-f40359318721#.54v56wei4
Based on the tflearn example located here:
https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_cifar10.py
"""
from __future__ import division, print_function, absolute_import

# Import tflearn and some helpers
import tflearn
from tflearn.data_utils import shuffle
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation
import pickle
import numpy as np
import os
from scipy.misc import imread, imresize, imsave
from sklearn.model_selection import train_test_split
import pandas as pd

## FUNCTIONS
# load images
def load_images_from_folder(folder):
    images = []
    for filename in os.listdir(folder):
        if filename.find('.jpg') != -1:  # if it contains .jpg
            img = imread(os.path.join(folder,filename))
            if img is not None:
                images.append(img)
    return images

## DEFINES
# Small 19 record dataset
#dir_ALB = '/Users/andrew/Documents/kaggle/fish/cnn-model-001/data/ALB_small'
#dir_OTHER = '/Users/andrew/Documents/kaggle/fish/cnn-model-001/data/OTHER_small'

# Large dataset
dir_ALB = '/Users/andrew/Documents/kaggle/fish/cnn-model-001/data/ALB/resized'
dir_OTHER = '/Users/andrew/Documents/kaggle/fish/cnn-model-001/data/OTHER/resized'

## Load the data set (replace this with something that loads our JPEGS)
# Load all the ALBs
ALB_images = load_images_from_folder(dir_ALB)
ALB_labels = np.repeat('ALB', len(ALB_images), axis=0)

# Load all the OTHERs
multiply_samples = 1
OTHER_images = load_images_from_folder(dir_OTHER)
OTHER_labels = np.repeat('OTHER', len(OTHER_images), axis=0)

# The ALB pics are 6 times more than the OTHER pics so duplicate OTHERS 6x
#OTHER_images = np.repeat(OTHER_images, multiply_samples, axis=0)

# Union datasets
df = []
df.extend(ALB_images); df.extend(OTHER_images)
ALB_images = []; OTHER_images = []
labels = []
labels.extend(ALB_labels); labels.extend(OTHER_labels)
ALB_labels = []; OTHER_labels = []

# Change labels into 1s and 0s
for i in range(len(labels)):
      # Converting 'ALB' field to float
      labels[i] = 1.0 if labels[i] == 'ALB' else 0.0

# Cast all image numbers to floats
df = np.asarray(df)
df = df.astype(float)

# Split into train and test
X, X_test, Y, Y_test = train_test_split(df, labels, test_size=0.33, random_state=42)

# Original load data code
# X, Y, X_test, Y_test = pickle.load(open("full_dataset.pkl", "rb"))

# Shuffle the data (shuffles both arrays by the same offsets)
X, Y = shuffle(X, Y)
X_test, Y_test = shuffle(X_test, Y_test)

# Make sure the data is normalized
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()

# Create extra synthetic training data by flipping, rotating and blurring the
# images on our data set.
img_aug = ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_rotation(max_angle=25.)
img_aug.add_random_blur(sigma_max=3.)

# Define our network architecture:

# Input is a 32x32 image with 3 color channels (red, green and blue)
network = input_data(shape=[None, 32, 32, 3],
                     data_preprocessing=img_prep,
                     data_augmentation=img_aug)

# Step 1: Convolution
network = conv_2d(network, 32, 3, activation='relu')

# Step 2: Max pooling
network = max_pool_2d(network, 2)

# Step 3: Convolution again
network = conv_2d(network, 64, 3, activation='relu')

# Step 4: Convolution yet again
network = conv_2d(network, 64, 3, activation='relu')

# Step 5: Max pooling again
network = max_pool_2d(network, 2)

# Step 6: Fully-connected 512 node neural network
network = fully_connected(network, 512, activation='relu')

# Step 7: Dropout - throw away some data randomly during training to prevent over-fitting
network = dropout(network, 0.5)

# Step 8: Fully-connected neural network with two outputs (0=isn't a bird, 1=is a bird) to make the final prediction
network = fully_connected(network, 2, activation='softmax')

# Tell tflearn how we want to train the network
network = regression(network, optimizer='adam',
                     loss='categorical_crossentropy',
                     learning_rate=0.001)

# Wrap the network in a model object
model = tflearn.DNN(network, tensorboard_verbose=0, checkpoint_path='fish-classifier.tfl.ckpt')

# Train it! We'll do 100 training passes and monitor it as it goes.
model.fit(X, Y, n_epoch=100, shuffle=True, validation_set=(X_test, Y_test),
          show_metric=True, batch_size=96,
          snapshot_epoch=True,
          run_id='fish-classifier')

# Save model when training is complete to a file
model.save("fish-classifier.tfl")
print("Network trained and saved as fish-classifier.tfl!")

## Test network performance

# Predict
prediction = model.predict(X_test)

# Output Scores to CSV
pred_df = pd.DataFrame(prediction, columns=["IS ALB","IS OTHER"])
obs_df = pd.DataFrame(Y_test, columns=["OBSERVATIONS"])
results = pd.concat([pred_df, obs_df], axis=1)
results.to_csv('/Users/andrew/Documents/kaggle/fish/cnn-model-001/results.csv', header=True)


from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
import numpy as np
from numpy.lib.format import open_memmap
import pickle

# IMPLEMENTS THE STACKING ALGORITHM
# TAKING AS INPUT A LIST OF MODELS TO USE TOGETHER
# THE reuse VARIABLE INDICATES IF THE MODEL HAS TO BE TRAINED
# AGAIN OR IF IT WILL JUST BE LOADED AND USED AS IS
# WARNING : MAKE SURE THAT A SAVED MODEL INSTANCE (.pkl) EXISTS
def f_STACK(X_train, Y_train, X_test, Y_test, models, reuse=False):
    if reuse:
        with open('Algos/STACK.pkl', 'rb') as filehandler:
            classifier = pickle.load(filehandler)
    # THE STACKING ALGORITHM IS A LOGISTIC CLASSIFIER
    else:
        classifier = LogisticRegression(multi_class='multinomial',
                                        solver='saga', verbose=0,
                                        class_weight='balanced', n_jobs=20)
        algorithms_predictions = np.empty(
                                 shape=(X_train.shape[0], len(models))
                                 )

        # LOADS THE MODEL AND SAVES THEIR PREDICTIONS
        for i, algo in enumerate(models):
            if algo.startswith('NN'):
                path = 'Algos/'+algo[:-1]+'_folder/'+algo+'.pkl'
                with open(path, 'rb') as filehandler:
                    model, model_history = pickle.load(filehandler)
                algorithms_predictions[:, i] = np.argmax(
                                               model.predict(X_train),
                                               axis=1)
            else:
                path = 'Algos/'+algo+'_folder/'+algo+'.pkl'
                with open(path, 'rb') as filehandler:
                    model = pickle.load(filehandler)
                algorithms_predictions[:, i] = model.predict(X_train)
        # TRAINS THE LOGISTIC CLASSIFIER TO CHOOSE OUTPUT DEPENDING
        # ON MODELS' DECISION
        classifier.fit(algorithms_predictions, Y_train)

    # NOW APPLIES ITSELF ON THE TEST SET, IT MUST FIRST OBTAIN THE
    # PREDICTIONS OF THE SUBMODELS
    algorithms_tests = np.empty(shape=(X_test.shape[0], len(models)))
    for i, algo in enumerate(models):
        if algo.startswith('NN'):
            path = 'Algos/'+algo[:-1]+'_folder/'+algo+'.pkl'
            with open(path, 'rb') as filehandler:
                model, model_history = pickle.load(filehandler)
            algorithms_tests[:, i] = np.argmax(
                                     model.predict(X_test),
                                     axis=1)
        else:
            path = 'Algos/'+algo+'_folder/'+algo+'.pkl'
            with open(path, 'rb') as filehandler:
                model = pickle.load(filehandler)
            algorithms_tests[:, i] = model.predict(X_test)
    expected = classifier.predict(algorithms_tests)

    # COMPUTES THE CONFUSION MATRIX
    conf_matrix = confusion_matrix(expected, Y_test)
    normalisation = np.sum(conf_matrix, axis=1, keepdims=True)
    conf_matrix = conf_matrix/normalisation
    print(conf_matrix)

    # SAVES THE MODEL
    with open('Algos/STACK.pkl', 'wb') as filehandler:
        pickle.dump(classifier, filehandler)
    print('Model saved')

    return conf_matrix


# ========================================================================
# Copyright 2017 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========================================================================

from mxnet import nd
from mxnet.gluon.loss import _apply_weighting, Loss

__author__ = 'Jinho D. Choi'

class BILOU:
    B = 'B'     # beginning
    I = 'I'     # inside
    L = 'L'     # last
    O = 'O'     # outside
    U = 'U'     # unit

    @classmethod
    def collect(cls, tags):
        """
        :param tags: a list of tags encoded by the BILOU format.
        :type tags: list of str
        :return: a dictionary where the keys represent the chunk spans and the values represent the tags
        """
        entities = {}
        begin = -1

        for i, tag in enumerate(tags):
            t = tag[0]

            if t == cls.B:
                begin = i
            elif t == cls.I:
                pass
            elif t == cls.L:
                if begin >= 0: entities[(begin, i+1)] = tag[2:]
                begin = -1
            elif t == cls.O:
                begin = -1
            elif t == cls.U:
                entities[(i, i+1)] = tag[2:]
                begin = -1

        return entities

    def quick_fix(self, tags):
        def fix(i, pt, ct, t1, t2):
            if pt == ct: tags[i][0] = t1
            else: tags[i-1][0] = t2

        def aux(i):
            p = tags[i-1][0]
            c = tags[i][0]
            pt = tags[i-1][1:]
            ct = tags[i][1:]

            if p == self.B:
                if   c == self.B: fix(i, pt, ct, self.I, self.U)  # BB -> BI or UB
                elif c == self.U: fix(i, pt, ct, self.L, self.U)  # BU -> BL or UU
                elif c == self.O: tags[i-1][0] = self.U           # BO -> UO
            elif p == self.I:
                if   c == self.B: fix(i, pt, ct, self.I, self.L)  # IB -> II or LB
                elif c == self.U: fix(i, pt, ct, self.I, self.L)  # IU -> II or LU
                elif c == self.O: tags[i-1][0] = self.L           # IO -> LO
            elif p == self.L:
                if   c == self.I: fix(i, pt, ct, self.I, self.B)  # LI -> II or LB
                elif c == self.L: fix(i, pt, ct, self.I, self.B)  # LL -> IL or LB
            elif p == self.O:
                if   c == self.I: tags[i][0] = self.B             # OI -> OB
                elif c == self.L: tags[i][0] = self.B             # OL -> OB
            elif p == self.U:
                if   c == self.I: fix(i, pt, ct, self.B, self.B)  # UI -> BI or UB
                elif c == self.L: fix(i, pt, ct, self.B, self.B)  # UL -> BL or UB

        for i in range(1, len(tags)): aux(i)
        p = tags[-1][0]
        if   p == self.B: tags[-1][0] = self.U
        elif p == self.I: tags[-1][0] = self.L

class MultiLabelSoftmaxCrossEntropyLoss(Loss):
    def __init__(self, axis=-1, sparse_label=True, from_logits=False, weight=None,
                 batch_axis=0, **kwargs):
        super(MultiLabelSoftmaxCrossEntropyLoss, self).__init__(weight, batch_axis, **kwargs)
        self._axis = axis
        self._sparse_label = sparse_label
        self._from_logits = from_logits

    def hybrid_forward(self, F, output, label, sample_weight=None):
        if not self._from_logits:
            output = F.log_softmax(output)
        if self._sparse_label:
            # loss = -F.pick(output, label, axis=self._axis, keepdims=True)
            l = -F.pick(output, label, axis=self._axis, keepdims=True)
            d = nd.array([0 if i.asscalar() < 0 else 1 for i in label]).reshape((-1, 1))
            loss = l * d
        else:
            loss = -F.sum(output*label, axis=self._axis, keepdims=True)
        loss = _apply_weighting(F, loss, self._weight, sample_weight)
        return F.mean(loss, axis=self._batch_axis, exclude=True)

# class ConvBlock(gluon.Block):
#     def __init__(self):
#         super(ConvBlock, self).__init__()
#         with self.name_scope():
#             self.fc0 = gluon.nn.Dense(5)
#             self.fc1 = gluon.nn.Dense(5)
#             self.fc2 = gluon.nn.Dense(5)
#             self.fcn = [self.fc0, self.fc1, self.fc2]
#
#     def forward(self, x):
#         t = [fc(x) for fc in self.fcn]
#         x = nd.concat(*t, dim=0)
#         return x
#
# X = nd.array([
#     [[1,0,0,0,0], [2,0,0,0,0], [3,0,0,0,0], [4,0,0,0,0]],
#     [[0,1,0,0,0], [0,2,0,0,0], [0,3,0,0,0], [0,4,0,0,0]],
#     [[0,0,1,0,0], [0,0,2,0,0], [0,0,3,0,0], [0,0,4,0,0]],
#     [[0,0,0,1,0], [0,0,0,2,0], [0,0,0,3,0], [0,0,0,4,0]],
#     [[0,0,0,0,1], [0,0,0,0,2], [0,0,0,0,3], [0,0,0,0,4]]
# ])
#
# Y = nd.array([[0,1,2],[0,1,2],[0,1,2],[0,-1,-1],[0,1,-1]])
#
# ctx = mx.cpu()
# net = ConvBlock()
# net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)
# trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})
#
# batch_size = 2
# L = MultiLabelSoftmaxCrossEntropyLoss()
# data = gluon.data.DataLoader(gluon.data.ArrayDataset(X, Y), batch_size=batch_size)
#
# for x, y in data:
#     x = x.as_in_context(ctx)
#     y = y.as_in_context(ctx)
#     y = nd.transpose(y).reshape((-1,1))
#     with autograd.record():
#         output = net(x)
#         print(output)
#         print(y)
#         loss = L(output, y)
#         loss.backward()
#
#
#     trainer.step(x.shape[0])
#
#
# @property
#     def x(self):
#         """
#         If the total number of words in the document is greater than max_len, the rest gets discarded.
#         :param document: a sentence or a list of sentences, where each sentence is represented by a dictionary.
#         :type document: Union[list of dict, dict]
#         :return: a list of feature vectors for the corresponding words across sentences.
#         :rtype: list of numpy.array -> max_len * (word_vsm.dim + ambi_vsm.dim + num_class)
#         """
#         def position(i, size):
#             return self.fst_word if i == 0 else self.lst_word if i+1 == size else self.mid_word
#
#         def aux(sentence):
#             word_emb = sentence.setdefault(WORD_EMB, self.word_vsm.get_list(sentence[TOKEN]))
#             ambi_emb = sentence.setdefault(AMBI_EMB, self.ambi_vsm.get_list(sentence[TOKEN]))
#             pos_scores = sentence.get(POS_OUT, None)
#
#             return [np.concatenate((
#                 position(i, len(sentence)),
#                 word_emb[i],
#                 ambi_emb[i],
#                 self.zero_scores if pos_scores is None else pos_scores[i]))
#                 for i in range(len(sentence))]
#
#         matrix = [v for s in document for v in aux(s)]
#
#         # zero padding
#         if len(matrix) < self.max_len:
#             matrix.extend([np.zeros(len(matrix[0]))] * (self.max_len - len(matrix)))
#         elif len(matrix) > self.max_len:
#             matrix = matrix[:self.max_len]
#
#         return np.array(matrix)
#
#     def gold_labels(self, document):
#         labels = np.concatenate([d[POS_GOLD] for d in document])
#
#         # zero padding
#         if len(labels) < self.max_len:
#             return np.append(labels, np.full(self.max_len - len(labels), -1))
#         elif len(labels) > self.max_len:
#             return labels[:self.max_len]
#         else:
#             return labels
#
#     def set_scores(self, document, output):
#         """
#         :param document: a sentence or a list of sentences, where each sentence is represented by a dictionary.
#         :type document: Union[list of dict, dict]
#         :param output: the output of the part-of-speech tag predictions.
#         :param output: numpy.array -> max_len * num_class
#         """
#         def index(i):
#             return (begin + i) * self.num_class
#
#         def get(sentence):
#             return [output[index(i):index(i+1)] for i in range(0, len(sentence))]
#
#         begin = 0
#
#         if isinstance(document, dict):
#             document[POS_OUT] = get(document)
#         else:
#             for d in document:
#                 sc = get(d)
#                 d[POS_OUT] = sc
#                 begin += sc
#
# def trunc_pads(labels, output):
#     idx = next((i for i, label in enumerate(labels) if label.asscalar() == -1), None)
#     return (labels[:idx], output[:idx]) if idx else (labels, output)

# def evaluate(dat_path, word_vsm, ambi_vsm, model_path):
#     word_vsm = FastText(word_vsm)
#     ambi_vsm = Word2Vec(ambi_vsm) if ambi_vsm else None
#     comp = POSTagger(mx.gpu(0), word_vsm, ambi_vsm, model_path=model_path)
#
#     cols = {TOKEN: 0, POS: 1}
#     dev_states = read_tsv(dat_path, cols, comp.create_state)
#     dev_eval = comp.evaluate(dev_states, 512, Accuracy())
#     print(dev_eval)


class Solution:
    def lengthOfLongestSubstring(self, s: str) -> int:
        from collections import defaultdict
        c_chars = defaultdict(int)
        l, r, res = 0, 0, 0
        contract = False
        while r < len(s):
            # Expand
            if s[r] in c_chars and c_chars[s[r]] > 0: # check if contract is needed
                contract = True
            c_chars[s[r]] += 1
            r += 1

            # Contract
            if contract:
                d_char = s[r-1] # Char to remove
                while s[l] != d_char and l < r:
                    c_chars[s[l]] -= 1
                    l += 1
                c_chars[s[l]] -= 1
                l += 1
                contract = False

            res = max(res, r-l)
        return res

leetcode/3. Longest Substring Without Repeating Characters/soln.py