from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QApplication, QWidget
from PyQt5.QtWidgets import QGridLayout, QLayout
from PyQt5.QtWidgets import QTextEdit, QLineEdit, QToolButton, QComboBox, QMessageBox
from PyQt5.QtGui import QColor, QFont

from Petstate import *
import random

class Dog_raising_game(QWidget):

    def __init__(self, parent=None):
        super().__init__(parent)

        # Status Layout creation
        gameLayout = QGridLayout()

        #pet state display window
        self.petstate = QTextEdit()
        self.petstate.setReadOnly(True)
        self.petstate.setAlignment(Qt.AlignLeft)
        font = self.petstate.font()
        font.setPointSize(font.pointSize())
        self.petstate.setFont(font)
        gameLayout.addWidget(self.petstate, 0, 0, 5, 2)

        #current pet display window
        self.currentpet = QTextEdit()
        self.currentpet.setReadOnly(True)
        gameLayout.addWidget(self.currentpet, 0, 2, 5, 1)

        #each function button (eat, walk, stroke, shower, sleep)
        self.eatButton = QToolButton()
        self.eatButton.setText('    Feeding    ')
        self.eatButton.clicked.connect(self.eatClicked)
        self.eatButton.setStyleSheet('QToolButton {font: bold; background-color: #B0E0E6;}')
        gameLayout.addWidget(self.eatButton, 0, 3)

        self.walkButton = QToolButton()
        self.walkButton.setText('    walking    ')
        self.walkButton.clicked.connect(self.walkClicked)
        self.walkButton.setStyleSheet('QToolButton {font: bold; background-color: #B0E0E6;}')
        gameLayout.addWidget(self.walkButton, 1, 3)

        self.strokeButton = QToolButton()
        self.strokeButton.setText('    stroking   ')
        self.strokeButton.clicked.connect(self.strokeClicked)
        self.strokeButton.setStyleSheet('QToolButton {font: bold; background-color: #B0E0E6;}')
        gameLayout.addWidget(self.strokeButton, 2, 3)

        self.showerButton = QToolButton()
        self.showerButton.setText('  showering  ')
        self.showerButton.clicked.connect(self.showerClicked)
        self.showerButton.setStyleSheet('QToolButton {font: bold; background-color: #B0E0E6;}')
        gameLayout.addWidget(self.showerButton, 3, 3)

        self.sleepButton = QToolButton()
        self.sleepButton.setText('    sleeping   ')
        self.sleepButton.clicked.connect(self.sleepClicked)
        self.sleepButton.setStyleSheet('QToolButton {font: bold; background-color: #B0E0E6;}')
        gameLayout.addWidget(self.sleepButton, 4, 3)

        # new game button
        self.newGameButton = QToolButton()
        self.newGameButton.setText('New game')
        self.newGameButton.clicked.connect(self.startgame)
        self.newGameButton.setStyleSheet('QToolButton {font: bold; background-color: #A3C1DA; color: blue;}')
        gameLayout.addWidget(self.newGameButton, 5, 0)

        # game rules button
        self.ruleButton = QToolButton()
        self.ruleButton.setText('?')
        self.ruleButton.clicked.connect(self.gameRules)
        self.ruleButton.setStyleSheet('QToolButton {font: bold; background-color: #A3C1DA; color: blue;}')
        gameLayout.addWidget(self.ruleButton, 5, 1)

        #pet Status Change message
        self.message = QLineEdit()
        self.message.setReadOnly(True)
        self.message.setAlignment(Qt.AlignCenter)
        gameLayout.addWidget(self.message, 5, 2)

        #job select box
        self.job = QComboBox()
        self.job.addItem('doctor')
        self.job.addItem("president")
        self.job.addItem("soccer player")
        self.job.addItem("teacher")
        gameLayout.addWidget(self.job, 5, 3)

        # Layout placement
        mainLayout = QGridLayout()
        mainLayout.setSizeConstraint(QLayout.SetFixedSize)
        mainLayout.addLayout(gameLayout, 0, 0)
        self.setLayout(mainLayout)
        self.setWindowTitle('Rasing your own Pet')

        #game start
        self.startgame()

    def startgame(self):
        self.Petname = input("Enter your pet name")

        self.colorChange()    # pet's color changes at random
        self.state = state(self.Petname)
        self.petstate.setText(self.state.displayCurrent())
        self.currentpet.setText(self.state.currentShape())
        self.message.clear()

        self.eatButton.setDisabled(False)
        self.showerButton.setDisabled(False)
        self.strokeButton.setDisabled(False)
        self.sleepButton.setDisabled(False)
        self.walkButton.setDisabled(False)

    def colorChange(self):    #pet color change
        color_list = [QColor(250, 140, 0), QColor(145, 210, 80), QColor(0, 180, 20), QColor(230, 30, 100),
                      QColor(155, 40, 175), QColor(105, 60, 185), QColor(255,70,50)]
        colorvar = random.choice(color_list)
        self.currentpet.setTextColor(colorvar)
        self.currentpet.setFontWeight(QFont.Bold)

    def gameRules(self):   #gamerule message box
        rule = """
        ✔ feeding: hungry-20, happy+15, dirty+15
        ✔ walking: hungry+20, happy+20, tired+30, dirty+25
        ✔ sleeping: happy+15, tired-20
        ✔ showering: happy-25, tired+15, dirty-30
        ✔ stroking: happy+15, dirty+10

        ✔ You can choose your own pet's job.
          When a pet's level reaches 10, it turns into a job!

        ✘ Keep your pet happiness index above zero!
        (Your pet may run away.)
        """
        QMessageBox.about(self, 'Game rules', rule)

    def eatClicked(self):
        if self.state.current['°⌓° hungry']<=20:
            self.message.setText("I\'m not hungry")
        else:
            self.state.current['°⌓° hungry'] -= 20  #pet state change
            self.state.current['･ᴗ･  happy'] += 15
            self.state.current['ʘ‿ʘ  dirty'] += 15
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('hungry: -20, happy, dirty: +15!')
        self.checkLevel()

    def walkClicked(self):
        if self.state.current['⌣_⌣́    tired']>=80:
            self.message.setText("I\'m very tired")
        elif self.state.current['°⌓° hungry']>=80:
            self.message.setText("I\'m very hungry")
        elif self.state.current['ʘ‿ʘ  dirty']>=70:
            self.message.setText('Let me take a shower first!')
        else:
            self.state.current['°⌓° hungry'] += 20  #pet state change
            self.state.current['･ᴗ･  happy'] += 20
            self.state.current['⌣_⌣́    tired'] += 30
            self.state.current['ʘ‿ʘ  dirty'] += 25
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('hungry, happy: +20, dirty: +25, tired: +30')
        self.checkLevel()

    def showerClicked(self):
        if self.state.current['⌣_⌣́    tired']>=80:
            self.message.setText("I\'m very tired")
        elif self.state.current['ʘ‿ʘ  dirty']<=30:
            self.message.setText("I\'m clean now!")
        else:
            self.state.current['･ᴗ･  happy'] -= 25  #pet state change
            self.state.current['⌣_⌣́    tired'] += 15
            self.state.current['ʘ‿ʘ  dirty'] -= 30
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('happy: -25, tired: +15, dirty: -30')
        self.checkLevel()

    def strokeClicked(self):
        if self.state.current['ʘ‿ʘ  dirty']>=90:
            self.message.setText('Let me take a shower first!')
        else:
            self.state.current['･ᴗ･  happy'] += 15   #pet state change
            self.state.current['ʘ‿ʘ  dirty'] += 10
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('happy: +15, dirty: +10')
            self.checkLevel()

    def sleepClicked(self):
        if self.state.current['⌣_⌣́    tired']<=20:
            self.message.setText("I\'m not sleepy")
        else:
            self.state.current['･ᴗ･  happy'] += 15    #pet state change
            self.state.current['⌣_⌣́    tired'] -= 20
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('happy: +15, tired: -20')
        self.checkLevel()

    def checkLevel(self):
        if self.state.current['･ᴗ･  happy'] <= 0:
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('Sorry. Your pet ran away')
            self.eatButton.setDisabled(True)    #button deactivation
            self.showerButton.setDisabled(True)
            self.strokeButton.setDisabled(True)
            self.sleepButton.setDisabled(True)
            self.walkButton.setDisabled(True)

        elif self.state.current['✔ Level']>=10:
            job = self.job.currentText()
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.futureShape(job))
            self.message.setText('Your pet got a job!: {}'.format(job))
            QMessageBox.about(self, "The end", "Congratulations!")
            self.eatButton.setDisabled(True)    #button deactivation
            self.showerButton.setDisabled(True)
            self.strokeButton.setDisabled(True)
            self.sleepButton.setDisabled(True)
            self.walkButton.setDisabled(True)

        elif self.state.current['･ᴗ･  happy']>=90:
            self.state.current['✔ Level']+=1
            self.state.current['･ᴗ･  happy'] = 30
            self.petstate.setText(self.state.displayCurrent())
            self.currentpet.setText(self.state.currentShape())
            self.message.setText('Level up!')

if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    game = Dog_raising_game()
    game.show()
    sys.exit(app.exec_())


import cflib.crtp
import time
import numpy as np
import math
import cv2
import time
from matplotlib import pyplot as plt
from distance_to_camera import DistanceCalculator
from myMotionCommander import MyMotionCommander
import threading
import urllib.request as urllib

from cflib.crazyflie import Crazyflie
from cflib.crazyflie.syncCrazyflie import SyncCrazyflie
from cflib.positioning.motion_commander import MotionCommander
from cflib.crazyflie.swarm import CachedCfFactory
from cflib.crazyflie.swarm import Swarm
from cflib.crazyflie.log import LogConfig

class Trajetoria():
    def __init__(self, mc):
        self.mc = mc
        self.calculator = DistanceCalculator()
        myMotionCommander = MyMotionCommander(self.mc)
        self.cap = cv2.VideoCapture(1)
        CenterLimitH = [305, 335]
        self.savedX = 50
        self.savedY = 0
        self.savedAlpha = 180
        self.savedBeta = 90

    def video(self):
        cont = 0
        while(self.cap.isOpened()):
            ret, frame = self.cap.read()

            if(frame is None):
                break

            if(cont%3==0):
                thread = threading.Thread(target=self.calculator.processImage, args=(frame, ))
                #set time
                thread.setDaemon(True)
                thread.start()

            if(cont%15==0):
                self.calculator.mediaDistance()

            self.calculator.writeDistance(frame)
            cv2.imshow('frame',frame)
            #markerCenter = self.calculator.center

            cont += 1
            if cv2.waitKey(10) & 0xFF == ord('q'):
                break

        cv2.waitKey()
        cv2.destroyAllWindows()

    def savePosition(self):
        self.savedX = self.calculator.distance_x
        self.savedY = self.calculator.distance_y
        self.savedAlpha = self.calculator.alpha
        self.savedBeta = self.calculator.beta

    def correctPosition(self):
        delta_x = self.savedX - self.calculator.distance_x
        delta_y = self.savedY - self.calculator.distance_y
        delta_alpha = self.savedAlpha - self.calculator.alpha

        print(delta_x, delta_y, self.calculator.alpha)
        alpha_rad = self.calculator.alpha*math.pi/180
        # mudança de coordenadas para o drone
        frente = delta_x*math.cos(alpha_rad) + delta_y*math.sin(alpha_rad)
        esquerda = -delta_x*math.sin(alpha_rad) + delta_y*math.cos(alpha_rad)
        print(frente, esquerda, delta_alpha)
        mc.move_distance(frente/100, esquerda/100, 0,velocity = 0.3)
        time.sleep(1)
        if(delta_alpha>0):
            mc.turn_left(delta_alpha)
        if(delta_alpha<0):
            mc.turn_right(-delta_alpha)
        time.sleep(1)

if __name__ == '__main__':
    #sincronização de cf
    cflib.crtp.init_drivers(enable_debug_driver=False)
    factory = CachedCfFactory(rw_cache='./cache')
    URI3 = 'radio://0/40/2M/E7E7E7E7E4'
    cf = Crazyflie(rw_cache='./cache')
    sync = SyncCrazyflie(URI3, cf=cf)
    sync.open_link()
    mc = MotionCommander(sync)
    trajetoria = Trajetoria(mc)

    thread = threading.Thread(target=trajetoria.video)
    thread.setDaemon(True)
    thread.start()

    mc.take_off()
    time.sleep(3)

    #trajetoria.savePosition()

    # mc.right(1, velocity = 0.3)
    # time.sleep(1)
    # mc.turn_right(90)
    # time.sleep(1)
    # mc.right(1, velocity = 0.3)
    # time.sleep(1)
    # mc.turn_right(90)
    # time.sleep(1)
    # mc.right(1, velocity = 0.3)
    # time.sleep(1)
    # mc.turn_right(90)
    # time.sleep(1)
    # mc.right(1, velocity = 0.3)
    # time.sleep(1)
    # mc.turn_right(90)
    # time.sleep(1)

    trajetoria.correctPosition()
    trajetoria.correctPosition()

    mc.land()
    sync.close_link()


from urllib.parse import urlencode
from urllib.request import urlopen
import json

from time import sleep
from datetime import datetime

import pandas as pd

NYT_ARTICLE_API_URL = 'https://api.nytimes.com/svc/search/v2/articlesearch.json'
COMMENTS_URL = 'http://www.nytimes.com/svc/community/V3/requestHandler?callback=NYTD.commentsInstance.drawComments&method=&cmd=GetCommentsAll&url='

def get_dataset(ARTICLE_API_KEY, page_lower=0, page_upper=30, begin_date=None, end_date=None,
                 max_comments=50000, sort='newest', query=None, save=False, printout=True):
    '''Collects the comments on the articles of NYT by first scraping the
    articles using NYT articles search API, calling on the customized function
    get_comments(url) to get comments on each article, processing the comments'
    and articles' data and returning two pandas dataframes - one each for articles
    and comments.'''

    params = {'api-key': ARTICLE_API_KEY}

    if page_lower<0:
        page_lower = 0

    if page_upper>=200:
        page_upper = 199

    if sort=='oldest':
        if begin_date is None:
            begin_date = '20081031'
    elif end_date is None:
        end_date = datetime.today().strftime('%Y%m%d')

    if begin_date: # Check begin_date is not None
        try:
            begin_date = pd.to_datetime(begin_date, errors='coerce').strftime('%Y%m%d')
        except:
            print("Error: Please try again with begin_date entered in the format %Y%m%d.")
        params['begin_date'] = begin_date

    if end_date: # Check end_date is not None
        try:
            end_date = pd.to_datetime(end_date, errors='coerce').strftime('%Y%m%d')
        except:
            print("Error: Please try again with end_date entered in the format %Y%m%d.")
        params['end_date'] = end_date

    if query:
        params['q'] = query

    params['sort'] = sort

    articles_list = []
    comments_df_list = []

    articles_df = pd.DataFrame()
    comments_df = pd.DataFrame()

    total_comments = 0

    for page in range(page_lower, page_upper):
        if total_comments < max_comments:
            params['page'] = page # Every page has 10 articles
            if printout:
                print("Page: ", page)
            try:
                # Using NYT API to scrap articles
                search_url = NYT_ARTICLE_API_URL + '?' + urlencode(params)
                file = urlopen(search_url).read() # Get the articles search data and read it into a string
                js = json.loads(file) # Load the articles search data as json

                if js['status'] == 'OK':
                    docs = js['response']['docs']
                    docs_length = len(docs)
                    if docs_length==0:
                        if printout:
                            print("No aricles found on page", page)
                        break
                    for i in range(docs_length):
                        if docs[i]['document_type'] != 'multimedia': # Ignore multimedia articles
                            article_url = docs[i]['web_url'] # Get the url for the article
                            comments, number_comments = get_comments(article_url) # Use the article url to get comments
                            total_comments += number_comments
                            if not comments.empty: # Check if the article has comments
                                if printout:
                                    print("Article url:", article_url)
                                article_id = docs[i]['_id']
                                article = docs[i]
                                articles_list.append(article)
                                comments['articleID'] = article_id
                                comments['sectionName'] = article.get('section_name', 'Unknown')
                                comments['newDesk'] = article.get('new_desk', 'Unknown')
                                comments['articleWordCount'] = article.get('word_count', 0)
                                comments['printPage'] = article.get('print_page', 0)
                                comments['typeOfMaterial'] = article.get('type_of_material', 'Unknown')
                                comments = preprocess_comments_dataframe(comments)
                                comments_df_list.append(comments)
            except:
                print('HTTP Error encountered midway. Retrieval terminated at page {}.'.format(page))
                break

    if comments_df_list: # Check that the list is not empty
        comments_df = pd.concat([df for df in comments_df_list])
        articles_df = pd.DataFrame(articles_list)
        articles_df = preprocess_articles_dataframe(articles_df)

    if printout:
        if page_lower > page_upper:
            print("page_lower value is greater than the page_upper. No articles returned.")
        elif (begin_date is not None) & (end_date is not None):
            if begin_date>end_date:
                print("begin_date is bigger than the end_date. No articles returned.")
            else:
                print()
                print("Total articles stored: ", articles_df.shape[0])
                print("Total comments retrieved: ", comments_df.shape[0])
        else:
            print()
            print("Total articles stored: ", articles_df.shape[0])
            print("Total comments retrieved: ", comments_df.shape[0])
    if save:
        articles_df.to_csv('Articles.csv', index=False)
        comments_df.to_csv('Comments.csv', index=False)
    return articles_df, comments_df

def get_comments(article_url, save=False, printout=True):
    '''Given the url of an articles from NYT, returns a dataframe of comments in that article'''

    article_url = article_url.replace(':','%253A') #convert the : to an HTML entity
    article_url = article_url.replace('/','%252F')

    offset = 0 #Start off at the very beginning
    total_comments = 0 # Initialize the count of comments in the article
    df_list = []
    comments_df = pd.DataFrame() # Set up a list to store the comments' data

    while True:
        try:
            sleep(1)
            url = COMMENTS_URL + article_url + '&offset='+str(offset)+'&sort=newest'
            file = urlopen(url).read().decode() # Get the comments data and read it into a string
            file = file.replace('NYTD.commentsInstance.drawComments(','')
            file = file.replace('      /**/ ','')
            file = file[:-2]
            js = json.loads(file) # Load the file as json
            if js['status'] == 'OK':
                results = js['results']
                total_comments_returned = results['totalCommentsReturned']
                total_comments += total_comments_returned # Store the total number of comments
                if total_comments_returned:
                    comments = results['comments']
                    df = pd.DataFrame(comments)
                    df_list.append(df)
                else:
                    break # Break when no comments are returned
            offset = offset + 25 # Increment the counter since 25 comments are scraped each time
        except:
            print('HTTP Error encountered midway. Retrieval terminated at page {}.'.format(page))
            break

    if total_comments:
        if printout:
            print('Retrieved ' + str(total_comments) + ' comments')
        comments_df = pd.concat([df for df in df_list])
        comments_df.drop_duplicates(subset=['commentID'], inplace=True)
        comments_df['inReplyTo'] = None
        comments_df = get_replies(comments_df)
    if save:
        comments_df.to_csv('Comments.csv', index=False)
    return comments_df, total_comments

def get_articles(ARTICLE_API_KEY, page_lower=0, page_upper=50, begin_date=None, end_date=None,
                 max_articles=100000, sort='newest', query=None, save=False, printout=True):
    '''Collects the data on the articles of NYT using NYT articles search API, processes the
    articles' data and returns a pandas dataframe for articles.'''

    params = {'api-key': ARTICLE_API_KEY}

    if page_lower<0:
        page_lower = 0

    if page_upper>=200:
        page_upper = 199

    if sort=='oldest':
        if begin_date is None:
            begin_date = '20081030'
    elif end_date is None:
        end_date = datetime.today().strftime('%Y%m%d')

    if begin_date: # Check begin_date is not None
        try:
            begin_date = pd.to_datetime(begin_date, errors='coerce').strftime('%Y%m%d')
        except:
            print("Error: Please try again with begin_date entered in the format %Y%m%d.")
        params['begin_date'] = begin_date

    if end_date: # Check end_date is not None
        try:
            end_date = pd.to_datetime(end_date, errors='coerce').strftime('%Y%m%d')
        except:
            print("Error: Please try again with end_date entered in the format %Y%m%d.")
        params['end_date'] = end_date

    if query:
        params['q'] = query

    params['sort'] = sort

    articles_list = []
    articles_df = pd.DataFrame()

    total_articles = 0

    for page in range(page_lower, page_upper):
        if total_articles < max_articles:
            try:
                sleep(1)

                params['page'] = page # Every page has 10 articles
                if printout:
                    print("Page: ", page)

                # Using NYT API to scrap articles
                search_url = NYT_ARTICLE_API_URL + '?' + urlencode(params)
                file = urlopen(search_url).read() # Get the articles search data and read it into a string
                js = json.loads(file) # Load the articles search data as json

                if js['status'] == 'OK':
                    docs = js['response']['docs']
                    docs_length = len(docs)
                    if docs_length==0:
                        if printout:
                            print("No articles found on page", page)
                        break
                    for i in range(docs_length):
                        article = docs[i]
                        article_url = article['web_url'] # Get the url for the article
                        if printout:
                            print("Article url:", article_url)
                        articles_list.append(article)
            except:
                print('HTTP Error encountered midway. Retrieval terminated at page {}.'.format(page))
                break

    articles_df = pd.DataFrame(articles_list)
    articles_df = preprocess_articles_dataframe(articles_df)

    if printout:
        if page_lower > page_upper:
            print("page_lower value is greater than the page_upper. No articles returned.")
        elif (begin_date is not None) & (end_date is not None):
            if begin_date>end_date:
                print("begin_date is bigger than the end_date. No articles returned.")
            else:
                print()
                print("Total articles stored: ", articles_df.shape[0])
        else:
            print()
            print("Total articles stored: ", articles_df.shape[0])
    if save:
        articles_df.to_csv('Articles.csv', index=False)
    return articles_df

def get_replies(df):
    '''Extracts the replies to the comments and returns the dataframe
    including the replies as comments'''

    if 'replyCount' in df.columns:
        selected_df = df.loc[df.replyCount>0]
        if selected_df.shape[0] > 0:
            df_list = []
            for idx, row in selected_df.iterrows():
                replies_df = pd.DataFrame(row.replies)
                replies_df['inReplyTo'] = row.commentID
                df_list.append(replies_df)
            all_replies_df = pd.concat([replies_df for replies_df in df_list])
            all_replies_df = get_replies(all_replies_df)
            df = pd.concat([df, all_replies_df])
    return df

def preprocess_comments_dataframe(df):
    '''Preprocesses the comments' dataframe'''

    df.reset_index(inplace=True, drop=True)
    df.drop(['replies'], inplace=True, axis=1)
    # Specify dtypes:
    df.approveDate = df.approveDate.astype('int64')
    df.articleID = df.articleID.astype('category')
    df.commentID = df.commentID.astype('category')
    df.commentTitle = df.commentTitle.astype('category')
    df.commentType = df.commentType.astype('category')
    df.createDate = df.createDate.astype('int64')
    df.inReplyTo = df.inReplyTo.fillna(0).astype('int32')
    df.newDesk = df.newDesk.astype('category')
    df.parentID = df.parentID.fillna(0).astype('category')
    df.parentUserDisplayName = df.parentUserDisplayName.astype('category')
    df.permID = df.permID.astype('category')
    df.picURL = df.picURL.astype('category')
    df.printPage = df.printPage.astype('int32')
    df.recommendations = df.recommendations.astype('int64')
    df.recommendedFlag  = df.recommendedFlag.astype('category')
    df.replyCount = df.replyCount.astype('int32')
    df.reportAbuseFlag = df.reportAbuseFlag.astype('category')
    df.sectionName = df.sectionName.astype('category')
    df.sharing = df.sharing.astype('category').cat.codes
    df.status = df.status.astype('category')
    df.timespeople = df.timespeople.astype('category').cat.codes
    df.trusted = df.trusted.astype('category').cat.codes
    df.updateDate = df.updateDate.astype('int64')
    df.userDisplayName = df.userDisplayName.astype('category')
    df.userID = df.userID.astype('category')#.cat.codes
    df.userLocation = df.userLocation.astype('category')
    df.userTitle = df.userTitle.astype('category')
    df.userURL = df.userURL.astype('category')
    return df

def preprocess_articles_dataframe(df):
    '''Preprocesses the articles' dataframe'''
    df.reset_index(inplace=True, drop=True)
    df.drop(['blog', 'score', 'uri'], axis=1, inplace=True)
    df = df.rename(columns = {'_id': 'articleID', 'document_type': 'documentType',
                'new_desk': 'newDesk', 'print_page': 'printPage', 'pub_date': 'pubDate',
                'section_name': 'sectionName', 'type_of_material': 'typeOfMaterial',
                'web_url': 'webURL', 'word_count': 'articleWordCount'})

    if 'printPage' in df.columns:
        df.printPage.fillna(0, inplace=True)
    else:
        df['printPage'] = 0

    if 'sectionName' in df.columns:
        df.sectionName.fillna('Unknown', inplace=True)
    else:
        df['sectionName'] = 'Unknown'

    if 'newDesk' in df.columns:
        df.newDesk.fillna('Unknown', inplace=True)
    else:
        df['newDesk'] = 'Unknown'

    if 'typeOfMaterial' in df.columns:
        df.typeOfMaterial.fillna('Unknown', inplace=True)
    else:
        df['typeOfMaterial'] = 'Unknown'

    if 'byline' in df.columns:
        df.loc[~df.byline.isnull(), 'byline'] = df.loc[~df.byline.isnull(),
                                'byline'].apply(lambda x: x['original']).fillna('By UNKNOWN')
        df.loc[df.byline.isnull(), 'byline'] = 'By UNKNOWN'
    else:
        df['byline'] = 'By UNKNOWN'

    if 'headline' in df.columns:
        df.headline = df.headline.apply(lambda x: x['print_headline']).fillna('Unknown').replace('', 'Unknown')
    else:
        df['headline'] = 'Unknown'

    df.keywords = df.keywords.apply(lambda keywords: [keyword['value'] for keyword in keywords])

    if 'multimedia' in df.columns:
        df.multimedia = df.multimedia.apply(lambda x: len(x))
    else:
        df['multimedia'] = 0

    if 'pubDate' in df.columns:
        df.pubDate = pd.to_datetime(df.pubDate, errors='coerce')

    # Specify dtypes:
    df.articleID = df.articleID.astype('category')
    df.byline = df.byline.astype('category')
    df.documentType = df.documentType.astype('category')
    df.newDesk = df.newDesk.astype('category')
    df.printPage = df.printPage.astype('int32')
    df.sectionName = df.sectionName.astype('category')
    df.source = df.source.astype('category')
    df.typeOfMaterial = df.typeOfMaterial.astype('category')
    df.webURL = df.webURL.astype('category')
    return df


from augment import augment_images
from datetime import datetime
from helpers import get_files, log_dir_or_None, write_log, LINE
import numpy as np
import os
import pandas as pd
from pathlib import Path
import re
import skimage.filters as filters
from skimage.measure import regionprops
from skimage.morphology._util import _offsets_to_raveled_neighbors
from tifffile import TiffWriter, imread
from time import time
import torch
from tqdm import tqdm
import zarr

# -------------------
# Generate Train Data
# -------------------
def get_train_data(
                   image_paths,
                   labels_paths,
                   out_dir,
                   shape=(10, 256, 256),
                   n_each=100,
                   channels=('z-1', 'y-1', 'x-1', 'centreness'),
                   scale=(4, 1, 1),
                   log=True
                   ):
    """
    Generate training data from whole ground truth volumes.

    Parameters
    ----------
    image_path: str
        path to the image zarr file, which must be in tczyx dim format (ome bioformats)
    labels_path: str
        path to the labels zarr file, which must be in tczyx dim format
    shape: tuple of int
        Shape of the test data to
    channels: tuple of str
        tuple of channels to be added to the training data.
            Affinities: 'axis-n' (pattern: r'[xyz]-\d+' e.g., 'z-1')
            Centreness: 'centreness'
    scale: tuple of numeric
        Scale of channels. This is used in calculating centreness score.
    log: bool
        Should print out be recorded in out_dir/log.txt?

    Returns
    -------
    xs: list of torch.Tensor
        List of images for training
    ys: list of torch.Tensor
        List of affinities for training
    ids: list of str
        ID strings by which each image and label are named.
        Eventually used for correctly labeling network output

    Notes
    -----
    It takes a very long time to obtain training data with sufficient
    information (as determined by min_affinity param).
    """
    assert len(image_paths) == len(labels_paths)
    for i in range(len(image_paths)):
        print(LINE)
        s = f'Generating training data from image: {image_paths[i]}, labels: {labels_paths[i]}'
        print(s)
        if log:
            write_log(LINE, out_dir)
            write_log(s, out_dir)
        im = zarr.open_array(image_paths[i])
        l = zarr.open_array(labels_paths[i])
        if i == 0:
            xs, ys, ids = get_random_chunks(im, l,
                                            out_dir,
                                            shape=shape,
                                            n=n_each,
                                            channels=channels,
                                            scale=scale,
                                            log=log)
        else:
            xs_n, ys_n, ids_n = get_random_chunks(im, l,
                                                  out_dir,
                                                  shape=shape,
                                                  n=n_each,
                                                  channels=channels,
                                                  scale=scale,
                                                  log=log)
            for j in range(len(xs_n)):
                xs.append(xs_n[j])
                ys.append(ys_n[j])
                ids.append(ids_n[j])
    return xs, ys, ids

def get_random_chunks(
                      image,
                      labels,
                      out_dir,
                      shape=(10, 256, 256),
                      n=25,
                      min_affinity=300,
                      channels=('z-1', 'y-1', 'x-1', 'centreness'),
                      scale=(4, 1, 1),
                      log=True
                      ):
    '''
    Obtain random chunks of data from whole ground truth volumes.

    Parameters
    ----------
    image: array like
        same shape as labels
    labels: array like
        same shape as image
    shape: tuple of int
        shape of chunks to obtain
    n: int
        number of random chunks to obtain
    min_affinity:
        minimum cut off sum of affinities for an image.
        As affinities as belong to {0, 1}, this param
        is the number of voxels that boarder labels.

    Returns
    -------
    xs: list of torch.Tensor
        List of images for training
    ys: list of torch.Tensor
        List of affinities for training
    ids: list of str
        ID strings by which each image and label are named.
        Eventually used for correctly labeling network output

    '''
    im = np.array(image)
    l = np.array(labels)
    assert len(im.shape) == len(shape)
    a = get_training_labels(l, channels=channels, scale=scale)
    xs = []
    ys = []
    labs = []
    i = 0
    df = {'z_start' : [],
          'y_start' : [],
          'x_start' : []}
    while i < n:
        dim_randints = []
        for j, dim in enumerate(shape):
            max_ = im.shape[j] - dim - 1
            ri = np.random.randint(0, max_)
            dim_randints.append(ri)
        # Get the network output: affinities
        s_ = [slice(None, None),] #
        for j in range(len(shape)):
            s_.append(slice(dim_randints[j], dim_randints[j] + shape[j]))
        s_ = tuple(s_)
        y = a[s_]
        if y.sum() > min_affinity * len(channels): # if there are a sufficient number of boarder voxels
            # add coords to output df
            for j in range(len(shape)):
                _add_to_dataframe(j, dim_randints[j], df)
            # Get the network input: image
            s_ = [slice(dim_randints[j], dim_randints[j] + shape[j]) for j in range(len(shape))]
            s_ = tuple(s_)
            x = im[s_]
            x = normalise_data(x)
            # get the GT labels so that later quatitative comparison can be made with final
            #   segmentation
            lab = l[s_] # that's right, be confused by my variable names!!
            # data augmentation for better generalisation
            x, y, lab = augment_images(x, y, lab)
            # add the affinities and image chunk to the training data
            y = torch.from_numpy(y.copy())
            ys.append(y)
            x = torch.from_numpy(x.copy())
            xs.append(x)
            labs.append(lab)
            # another successful addition, job well done you crazy mofo
            i += 1
    print(LINE)
    s = f'Obtained {n} {shape} chunks of training data'
    print(s)
    if log:
        write_log(LINE, out_dir)
        write_log(s, out_dir)
    log_dir = log_dir_or_None(log, out_dir)
    print_labels_info(channels, out_dir=log_dir)
    ids = save_random_chunks(xs, ys, labs, out_dir)
    now = datetime.now()
    d = now.strftime("%y%m%d_%H%M%S")
    df['data_ids'] = ids
    df = pd.DataFrame(df)
    df.to_csv(os.path.join(out_dir, 'start_coords' + d + '.csv'))
    return xs, ys, ids

def _add_to_dataframe(dim, start, df):
    if dim == 0:
        df['z_start'].append(start)
    if dim == 1:
        df['y_start'].append(start)
    if dim == 2:
        df['x_start'].append(start)

# --------------------------
# Lable Generating Functions
# --------------------------

def get_training_labels(
                        l,
                        channels=('z-1', 'y-1', 'x-1', 'centreness'),
                        scale=(4, 1, 1)):
    labels = []
    for chan in channels:
        if chan.startswith('z'):
            axis = 0
        elif chan.startswith('y'):
            axis = 1
        elif chan.startswith('x'):
            axis = 2
        n = re.search(r'\d+', chan)
        if n is not None:
            # get the nth affinity
            n = int(n[0])
            lab = nth_affinity(l, n, axis)
        elif chan == 'centreness':
            # get the centreness score
            lab = get_centreness(l, scale=scale)
        elif chan == 'centreness-log':
            lab = get_centreness(l, scale=scale, log=True)
        elif chan == 'centroid-gauss':
            lab = get_gauss_centroids(l)
        else:
            m = f'Unrecognised channel type: {chan} \n'
            m = m + 'Please enter str of form axis-n for nth affinity \n'
            m = m + 'or centreness for centreness score.'
            raise ValueError(m)
        if chan.endswith('-smooth'):
            lab = smooth(lab)
        labels.append(lab)
    labels = np.stack(labels, axis=0)
    return labels

def nth_affinity(labels, n, axis):
    affinities = []
    labs_pad = np.pad(labels, n, mode='reflect')
    for i in range(labels.shape[axis]):
        s_0 = [slice(None, None)] * len(labs_pad.shape)
        s_0[axis] = slice(i, i + 1)
        s_0 = tuple(s_0)
        s_n = [slice(None, None)] * len(labs_pad.shape)
        s_n[axis] = slice(i + n, i + n + 1)
        s_n = tuple(s_n)
        new = labs_pad[s_0] - labs_pad[s_n]
        new = np.squeeze(new)
        if len(new) > 0:
            affinities.append(new)
    affinities = np.stack(affinities, axis=axis)
    s_ = [slice(n, -n)] * len(labs_pad.shape)
    s_[axis] = slice(None, None)
    s_ = tuple(s_)
    affinities = affinities[s_]
    affinities = np.where(affinities != 0, 1., 0.)
    return affinities

def get_centreness(labels, scale=(4, 1, 1), log=False, power=False):
    """
    Obtains a centreness score for each voxel belonging to a labeled object.
    Values in each object sum to one. Values are inversely proportional
    to euclidian distance from the object centroid.

    Notes
    -----
    Another possible implementation would involve the medioid, as in:
    Lalit, M., Tomancak, P. and Jug, F., 2021. Embedding-based Instance
    Segmentation of Microscopy Images. arXiv.

    Unfortunately, skimage doesn't yet have a method for finding the
    medioid (more dev, *sigh*).
    """
    scale = np.array(scale)
    def dist_score(mask):
        output = np.zeros_like(mask, dtype=np.float32)
        c = np.mean(np.argwhere(mask), axis=0)
        indices, values = inverse_dist_score(
                mask, c, scale, log=log, power=power
                )
        output[indices] = values
        return output
    t = time()
    props = regionprops(labels, extra_properties=(dist_score,))
    new = np.zeros(labels.shape, dtype=np.float32)
    for i, prop in tqdm(enumerate(props), desc='Score centreness'):
        new[prop.slice] += prop.dist_score
    new = np.nan_to_num(new)
    print('------------------------------------------------------------')
    print(f'Obtained centreness scores in {time() - t} seconds')
    return new

def inverse_dist_score(mask, centroid, scale, log, power):
    '''
    Compute euclidian distances of each index from a mask
    representing a single object from the centroid of said object

    Uses scale to account for annisotropy in image
    '''
    indices = np.argwhere(mask > 0)
    distances = []
    centre = centroid
    for i in range(indices.shape[0]):
        ind = indices[i, ...]
        diff = (centre - ind) * scale
        dist = np.linalg.norm(diff)
        if log and abs(dist) > 0:
            m = f'Infinite value with distance of {dist}'
            dist = np.log(dist)
            assert not np.isinf(dist), m
        if power:
            dist = 2 ** dist
        distances.append(dist)
    distances = np.array(distances)
    if log:
        distances = distances + np.abs(distances.min()) # bring min value to 0
    norm_distances = distances / distances.max()
    values = (1 - norm_distances)
    indices = tuple(indices.T.tolist())
    return indices, values

# not used
def get_gauss_centroids(labels, sigma=1, z=0):
    centroids = [prop['centroid'] for prop in regionprops(labels)]
    centroids = tuple(np.round(np.stack(centroids).T).astype(int))
    centroid_image = np.zeros(labels.shape, dtype=float)
    centroid_image[centroids] = 1.
    gauss_cent = []
    for i in range(labels.shape[z]):
        s_ = [slice(None, None)] * labels.ndim
        s_[z] = slice(i, i+1)
        s_ = tuple(s_)
        plane = np.squeeze(centroid_image[s_])
        gauss_cent.append(filters.gaussian(plane, sigma=sigma))
    out = np.stack(gauss_cent, axis=z)
    out = out - out.min()
    out = out / out.max()
    #print(out.dtype, out.shape, out.max(), out.min())
    return out

def smooth(image, z=0, sigma=1):
    out = []
    for i in range(image.shape[z]):
        s_ = [slice(None, None)] * image.ndim
        s_[z] = slice(i, i+1)
        s_ = tuple(s_)
        plane = np.squeeze(image[s_])
        out.append(filters.gaussian(plane, sigma=sigma))
    out = np.stack(out, axis=z)
    return out

# not currently referenced, uses nth_affinity() for generality
def get_affinities(image):
    """
    Get short-range voxel affinities for a segmentation. Affinities are
    belonging to {0, 1} where 1 represents a segment boarder voxel in a
    particular direction. Affinities are produced for each dimension of
    the labels and each dim has its own channel (e.g, (3, z, y, x)).

    Note
    ----
    Others may represent affinities with {-1, 0}, because technically...
    My network wasn't designed for this :)
    """
    padded = np.pad(image, 1, mode='reflect')
    affinities = []
    for i in range(len(image.shape)):
        a = np.diff(padded, axis=i)
        a = np.where(a != 0, 1.0, 0.0)
        a = a.astype(np.float32)
        s_ = [slice(1, -1)] * len(image.shape)
        s_[i] = slice(None, -1)
        s_ = tuple(s_)
        affinities.append(a[s_])
    affinities = np.stack(affinities)
    return affinities

# -------------
# Log and Print
# -------------

def print_labels_info(channels, out_dir=None, log_name='log.txt'):
    print(LINE)
    s = f'Training labels have {len(channels)} output channels: \n'
    print(s)
    if out_dir is not None:
            write_log(LINE, out_dir, log_name)
            write_log(s, out_dir, log_name)
    for i, chan in enumerate(channels):
        affinity_match = re.search(r'[xyz]-\d*', chan)
        if affinity_match is not None:
            n = f'{affinity_match[0]} affinities'
        elif chan == 'centreness':
            n = 'centreness score'
        elif chan == 'centreness-log':
            n = 'log centreness score'
        elif chan == 'centroid-gauss':
            n = 'gaussian centroids'
        else:
            n = 'Unknown channel type'
        s = f'Channel {i}: {n}'
        print(s)
        if out_dir is not None:
            write_log(s, out_dir, log_name)

# -----------
# Save Output
# -----------

def save_random_chunks(xs, ys, labs, out_dir):
    '''
    Save the random chunks as they are sampled
    '''
    os.makedirs(out_dir, exist_ok=True)
    assert len(xs) == len(ys)
    ids = []
    # iterate over the sample
    for i in range(len(xs)):
        # get the datetime to give the samples unique names
        now = datetime.now()
        d = now.strftime("%y%m%d_%H%M%S") + '_' + str(i)
        ids.append(d)
        # save the image
        i_name = d + '_image.tif'
        i_path = os.path.join(out_dir, i_name)
        with TiffWriter(i_path) as tiff:
            tiff.write(xs[i].numpy())
        # save the labels
        l_name = d + '_labels.tif'
        l_path = os.path.join(out_dir, l_name)
        with TiffWriter(l_path) as tiff:
            tiff.write(ys[i].numpy())
        # save the ground truth segmentation
        s_name = d + '_GT.tif'
        s_path = os.path.join(out_dir, s_name)
        with TiffWriter(s_path) as tiff:
            tiff.write(labs[i]) # this is already ndarray not tensor
    assert len(ids) == len(ys)
    print('------------------------------------------------------------')
    print('Training data saved at:')
    print(out_dir)
    return ids

# ---------------
# Load Train Data
# ---------------

def load_train_data(
                    data_dir,
                    id_regex=r'\d{6}_\d{6}_\d{1,3}',
                    x_regex=r'\d{6}_\d{6}_\d{1,3}_image.tif',
                    y_regex=r'\d{6}_\d{6}_\d{1,3}_labels.tif'
                    ):
    '''
    Load train data from a directory according to a naming convention

    Parameters
    ----------
    data_dir: str
        Directory containing data
    id_regex: r string
        regex that will be used to extract IDs that will
        be used to label network output
    x_regex: r string
        regex that represents image file names, complete with
        extension (tiff please)
    y_regex: r string
        regex that represents training label files, complete
        with extension (tiff please)

    Returns
    -------
    xs: list of torch.Tensor
        List of images for training
    ys: list of torch.Tensor
        List of affinities for training
    ids: list of str
        ID strings by which each image and label are named.
        Eventually used for correctly labeling network output

    '''
    # Get file names for images and training labels
    x_paths, y_paths = get_files(
                                 data_dir,
                                 x_regex=x_regex,
                                 y_regex=x_regex
                                 )
    # Get IDs
    id_pattern = re.compile(id_regex)
    ids = []
    x_paths.sort()
    y_paths.sort()
    for i in range(len(x_paths)):
        xn = Path(x_paths[i]).stem # this could have been avoided
        yn = Path(y_paths[i]).stem # why would I bother now though?!
        # assumes there will be a match for each
        xid = id_pattern.search(xn)[0]
        yid = id_pattern.search(yn)[0]
        m = 'There is a mismatch in image and label IDs'
        assert xid == yid, m
        ids.append(xid)
    # Get images and training labels in tensor form
    xs = []
    ys = []
    for i in range(len(x_paths)):
        xp = x_paths[i]
        yp = y_paths[i]
        x = imread(xp)
        x = normalise_data(x)
        y = imread(yp)
        xs.append(torch.from_numpy(x))
        ys.append(torch.from_numpy(y))
    # returns objects in the same manner as get_train_data()
    print('------------------------------------------------------------')
    print(f'Loaded {len(xs)} sets of training data')
    print_labels_info(ys[0].shape)
    return xs, ys, ids

# ------------------------
# General Helper Functions
# ------------------------

def normalise_data(image):
    '''
    Bring image values to between 0-1

    Parameters
    ----------
    image: np.array
        Image data. Dtype should be float.
    '''
    im = image / image.max()
    return im

if __name__ =="__main__":
    #
    pass


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/6/6 15:45
# @Author  : snaps

from __future__ import absolute_import, unicode_literals
from celery import Celery

app = Celery('runcelery',
             broker='redis://localhost:6379/0',
             backend='redis://localhost:6379/0',
             include = ['tasks'])
# app.config_from_object('celeryconfig')

app.conf.update(
    result_expres=3600,
)

if __name__ == '__main__':
    app.start()


from django.shortcuts import render, reverse, HttpResponseRedirect
from django.contrib.auth.decorators import login_required
from notification.models import Notification
from twitteruser.models import TwitterUser

@login_required
def notif(request, username):
    notif_count = Notification.objects.filter(
        target_user=request.user).filter(tweet_visibility=True).count()
    actual_notifs = Notification.objects.filter(
        target_user=request.user).filter(tweet_visibility=True)
    for notif in actual_notifs:
        notif.tweet_visibility = False
        notif.save()
    return render(request, 'notifications.html', {
        'actual_notifs': actual_notifs,
        'notif_count': notif_count})


import simpy
from math import sqrt
from Framework import PropagationModel
from Framework.AirInterface import AirInterface
from Framework.EnergyProfile import EnergyProfile
from Framework.Gateway import Gateway
from Framework.MessagePacket import MessagePacket
from Framework.LoRaParameters import LoRaParameters
from Framework.Node import Node
from Framework.SNRModel import SNRModel
from Simulations.GlobalConfig import *
from Framework.Router import Router

tx_power_mW = {2: 91.8, 5: 95.9, 8: 101.6, 11: 120.8, 14: 146.5}
rx_measurements = {'pre_mW': 8.2, 'pre_ms': 3.4, 'rx_lna_on_mW': 39,
                   'rx_lna_off_mW': 34,
                   'post_mW': 8.3, 'post_ms': 10.7}

def run_helper(args):
    return run(*args)

def run(message_packet, locs, p_size, sigma, sim_time, gateway_location, num_nodes, num_routers,transmission_rate, confirmed_messages, adr):
    sim_env = simpy.Environment()
    gateway = Gateway(sim_env, gateway_location, max_snr_adr=True, avg_snr_adr=False)
    nodes = []
    routers = []
    node_coordinates = {}
    router_coordinates = {}
    # {router_A: [6,12]}
    air_interface = AirInterface(gateway, PropagationModel.LogShadow(std=sigma), SNRModel(), sim_env)

    # for router_id in range(num_routers):
    #     energy_profile = EnergyProfile(5.7e-3, 15, tx_power_mW,
    #                                    rx_power=rx_measurements)
    #
    #
    #     _sf = np.random.choice(LoRaParameters.SPREADING_FACTORS)
    #     if start_with_fixed_sf:
    #         _sf = start_sf
    #     lora_param = LoRaParameters(freq=np.random.choice(LoRaParameters.DEFAULT_CHANNELS),
    #                                 sf=_sf,
    #                                 bw=125, cr=5, crc_enabled=1, de_enabled=0, header_implicit_mode=0, tp=14)
    #     router = Router(message_packet, router_id, energy_profile, lora_param, sleep_time=(8 * p_size / transmission_rate),
    #                 processs_time=5,
    #                 adr=adr,
    #                 location=locs[router_id],
    #                 base_station=gateway, env=sim_env, payload_size= p_size, coordinates=[0, 0], air_interface=air_interface,
    #                 confirmed_messages=confirmed_messages)
    #     routers.append(router)
    #     router_coordinates[router_id] = router.coordinates
    #     sim_env.process(router.run())

    for node_id in range(10):
        energy_profile = EnergyProfile(5.7e-3, 15, tx_power_mW,
                                       rx_power=rx_measurements)

        _sf = np.random.choice(LoRaParameters.SPREADING_FACTORS)
        if start_with_fixed_sf:
            _sf = start_sf
        lora_param = LoRaParameters(freq=np.random.choice(LoRaParameters.DEFAULT_CHANNELS),
                                    sf=_sf,
                                    bw=125, cr=5, crc_enabled=1, de_enabled=0, header_implicit_mode=0, tp=14)
        node = Node(message_packet, node_id, energy_profile, lora_param, sleep_time=(8 * p_size / transmission_rate),
                    process_time=5,
                    adr=adr,
                    location=locs[node_id],
                    base_station=gateway, env=sim_env, payload_size= p_size, coordinates=[0, 0], air_interface=air_interface,
                    confirmed_messages=confirmed_messages)
        nodes.append(node)
        node_coordinates[node_id] = node.coordinates
        sim_env.process(node.run())

        # def closest_router():
        #     distances = []
        #
        #     for key, value in router_coordinates.items():
        #         dist = sqrt((router_coordinates[key][0] - node.coordinates[0])**2 + (router_coordinates[key][1] - node.coordinates[1])**2)
        #         distances.append(dist)
        #     distances.sort()
        #     node.base_station = 0
        # closest_router()

    sim_env.run(until=sim_time)

    # Simulation is done.
    # process data

    mean_energy_per_bit_list = list()
    for n in nodes:
        mean_energy_per_bit_list.append(n.energy_per_bit())

    data_mean_nodes = Node.get_mean_simulation_data_frame(nodes, name=sigma) / (
        num_nodes)

    data_gateway = gateway.get_simulation_data(name=sigma) / num_nodes
    data_air_interface = air_interface.get_simulation_data(name=sigma) / (
        num_nodes)

    # eff_en = data_mean_nodes['TotalEnergy'][sigma] / (p_size*data_mean_nodes['UniquePackets'][sigma])
    # print('Eb {} for Size:{} and Sigma:{}'.format(eff_en, p_size, sigma))

    return {
        'message_as_packet': MessagePacket.message_packets,
        'Custom Payload': MessagePacket.packets,
        'mean_nodes': data_mean_nodes,
        'gateway': data_gateway,
        'air_interface': data_air_interface,
        'path_loss_std': sigma,
        'payload_size': p_size,
        'mean_energy_all_nodes': mean_energy_per_bit_list

    }


# imports
from django.conf.urls import patterns, include, url
from django.contrib import admin
from rest_framework import routers
from app import views

admin.autodiscover()

# router config
router = routers.DefaultRouter()
router.register(r'users',views.UserViewSet)
router.register(r'useraccount',views.UserAccountViewSet)
router.register(r'useraccountinfo',views.UserAccountInfoViewSet)
router.register(r'addresses',views.AddressesViewSet)
router.register(r'customeraccountinfo',views.CustomerAccountInfoViewSet)
router.register(r'artistaccountinfo',views.ArtistAccountInfoViewSet)

# url patterns
urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'tutorial.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    url(r'^admin/', include(admin.site.urls)),
	url(r'^', include(router.urls)),
	url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue May 25 12:54:36 2021

@author: nicholasmcquibban
"""

import os
import regex as re
import datetime

import pandas as pd
from tqdm import tqdm
import spacy
from spacy.tokens import DocBin

start_time = datetime.datetime.now() #grabs start time

message = '' #empty string to hold message with time in it

PMC_files=[] #empty list to hold all PMC filenames in it for iteration

all_files = os.listdir() #make sure to run this algorithm in the directory containing the PMCID.txt files

for n in all_files:

    if n.startswith('PMC'):  #gets a list of every PMC file in the directory
        PMC_files.append(n)

content = ()

tuple_list = []

all_sentences = []

for in_file in PMC_files: #for each PMC M&M file do

    with open(in_file) as m_file:

        data = m_file.read()

        illumina_sentence = re.findall(r"([^.]*?Illumina[^.]*\d.\d*[^.]*.)", data)

        all_sentences += illumina_sentence

        affymetrix_sentence = re.findall(r"([^.]*?Affymetrix[^.]*\d.\d*[^.]*.)", data)

        all_sentences += affymetrix_sentence

        perlegen_sentence = re.findall(r"([^.]*?Perlegen[^.]*\d.\d*[^.]*.)", data)

        all_sentences += perlegen_sentence

#all_sentences = impute_sentences + beagle_sentences + mach_sentences

        #content.append()

#print(all_sentences)

all_sentences_nodupes = list(set(all_sentences))

#print(all_sentences_nodupes)

with open('TEST2_DATA', 'a') as out_file:

    #/project/home20/nam220/NER/NLP/

    for n in all_sentences_nodupes:

        if " or " in n:

            continue

        if " Array Sets " in n:

            continue

        if "(Illumina)" in n:

            continue

        if "(Affymetrix)" in n:

            continue

        if "(Supplementary Table 4)" in n:

            continue

        if "Illumina" and "Affymetrix" in n:

            continue

        entity_list = []

        match_Iplat = re.search(r"(Illumina.*?platform[s]?)", n, re.IGNORECASE)
        match_IaN = re.search(r"(Illumina.*?array[s]?.\d[.]?\d)", n, re.IGNORECASE)
        match_Ia = re.search("(Illumina.*?array[s]?)", n, re.IGNORECASE)
        match_Ic = re.search(r"(Illumina.*? chip[s]?)", n, re.IGNORECASE)
        match_IBC = re.search(r"(Illumina.*?beadchip[s]?)", n, re.IGNORECASE)
        match_IK = re.search(r"(Illumina.\d+.?[K])", n, re.IGNORECASE)
        match_IW = re.search(r"(Illumina.\d+.?[W])", n, re.IGNORECASE)

        match_Ac = re.search(r"(Affymetrix.*? chip[s]?)", n, re.IGNORECASE)
        match_AaN = re.search(r"(Affymetrix.*?array[s]?.\d[.]?\d)", n, re.IGNORECASE)
        match_Aa = re.search(r"(Affymetrix.*?array[s]?)", n, re.IGNORECASE)
        match_Aplat = re.search(r"(Affymetrix.*?platform[s]?)", n, re.IGNORECASE)
        match_AK = re.search(r"(Affymetrix.\d+.?[K])", n, re.IGNORECASE)

        match_Pc = re.search(r"(Perlegen.*? chip[s]?)", n, re.IGNORECASE)
        match_PK = re.search(r"(Perlegen.\d+.?[K])", n, re.IGNORECASE)

        match_V = re.search(r"(v\d+[.]?\d+)", n, re.IGNORECASE)

        if match_Iplat:
            #print(match)

            info = (match_Iplat.start(), match_Iplat.end(), 'Illumina assay')
            entity_list.append(info)

        elif match_IBC:

            info = (match_IBC.start(), match_IBC.end(), 'Illumina assay')
            entity_list.append(info)

        elif match_IaN:

            info = (match_IaN.start(), match_IaN.end(), 'Illumina assay')
            entity_list.append(info)

        elif match_Ia:

            info = (match_Ia.start(), match_Ia.end(), 'Illumina assay')
            entity_list.append(info)

        elif match_Ic:

            info = (match_Ic.start(), match_Ic.end(), 'Illumina assay')
            entity_list.append(info)

        elif match_IK:

            info = (match_IK.start(), match_IK.end(), 'Illumina assay')
            entity_list.append(info)

        elif match_IW:

            info = (match_IW.start(), match_IW.end(), 'Illumina assay')
            entity_list.append(info)

        if match_Ac:

            info = (match_Ac.start(), match_Ac.end(), 'Affymetrix assay')
            entity_list.append(info)

        elif match_AaN:

            info = (match_AaN.start(), match_AaN.end(), 'Affymetrix assay')
            entity_list.append(info)

        elif match_Aa:

            info = (match_Aa.start(), match_Aa.end(), 'Affymetrix assay')
            entity_list.append(info)

        elif match_Aplat:

            info = (match_Aplat.start(), match_Aplat.end(), 'Affymetrix assay')
            entity_list.append(info)

        elif match_AK:

            info = (match_AK.start(), match_AK.end(), 'Affymetrix assay')
            entity_list.append(info)

        if match_Pc:

            info = (match_Pc.start(), match_Pc.end(), 'Perlegen assay')
            entity_list.append(info)

        elif match_PK:

            info = (match_PK.start(), match_PK.end(), 'Perlegen assay')
            entity_list.append(info)

        content = (n, {'entities': entity_list})

        tuple_list.append(content)

    print(tuple_list)

    for element in tuple_list:

        if element == tuple_list[0]:

            out_file.write('[')
            out_file.write(str(element))
            out_file.write(',')
            out_file.write('\n')

        elif element == tuple_list[-1]:

            out_file.write(' ')
            out_file.write(str(element))
            out_file.write(']')

        else:

            out_file.write(' ')
            out_file.write(str(element))
            out_file.write(',')
            out_file.write('\n')

    #print(content[0])

#('Full sentence appears here.', {'entities': [(start, stop, 'platform')]})

nlp = spacy.blank("en") # load a new spacy model
db = DocBin() # create a DocBin object

DEV_DATA = tuple_list

for text, annot in tqdm(DEV_DATA): # data in previous format
    doc = nlp.make_doc(text) # create doc object from text
    ents = []
    for start, end, label in annot["entities"]: # add character indexes
        span = doc.char_span(start, end, label=label, alignment_mode="contract")
        if span is None:
            print("Skipping entity")
        else:
            ents.append(span)
    doc.ents = ents # label the text with the ents
    db.add(doc)

db.to_disk("/project/home20/nam220/NER/NLP/test.spacy") # save the docbin object

#"/project/home20/nam220/NER/NLP/train.spacy"

stop_time = datetime.datetime.now() #grabs stop time

message = 'Start time is ' + str(start_time) + '\n' + 'Stop time is ' + str(stop_time)

with open('/project/home20/nam220/NER/NLP/TEST2_DATA_runtime.log', 'a') as time_file:   #start and stop time are written into a runtime.log file

    #'/project/home20/nam220/NER/NLP/
    time_file.write(message)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 23 12:00:22 2020.

@author: myo
"""
from nptdms import TdmsFile
import numpy as np
from pathlib import Path
import matplotlib.pyplot as plt
import pickle
from scipy import signal
import AxZoom
from scipy.ndimage import uniform_filter1d

class ExfoJobj(object):
    """
    Object to create and manage exfoliation tool data.

    Parameters
    ----------
        Paths
    Attributes
    ----------
        Scan Vectors
        Load Vector
    """

    scans = {'b': 'base',
             'w': 'wafer',
             'n': 'nickel',
             's': 'stress',
             'l': 'load',
             'e': 'exfo'}

    def __init__(self, name):
        """
        Init.

        Parameters
        ----------
        name : TYPE
            DESCRIPTION.

        Returns
        -------
        None.

        """
        self.name = name

    def Load_Scan(self, filepath, file, trim=False, scan=False, filt=None, llim=26, ulim=74):
        """
        Load scan.

        Parameters
        ----------
        filepath : String
            /path/to/file/
        file : String
            file (.tdms or .dat)
        scan : String
            'base'
            'wafer'
            'nickel'
            'stress'
            'load'
            'exfo'
        filt : Function, optional
            filtering function lzr = filt(lzr)

        Returns
        -------
        None.
        """
        if not scan:
            scan = self.scans.get(Path(file).name[0], 'base')
        print(scan)

        if Path(file).suffix == '.tdms':
            # import tdms - SLOW
            print('Loading ', Path(file).name)
            tdms_file = TdmsFile(Path(Path(filepath).joinpath(file)).with_suffix(".tdms"))
            group = tdms_file['Untitled']
            channel1 = group['encoder']
            enc = channel1.data
            # zero data and convert to mm
            # enc=(enc-min(enc)+1)/100

            time = group['Untitled']
            # elapsed time in seconds
            time = time - time[0]

            channel2 = group['laser']
            lzr = channel2.data
            # zero data and convert to mm
            # enc = (enc-min(enc)+1) / 100
            # convert to mm, but leave enc at lv zero
            # enc = enc / 100
            channel2 = group['load']
            lod = channel2.data

        elif 'trim' in Path(file).name:
            [time, enc, lzr, trim] = pickle.load(
                open(Path(Path(filepath).joinpath(file)).with_suffix(".dat"), 'rb'))
            print('Loading ', Path(file).name)

        elif Path(file).suffix == '.dat':
            if scan == 'load':
                [time, lod, enc, lzr] = pickle.load(
                    open(Path(Path(filepath).joinpath(file)).with_suffix(".dat"), 'rb'))
            else:
                [time, enc, lzr] = pickle.load(
                    open(Path(Path(filepath).joinpath(file)).with_suffix(".dat"), 'rb'))
            print('Loading ', Path(file).name)

        else:
            raise ValueError('file load error')

        if scan == 'load':
            self.Load_Load_Scan(time, lod, enc, lzr, filt=filt, llim=llim, ulim=ulim)
        else:
            setattr(self, scan, Scan(time, enc, lzr, trim=trim, filt=filt, llim=llim, ulim=ulim))

    def Load_Load_Scan(self, time, lod, enc, lzr, filt=None, llim=26, ulim=74):
        self.load = Lscan()
        tempst = np.stack([enc, lod, lzr, time])
        y = np.delete(tempst, (tempst[0, :] > ulim) | (tempst[0, :] < llim), 1)
        self.load.time = y[3, :]
        self.load.encr = y[0, :].copy()
        enc = signal.medfilt(y[0, :], 21)
        # convert to mm
        enc = enc / 100
        # shift from laser to roller
        enc = enc-32.6
        self.load.enc = y[0, :]
        self.load.lzrr = y[2, :].copy()
        self.load.lzr = y[2, :].copy()
        if filt:
            print('Filtering Laser')
            self.load.lzr = filt(lzr)
        # filters
        # lzr = signal.medfilt(lzr, 21)
        # self.trim = trim
        self.load.lod = y[1, :]
        # self.load.time = time
        self.load.enc = enc
        # self.load.lzr = lzr

    def Load_Batch(self, directory, trim=False, filt=None, llim=26, ulim=74):
        """
        Load a batch of scan data thats describes one exfoliation

        Parameters
        ----------
        directory : String
            'directory full of a set of scans. tmds and/or dat'
        trim : array or slice, optional
            Specify which scans to skip. The default is False.
        filt : Function, optional
            filtering function lzr = filt(lzr)

        Returns
        -------
        None.
        """

        if type(filt) is int:
            filt = exFilter(filt)

        names = set([Path(d.stem).stem for d in [*Path(directory).iterdir()]])
        for n in names:
            filepath = directory
            if Path(directory).joinpath(n).with_suffix('.trim.dat').exists():
                file = Path(n).with_suffix('.trim.dat')
            elif Path(directory).joinpath(n).with_suffix('.dat').exists():
                file = Path(n).with_suffix('.dat')
            elif Path(directory).joinpath(n).with_suffix('.tdms').exists():
                file = Path(n).with_suffix('.tdms')
                print(file)
                self.Load_Scan(filepath, file, trim, scan=False, filt=None, llim=llim, ulim=ulim)
                scan = self.scans.get(Path(file).name[0], 'base')
                getattr(self, scan).Pickle_TDMS(directory, file)
            else:
                print('load error')
                print(n)
                continue
            self.Load_Scan(filepath, file, trim, scan=False, filt=filt, llim=llim, ulim=ulim)

    def Get_Stress(self, win=50, wafer_thickness=580E-6, filt=None):

        if type(filt) is int:
            filt = sFilter(self, filt)

        MSi = 1.803E11
        ENi = 180E9
        nuNi = .31
        # wafer_thickness = 500E-6
        # s = filt(self.sync_stress)
        z = np.polyfit(self.sync_stress_x, self.sync_stress, 2)
        p = np.poly1d(z)
        # s = p(self.sync_stress_x)
        dp = np.polyder(p)
        ds = dp(self.sync_stress_x)
        ddp = np.polyder(p, 2)
        dds = ddp(self.sync_stress_x)
        R = abs(((1+ds**2)**1.5)/dds)
        # mask = np.isnan(R)
        # R[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), R[~mask])
        ni = self.ni
        if filt:
            # R = filt(R)
            ni = filt(self.ni)
        # Convert to meters
        R = R*1E-3
        self.S = (MSi*wafer_thickness**2)/(6*ni*R)
        +(2*ni*ENi)/(3*R*(1-nuNi))
        self.R = R

    def Wafer_Sync(self, d='f', resamp=False):
        """
        Take scan means and sync them togther to process as a wafer

        Parameters
        ----------
        d : string, optional
            specify scan direction forward 'f' or backward 'b'. The default is 'f'.

        Returns
        -------
        None.
        """

        scans = [scan for scan in self.scans.values() if getattr(self, scan, False)]
        if 'load' in scans:
            scans.remove('load')

        if resamp:
            X = np.arange(resamp[0], resamp[1], .01)
            for scan in scans:
                m = [*getattr(getattr(self, scan), d+'means')]
                x = m[4]
                y = m[0]
                Y = np.interp(X, x, y)
                m[0] = Y
                m[4] = X

                setattr(getattr(self, scan), d+'means', m)

        trim_min = max([getattr(getattr(self, scan), d+'means')[4].min() for scan in scans])
        trim_max = min([getattr(getattr(self, scan), d+'means')[4].max() for scan in scans])

        idx_min = []
        idx_max = []
        for scan in scans:
            scan_x = getattr(getattr(self, scan), d+'means')[4]
            idx_min.append(np.where(scan_x == trim_min)[0][0])
            idx_max.append(np.where(scan_x == trim_max)[0][0])

        x_len = min(idx_max)-max(idx_min)

        for scan in scans:
            scan_mean = getattr(getattr(self, scan), d+'means')[0]
            scan_x = getattr(getattr(self, scan), d+'means')[4]
            x_min = np.where(scan_x == trim_min)[0][0]
            setattr(self, 'sync_'+scan, scan_mean[x_min:x_min+x_len])
            setattr(self, 'sync_'+scan+'_x', scan_x[x_min:x_min+x_len])

        # need to add scaling if enc is not uniform
    def Wafer_Make(self, w=.545, wo=.017, glass=1.8631, k=.101, filt=None):

        if type(filt) is int:
            filt = wFilter(filt)

        self.x = self.sync_base_x
        self.w = glass+w+k-wo
        toolfloor = self.sync_base-self.w
        self.ni = (self.sync_nickel-toolfloor)*1E-3
        # self.ni = ((self.sync_base-self.sync_nickel)-toolfloor)*1E-3
        if hasattr(self, 'sync_exfo'):
            # self.ex = ((self.sync_base-self.sync_exfo)-toolfloor)*1E-3
            self.ex = (self.sync_exfo-toolfloor+wo)*1E-3
            if filt:
                self.ex = filt(self.ex)
        # self.Get_Stress()
        # self.Wafer = np.vstack((ni, ex, S)).T
        if filt:
            self.ni = filt(self.ni)
        if hasattr(self, 'load'):
            self.load.enc = self.load.enc-self.x[0]
        self.x = self.x-self.x[0]

    def Pickle_Wafer(self, filepath, file):
        """Pickle the wafer data so it can be loaded faster."""
        print('Pickling ', Path(file).name)
        pickle.dump([self.x, self.w, self.ni, self.ex, self.S, self.Load, self.load], open(
            Path(filepath).joinpath(file).with_suffix(".dat"), 'wb'))

    def Load_Wafer(self, filepath, file):
        [self.x, self.w, self.ni, self.ex, self.S, self.Load, self.load] = pickle.load(
            open(Path(Path(filepath).joinpath(file)).with_suffix(".dat"), 'rb'))
        print('Loading ', Path(file).name)

    def Make_Meta(self, file, i=2):
        """
        i: 0-Ni (um),1-Si (um), 2-T (N), 3-S (MPa),4-a (um), 5-h (um), 6-K2, 7-K1
        """
        from sklearn.gaussian_process import GaussianProcessRegressor
        from sklearn.gaussian_process.kernels import DotProduct
        from sklearn.gaussian_process.kernels import ConstantKernel as C
        from sklearn import preprocessing

        kernel = C(.1, (0.01, 10.0)) * (DotProduct(sigma_0=7.74, sigma_0_bounds=(0.01, 10.0)) ** 2)

        adat = np.genfromtxt(file, delimiter=",")[1:, :]
        X = np.concatenate((adat[:, :i], adat[:, i+1:]), 1)
        self.scalerX = preprocessing.StandardScaler().fit(X)
        y = np.expand_dims(adat[:, i], 1)
        self.scalery = preprocessing.StandardScaler().fit(y)

        self.gpr = GaussianProcessRegressor(
            kernel=kernel, random_state=0, n_restarts_optimizer=0, alpha=.001).fit(
                self.scalerX.transform(X), self.scalery.transform(y))

    def Pickle_Meta(self, filepath, file):
        """Pickle the meta so it can be loaded faster."""
        print('Pickling ', Path(file).name)
        pickle.dump([self.scalerX, self.scalery, self.gpr], open(
            Path(filepath).joinpath(file).with_suffix(".dat"), 'wb'))

    def Load_Meta(self, filepath, file):
        [self.scalerX, self.scalery, self.gpr] = pickle.load(
            open(Path(Path(filepath).joinpath(file)).with_suffix(".dat"), 'rb'))
        print('Loading ', Path(file).name)

    def Make_Load(self, ex=8, a=1500, h=150, k2=0, k1=730, filt=None, floor=5):
        """
        Make compensated Load profile

        Parameters
        ----------
        ex : int, optional
            Desired exfoliated film thickness in um. The default is 8.
        a : int, optional
            Crack length in um. The default is 1500.
        h : int, optional
            Roller height in um. The default is 150.
        k1 : int, optional
            Mode I stress intensity target in MPa*mm*(.5). The default is 730.
        k2 : int, optional
            Mode II stress intensity target in MPa*mmd*(.5). The default is 0.

        Returns
        -------
        None.
        """

        if type(filt) is int:
            filt = nFilter(self, filt)

        const = np.array([0, ex, 0, a, h, k2, k1])
        C = np.ones((self.ni.shape[0], 7))*const

        ni = self.ni
        if filt:
            ni = filt(self.ni)
        Xtest = C
        Xtest[:, 0] = ni*1E6
        Xtest[:, 2] = self.S*1E-6

        # Compute load with scaling
        self.Load = self.scalery.inverse_transform(self.gpr.predict(self.scalerX.transform(Xtest)))
        if floor:
            self.Load[self.Load < floor] = floor

    def Export_Load(self, filepath, file, wafer_points=3000, ramp=False, window=False):
        # Interp load to fit export
        Load = np.interp(np.linspace(0, len(self.Load)-1, wafer_points),
                         np.arange(0, len(self.Load)), np.squeeze(self.Load))
        # Convert to N?
        Load = Load/9.81
        if window:
            pad = np.ones(int((window-wafer_points)/2))
            Load = np.concatenate((pad*Load[0], Load, pad*Load[-1]))
        if ramp:
            Load = np.concatenate((np.linspace(0, Load[0], ramp), Load.flatten()))
        np.savetxt(Path(Path(filepath).joinpath(file)).with_suffix(".csv"),
                   Load, delimiter=',')

    def Load_exLoad(self, filepath, file, trim=False, filt=None):
        tdms_file = TdmsFile(Path(Path(filepath).joinpath(file)).with_suffix(".tdms"))
        group = tdms_file['Untitled']
        channel1 = group['encoder']
        enc = channel1.data
        # zero data and convert to mm
        # enc=(enc-min(enc)+1)/100

        time = group['Untitled']
        # elapsed time in seconds
        time = time - time[0]
        channel3 = group['load']
        load = channel3.data
        self.exLoad = Scan(time, enc, load, trim=trim, filt=filt)

    def Load_exLoadalt(self, filepath, file):
        tdms_file = TdmsFile(Path(Path(filepath).joinpath(file)).with_suffix(".tdms"))
        group = tdms_file['Untitled']
        channel1 = group['encoder']
        enc = channel1.data
        channel3 = group['load']
        load = channel3.data
        self.exLoad = enc, load
        # %% plot zone

    def Plot_Wafer(self):

        color = plt.rcParams['axes.prop_cycle'].by_key()['color'][0]
        fig, ax1 = plt.subplots(figsize=(8, 4))
        ax1.plot(self.x, self.ni*1E6, color=color, linestyle='--', label='Ni Thickness')
        if hasattr(self, 'ex'):
            ax1.plot(self.x, self.ex*1E6, color=color, linestyle='-', label='Si Thickness')
        ax1.tick_params(axis='y', labelcolor=color)
        # ax1.set_title('Wafer Profile')
        ax1.set_xlabel('Chuck (mm)')
        ax1.set_ylabel('Si and Ni Film Thickness (µm)', color=color)

        color = plt.rcParams['axes.prop_cycle'].by_key()['color'][1]
        ax2 = ax1.twinx()
        ax2.plot(self.x, self.S/1E8, color=color, linestyle='-.', label='Ni Stress')
        ax2.plot(self.x, self.Load/9.81, color=color, linestyle='-', label='Handle Tension')
        if hasattr(self, 'load'):
            x, _, y = syncstuff(self.x, self.Load, self.load.enc, self.load.lod, dm=False)
            ax2.plot(x, y, color=color, linestyle=':', label='Measured Tension')
        ax2.tick_params(axis='y', labelcolor=color)
        ax2.set_ylabel(r'Ni Stress (Pa$\cdot 10^8$) and Handle Tension (Kg)', color=color)
        # lines, labels = ax1.get_legend_handles_labels()
        # lines2, labels2 = ax2.get_legend_handles_labels()
        # ax2.legend(lines + lines2, labels + labels2, loc=0)
        fig.legend(bbox_to_anchor=(0.03, 0.5, .93, 0.5), loc='upper left', ncol=5, mode="expand", borderaxespad=0., handletextpad=0.1)

        fig.tight_layout(pad=0.1)
        # ax1.set_xlim(left=0, right=25)
        ax1.margins(x=0)
        ax1.set_title('')
        ax1.set_xlabel('Distance from Stable Crack Start (mm)')
        return fig, ax1, ax2

    def Plot_Batch(self):
        scans = [scan for scan in self.scans.values() if getattr(self, scan, False)]
        [(getattr(self, scan).plot_check(), plt.suptitle(scan)) for scan in scans]
        [(getattr(self, scan).plot_scans_sub(), plt.suptitle(scan)) for scan in scans]

# %%

class Scan(object):
    """Object to stor LabView scans."""

    def __init__(self, time, enc, lzr, llim=41, ulim=65, trim=False, filt=None):
        """
        Init.

        Parameters
        ----------
        time : np.array from tdms import
            Time in unix format
        enc : np.array from tdms import
            Encoder data 10um unit
        lzr : np.array from tdms import
            Laser measurement data (mm)
        llim : int, optional
            Lower scan cutoff in mm. The default is 20.
        ulim : int, optional
            Upper scan cutoff in mm. The default is 80.
        trim : array or slice, optional
            Specify which scans to skip. The default is False.

        Raises
        ------
        ValueError
            When encoder data is really bad.
        """
        # zero data and convert to mm
        # enc = (enc-min(enc)+1) / 100
        # convert to mm, but leave enc at lv zero

        self.time = time
        self.encr = enc.copy()
        enc = signal.medfilt(enc, 21)
        enc = enc / 100
        self.enc = enc
        self.lzrr = lzr.copy()
        self.lzr = lzr.copy()
        # if filt:
        #     print('Filtering Laser')
        #     self.lzr = filt(lzr)
        # filters
        # lzr = signal.medfilt(lzr, 21)
        self.trim = trim

        self.g = np.gradient(self.enc/self.enc.max())
        if any(*np.where(self.g > .005)):
            print('JUMPS AT', np.where(self.g > .005))

        def zero_runs(a):
            """Create an array that is 1 where a is 0, and pad each end with an extra 0."""
            iszero = np.concatenate(
                ([0], np.logical_not(np.equal(a, 0).view(np.int8)), [0]))
            absdiff = np.abs(np.diff(iszero))
            # Runs start and end where absdiff is 1.
            ranges = np.where(absdiff == 1)[0].reshape(-1, 2)
            return ranges

        # Stack data to trim into individual scans
        tempst = np.stack([self.enc, self.lzr])
        # trim edges of scans
        tempst[:, ((tempst[0, :] > ulim) | (tempst[0, :] < llim))] = 0
        # find indices of runs start stop
        self.idx = zero_runs(tempst[0, :])
        print('Loaded ', self.idx.shape[0] / 2, ' scans')

        if np.asarray(trim).any():
            self.idx = np.delete(self.idx, trim, 0)
            print(self.idx.shape)

        if self.idx.size == 0:
            raise ValueError('encoder data is very bad')

        # %% Stack scans
        # find shortest scan and trim others to match. then stack them
        def stack_scans(idx, enc, lzr, fb):
            minL = min(abs(idx[:, 0]-idx[:, 1])[fb:][::2])
            estack = np.empty((0, minL))
            lstack = estack.copy()
            for ii in idx[fb:][::2, :]:
                ef = enc[range(*ii.tolist())]
                if filt:
                    # print('Filtering Laser')
                    lf = filt(lzr[range(*ii.tolist())])
                else:
                    lf = lzr[range(*ii.tolist())]
                estack = np.append(estack, np.expand_dims(ef[:minL], axis=0), axis=0)
                lstack = np.append(lstack, np.expand_dims(lf[:minL], axis=0), axis=0)
            return [estack, lstack]

        self.eforward, self.lforward = stack_scans(self.idx, self.enc, self.lzr, 0)
        self.ebackward, self.lbackward = stack_scans(self.idx, self.enc, self.lzr, 1)

        # %%Calculate measurements per position
        def lasers_per_enc(ence, lzre):
            from collections import defaultdict
            fd = []
            for ii, jj in zip(ence, lzre):
                fd.append(dict(zip(ii, jj)))

            p_error = defaultdict(list)
            # you can list as many input dicts as you want here
            for d in fd:
                for key, value in d.items():
                    p_error[key].append(value)
            return p_error

        self.flper_enc = lasers_per_enc(self.eforward, self.lforward)
        self.blper_enc = lasers_per_enc(self.ebackward, self.lbackward)

        # %% Calculate errors relative to position
        def err_by_pos(error_dict):
            x = []
            for key, value in error_dict.items():
                value = np.array(value)
                value = value-value.mean()
                error_dict.update({key: value})
                x.append(abs(value).mean())
            x = np.array(x)
            x = x[~np.isnan(x)]
            return x

        self.error_byp_f = err_by_pos(self.flper_enc.copy())
        self.error_byp_b = err_by_pos(self.blper_enc.copy())

        # %% Get mean scan and std
        def get_mean(lzr_per_enc):
            mn = np.array([])
            std = np.array([])
            n = np.array([])
            encm = np.array([])
            for key, value in lzr_per_enc.items():
                mn = np.append(mn, np.array(value).mean())
                std = np.append(std, np.std(value))
                n = np.append(n, len(value))
                encm = np.append(encm, key)
            SEM = std/np.sqrt(n)
            t = np.vstack((mn, std, n, SEM, encm)).T
            x = t[t[:, 4].argsort()].T
            mn, std, n, SEM, encm = [*x]
            return mn, std, n, SEM, encm

        self.fmeans = get_mean(self.flper_enc)
        self.bmeans = get_mean(self.blper_enc)

        # %% Get error along scan for each scan
        def scan_error(estack, lstack, means):
            [mn, std, n, SEM, encm] = means
            M = dict(zip(encm, mn))
            # covert to microns
            scan_err = [1000 * np.array([l-M[e] for e, l in zip(*scan)])
                        for scan in zip(estack, lstack)]
            smean, sstd, smed = zip(*[
                [scan.std(), abs(scan).mean(), np.median(abs(scan))]
                for scan in scan_err])
            return [scan_err, smean, smed, sstd]

        [self.fscan_error, self.fsmean, self.fsmed, self.fsstd] = scan_error(
            self.eforward, self.lforward, self.fmeans)
        [self.bscan_error, self.bsmean, self.bsmed, self.bsstd] = scan_error(
            self.ebackward, self.lbackward, self.bmeans)

        # %%

    def Pickle_TDMS(self, filepath, file, Trim=False):
        """Pickle the TDMS so it can be loaded faster."""
        print('Pickling ', Path(file).name)
        if Trim is not False:
            pickle.dump([self.time, self.encr, self.lzrr, self.trim], open(
                Path(filepath).joinpath(file).with_suffix(".trim.dat"), 'wb'))
        elif Path(file).name[0] == 'l':
            pickle.dump([self.time, self.lod, self.encr, self.lzrr], open(
                Path(filepath).joinpath(file).with_suffix(".dat"), 'wb'))
        else:
            pickle.dump([self.time, self.encr, self.lzrr], open(
                Path(filepath).joinpath(file).with_suffix(".dat"), 'wb'))
        # %% plot zone

    def plot_check(self):
        """
        Plot quick check of enc data vs time to check for jumps and junk.

        Returns
        -------
        fig : TYPE
            DESCRIPTION.
        ax1 : TYPE
            DESCRIPTION.
        ax2 : TYPE
            DESCRIPTION.
        """
        fig, ax1 = plt.subplots()
        for ii in self.idx:
            ax1.scatter(self.idx[:, 0], self.enc[self.idx[:, 0]], marker='x')
        for ii in self.idx:
            ax1.scatter(self.idx[:, 1]-1, self.enc[self.idx[:, 1]-1], marker='x', color='blue')
        ax1.plot(self.enc)
        plt.title('Check Start and Stop')
        plt.xlabel('time steps')
        plt.ylabel('encoder distance')

        ax2 = ax1.twinx()
        c = plt.rcParams['axes.prop_cycle'].by_key()['color'][2]
        ax2.plot(self.g/self.g.max(), color=c)
        plt.ylabel('encoder gradient')
        # check errors
        fx = np.array(self.idx[1:][::2, :].mean(axis=1), dtype='int')
        bx = np.array(self.idx[0:][::2, :].mean(axis=1), dtype='int')
        # median or mean?
        ax2.plot(fx, self.fsmean)
        ax2.plot(bx, self.bsmean)
        return fig, ax1, ax2

    def plot_scans(self):
        """
        Plot forwards and backwards scans calculated from idx.

        Returns
        -------
        fig2 : TYPE
            DESCRIPTION.
        ax2 : TYPE
            DESCRIPTION.
        fig3 : TYPE
            DESCRIPTION.
        ax3 : TYPE
            DESCRIPTION.

        """
        fig2, ax2 = plt.subplots()
        # grab every other scan from idices
        # forward
        for ii in self.idx[0:][::2, :]:
            ef = self.enc[range(*ii.tolist())]
            lf = self.lzr[range(*ii.tolist())]
            ax2.plot(ef, lf)
        ax2.set_title('Forward Chuck Scans')
        ax2.set_xlabel('Chuck (mm)')
        ax2.set_ylabel('Laser Distance (mm)')

        fig3, ax3 = plt.subplots()
        for ii in self.idx[1:][::2, :]:
            eb = self.enc[range(*ii.tolist())]
            lb = self.lzr[range(*ii.tolist())]
            ax3.plot(eb, lb)
        ax3.set_title('Backward Chuck Scans')
        ax3.set_xlabel('Chuck (mm)')
        ax3.set_ylabel('Laser Distance (mm)')
        return fig2, ax2, fig3, ax3

    def plot_scans2(self, trim=np.s_[:]):
        """
        Plot forwards and backwards scans.

        Parameters
        ----------
        trim : TYPE, optional
            DESCRIPTION. The default is np.s_[50:-500].

        Returns
        -------
        fig2 : TYPE
            DESCRIPTION.
        ax2 : TYPE
            DESCRIPTION.
        fig3 : TYPE
            DESCRIPTION.
        ax3 : TYPE
            DESCRIPTION.

        """
        fig2, ax2 = plt.subplots()
        # grab every other scan from idices using attributes
        # forward
        for ii, jj in zip(self.eforward, self.lforward):
            plt.plot(ii[trim], jj[trim])
        ax2.set_title('Forward Chuck Scans')
        ax2.set_xlabel('Chuck (mm)')
        ax2.set_ylabel('Laser Distance (mm)')

        fig3, ax3 = plt.subplots()
        for ii, jj in zip(self.ebackward, self.lbackward):
            plt.plot(ii[trim], jj[trim])
        ax3.set_title('Backward Chuck Scans')
        ax3.set_xlabel('Chuck (mm)')
        ax3.set_ylabel('Laser Distance (mm)')
        return fig2, ax2, fig3, ax3

    def plot_scans_sub(self, trim=np.s_[:]):
        fig, (ax1, ax2) = plt.subplots(2, 1, sharey=True, sharex=True)
        for ii, jj in zip(self.eforward, self.lforward):
            ax1.plot(ii[trim], jj[trim])
        ax1.set_title('Forward Chuck Scans')
        ax1.set_xlabel('Chuck (mm)')
        # ax1.set_ylabel('Laser Distance (mm)')

        for ii, jj in zip(self.ebackward, self.lbackward):
            ax2.plot(ii[trim], jj[trim])
        ax2.set_title('Backward Chuck Scans')
        ax2.set_xlabel('Chuck (mm)')
        # ax2.set_ylabel('Laser Distance (mm)')

        AxZoom.suplabel(ax1, 'Laser Distance (mm)', labelpad=10)
        return ax1, ax2, fig

    def plot_mean(self, trim=np.s_[:]):
        """
        Plot the means of F-B scans with 1 std window.

        Parameters
        ----------
        trim : TYPE, optional
            DESCRIPTION. The default is np.s_[50:-500].

        Returns
        -------
        fig : TYPE
            DESCRIPTION.
        ax : TYPE
            DESCRIPTION.

        """
        def mean_plot(means):
            means = [ii[trim] for ii in means]
            mn, std, n, SEM, encm = means
            fig, ax = plt.subplots()
            # ax.fill_between(x, mn - SEM, mn + SEM, alpha=0.2)
            ax.fill_between(encm, mn - std, mn + std, alpha=0.2)
            ax.plot(encm, mn)
            return fig, ax

        fig, ax = mean_plot(self.fmeans)
        ax.set_title('Forward Scan Mean')
        ax.set_xlabel('Chuck (mm)')
        ax.set_ylabel('Laser Distance (mm)')

        fig2, ax2 = mean_plot(self.bmeans)
        ax2.set_title('Backward Scan Mean')
        ax2.set_xlabel('Chuck (mm)')
        ax2.set_ylabel('Laser Distance (mm)')

    def plot_mean_sub(self, trim=np.s_[:]):
        def mean_plot(means, ax):
            means = [ii[trim] for ii in means]
            mn, std, n, SEM, encm = means
            ax.fill_between(encm, mn - std, mn + std, alpha=0.2)
            ax.plot(encm, mn)

        fig, (ax1, ax2) = plt.subplots(2, 1, sharey=True, sharex=True)
        mean_plot(self.fmeans, ax1)
        mean_plot(self.bmeans, ax2)
        # fig.tight_layout()
        ax1.set_title('Scan Means')
        # fig.suptitle('Scan Means')
        # ax1.set_title('Forward')
        # ax2.set_title('Backward')
        AxZoom.suplabel(ax1, 'Laser Distance (mm)', labelpad=10)
        ax1.set_ylabel('Forward')
        ax2.set_ylabel('Backward')
        ax2.set_xlabel('Chuck (mm)')
        return ax1, ax2, fig

    def plot_error_byp(self, m=2, bins=50):
        """
        Plot histograms of error from mean for each unique encoder position for F-B and combined.
        Plot error to mean for each unique encoder value.

        Trim outliers for more legible axis.

        Parameters
        ----------
        m : int, optional
            exclude outliers m standard deviations out. The default is 2.
        bins : int, optional
            Histogram bins. The default is 500.

        Returns
        -------
        ax and fig handles
            forward, backwards, and combined.

        """
        def reject_outliers(data, m=m):
            return data[abs(data - np.mean(data)) < m * np.std(data)]

        fig1, (ax1, ax2) = plt.subplots(2, 1, sharey=True, sharex=True, constrained_layout=True)
        # fig1.tight_layout()
        # fig1, ax1 = plt.subplots()
        ax1.hist(reject_outliers(self.error_byp_f * 1000), bins)
        ax1.set_title('Forward Height Error by Position')
        ax1.set_ylabel('Counts')
        # ax1.set_xlabel('Laser Distance Error (µm)')

        # fig2, ax2 = plt.subplots()
        ax2.hist(reject_outliers(self.error_byp_b * 1000), bins)
        ax2.set_title('Backward Height Error by Position')
        ax2.set_ylabel('Counts')
        ax2.set_xlabel('Laser Distance Error (µm)')

        fig3, ax3 = plt.subplots()
        x = np.concatenate((self.error_byp_f, self.error_byp_b)) * 1000
        ax3.hist(reject_outliers(x[~np.equal(x, 0)]), bins)
        ax3.set_title('Combined F-B Height Error by Position')
        ax3.set_ylabel('Counts')
        ax3.set_xlabel('Laser Distance Error (µm)')
        return ax1, fig1, ax2, ax3, fig3

    def plot_scan_error(self):
        """Plot median error and std for each scan."""
        fig, ax = plt.subplots()
        ax.plot(self.fsmed, label='Forward Median')
        ax.plot(self.bsmed, label='Backward Median')
        ax.legend()
        ax.set_title('Per Scan Error')
        ax.set_xlabel('Scan')
        ax.set_ylabel('Laser Median Error (µm)')

        ax2 = ax.twinx()
        ax2.plot(self.fsstd, label='Forward Std', linestyle='--')
        ax2.plot(self.bsstd, label='Backward Std', linestyle='--')
        ax2.set_ylabel('Laser Error Std (µm)')
        ax2.legend()
        # fig.legend(loc="upper right", bbox_to_anchor=(1,1), bbox_transform=ax.transAxes)
        return fig, ax, ax2

    def plot_densities(self):
        """Plot freq domain stuff."""
        # plt.csd(mn, mnb, NFFT = 1024, detrend='mean')
        # plt.csd(mn, mnb, NFFT = 64, Fs = 1, noverlap = 32, pad_to=150, detrend='mean')
        fig, ax = plt.subplots()
        ax.csd(self.fmeans[0], self.bmeans[0],
               NFFT=129, Fs=2, noverlap=12, pad_to=900, detrend='mean')
        ax.set_xlabel('Normalized Frequency')
        ax.set_xlabel('Combined F-B Spectral Density')
        ax.set_title('CSD')

        # plt.set_cmap('inferno')
        # fig2, ax2 = plt.subplots()
        # ax2.specgram(self.fmeans[0],
        #              NFFT=65, Fs=1, noverlap=32, detrend='mean', pad_to=200, mode='magnitude')
        # t = signal.butter(10, .0005, btype='lowpass', output='sos')
        # tt = signal.sosfilt(t, self.fmeans[0]-self.fmeans[0].mean())
        # ttb = signal.sosfilt(t, self.bmeans[0]-self.bmeans[0].mean())
        # fig3, ax3 = plt.subplots()
        # plt.plot(tt)
        # plt.plot(ttb)

    # %%
    def plot_all(self):
        """Run all the plots."""
        self.plot_check()
        self.plot_scans2()
        self.plot_error_byp(bins=50)
        self.plot_mean()
        self.plot_scan_error()
        # self.plot_densities()

# %%
class Lscan(Scan):
    def __init__(self):
        pass

# %%
def syncstuff(x1, y1, x2, y2, dm=False, r=2):
    if r:
        x1 = x1.round(2)
        x2 = x2.round(2)

    if dm:
        f = dict(zip(x1, y1-y1.mean()))
        b = dict(zip(x2, y2-y2.mean()))
    else:
        f = dict(zip(x1, y1))
        b = dict(zip(x2, y2))

    X = [*set.intersection(set(x1), set(x2))]
    X.sort()

    Y1 = []
    Y2 = []

    for i in X:
        if i in f.keys() and i in b.keys():
            Y1.append(f[i])
            Y2.append(b[i])
    return map(lambda x: np.array(x), [X, Y1, Y2])

def exFilter(size):
    # make filter for scans
    def filt(lzr):
        lzr = signal.medfilt(lzr, 65)
        lzr = uniform_filter1d(lzr, size)
        return lzr
    return filt

def wFilter(size):
    # make filter for processed scans
    def filt(lzr):
        lzr = signal.medfilt(lzr, size)
        return lzr
    return filt

def sFilter(Ex, size):
    # make filter for calculating stress
    def filt(lzr):
        lzr = signal.medfilt(lzr, 129)
        z = np.polyfit(Ex.sync_stress_x, lzr, size)
        p = np.poly1d(z)
        lzr = p(Ex.sync_stress_x)
        return lzr
    return filt

def nFilter(Ex, size):
    # make filter for calculating load
    def filt(lzr):
        z = np.polyfit(Ex.sync_stress_x, lzr, size)
        p = np.poly1d(z)
        lzr = p(Ex.sync_stress_x)
        return lzr
    return filt


import numpy as np
import cv2

def my_orb(img):
    orb1 = cv2.ORB_create()
    kp1 = orb1.detect(img, None)
    return orb1.compute(img, kp1)

def my_Matcher(des1, des2):
    matches = []
    row_score1 = np.zeros((len(des1)))
    idx1 = np.zeros((len(des1), 2)).astype('uint64')
    score1 = np.zeros(len(des2))
    for i in range(len(des1)):
        for j in range(len(des2)):
            score1[j] = cv2.norm(des1[i], des2[j], cv2.NORM_HAMMING)
        row_score1[i] = min(score1)
        idx1[i] = (i, np.argmin(score1))

    rank = row_score1.argsort()
    # print("rank (src->dest)",row_score1[rank[:top]])
    # print(idx1[rank[:top]])

    row_score2 = np.zeros((len(des2)))
    idx2 = np.zeros((len(des2), 2)).astype('uint64')
    score2 = np.zeros(len(des1))
    for i in range(len(des2)):
        for j in range(len(des1)):
            score2[j] = cv2.norm(des1[j], des2[i], cv2.NORM_HAMMING)
        row_score2[i] = min(score2)
        idx2[i] = (np.argmin(score2), i)

    rank2 = row_score2.argsort()
    # print("rank2 (dest->src)", row_score2[rank2[:top]])
    # print(idx2[rank2[:top]])

    markedSrc = np.zeros((len(des1) + 1))
    markedDest = np.zeros((len(des2) + 1))
    matches = []
    i = j = 0
    while i < len(des1) and j < len(des2):
        matches.append(cv2.DMatch(idx1[rank[i], 0], idx1[rank[i], 1], row_score1[rank[i]]))
        markedDest[idx1[rank[i], 1]] = markedSrc[idx1[rank[i], 0]] = 1
        i += 1
        j += 1
        if i < len(rank) - 1:
            while (markedSrc[idx1[rank[i], 0]] or markedDest[idx1[rank[i], 1]]):
                i += 1
        if j < len(rank2) - 1:
            while (markedSrc[idx2[rank2[j], 0]] or markedDest[idx2[rank2[j], 1]]) and j < len(rank2) - 1:
                j += 1
    if i < len(des1):
        matches.append(cv2.DMatch(idx1[rank[i], 0], idx1[rank[i], 1], row_score1[rank[i]]))
        #print(i, j, "(", idx1[rank[i], 0], ",", idx1[rank[i], 1], ",", row_score1[rank[i]], ")", idx2[rank2[j], 0],     markedDest[idx2[rank2[j], 1]])
        markedDest[idx1[rank[i], 1]] = markedSrc[idx1[rank[i], 0]] = 1
        i += 1
        if i < len(rank) - 1:
            while (markedSrc[idx1[rank[i], 0]] or markedDest[idx1[rank[i], 1]]):
                i += 1
    if j < len(des2):
        matches.append(cv2.DMatch(idx2[rank2[j], 0], idx2[rank2[j], 1], row_score2[rank2[j]]))
        markedDest[idx2[rank2[j], 1]] = markedSrc[idx2[rank2[j], 0]] = 1
        #print(j)
        j += 1
        if j < len(rank2) - 1:
            while (markedSrc[idx2[rank2[j], 0]] or markedDest[idx2[rank2[j], 1]]):
                j += 1

    return matches,idx1,rank

def my_kp(kp1,kp2,top,idx1,rank):

    kp1pt = np.zeros((top,2))
    kp2pt = np.zeros((top,2))
    for i in range(top):
        kp1pt[i]=kp1[idx1[rank[i],0]].pt
        kp2pt[i]=kp2[idx1[rank[i],1]].pt
    return kp1pt,kp2pt