"""
Work with datasets

Written by Evgeniy Ryzhkov

------------------------------------------------------------

Usage:

    # Split datasets (datasets/filtered dir) into train and val datasets
    python - m scripts.dataset --action=split_train_val_images

"""

import http.client, urllib.parse, json
import config.main as config
import config.access as config_access
import requests
import time
import os
import shutil
import argparse
from imutils import paths
import random

class Dataset:

    def __init__(self):
        self.subscription_key = config_access.AZURE_SUBSCRIPTION_KEY
        self.search_url = 'https://api.cognitive.microsoft.com/bing/v7.0/images/search'

        self.downloaded_images_path = config.DOWNLADED_IMG_DIR
        self.search_term = ['card visa'] # for diversity use different term 'debit card', 'card bank of america' and so on
        self.images_amount_for_class = 300

    def run(self):
        try:
            user_arguments = self._get_command_line_arguments()

            if user_arguments['action'] == 'download_images':
                self._download_images_for_train()
            elif user_arguments['action'] == 'split_train_val_images':
                self._split_train_val_images()
            else:
                print('Incorrect action type. You should choose one of possible values.')
        except:
            print('Argument --action is required. Possible values: "download_images", "split_train_val_images"')

    @staticmethod
    def _get_command_line_arguments():
        ap = argparse.ArgumentParser()
        ap.add_argument("--action", required=True,
                        help="What's action you need? Possible values: 'download_images', 'split_train_val_images' ")
        args = vars(ap.parse_args())
        return args

    def _download_images_for_train(self):
        print('[INFO] Creating dataset...')
        self._clear_directory(self.downloaded_images_path)
        search_results = self._get_json(self.search_term)

        url_images = gd_o.get_first_n_image_urls(search_results, n_images=self.images_amount_for_class)
        self._parse_and_save_images(url_images, self.downloaded_images_path)

        print('[OK] Images for train has been saved successfully into dir {}.'.format(self.downloaded_images_path))

    @staticmethod
    def _clear_directory(path):
        for root, dirs, files in os.walk(path):
            for f in files:
                os.unlink(os.path.join(root, f))
            for d in dirs:
                shutil.rmtree(os.path.join(root, d))

    def _get_json(self, search_term):
        headers = {"Ocp-Apim-Subscription-Key": self.subscription_key}
        offset = 0
        n_grab_images = 0
        value_results = []

        while n_grab_images < self.images_amount_for_class:
            params = {"q": search_term, "imageType": "photo", "offset": offset}
            response = requests.get(self.search_url, headers=headers, params=params)
            response.raise_for_status()
            search_results = response.json()

            value_field = search_results['value']
            n_grab_images += len(value_field)
            offset = search_results['nextOffset']
            value_results += value_field

            # print(json.dumps(search_results, indent=4, sort_keys=True))
            # There is a limit on requests per sec for free azure account
            time.sleep(1)

        return value_results

    @staticmethod
    def _parse_and_save_images(url_images, dir_path):

        file_name_pattern = 'img_'
        file_name_ext = '.jpg'
        start_idx = 122 # for increasing amount of dataset. take last number of image from current dataset

        for idx, img in enumerate(url_images):
            file = requests.get(img, timeout=30)
            file_name = file_name_pattern + str(start_idx + idx) + file_name_ext
            file_path = dir_path + file_name
            print('[INFO] saving ' + file_name)

            # write the image to disk
            f = open(file_path, "wb")
            f.write(file.content)
            f.close()

    @staticmethod
    def get_first_n_image_urls(search_results, n_images):
        image_urls = [img["contentUrl"] for img in search_results[:n_images]] # using big image as on small can't see guitars
        return image_urls

    def _split_train_val_images(self):
        print('[INFO] Splitting filtered images to train and valid sets...')
        list_filtered_image_paths = list(paths.list_images(config.FILTERED_IMG_DIR))
        n_full_dataset = len(list_filtered_image_paths)
        print('[INFO] Full dataset amount = ', n_full_dataset)

        # shuffle images for better model learning and simplier split data by train / valid
        random.Random(4).shuffle(list_filtered_image_paths)
        val_size = 0.2
        n_val_size = round(n_full_dataset * val_size)
        n_train_size = n_full_dataset - n_val_size
        train_images_paths = list_filtered_image_paths[:n_train_size]
        val_images_paths = list_filtered_image_paths[-n_val_size:]

        self._copy_images(train_images_paths, config.TRAIN_IMG_DIR)
        self._copy_images(val_images_paths, config.VAL_IMG_DIR)
        print('[OK] Train ({}) and validation ({}) images were copied.'.format(n_train_size, n_val_size))

    @staticmethod
    def _copy_images(img_list, new_dir_path):
        for img in img_list:
            shutil.copy(img, new_dir_path)

    @staticmethod
    def _create_train_test_txt():
        print('[INFO] Creating train and test.txt...')

        list_labeled_image_paths = list(paths.list_images(config.LABELED_TRAIN_IMG_DIR))
        n_full_dataset = len(list_labeled_image_paths)
        random.Random(4).shuffle(list_labeled_image_paths) # shuffle list for better model learning and simplier split data by train / test

        test_size = 0.2
        n_test_size = round(n_full_dataset * test_size)
        n_train_size = n_full_dataset - n_test_size
        train_images_paths = list_labeled_image_paths[:n_train_size]
        test_images_paths = list_labeled_image_paths[-n_test_size:]

        # creating and saving txt files for darknet training
        # we need full path to images in order to darknet could fild them
        file_train = open(config.TXT_TRAIN_FILE, 'w')
        for img in train_images_paths:
            file_train.write(os.path.abspath(img) + '\n')

        file_test = open(config.TXT_TEST_FILE, 'w')
        for img in test_images_paths:
            file_test.write(os.path.abspath(img) + '\n')

        print('[OK] ' + config.TXT_TRAIN_FILE + ' and ' + config.TXT_TEST_FILE + ' have been created successfully.')

# ----------------------------------------------------
gd_o = Dataset()
gd_o.run()


#!/usr/bin/env python
# coding: utf-8

import requests
from bs4 import BeautifulSoup
import pandas as pd
from splinter import Browser
from bs4 import BeautifulSoup as bs
import time

def init_browser():
    # @NOTE: Replace the path with your actual path to the chromedriver
    executable_path = {"executable_path": "/usr/local/bin/chromedriver"}
    return Browser("chrome", **executable_path, headless=False)

def scrape():

    r = requests.get('https://mars.nasa.gov/news/?page=0&per_page=40&order=publish_date+desc%2Ccreated_at+desc&search=&category=19%2C165%2C184%2C204&blank_scope=Latest')

    soup = BeautifulSoup(r.text, 'html.parser')

    result1 = soup.find_all('div', attrs={'class':'content_title'})

    news_title = result1[0].a.text

    result2 = soup.find_all('div', attrs={'class':'rollover_description_inner'})

    news_p = result2[0].text

    # # Nasa .gov Site

    short_url = "https://www.jpl.nasa.gov/"
    url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
    r2 = requests.get(url)

    soup2 = BeautifulSoup(r2.text, 'html.parser')

    result3 = soup2.find_all('article', attrs={'class':'carousel_item'})

    split1 = result3[0]["style"].split("(")

    short_url_2 = split1[1].split("'")[1]

    featured_image_url = short_url + short_url_2

    from IPython.display import Image
    Image(featured_image_url)

    # #  Nasa Twitter Weather
    #

    url_tw = "https://twitter.com/marswxreport?lang=en"
    r3 = requests.get(url_tw)
    soup3 = BeautifulSoup(r3.text, 'html.parser')

    result4 = soup3.find_all('div', attrs={'class':'js-tweet-text-container'})

    mars_weather = result4[0].p.text.split("pic")[0]

    # # Mars Facts

    fact_url = "https://space-facts.com/mars/"
    r4 = requests.get(fact_url)
    soup4 = BeautifulSoup(r4.text, 'html.parser')

    result5 = soup4.find_all(attrs={'class':'column-2'})
    result6 = soup4.find_all(attrs={'class':'column-1'})

    data_list1 = []
    data_list2 = []
    for x in list(range(7,16)):
        data_list1.append(result5[x].text)
        data_list2.append(result6[x].text)

    df1 = pd.DataFrame({"Value":data_list1},index=data_list2)

    df1.to_html('mars_weather.html')

    # # Mars Image

    img_url1 = "https://astrogeology.usgs.gov/cache/images/cfa62af2557222a02478f1fcd781d445_cerberus_enhanced.tif_full.jpg"
    Image(img_url1)

    title1 = "Cerberus Hemisphere"

    img_url2 = "https://astrogeology.usgs.gov/cache/images/3cdd1cbf5e0813bba925c9030d13b62e_schiaparelli_enhanced.tif_full.jpg"
    Image(img_url2)

    title2 = "Schiaparelli Hemisphere"

    img_url3 = "https://astrogeology.usgs.gov/cache/images/ae209b4e408bb6c3e67b6af38168cf28_syrtis_major_enhanced.tif_full.jpg"
    Image(img_url3)

    title3 = "Syrtis Major Hemisphere"

    img_url4 = "https://astrogeology.usgs.gov/cache/images/7cf2da4bf549ed01c17f206327be4db7_valles_marineris_enhanced.tif_full.jpg"
    Image(img_url4)

    title4 = "Valles Marineris Hemisphere"

    # Store data in a dictionary
    mars_data = {
       "news_title":news_title,
        "news_p":news_p,
        "featured_image_url":featured_image_url,
        "value":"Value",
        "mars_weather":mars_weather,
        "df1i1":df1.index[0],
        "df1i2":df1.index[1],
        "df1i3":df1.index[2],
        "df1i4":df1.index[3],
        "df1i5":df1.index[4],
        "df1i6":df1.index[5],
        "df1i7":df1.index[6],
        "df1i8":df1.index[7],
        "df1i9":df1.index[8],
        "df1v1":df1.Value[0],
        "df1v2":df1.Value[1],
        "df1v3":df1.Value[2],
        "df1v4":df1.Value[3],
        "df1v5":df1.Value[4],
        "df1v6":df1.Value[5],
        "df1v7":df1.Value[6],
        "df1v8":df1.Value[7],
        "df1v9":df1.Value[8],
        "df2t1":title1,
        "df2t2":title2,
        "df2t3":title3,
        "df2t4":title4,
        "df2i1":img_url1,
        "df2i2":img_url2,
        "df2i3":img_url3,
        "df2i4":img_url4

    }

 # Return results
    return mars_data


import hashlib
import hmac
import json

import requests

from time import time

class TxbitResponse:
    def __init__(self, success, message, result):
        self.success = success
        self.message = message
        self.result = result

    def __str__(self):
        d = {
            'success': self.success,\
            'message': self.message,\
            'result': self.result
        }
        return str(json.dumps(d, indent = 4))

class Txbit:
    """A class to interact with the Txbit.io API

    Attributes
    ----------
    endpoint : str
        the base url for API calls
    APIKey : str
        key for working with the Market and Account methods
    Secret : str
        secret for working with the Market and Account methods

    Methods
    -------
    expandPathToUrl(path, params={}):
       adds onto the base url for specific methods
    request(path, params={}):
        uses `expandPathToUrl()` to make the API call
    authenticatedRequest(path, params={})
        authenticated API call with APIKey and Secret for Market and Account methods
    getMarkets():
        get the open and available trading markets along with other meta data
    getCurrencies():
        get all supported assets along with other meta data
    getMarketSummaries():
        get the last 24 hour summary of all active markets
    getExchangePairs():
        get list of all pairs that form markets
    getSystemStatus():
        get the system related status for all currencies
    getOrderBook(market, bookType='both'):
        get the orderbook for a given market
    getTicker(market):
        get current tick values for a market
    getMarketHistory(market):
        get the latest trades that have occurred for a specific market
    getCurrencyInformation(currency):
        get specific information and metadata about the listed currency
    getCurrencyBalanceSheet(currency):
        get solvency information for listed currencies
    getBalances():
        get all balances from your account
    getBalanceFor(currency):
        get the balance from your account for a specific asset
    getDepositAddress(currency):
        get or generate an address for a specific currency
    withdraw(currency, quantity, address, paymentid=None):
        withdraw funds from your account
    getOrder(uuic):
        get a single order by uuid
    getOrderHistory(market):
        get your order history
    getWithdrawlHistory(currency):
        get your withdrawal history
    getDepositHistory(currency):
        get your deposit history
    butLimit(market, quantity, rate):
        place a Buy Limit order in a specific market
    sellLimit(market, quantity, rate):
        place a Sell Limit order in a specific market
    cancel(uuid):
        cancel a buy or sell order
    getOpenOrders(market):
        get all orders that you currently have opened

    Notes
    -----
    Public methods can be run without supplying APIKey or Secret
    Market and Account methods need APIKey and Secret for authentification
    Market methods need 'ALLOW TRADING' permission set up on the APIKey
    Account methods need 'ALLOW READING' permission set up on the APIKey
    For more information, see: https://apidocs.txbit.io/#txbit-io-api
    """
    endpoint = 'https://api.txbit.io/api/'

    def __init__(self, APIKey, Secret):
        self.APIKey = APIKey
        self.Secret = Secret

    def expandPathToUrl(path, params={}):
        """adds onto the base url for specific methods"""
        url = Txbit.endpoint + path
        url += '?' if params else ''

        for key in params:
            url += key + '=' + params[key] + '&'

        return url

    def request(path, params={}):
        """uses `expandPathToUrl()` to make the API call"""
        url = Txbit.expandPathToUrl(path, params)
        return requests.get(url)

    def authenticatedRequest(self, path, params={}):
        """authenticated API call with APIKey and Secret for Market and Account methods"""
        params['apikey'] = self.APIKey
        params['nonce'] = str(int(time()))

        url = Txbit.expandPathToUrl(path, params)

        apisign = hmac.new(bytes(self.Secret, 'utf-8'),
                           bytes(url, 'utf-8'),
                           hashlib.sha512).hexdigest().upper()

        headers = {'apisign': apisign}
        return requests.get(url, headers=headers)

    ## PUBLIC FUNCTIONS ---------------

    def getMarkets():
        """(P) get the open and available trading markets along with other meta data"""
        res = Txbit.request('public/getmarkets')
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getCurrencies():
        """(P) get all supported assets along with other meta data"""
        res = Txbit.request('public/getcurrencies')
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getMarketSummaries():
        """(P) get the last 24 hour summary of all active markets"""
        res = Txbit.request('public/getmarketsummaries')
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getExchangePairs():
        """(P) get list of all pairs that form markets"""
        res = Txbit.getMarketSummaries()
        res.result = [pair['MarketName'] for pair in res.result]
        return res

    def getSystemStatus():
        """(P) get the system related status for all currencies"""
        res = Txbit.request('public/getsystemstatus')
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getOrderBook(market, bookType='both'):
        """(P) get the orderbook for a given market"""
        params = {'market': market, 'type': bookType}
        res = Txbit.request('public/getorderbook', params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getTicker(market):
        """(P) get current tick values for a market"""
        params = {'market': market}
        res = Txbit.request('public/getticker', params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getMarketHistory(market):
        """(P) get the latest trades that have occurred for a specific market"""
        params = {'market': market}
        res = Txbit.request('public/getmarkethistory', params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getCurrencyInformation(currency):
        """(P) get specific information and metadata about the listed currency"""
        params = {'currency': currency}
        res = Txbit.request('public/getcurrencyinformation', params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getCurrencyBalanceSheet(currency):
        """(P) get solvency information for listed currencies"""
        params = {'currency': currency}
        res = Txbit.request('public/getcurrencybalancesheet', params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    ## ACOUNT FUNCTIONS ---------------

    def getBalances(self):
        """(A) get all balances from your account"""
        path = 'account/getbalances'
        res = self.authenticatedRequest(path)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getBalanceFor(self, currency):
        """(A) get the balance from your account for a specific asset"""
        path = 'account/getbalance'
        params = {'currency': currency}
        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getDepositAddress(self, currency):
        """(A) get or generate an address for a specific currency

        Notes
        -----
        will return ADDRESS_GENERATING until one is available
        """
        path = 'account/getdepositaddress'
        params = {'currency': currency}
        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def withdraw(self, currency, quantity, address, paymentid=None):
        """(A) withdraw funds from your account

        Notes
        -----
        account for txfee
        """
        path = 'account/withdraw'
        params = {'currency': currency, 'quantity': quantity, 'address': address}

        if paymentid is not None:
            params['paymentid'] = paymentid

        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getOrder(self, uuid):
        """(A) get a single order by uuid"""
        path = 'account/getorder'
        params = {'uuid': uuid}
        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getOrderHistory(self, market=None):
        """(A) get your order history"""
        path = 'account/getorderhistory'
        params = {}

        if market is not None:
            params['market'] = market

        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getWithdrawlHistory(self, currency=None):
        """(A) get your withdrawal history"""
        path = 'account/getwithdrawalhistory'
        params = {}

        if currency is not None:
            params['currency'] = currency

        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getDepositHistory(self, currency=None):
        """(A) get your deposit history"""
        path = 'account/getdeposithistory'
        params = {}

        if currency is not None:
            params['currency'] = currency

        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    ## MARKET FUNCTIONS ---------------

    def buyLimit(self, market, quantity, rate):
        """(M) place a Buy Limit order in a specific market"""
        path = 'market/buylimit'
        params = {'market': market, 'quantity': quantity, 'rate': rate}
        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def sellLimit(self, market, quantity, rate):
        """(M) place a Sell Limit order in a specific market"""
        path = 'market/selllimit'
        params = {'market': market, 'quantity': quantity, 'rate': rate}
        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def cancel(self, uuid):
        """(M) cancel a buy or sell order"""
        path = 'market/cancel'
        params = {'uuic': uuic}
        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)

    def getOpenOrders(self, market=None):
        """(M) get all orders that you currently have opened"""
        path = 'getopenorders'
        params = {}

        if market is not None:
            params['market'] = market

        res = self.authenticatedRequest(path, params)
        result = res.json()['result'] if res.ok and res.json()['success'] else res.status_code
        return TxbitResponse(res.ok and res.json()['success'], "", result)


from typing import List

import gym
from game_env import Game
import numpy as np
from config import MuZeroConfig

class CartPole(Game):
    def __init__(self, action_space_size, discount):
        super().__init__(action_space_size, discount)
        self.env = gym.make('CartPole-v1')
        self.done = False

    def step(self, action):
        """Execute one step of the game conditioned by the given action."""
        observation, reward, done, _ = self.env.step(action)
        self.done = done
        return np.array([[observation]]), reward

    def terminal(self):
        """Is the game is finished?"""
        return self.done

    def legal_actions(self):
        """Return the legal actions available at this instant."""
        return [i for i in range(2)]

    def reset(self):
        """
        Reset the game for a new game.

        Returns:
            Initial observation of the game.
        """
        return np.array([[self.env.reset()]])

    def close(self):
        """
        Properly close the game.
        """
        self.env.close()

def make_cartpole_config():
    def visit_softmax_temperature(num_moves, training_steps):
        return 1.0

    return MuZeroConfig(
        game=CartPole,
        action_space_size=2,
        max_moves=500,
        discount=0.997,
        dirichlet_alpha=0.25,
        num_simulations=50,
        num_training_loop=50,
        num_epochs=5000,
        batch_size=128,
        td_steps=50,
        num_train_episodes=20,
        num_eval_episodes=1,
        lr_init=0.05,
        lr_decay_steps=1000,
        max_priority=False,
        visit_softmax_temperature_fn=visit_softmax_temperature,
        network_args={'support_size': 10,
                      'encoding_size': 8,
                      'rep_hidden': [],
                      'dyn_hidden': [16],
                      'rew_hidden': [16],
                      'val_hidden': [],
                      'pol_hidden': [],
                      'observation_shape': (1, 1, 4),
                      },
        result_path="cartpole.weights"
        )


import time
import threading
from diagram_classes import *
from dropdowns import *
from IPython.display import display
import plot_funcs as pf
from API_Anrop import YEARS

def alter_widget_placement(widget, cols):
    """
    Placerar alla widgets bredvid varandra, på specificerat antal kolumner.
    Plottar grafen under dessa.

    Arguments:
    widget - interactive datastruktur, innehåller den nödvändiga datan för att skapa en plot med interaktiva widgets.
    cols   - Specificerar antalet kolumner som widgetsarna placeras på.
    """

    center = widgets.Layout(align_items = "center")                                 # Centrera plotten och dropdownmenyerna.
    widget.update()                                                                 # Behövs för att visa plotten vid uppstart.
    output = widget.children[-1]                                                    # Hämtar plotten
    sub_boxes = []
    widget_count = len(widget.children)-1
    for i in range(0,widget_count,cols):
        tmp = []
        for j in range(cols):
            if i+j == widget_count:
                break
            tmp.append(widget.children[i+j])
        sub_boxes.append(widgets.HBox(tmp))
    display(widgets.VBox([*sub_boxes,output],layout=center))                        # Centrerar och visar plotten.

class interactive_diagrams:

    def __init__(self):

        # Skapande av listorna som utgör valen i dropdownmenyerna
        self._munis = ['Ej vald'] + [k for k in pf.mdata.keys()]
        self._sekom = ["Nej","Ja"]
        self._years = YEARS.strip().split(",")

        self._1_keywords = ["N15419", "N15505", "N15436"]
        self._1_keydesc = [pf.key_to_desc[k] for k in self._1_keywords]
        self._1_drop_keys = Dropdown(self._1_keydesc, 'Nyckeltal: ')
        self._1_drop_years = Dropdown(self._years, 'År: ')
        self._1_drop_munis = Dropdown(self._munis,'Kommun: ')
        self._1_drop_sekom = Dropdown(self._sekom, "Kommungrupp: ")
        self._1 = diagram_1()

        self._2_keywords = ["N15419", "N15505", "N15436"]
        self._2_keydesc = [pf.key_to_desc[k] for k in self._2_keywords]
        self._2_drop_keys = Dropdown(self._2_keydesc, 'Nyckeltal: ')
        self._2_drop_years = Dropdown(self._years,'År:')
        self._2_drop_munis = Dropdown(self._munis,'Kommun: ')
        self._2_drop_sekom = Dropdown(self._sekom, "Kommungrupp: ")
        self._2 = diagram_2()

        self._3_drop_years = Dropdown(self._years,'År: ')
        self._3_drop_munis = Dropdown(self._munis,'Kommun: ')
        self._3_drop_subj = Dropdown(["Matematik", "Svenska", "Engelska"],"Ämne: ")
        self._3 = diagram_3()

        self._4_drop_years = Dropdown(self._years, 'År: ')
        self._4_drop_munis = Dropdown(self._munis,'Kommun: ')
        self._4_drop_subj = Dropdown(["Matematik", "Svenska", "Engelska"],"Ämne: ")
        self._4_drop_over_under = Dropdown(["Betyg över NP-resultat", "Betyg under NP-resultat"],"högre/lägre: ")
        self._4 = diagram_4()

        self._5_keywords = ["N15419", "N15505", "N15436"]
        self._5_keydesc = [pf.key_to_desc[k] for k in self._5_keywords]
        self._5_drop_years = Dropdown(self._years, 'År: ')
        self._5_drop_munis = Dropdown(self._munis,'Kommun: ')
        self._5_drop_keyword = Dropdown(self._5_keydesc,"Nyckeltal: ")
        self._5 = diagram_5()

    def plot1(self):
        """
        Binder Dropdown-menyerna till plottar, och visar dem.
        """
        alter_widget_placement(widgets.interactive(self._1.update,keyword_desc=self._1_drop_keys.get(),year=self._1_drop_years.get(),
        kommun=self._1_drop_munis.get(), sekom=self._1_drop_sekom.get()),cols=2)

    def plot2(self):
        alter_widget_placement(widgets.interactive(self._2.update,keyword_desc=self._2_drop_keys.get(),year=self._2_drop_years.get(),
        kommun=self._2_drop_munis.get(),sekom=self._2_drop_sekom.get()),cols=2)

    def plot3(self):
        alter_widget_placement(widgets.interactive(self._3.update,year=self._3_drop_years.get(),
        kommun=self._3_drop_munis.get(),subject=self._3_drop_subj.get()),cols=3)

    def plot4(self):
        alter_widget_placement(widgets.interactive(self._4.update,year=self._4_drop_years.get(),
        kommun=self._4_drop_munis.get(),subject=self._4_drop_subj.get(),overUnder=self._4_drop_over_under.get()),cols=2)

    def plot5(self):
        alter_widget_placement(widgets.interactive(self._5.update,year=self._5_drop_years.get(),
        kommun=self._5_drop_munis.get(),keyword_desc=self._5_drop_keyword.get()),cols=3)

class ThreadingCounter(object):
    """ Threading example class
    The run() method will be started and it will run in the background
    until the application exits.
    """

    def __init__(self, interval=900): #Runns every 15 minutes
        """ Constructor
        :type interval: int
        :param interval: Check interval, in seconds
        """
        self.interval = interval

        thread = threading.Thread(target=self.run, args=())
        thread.daemon = True                            # Daemonize thread
        thread.start()                                  # Start the execution

    def run(self):
        x= 0
        """ Method that runs forever """
        X= True
        while X:
            x +=0.1
            print('')
            time.sleep(self.interval)

Threading = ThreadingCounter()


import os
import time

import numpy as np
import scipy.io
import tensorflow as tf
import tensorflow.keras.layers as layers
import tensorflow_addons as tfa
from skimage.transform import resize
from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import to_categorical

log_dir = "logs/"
BATCH_SIZE = 64
BUFFER_SIZE = 1000
images_shape = [256, 256, 3]
images_size = (256, 256)
embedding_size = 64
class_count = 2
EPOCHS = 50
margin = 0.5
abnormal_path = "/home/alireza/projects/python/triplet loss/dataset/ab.mat"
normal_path = "/home/alireza/projects/python/triplet loss/dataset/norm.mat"

def load_data_set():
    abnormal_data = np.array(scipy.io.loadmat(abnormal_path)['arr'], np.uint16)
    normal_data = np.array(scipy.io.loadmat(normal_path)['arr'], np.uint16)

    y = np.concatenate([np.zeros(normal_data.shape[0]), np.ones(abnormal_data.shape[0])])
    x = np.concatenate([normal_data, abnormal_data])
    x = x.astype("float32")
    x = np.divide(x, x.max())
    x = np.multiply(x, 255.0)
    new_x = []
    for image in x:
        trim_image = image[~np.all(image == 0, axis=1)]
        resized_image = resize(trim_image, images_size, anti_aliasing=True)
        new_x.append(resized_image)

    x = np.array(new_x)
    x = np.repeat(x[..., np.newaxis], 3, -1)
    y_categorical = to_categorical(y, class_count)

    x_train, x_test, y_train_categorical, y_test_categorical, y_train, y_test = train_test_split(x, y_categorical, y,
                                                                                test_size=0.2)

    return tf.data.Dataset.from_tensor_slices((x_train, y_train_categorical, y_train)).shuffle(BUFFER_SIZE).batch(
        BATCH_SIZE), tf.data.Dataset.from_tensor_slices(
        (x_test, y_test_categorical, y_test)).batch(BATCH_SIZE)

def make_model():
    input_layer = layers.Input(shape=images_shape)
    base_model = tf.keras.applications.VGG16(input_shape=images_shape,
                                             include_top=False,
                                             weights='imagenet',
                                             input_tensor=input_layer)
    base_model.trainable = False
    global_average_layer = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)
    output_layer = layers.Dense(256)(global_average_layer)
    output_layer = layers.ReLU()(output_layer)
    output_layer = layers.Dropout(0.3)(output_layer)
    output_layer = layers.Dense(128)(output_layer)
    output_layer = layers.ReLU()(output_layer)
    output_layer = layers.Dropout(0.3)(output_layer)
    embedding_layer = layers.Dense(embedding_size)(output_layer)
    embedding_layer = layers.ReLU()(embedding_layer)
    output_layer = layers.Dense(class_count)(embedding_layer)
    output_layer = layers.Softmax()(output_layer)
    return tf.keras.Model(inputs=input_layer, outputs=[embedding_layer, output_layer])

def print_models():
    model = make_model()
    tf.keras.utils.plot_model(
        model, to_file='model.png', show_shapes=True, show_layer_names=True,
        rankdir='TB', expand_nested=True, dpi=96
    )

def triplet_loss(y, model_output):
    return triplet_loss_object(y, model_output)

def classification_loss(y, output):
    return loss_object(y, output)

train_data_gen, test_data_gen = load_data_set()
model_optimizer = tf.keras.optimizers.Adam()
loss_object = tf.keras.losses.CategoricalCrossentropy()
triplet_loss_object = tfa.losses.TripletSemiHardLoss()
model = make_model()
checkpoint_dir = './training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
checkpoint = tf.train.Checkpoint(generator_optimizer=model_optimizer, model=model)

train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.CategoricalAccuracy(name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy')

@tf.function
def test_step(x, y, z):
    feature_output_anchor, model_output_anchor = model(x, training=False)

    loss = [classification_loss(y, model_output_anchor),
            triplet_loss(z, feature_output_anchor)]
    test_loss(loss)
    test_accuracy(y, model_output_anchor)

@tf.function
def train_step(x, y, z):
    with tf.GradientTape() as tape:
        feature_output_anchor, model_output_anchor = model(x, training=True)

        loss = [triplet_loss(z, feature_output_anchor),
                classification_loss(y, model_output_anchor)]

    gradients_of_model = tape.gradient(loss, model.trainable_variables)

    model_optimizer.apply_gradients(zip(gradients_of_model, model.trainable_variables))
    train_loss(loss)
    train_accuracy(y, model_output_anchor)

def train(train_data_set, test_data_set, epochs=EPOCHS):
    for epoch in range(epochs):
        start = time.time()

        for x, y, z in train_data_set:
            train_step(x, y, z)

        for x, y, z in test_data_set:
            test_step(x, y, z)

        if (epoch + 1) % 15 == 0:
            checkpoint.save(file_prefix=checkpoint_prefix)

        template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}, time: {}'
        print(template.format(epoch + 1, train_loss.result(),
                              train_accuracy.result() * 100,
                              test_loss.result(),
                              test_accuracy.result() * 100,
                              time.time() - start))

train(train_data_gen, test_data_gen, EPOCHS)


###########################################################################
# Created by: CASIA IVA
# Email: jliu@nlpr.ia.ac.cn
# Copyright (c) 2018
###########################################################################
from __future__ import division
import os
import numpy as np
import torch
import torch.nn as nn
from torch.nn.functional import upsample, normalize
from ..nn import PAM_Module, topk_PAM_Module
from ..nn import CAM_Module , SE_CAM_Module, selective_aggregation_ASPP_Module, reduce_PAM_Module,SE_CAM_Module2, selective_aggregation_ASPP_Module2
from ..models import BaseNet
from ..nn import ASPP_Module

__all__ = ['GLCNet5_topkpam', 'get_glcnet5_topkpam']

class GLCNet5_topkpam(BaseNet):
    r"""Fully Convolutional Networks for Semantic Segmentation

    Parameters
    ----------
    nclass : int
        Number of categories for the training dataset.
    backbone : string
        Pre-trained dilated backbone network type (default:'resnet50'; 'resnet50',
        'resnet101' or 'resnet152').
    norm_layer : object
        Normalization layer used in backbone network (default: :class:`mxnet.gluon.nn.BatchNorm`;

    Reference:

        Long, Jonathan, Evan Shelhamer, and Trevor Darrell. "Fully convolutional networks
        for semantic segmentation." *CVPR*, 2015

    """

    def __init__(self, nclass, backbone, aux=False, se_loss=False, norm_layer=nn.BatchNorm2d, **kwargs):
        super(GLCNet5_topkpam, self).__init__(nclass, backbone, aux, se_loss, norm_layer=norm_layer, **kwargs)
        self.head = GLCNet5_topkpamHead(2048, nclass, norm_layer)

        self.dsn = nn.Sequential(
            nn.Conv2d(1024, 512, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(512), nn.ReLU(inplace=True),
            nn.Dropout2d(0.1),
            nn.Conv2d(512, nclass, kernel_size=1, stride=1, padding=0, bias=True)
        )
    def forward(self, x):
        imsize = x.size()[2:]
        _, _, c3, c4 = self.base_forward(x)
        # x_dsn = self.dsn(c3)
        x = self.head(c4)
        x = list(x)
        x[0] = upsample(x[0], imsize, **self._up_kwargs)
        # x[1] = upsample(x[1], imsize, **self._up_kwargs)
        # x[2] = upsample(x[2], imsize, **self._up_kwargs)
        # x[3] = upsample(x[3], imsize, **self._up_kwargs)
        # x_dsn = upsample(x_dsn, imsize, **self._up_kwargs)

        outputs = [x[0]]
        # outputs.append(x[1])
        # outputs.append(x[2])
        # outputs.append(x[3])
        # outputs.append(x_dsn)

        return tuple(outputs)

class GLCNet5_topkpamHead(nn.Module):
    def __init__(self, in_channels, out_channels, norm_layer):
        super(GLCNet5_topkpamHead, self).__init__()
        inter_channels = in_channels // 4

        self.conv5a = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.conv5as = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.sa = topk_PAM_Module(inter_channels, 256, inter_channels, 10)
        self.conv51 = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())

        self.conv5s = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 1, padding=0, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.aspp = selective_aggregation_ASPP_Module2(inter_channels, inner_features=256, out_features=256,
                                                      dilations=(12, 24, 36))
        self.conv52 = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())

        self.conv5c = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.sc = SE_CAM_Module2(inter_channels)
        self.conv53 = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())

        self.conv5 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))
        self.conv6 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))
        self.conv7 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))
        self.conv8 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))

    def forward(self, x):
        #ssa
        feat1 = self.conv5a(x)
        sa_feat = self.sa(feat1)
        sa_conv = self.conv51(sa_feat)

        sa_output = self.conv5(sa_conv)

        output =[sa_output]
        #aaspp
        #
        # feat2 = self.conv5s(x)
        # feat_fuse = sa_conv + feat2

        # feat_as = self.conv5as(x)
        # aspp_feat,_ = self.aspp(feat_as)
        # aspp_conv = self.conv52(aspp_feat)
        #
        # aspp_output = self.conv6(aspp_conv)
        #
        # #sec
        # feat2 = self.conv5c(x)
        # sc_feat = self.sc(feat2)
        # sc_conv = self.conv53(sc_feat)
        #
        # sc_output = self.conv7(sc_conv)
        #
        # #fuse
        # feat_sum = aspp_conv + sc_conv + sa_conv
        # sasc_output = self.conv8(feat_sum)

        # output = [sasc_output]
        # output.append(sa_output)
        # output.append(sc_output)
        # output.append(aspp_output)

        return tuple(output)

def get_glcnet5_topkpam(dataset='pascal_voc', backbone='resnet50', pretrained=False,
              root='./pretrain_models', **kwargs):
    r"""DANet model from the paper `"Dual Attention Network for Scene Segmentation"
    <https://arxiv.org/abs/1809.02983.pdf>`
    """
    acronyms = {
        'pascal_voc': 'voc',
        'pascal_aug': 'voc',
        'pcontext': 'pcontext',
        'ade20k': 'ade',
        'cityscapes': 'cityscapes',
    }
    # infer number of classes
    from ..datasets import datasets, VOCSegmentation, VOCAugSegmentation, ADE20KSegmentation
    model = GLCNet5_topkpam(datasets[dataset.lower()].NUM_CLASS, backbone=backbone, root=root, **kwargs)
    if pretrained:
        from .model_store import get_model_file
        model.load_state_dict(torch.load(
            get_model_file('fcn_%s_%s' % (backbone, acronyms[dataset]), root=root)),
            strict=False)
    return model


"""
Message producer.

"""
from collections import defaultdict

from boto3 import client
from microcosm.api import defaults

from microcosm_pubsub.errors import TopicNotDefinedError

class SNSProducer(object):
    """
    Produces messages to SNS topics.

    """
    def __init__(self, sns_client, sns_topic_arns, pubsub_message_codecs):
        self.sns_client = sns_client
        self.sns_topic_arns = sns_topic_arns
        self.pubsub_message_codecs = pubsub_message_codecs

    def produce(self, media_type, dct=None, **kwargs):
        """
        Produce a message.

        :returns: the message id

        """
        topic_arn = self.choose_topic_arn(media_type)
        content = self.pubsub_message_codecs[media_type].encode(dct, **kwargs)
        result = self.sns_client.publish(
            TopicArn=topic_arn,
            Message=content,
        )
        return result["MessageId"]

    def choose_topic_arn(self, media_type):
        """
        Choose a topic for this type of message.

        """
        try:
            topic_arn = self.sns_topic_arns[media_type]
        except KeyError:
            topic_arn = None

        if topic_arn is None:
            raise TopicNotDefinedError("No topic arn was registered for messages of type: {}".format(
                media_type,
            ))
        return topic_arn

@defaults(
    default=None,
    mappings={},
)
def configure_sns_topic_arns(graph):
    """
    Configure a mapping from message types to topic ARNs.

    """
    if graph.config.sns_topic_arns.default is None:
        sns_topic_arns = dict()
    else:
        sns_topic_arns = defaultdict(lambda: graph.config.sns_topic_arns.default)

    sns_topic_arns.update(graph.config.sns_topic_arns.mappings)

    return sns_topic_arns

def configure_sns_producer(graph):
    """
    Configure an SNS producer.

    """
    if graph.metadata.testing:
        from mock import MagicMock
        sns_client = MagicMock()
    else:
        sns_client = client("sns")

    return SNSProducer(
        sns_client=sns_client,
        sns_topic_arns=graph.sns_topic_arns,
        pubsub_message_codecs=graph.pubsub_message_codecs,
    )


from fit import sliding_window_detector, polynomial_detector
from sklearn.metrics import mean_squared_error

MSE_THRESHOLD = 1000

class FitPolicy:
    def __init__(self, tolerance, scale):
        def mse_threshold(l, r, base_l, base_r, th):
            return (mean_squared_error(*x) < th for x in ((l, base_l), (r, base_r)))

        self._max_tolerance = tolerance
        self._tolerance = 0
        self._l_base = [None]
        self._r_base = [None]
        self._l_poly = None
        self._r_poly = None

        self._scaled_polynomial_detector = lambda lp, rp: polynomial_detector(lp, rp, *scale)
        self._mse_threshold_acceptor = lambda lb, rb: lambda l, r:  mse_threshold(l, r, lb, rb, MSE_THRESHOLD)
        self._force_acceptor = lambda l, r: (True, True)

    def __next__(self):
        if self._tolerance <= 0:
            self._tolerance = self._max_tolerance
            return (
                sliding_window_detector(self._l_base[-1], self._r_base[-1]),
                self._force_acceptor
            )
        else:
            return (
                self._scaled_polynomial_detector(self._l_poly, self._r_poly),
                self._mse_threshold_acceptor(self._l_base, self._r_base)
            )

    def accepted(self, l_base, r_base, l_poly, r_poly):
        self._tolerance = min(self._max_tolerance, self._tolerance + 1)
        self._l_base, self._r_base = l_base, r_base
        self._l_poly, self._r_poly = l_poly, r_poly

    def rejected(self, l_base, r_base):
        self._tolerance -= 1
        self._l_base, self._r_base = l_base, r_base

if __name__ == '__main__':
    fp = FitPolicy(1, (1, 1))
    d, a = next(fp)
    fp.rejected([1], [2])
    next(fp)
    fp.accepted([1], [2], [3], [4])
    next(fp)


from dos_monitor import DOSmonitor
from qlf_models import QLFModels
from multiprocessing import Process, Event, Value, Manager, Lock
import Pyro4
import configparser
import sys
import os
import errno
from socket import error as socket_error
from log import setup_logger
from procutil import kill_proc_tree
from qlf_pipeline import Jobs as QLFPipeline
from scalar_metrics import LoadMetrics

qlf_root = os.getenv('QLF_ROOT')
cfg = configparser.ConfigParser()

cfg.read('%s/framework/config/qlf.cfg' % qlf_root)
logfile = cfg.get("main", "logfile")
logpipeline = cfg.get("main", "logpipeline")
loglevel = cfg.get("main", "loglevel")

logger = setup_logger("icslogger", logfile, loglevel)
mainlogger = setup_logger("main_logger", "main_daemon.log", loglevel)

class QLFAutoRun(Process):

    def __init__(self, exposures=None):
        super().__init__()
        self.running = Event()
        self.exit = Event()
        self.dos_monitor = DOSmonitor()
        self.process_id = Value('i', 0)
        self.base_night = self.dos_monitor.get_last_night()
        self.list_lock = Lock()
        self.exposures = Manager().list()

        if not exposures:
            exposures = self.dos_monitor.get_exposures_by_night(
                self.base_night)

            for item in exposures:
                item['discovery'] = True
                self.exposures.append(item)
        else:
            self.add_exposures(exposures)

    def add_exposures(self, exposures):
        with self.list_lock:
            for exposure in exposures:
                item = self.dos_monitor.get_exposure(self.base_night, exposure)
                self.exposures.append(item)

    def run(self):
        while not self.get_exit():

            with self.list_lock:
                exposure = self.exposures.pop(0) if self.exposures else None

            if self.get_exit():
                mainlogger.info('Exit is set')
                break

            if not exposure:
                continue

            if exposure.get('discovery', None):
                logger.info('Found expID {}'.format(exposure.get('expid')))

            self.running.set()

            try:
                ql = QLFPipeline(exposure)
                self.process_id.value = ql.start_process()
                ql.start_jobs()
                ql.finish_process()
            except socket_error as serr:
                if serr.errno != errno.ECONNREFUSED:
                    mainlogger.exception('Daemon Error')
                    raise

            self.running.clear()
            self.process_id.value = 0

    def get_current_process_id(self):
        """ """
        return self.process_id.value

    def set_exit(self, value=True):
        """ """
        if value:
            self.exit.set()
        else:
            self.exit.clear()

    def get_exit(self):
        """ """
        return self.exit.is_set()

# TODO: refactor QLFManualRun
class QLFManualRun(Process):

    def __init__(self, exposures):
        super().__init__()
        self.running = Event()
        self.exit = Event()
        self.current_exposure = None

        # TODO: improve the method for obtaining exposures
        dos_monitor = DOSmonitor()
        night = dos_monitor.get_last_night()
        self.exposures = list()

        for exposure in exposures:
            self.exposures.append(dos_monitor.get_exposure(night, exposure))

    def run(self):
        self.exit.clear()

        for exposure in self.exposures:
            if self.exit.is_set():
                mainlogger.info('Execution stopped')
                break

            self.running.set()
            self.current_exposure = exposure
            ql = QLFPipeline(self.current_exposure)
            mainlogger.info('Executing expid {}...'.format(exposure.get('expid')))
            ql.start_process()
            ql.start_jobs()
            ql.finish_process()

        self.running.clear()
        self.current_exposure = None

        logger.info("Bye!")
        self.shutdown()

    def clear(self):
        self.exit.clear()

    def shutdown(self):
        self.exit.set()

@Pyro4.expose
@Pyro4.behavior(instance_mode="single")
class QLFAutomatic(object):
    def __init__(self):
        self.process = False

    def start(self, exposures=None):
        if self.process and self.process.is_alive():
            self.process.set_exit(False)
            mainlogger.info("Monitor is already initialized (pid: %i)." % self.process.pid)
        else:
            self.process = QLFAutoRun(exposures)
            self.process.start()
            mainlogger.info("Starting pid %i..." % self.process.pid)

    def stop(self):
        if self.process and self.process.is_alive():
            self.process.set_exit()
            mainlogger.info("Stop pid %i" % self.process.pid)

            process_id = self.process.get_current_process_id()
            pid = self.process.pid

            kill_proc_tree(pid, include_parent=False)
            truncate_logs([logpipeline, logfile])

        else:
            mainlogger.info("Monitor is not initialized.")

    def reset(self):
        self.stop()
        truncate_logs([logpipeline, logfile])

    def add_exposures(self, exposures):

        if not isinstance(exposures, list):
            mainlogger.info("add_exposures function requires a list of exposures")
            return

        if self.process and self.process.is_alive():
            self.process.add_exposures(exposures)
        else:
            self.start(exposures)

    def get_status(self):
        status = False

        if self.process and not self.process.get_exit():
            status = True

        return status

    def is_running(self):
        running = False

        if self.process and self.process.running.is_set():
            running = True

        return running

    def qa_tests(self, process_id):
        qa_tests = list()
        for job in QLFModels().get_jobs_by_process_id(process_id):
            try:
                process = job.process
                exposure = process.exposure
                lm = LoadMetrics(process_id, job.camera_id, process.exposure_id, exposure.night)
                qa_tests.append({job.camera_id: lm.load_qa_tests()})
            except:
                mainlogger.error('qa_tests error camera %s' % job.camera)
        return qa_tests

    def load_scalar_metrics(self, process_id, cam):
        scalar_metrics = dict()
        try:
            process = QLFModels().get_process_by_process_id(process_id)
            exposure = process.exposure
            lm = LoadMetrics(process_id, cam, process.exposure_id, exposure.night)
            scalar_metrics['metrics'] = lm.metrics
            scalar_metrics['tests'] = lm.tests
        except:
            print('load_scalar_metrics error')
        return scalar_metrics

# TODO: refactor QLFManual
@Pyro4.expose
@Pyro4.behavior(instance_mode="single")
class QLFManual(object):

    def __init__(self):
        self.process = False
        self.exposures = list()

    def start(self, exposures):
        if self.process and self.process.is_alive():
            self.process.clear()
            mainlogger.info("Monitor is already initialized (pid: %i)." % self.process.pid)
        else:
            self.process = QLFManualRun(exposures)
            self.process.start()
            mainlogger.info("Starting pid %i..." % self.process.pid)

    def stop(self):
        if self.process and self.process.is_alive():
            mainlogger.info("Stop pid %i" % self.process.pid)
            self.process.shutdown()
        else:
            mainlogger.info("Monitor is not initialized.")

    def get_status(self):
        status = False

        if self.process and not self.process.exit.is_set():
            status = True

        mainlogger.info("QLF Manual status: {}".format(status))
        return status

    def get_current_run(self):
        return self.process.current_exposure

def truncate_logs(logs):
    for log in logs:
        with open(log, 'r+') as filelog:
            filelog.truncate()

def main():
    try:
        auto_mode = os.environ.get('QLF_DAEMON_NS', 'qlf.daemon')
        manual_mode = os.environ.get('QLF_MANUAL_NS', 'qlf.manual')
        host = os.environ.get('QLF_DAEMON_HOST', 'localhost')
        port = int(os.environ.get('QLF_DAEMON_PORT', '56005'))
    except Exception as err:
        logger.error(err)
        sys.exit(1)

    Pyro4.Daemon.serveSimple(
        {QLFAutomatic: auto_mode, QLFManual: manual_mode},
        host=host,
        port=port,
        ns=False
    )

if __name__ == "__main__":
    main()


from PyQt5 import (
    QtWidgets,
    QtCore,
)

from slot_signal_ui import Ui_Form

QtCore.pyqtSlot(str)
def emit_but_slot(name):
    print(f'emitted: {name}')

class MyForm(QtWidgets.QWidget):
    my_signal = QtCore.pyqtSignal(str)

    def __init__(self):
        super().__init__()
        self.ui = Ui_Form()
        self.ui.setupUi(self)
        self.ui.emit_but.clicked.connect(self.emit_but_cb)
        self.th = MyThread()
        self.my_signal.connect(self.th.emit_but_slot)

    def emit_but_cb(self):
        self.my_signal.emit('abc')

class MyThread(QtCore.QThread):

    def __init__(self):
        super().__init__()

    @QtCore.pyqtSlot(str)
    def emit_but_slot(self, name):
        print(f'another thread: {name}')

if __name__ == '__main__':
    app = QtWidgets.QApplication([])
    form = MyForm()
    form.my_signal.connect(emit_but_slot)
    form.show()
    app.exec()


#!/bin/env python3

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import sys, getopt

inputfile = ""
outputfile = ""
start = 0
end = -1

try:
	opts, args = getopt.getopt(sys.argv[1:],"hi:o:s:e:",["ifile=","ofile="])
except getopt.GetoptError:
	print("plot.py -i <inputfile> -o <outputfile>")
	sys.exit(2)

for opt, arg in opts:
	if opt == '-h':
		print("plot.py -i <inputfile> -o <outputfile>")
		sys.exit()
	elif opt in ("-i", "--ifile"):
		inputfile = arg
	elif opt in ("-o", "--ofile"):
		outputfile = arg
	elif opt == "-s":
		start = int(arg)
	elif opt == "-e":
		end = int(arg)

if not inputfile or not outputfile:
	print("plot.py -i <inputfile> -o <outputfile>")
	sys.exit(2)

f = open(inputfile, "rb")
data = f.read(5)

timestamps = []
values = []
d1 = []
d2 = []
median = []

i = 0
while True:
	i += 1
	t = f.read(8)
	v = f.read(2)

	timestamp = int.from_bytes(t, byteorder="little")
	value = int.from_bytes(v, byteorder="little")
	timestamps.append(timestamp)
	values.append(value)

	if (len(v) < 2 or len(t) < 8 or i > 50000):
		break

MIN_VALIDITY_MICROS = 	60000
INTEGRATE_MICROS = 		55000
MEDIAN_MICROS = 	   120000
RESET_INTEGRATION =		90000
INTEGRAL_THRESHOLD = 	 5000

MIN_LEVEL_MAXIMUM = 	  200
MAX_MICROS_UPFLANK = 	40000
MAX_MICROS_DOWNFLANK =  40000

values = np.asarray(values)
timestamps = np.asarray(timestamps)
md = np.zeros(len(values))
integral = np.zeros(len(values))

t0 = timestamps[0]
timestamps[0] -= t0

median_backptr = 0

for i in range(1, len(values)):
	timestamps[i] -= t0

	while (timestamps[i] - timestamps[median_backptr] > MEDIAN_MICROS):
		median_backptr += 1

	md[i] = np.median(values[median_backptr:i+1])

for i in range(1, len(values)):
	values[i] = values[i] - md[i] - 10

beats = []
armed = False

invalidity_backptr = 0
integration_backptr = 0
last_negative1 = 0
last_negative2 = 0

for i in range(1, len(values)):

	t_diff = timestamps[i] - timestamps[i-1]

	if t_diff > 3000 or t_diff < 0:
		print("new invalidity at " + str(i))
		invalidity_backptr = i

	if values[i] < 0:
		integration_backptr = i
		last_negative2 = last_negative1
		last_negative1 = i

	if timestamps[i] - timestamps[last_negative1] > RESET_INTEGRATION:
		integration_backptr = i

	while (timestamps[i] - timestamps[integration_backptr]) > INTEGRATE_MICROS:
		integration_backptr += 1

	if (integration_backptr < i+1):
		integral[i] = np.sum(values[integration_backptr:i+1]) /  25
	else:
		integral[i] = 0

	if (integral[i]*25 > INTEGRAL_THRESHOLD) and not armed:
		armed = True
		print("armed at " + str(i))

	if armed and last_negative1 == i:
		armed = False
		print("check at " + str(i))
		print(invalidity_backptr)
		print(timestamps[i])
		print(timestamps[invalidity_backptr])

		if timestamps[i] - timestamps[invalidity_backptr] > 60000:

			if (timestamps[i] - timestamps[last_negative2]) <= 60000:

				m = np.amax(values[last_negative2:i+1])
				k1 = last_negative2
				k2 = i
				while (values[k1] < 0.7*m):
					k1 += 1
				while (values[k2] < 0.7*m):
					k2 -= 1

				#print("m = " +str(m))
				#print(str(k1-j))
				#print(str(i -k2))

				if m > MIN_LEVEL_MAXIMUM and timestamps[k1] - timestamps[last_negative2] < MAX_MICROS_UPFLANK and timestamps[last_negative1] - timestamps[k2] < MAX_MICROS_DOWNFLANK:
					beats.append(i)
		else:
			print("invalidity too close")

print(timestamps[end] - timestamps[start])

#for i in range(0, len(values)):
#	d1[i] = d1[i]*2
#	d2[i] = d2[i]*2

if end == -1:
	end = len(values)

x = np.arange(start, end)
#x = timestamps[start:end]
fig, ax = plt.subplots()
fig.set_size_inches(200,10)
ax.plot(x, values[start:end], ".-")
#ax.plot(x, d1[start:end], ".-")
#ax.plot(x, d2[start:end], ".-")
ax.plot(x, md[start:end], ".-")
#ax.plot(x, sig1[start:end], ".-")
#ax.plot(x, sig2[start:end], ".-")
#ax.plot(x, sig3[start:end], ".-")
ax.plot(x, integral[start:end], ".-")
ax.plot([start, end], [0, 0], 'k-', lw=1)
ax.plot([start, end], [100, 100], 'k-', lw=1)
ax.plot([start, end], [200, 200], 'k-', lw=1)
ax.plot([start, end], [300, 300], 'k-', lw=1)
ax.plot([start, end], [400, 400], 'k-', lw=1)
ax.plot([start, end], [500, 500], 'k-', lw=1)

for b in beats:
	if b >= start and b <= end:
		ax.plot([b,b], [0, 500], 'b-', lw=2)

fig.savefig(outputfile, dpi=100)
plt.show


import cv2
import face_recognition
import os

#https://stackoverflow.com/questions/17967320/python-opencv-convert-image-to-byte-string

def main_encoding_face(PATH_PROFILE):
    '''
    PATH_DISK - путь где лежат фотографии
    path - путь где лежат фотографии
    image_paths - список всех ссылок фотографиий
    image_name - имя фотоки
    resilt_dic - список {'имя фото': 'массив фото'}

    определяет массив лица и записывает в result_dic имя фотки и его массив

    return список {'имя фото': 'массив фото'}
    '''
    result_dic = {}

    def get_preview_face_list(image):
        face_list = []
        fac_loc = face_recognition.face_locations(image)
        for el_face in fac_loc:
            cropped = image[el_face[0]:el_face[2], el_face[3]:el_face[1]]
            cropped = cv2.resize(cropped, (200, 200), interpolation=cv2.INTER_AREA)
            img_to_byte = cv2.imencode('.jpg', cropped)[1].tostring()
            face_list.append(img_to_byte)
        return face_list

    def get_preview_photo(image):
        image = cv2.resize(image, (200, 200), interpolation=cv2.INTER_AREA)
        img_to_byte = cv2.imencode('.jpg', image)[1].tostring()
        return img_to_byte

    def get_encode_image(image):
        image_encode = face_recognition.face_encodings(image)
        return image_encode

    # перебор всех файлов из папки
    path = f'{PATH_PROFILE}/photo/'
    image_paths = [os.path.join(path, f) for f in os.listdir(path)]
    for image in image_paths:
        if image.endswith('.jpg') or image.endswith('.jpeg'):
            try:  # загрузилась сшиком большая фотография
                image_name = image
                image = cv2.cvtColor(face_recognition.load_image_file(f'{image_name}'), cv2.COLOR_BGR2RGB)
                encode_image = get_encode_image(image)
                if encode_image != []:
                    preview_photo = get_preview_photo(image)
                    preview_face_list = get_preview_face_list(image)
                    result_dic[image_name] = {'encode': encode_image,
                                              'preview_photo': preview_photo,
                                              'preview_face_list': preview_face_list
                                              }
            except:
                continue
        else:
            continue
    return result_dic

if __name__ == '__main__':
    print(main_encoding_face('C:/inst_test/3170429672'))


import numpy as np
import matplotlib.pyplot as plt

rounds1 = ()
rounds2 = ()
rounds3 = ()

fig, ax = plt.subplots()
plt.rc('font',**{'family':'Times'})
plt.rc('text', usetex=True)
index = np.arange(4)

plt.plot(index, rounds1, '-o', color='black', linewidth=3, markersize = 8, label = r"\beta=1")
plt.plot(index, rounds2, '-d', color='grey', linewidth=3, markersize = 8, label = r"\beta=2")
plt.plot(index, rounds3, '-^', color='silver', linewidth=3, markersize = 8, label = r"\beta=3")

plt.xlabel(r'SINR path-loss exponent ($\alpha$)', fontsize = 24)
plt.ylabel('Running time (rounds)', fontsize = 24)
#ax.set_xlim(-0.5, 4.5)
#ax.set_ylim(0, 450)
ax.tick_params(axis='x', which='major', pad=8)
plt.xticks(index, (3, 4, 5, 6), fontsize = 20)
plt.yticks(fontsize = 20)
plt.legend(loc = 'upper left', numpoints = 1, prop = {'size':24})

fig.set_tight_layout(True)
fig.savefig('alpha-beta.pdf', format='pdf', dpi = 500)


import os
import glob
import unittest
import sys

import torch
import torchvision
from PIL import Image
from torchvision.io.image import read_png, decode_png, read_jpeg, decode_jpeg
import numpy as np

IMAGE_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets")
IMAGE_DIR = os.path.join(IMAGE_ROOT, "fakedata", "imagefolder")
DAMAGED_JPEG = os.path.join(IMAGE_ROOT, 'damaged_jpeg')

def get_images(directory, img_ext):
    assert os.path.isdir(directory)
    for root, _, files in os.walk(directory):
        if os.path.basename(root) == 'damaged_jpeg':
            continue

        for fl in files:
            _, ext = os.path.splitext(fl)
            if ext == img_ext:
                yield os.path.join(root, fl)

class ImageTester(unittest.TestCase):
    def test_read_jpeg(self):
        for img_path in get_images(IMAGE_ROOT, ".jpg"):
            img_pil = torch.load(img_path.replace('jpg', 'pth'))
            img_ljpeg = read_jpeg(img_path)
            self.assertTrue(img_ljpeg.equal(img_pil))

    def test_decode_jpeg(self):
        for img_path in get_images(IMAGE_ROOT, ".jpg"):
            img_pil = torch.load(img_path.replace('jpg', 'pth'))
            size = os.path.getsize(img_path)
            img_ljpeg = decode_jpeg(torch.from_file(img_path, dtype=torch.uint8, size=size))
            self.assertTrue(img_ljpeg.equal(img_pil))

        with self.assertRaisesRegex(ValueError, "Expected a non empty 1-dimensional tensor."):
            decode_jpeg(torch.empty((100, 1), dtype=torch.uint8))

        with self.assertRaisesRegex(ValueError, "Expected a torch.uint8 tensor."):
            decode_jpeg(torch.empty((100, ), dtype=torch.float16))

        with self.assertRaises(RuntimeError):
            decode_jpeg(torch.empty((100), dtype=torch.uint8))

    def test_damaged_images(self):
        # Test image with bad Huffman encoding (should not raise)
        bad_huff = os.path.join(DAMAGED_JPEG, 'bad_huffman.jpg')
        try:
            _ = read_jpeg(bad_huff)
        except RuntimeError:
            self.assertTrue(False)

        # Truncated images should raise an exception
        truncated_images = glob.glob(
            os.path.join(DAMAGED_JPEG, 'corrupt*.jpg'))
        for image_path in truncated_images:
            with self.assertRaises(RuntimeError):
                read_jpeg(image_path)

    def test_read_png(self):
        # Check across .png
        for img_path in get_images(IMAGE_DIR, ".png"):
            img_pil = torch.from_numpy(np.array(Image.open(img_path)))
            img_lpng = read_png(img_path)
            self.assertTrue(img_lpng.equal(img_pil))

    def test_decode_png(self):
        for img_path in get_images(IMAGE_DIR, ".png"):
            img_pil = torch.from_numpy(np.array(Image.open(img_path)))
            size = os.path.getsize(img_path)
            img_lpng = decode_png(torch.from_file(img_path, dtype=torch.uint8, size=size))
            self.assertTrue(img_lpng.equal(img_pil))

            with self.assertRaises(ValueError):
                decode_png(torch.empty((), dtype=torch.uint8))
            with self.assertRaises(RuntimeError):
                decode_png(torch.randint(3, 5, (300,), dtype=torch.uint8))

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
import roslib
roslib.load_manifest('collision_avoidance')
import rospy
import numpy as np
import cv2

from geometry_msgs.msg import Point32
from geometry_msgs.msg import Twist
from sensor_msgs.msg import Image #For receiving video feed
roslib.load_manifest('ardrone_autonomy')
from ardrone_autonomy.msg import *

from threading import Lock
from cv_bridge import CvBridge, CvBridgeError

final_pos = []

rospy.init_node('recorder')

global command_vel
command_vel = Twist()

class Recorder:
	def __init__(self):
		subPosition = rospy.Subscriber('obstacle_pos', Point32, self.callback_pos)
		subVelocity = rospy.Subscriber('/cmd_vel',Twist,self.callback_vel)
		subState = rospy.Subscriber('/ardrone/navdata', Navdata, self.callback_state)
		self.cap = rospy.Subscriber('/ardrone/front/image_raw',Image,self.callback_image)
		self.imageLock = Lock()
		self.bridge = CvBridge()
		self.state = -1
		self.altitude = -1

		self.obstacle_pos = Point32()
	def callback_image(self,data):
		self.imageLock.acquire()
		try:
			self.frame = self.bridge.imgmsg_to_cv2(data,"bgr8") # Save the ros image for processing by the display thread

			orig_img = np.asarray(self.frame[:,:])
			cv2.putText(orig_img, "x_coordinate: "+str(self.state), (10, 150), cv2.FONT_HERSHEY_PLAIN, 1,(0,0,0),2)
			cv2.imshow('',orig_img)

		except CvBridgeError as e:
			print(e)
		finally:
			self.imageLock.release()

	def callback_state(self,navdata):
		self.state = navdata.state
		self.altitude = navdata.altd

	def callback_pos(self,data):
		self.obstacle_pos.x, self.obstacle_pos.y, self.obstacle_pos.z = data.x, data.y, data.z

	def callback_vel(self,data):
		command_vel.linear.x, command_vel.linear.y,command_vel.linear.z, = data.linear.x, data.linear.y, data.linear.z
		self.run(command_vel)

	def run(self,data):
		command_vel.linear.x, command_vel.linear.y,command_vel.linear.z, = data.linear.x, data.linear.y, data.linear.z

		final_pos.append([self.obstacle_pos.x, self.obstacle_pos.y, self.obstacle_pos.z,command_vel.linear.x, command_vel.linear.y, command_vel.linear.z,  self.state, self.altitude])
		np.savetxt('position.txt',np.c_[final_pos],delimiter=",")

def main():
	Recorder().__init__()
	rospy.spin()

if __name__ == '__main__':
	main()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

from subprocess import Popen, PIPE
import os
from time import time
import requests
from bs4 import BeautifulSoup

urls = [
    'https://emojipedia.org/people/',
    'https://emojipedia.org/nature/',
    'https://emojipedia.org/food-drink/',
    'https://emojipedia.org/activity/',
    'https://emojipedia.org/travel-places/',
    'https://emojipedia.org/objects/',
    'https://emojipedia.org/symbols/',
    'https://emojipedia.org/flags/']

emoji_cache = os.path.join(os.path.expanduser('~'), '.cache/rofimojis')

def refresh_cache():
    text = ''
    for url in urls:
        data = requests.get(url)  # type: requests.Response
        soup = BeautifulSoup(data.content, 'lxml')  # type: BeautifulSoup

        soup = soup.find("div", attrs={"class": "content"})
        for table_row in soup.find_all('li'):
            emoji_row = table_row.find('a')
            text += emoji_row.get_text() + '\n'
    with open(emoji_cache, 'w') as f:
        f.write(text)

    return text

def open_cache():
    with open(emoji_cache, 'r') as f:
        return f.read()

def get_emojis():
    if not os.path.exists(emoji_cache):
        return refresh_cache()

    last_modified = os.stat(emoji_cache).st_mtime
    if (time() - last_modified >= 60 * 60 * 24 * 30):
        return refresh_cache()

    return open_cache()

emojis = get_emojis()

rofi = Popen(
    args=[
        'rofi',
        '-dmenu',
        '-i',
        '-p',
        ' 😀   ',
        '-kb-custom-1',
        'Alt+c'
    ],
    stdin=PIPE,
    stdout=PIPE
)
(stdout, stderr) = rofi.communicate(input=emojis.encode('utf-8'))

if rofi.returncode == 1:
    exit()
else:
    emoji = stdout.split()[0]
    if rofi.returncode == 0:
        Popen(
            args=[
                'xdotool',
                'type',
                '--clearmodifiers',
                emoji.decode('utf-8')
            ]
        )
    elif rofi.returncode == 10:
        xsel = Popen(
            args=[
                'xsel',
                '-i',
                '-b'
            ],
            stdin=PIPE
        )
        xsel.communicate(input=emoji)


from django.http import HttpResponse, HttpResponseRedirect, HttpRequest
from django.template import Context, loader
from stats.models import SearchIP

def time(request):
    if request.method == 'GET':
        tm = SearchIP().Get_date()
        return HttpResponse(tm)

def myip(request):
    if request.method == 'GET':
        ipaddr = request.META['REMOTE_ADDR']
        return HttpResponse(ipaddr)

def index(request):
    template = loader.get_template('index.html')
    return HttpResponse(template.render())

def search(request):
    if 'ip' in request.GET:
        ip = request.GET['ip']
        tdt = request.GET['tdt']
        data = SearchIP().get_like_ip(ip, tdt)
        template = loader.get_template('result.html')
        if data == u'NotData' or data == u'NotConnect' or data == u'IPaddressNotValid' or data == u'DateNotValid':
            context = dict({
                'data': data,
            })
            template = loader.get_template('error.html')
            return HttpResponse(template.render(context))
        context = dict({
            'data': data,
        })
        return HttpResponse(template.render(context))
    else:
        template = loader.get_template('error.html')
        return HttpResponse(template.render())


"""SDRF URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path,include
from SDRFAPP import views
from rest_framework.routers import DefaultRouter
router =DefaultRouter()

#regsiter studentvieset with routre

#router.register('studenapi',views.StudentViewSet,basename='student'),
#router.register('studenapi',views.StudenModeltViewSet,basename='student'),
router.register('studenapi',views.StudenReadOnlyModeltViewSet,basename='student'),

urlpatterns = [
    path('admin/', admin.site.urls),
    path('studinfo/<int:pk>',views.StudentDetails),#this url helps to get single data
    path('studinfo/',views.StudentDetailsList),#this one by all data
    path('studcreate/', views.StudentCreate),
    path('studapi/', views.student_api),
    path('staffapi/', views.StaffAPI.as_view()),
    path('empapi/', views.EmployeeCreate),
    path('EmployeeAPI/', views.EmployeeAPI),
    path('EmployeeAPI/<int:pk>', views.EmployeeAPI),
    path('EmployeeAPIView/', views.EmployeeAPIView.as_view()),
    path('EmployeeAPIView/<int:pk>', views.EmployeeAPIView.as_view()),
    # path('studentapi/',views.StudentList.as_view()),
    # path('studentapi/',views.StudentCreate.as_view()),
    # path('studentapi/<int:pk>',views.StudentRetrieve.as_view()),
    # path('studentapi/<int:pk>',views.StudentUpdate.as_view()),
    # path('studentapi/<int:pk>',views.StudentDestroy.as_view()),
    # path('studentapi/',views.StudentListCreateAPIVIEW.as_view()),
    # path('studentapi/<int:pk>',views.StudentUDRAPIVIEW.as_view()),
    # path('studentapi/',views.studList.as_view()),
    # path('studentapi/',views.studCreateList.as_view()),
    # path('studentapi/<int:pk>',views.studRetrieveList.as_view()),
    # path('studentapi/<int:pk>',views.studUpdateList.as_view()),
    path('studentapi/<int:pk>', views.studDestroyList.as_view()),
    path('api/',include(router.urls)),

]


import matplotlib.pyplot as plt
import numpy as np
import sys

with open('./results/'+sys.argv[2],'r') as f:
	lines = f.readlines()
#avg = int(lines.pop(-1))
noise = [int(i) for i in lines]

with open('./results/'+sys.argv[1],'r') as f:
	lines = f.readlines()
#avg = int(lines.pop(-1))
times = [int(i) for i in lines]
m = np.mean(times)
s = np.std(times)
sigma = 1
print(m,s,m+s*sigma)
#outlier = 350

#times = [t for t in times if t<m+s*sigma and t<outlier]

#plt.plot(range(len(times)),times,color='red')
plt.scatter(range(len(times)),times,color='blue',marker='.')
plt.scatter(range(len(noise)),noise,color='red',marker='.')
#for xc in range(0,401,20):
#	plt.axvline(x=xc)
#plt.ylim(0,500)

plt.ylabel('latency (cycles)')
plt.xlabel('trials (20 per second)')
plt.ylim(m-s*sigma,m+s*sigma)
plt.savefig('./figs/{}.png'.format(sys.argv[1]))
plt.show()


import os
import gzip
import numpy as np
import struct
import random
import math
import scipy.signal
import librosa
import torch
import scipy.io as sio
#import torchaudio
import scipy.io.wavfile as wav
from scipy import signal

from py_utils.wav_io import readwav, writewav
from py_utils.utils import convfft

windows = {'hamming': scipy.signal.hamming, 'hann': scipy.signal.hann, 'blackman': scipy.signal.blackman,
           'bartlett': scipy.signal.bartlett}
audio_conf_example = dict(sample_rate=16000,
                          num_channel=5,
                          num_uChannel=1,
                          window_size=512,
                          window_shift=256,
                          window='hamming',
                          mat_key='wav')

def FileSize(audio_path):
    return os.path.getsize(audio_path)

'''
def load_audio(path):
    try:
        if os.path.exists(path) and FileSize(path) > 15000:
            rate, sound = wav.read(path)
            if len(sound.shape) > 1:
                if sound.shape[1] == 1:
                    sound = sound.squeeze()
                else:
                    sound = sound.mean(axis=1)
            if rate != 16000:
                sound = signal.resample(sound, len(sound) * 16000 / rate)
        else:
            return None

        return sound
    except:
        print('load_audio {} failed'.format(path))
        return None
'''

def load_audio(path):
    try:
        if os.path.exists(path) and FileSize(path) > 4000:
            #rate, sound = wav.read(path)
            sound, rate = librosa.load(path, sr=16000)

            if len(sound.shape) > 1:
                if sound.shape[1] == 1:
                    sound = sound.squeeze()
                else:
                    sound = sound.mean(axis=1)
            if rate != 16000:
                #sound = signal.resample(sound, len(sound) * 16000 / rate)
                sound = librosa.resample(sound, rate, 16000)
        else:
            return None

        ##sound = sound / 65536.
        return sound
    except:
        print('load_audio {} failed'.format(path))
        return None

def load_mat(filename, key='feat'):
    if os.path.exists(filename):
        key = sio.whosmat(filename)[0][0]
        ##print('key = %s' % key)
        data = sio.loadmat(filename)
        if key in data:
            mat = data[key]
        else:
            return None
        '''
        if mat.shape[0] > 1:
            mat = mat[:, 0];
            mat = mat.reshape(mat.shape[0])
        elif mat.shape[1] > 1:
            mat = mat[0, :];
            mat = mat.reshape(mat.shape[1])
        '''
        if np.isnan(mat).sum() > 0:
            return None
        return mat
    else:
        print('load_mat {} failed'.format(filename))
        return None

def load_audio_feat_len(audio_path):
    path, pos = audio_path.split(':')
    ark_read_buffer = open(path, 'rb')
    ark_read_buffer.seek(int(pos), 0)
    header = struct.unpack('<xcccc', ark_read_buffer.read(5))
    if header[0].decode('utf-8') != "B":
        raise Exception("Input .ark file is not binary")
    if header[1].decode('utf-8') == "C":
        raise Exception("Input .ark file is compressed")

    _, rows = struct.unpack('<bi', ark_read_buffer.read(5))

    return rows

def load_npy(filename):
    if os.path.exists(filename):
        mat = np.load(filename)
        if np.isnan(mat).sum() > 0:
            return None
        return mat
    else:
        print('load_npy {} failed'.format(filename))
        return None

def target_trans(target, frame_num, word_dict):
    target_size = len(target)
    if target_size == frame_num:
        final_target = list(int(x) for x in list(target) if x is not None)
    else:
        final_target = list(int(target[int(x * target_size / frame_num)]) for x in range(frame_num))

    return final_target

class Targetcounter(object):
    def __init__(self, target_path, label_num):
        self.target_dict = read_target_file(target_path)
        self.label_num = label_num

    def compute_target_count(self):
        encoded_targets = np.concatenate(
            [self.encode(targets)
             for targets in self.target_dict.values()])

        #  count the number of occurences of each target
        count = np.bincount(encoded_targets,
                            minlength=self.label_num)
        return count

    def encode(self, targets):
        """ encode a target sequence """
        encoded_targets = list([int(x) for x in list(targets)])
        return np.array(encoded_targets, dtype=np.int)

def read_target_file(target_path):
    """
    read the file containing the state alignments
    Args:
        target_path: path to the alignment file
    Returns:
        A dictionary containing
            - Key: Utterance ID
            - Value: The state alignments as a space seperated string
    """

    target_dict = {}

    with gzip.open(target_path, 'rb') as fid:
        for line in fid:
            split_line = line.decode('utf-8').strip().split(' ')
            target_dict[split_line[0]] = split_line[1:]
    return target_dict

def splice(utt, left_context_width, right_context_width):
    """
    splice the utterance
    Args:
        utt: numpy matrix containing the utterance features to be spliced
        context_width: how many frames to the left and right should
            be concatenated
    Returns:
        a numpy array containing the spliced features, if the features are
        too short to splice None will be returned
    """
    # return None if utterance is too short
    if utt.shape[0] < 1 + left_context_width + right_context_width:
        return None

    #  create spliced utterance holder
    utt_spliced = np.zeros(
        shape=[utt.shape[0], utt.shape[1] * (1 + left_context_width + right_context_width)],
        dtype=np.float32)

    #  middle part is just the utterance
    utt_spliced[:, left_context_width * utt.shape[1]:
                   (left_context_width + 1) * utt.shape[1]] = utt

    for i in range(left_context_width):
        #  add left context
        utt_spliced[i + 1:utt_spliced.shape[0],
        (left_context_width - i - 1) * utt.shape[1]:
        (left_context_width - i) * utt.shape[1]] = utt[0:utt.shape[0] - i - 1, :]

    for i in range(right_context_width):
        # add right context
        utt_spliced[0:utt_spliced.shape[0] - i - 1,
        (left_context_width + i + 1) * utt.shape[1]:
        (left_context_width + i + 2) * utt.shape[1]] = utt[i + 1:utt.shape[0], :]

    return utt_spliced

def add_delta(utt, delta_order):
    num_frames = utt.shape[0]
    feat_dim = utt.shape[1]

    utt_delta = np.zeros(
        shape=[num_frames, feat_dim * (1 + delta_order)],
        dtype=np.float32)

    #  first order part is just the utterance max_offset+1
    utt_delta[:, 0:feat_dim] = utt

    scales = [[1.0], [-0.2, -0.1, 0.0, 0.1, 0.2],
              [0.04, 0.04, 0.01, -0.04, -0.1, -0.04, 0.01, 0.04, 0.04]]

    delta_tmp = np.zeros(shape=[num_frames, feat_dim], dtype=np.float32)
    for i in range(1, delta_order + 1):
        max_offset = (len(scales[i]) - 1) / 2
        for j in range(-max_offset, 0):
            delta_tmp[-j:, :] = utt[0:(num_frames + j), :]
            for k in range(-j):
                delta_tmp[k, :] = utt[0, :]
            scale = scales[i][j + max_offset]
            if scale != 0.0:
                utt_delta[:, i * feat_dim:(i + 1) * feat_dim] += scale * delta_tmp

        scale = scales[i][max_offset]
        if scale != 0.0:
            utt_delta[:, i * feat_dim:(i + 1) * feat_dim] += scale * utt

        for j in range(1, max_offset + 1):
            delta_tmp[0:(num_frames - j), :] = utt[j:, :]
            for k in range(j):
                delta_tmp[-(k + 1), :] = utt[(num_frames - 1), :]
            scale = scales[i][j + max_offset]
            if scale != 0.0:
                utt_delta[:, i * feat_dim:(i + 1) * feat_dim] += scale * delta_tmp

    return utt_delta

class AudioParser(object):
    '''def parse_transcript(self, transcript_path):
        """
        :param transcript_path: Path where transcript is stored from the manifest file
        :return: Transcript in training/testing format
        """
        raise NotImplementedError'''

    def WaveData(self, audio_path):
        y = load_audio(audio_path)
        return y

    def WriteWave(self, audio_path, data, sample_rate=16000):
        if data is not None:
            wav.write(audio_path, sample_rate, data)

    def RMRData(self, rmr_path, mat_key='rir', mic = 0):
        y = load_mat(rmr_path, mat_key)
        if y.shape[0] > 100:
            y = y[:, mic]
        elif y.shape[1] > 100:
            y = y[mic, :]
        y = y.squeeze()
        return y

    def MakeMixture(self, speech, noise, db):
        if speech is None or noise is None:
            # print("MakeMixture: speech is None or noise is None")
            return None
        if np.sum(np.square(noise)) < 1.0e-13:
            #print("MakeMixture: np.sum(np.square(noise)) < 1.0e-6")
            return None

        spelen = speech.shape[0]

        exnoise = noise
        while exnoise.shape[0] < spelen:
            exnoise = np.concatenate([exnoise, noise], 0)
        noise = exnoise
        noilen = noise.shape[0]

        elen = noilen - spelen - 1
        if elen > 1:
            s = int(np.round(random.randint(0, elen - 1)))
        else:
            s = 0
        e = s + spelen

        noise = noise[s:e]

        try:
            noi_pow = np.sum(np.square(noise))
            if noi_pow > 0:
                noi_scale = math.sqrt(np.sum(np.square(speech)) / (noi_pow * (10 ** (db / 10.0))))
            else:
                return None
        except:
            return None

        nnoise = noise * noi_scale
        mixture = speech + nnoise
        mixture = mixture.astype('float32')
        return mixture

    def MixVoice(self, speech, noise, mix_rate = 0.5):
        if speech is None or noise is None:
            print("MakeMixture: speech is None or noise is None")
            return None
        #if np.sum(np.square(noise)) < 1.0e-20:
        #    print("MakeMixture: np.sum(np.square(noise)) < 1.0e-6")
        #    return None

        spelen = speech.shape[0]

        exnoise = noise
        while exnoise.shape[0] < spelen:
            exnoise = np.concatenate([exnoise, noise], 0)
        noise = exnoise
        noilen = noise.shape[0]

        elen = noilen - spelen - 1
        if elen > 1:
            s = int(np.round(random.randint(0, elen - 1)))
        else:
            s = 0
        e = s + spelen

        noise = noise[s:e]

        nnoise = noise * mix_rate
        mixture = speech + nnoise
        mixture = mixture.astype('float32')
        return mixture

    def Make_Reverberation(self, speech, rmr, use_fast=False):

        if speech is None or rmr is None:
            # print("Make_Reverberation: speech is None or rmr is None")
            return None
        nsample = len(speech)
        #speech = signal.convolve(speech, rmr, mode='same')
        speech = signal.convolve(speech, rmr, mode='full')
        speech = speech[0:nsample]
        return speech

    def Gain_Control(self, wave, Gain):

        if wave is None:
            # print("Gain_Control: wave is None")
            return None

        max_sample = np.max(np.fabs(wave))

        if max_sample <= 0:
            # print("Gain_Control: np.fabs(wave) is 0")
            return None

        wave = wave / max_sample

        wave = wave * Gain

        wave = wave.astype('float32')

        return wave

    def Make_Noisy_Wave(self, speech, noise, spe_rmr, noi_rmr, SNR):
        if speech is None or noise is None:
            # print("Make_Noisy_Wave:speech is None or noise is None")
            return None

        if spe_rmr is not None:
            speech = self.Make_Reverberation(speech, spe_rmr)
        if noi_rmr is not None:
            noise = self.Make_Reverberation(noise, noi_rmr)

        noisy = self.MakeMixture(speech, noise, SNR)

        if noisy is not None:
            noisy = noisy.astype('float32')
            return noisy
        else:
            return None


# -*- coding: utf-8 -*-
import random
from functools import reduce

list_merge = lambda lst: reduce(lambda d, el: d.extend(el) or d, lst, [])
get_xy = lambda xy: [int(i)-1 for i in str(xy)]
get_hunt = lambda lst: int(''.join(str(x+1) for x in lst))

def make_matrix(n, m):
    if 2 <= m < 10 and 2 <= n < 10:
        return [[y+(x+1)*10+1 for y in range(m) if y < 9] for x in range(n)]
    return []

def make_hunt_matrix(lst):
    n = len(lst)
    m = len(lst[0])
    merged_data = list_merge(lst)
    random.shuffle(merged_data)
    data = [[0] * m for _ in range(n)]

    for x in range(n):
        for y in range(m):
            v = merged_data.pop()
            # set hunt instead of 11
            data[x][y] = v if v != 11 else 21
    return data

def find_hunt(matrix):
    x, y = 0, 0
    chain_data = []
    while True:
        value = matrix[x][y]
        hunt = get_hunt([x, y])
        if value == hunt:
            break
        if value in chain_data:
            chain_data.append(0)
            break
        chain_data.append(value)
        s = get_xy(value)
        x, y = s[0], s[1]

    return chain_data

def main(n, m):
    matrix = make_matrix(n, m)
    hunt_matrix = (make_hunt_matrix(matrix))
    print(hunt_matrix)
    print(find_hunt(hunt_matrix))

if __name__ == '__main__':
    main(3, 2)


import socket
from matplotlib import pyplot as plt
import hashlib
import threading
import os
import sys
import timeit
from queue import PriorityQueue

host = sys.argv[1]
file = sys.argv[2]
port = 80
chunkSize = (int)(sys.argv[3])
numTCP = (int)(sys.argv[4])
output_file = sys.argv[5] + '.txt'
start = 0
data = PriorityQueue()
start_time = timeit.default_timer()

print("Hostname: {}".format(host))
print("Accessing File: {}".format(file))
print("chunkSize = {}".format(chunkSize))
print("Number of Parallel TCP connections = {}".format(numTCP))

fb = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
fb.connect((host, port))
fbrequest = 'GET /big.txt HTTP/1.1\r\nHost:%s\r\nConnection: keep-alive\r\nRange: bytes=%s\r\n\r\n' % (host, '0-1')
fb.send(fbrequest.encode())

header = ''
while True:
	msg = fb.recv(1)
	header += msg.decode()
	if '\r\n\r\n' in header:
		break

l = header.find("Content-Range")
fileSize = (int)(header[l:].split('\r\n')[0].split("/")[1])

def getRequest():
    global start
    a = start
    b = start + chunkSize
    if b > fileSize:
        b = fileSize
        start = fileSize
        return a, b
    start += chunkSize + 1
    return a, b

loglist = []

def sendRequest(l, threadName):
    global start
    global data
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((host, port))
    while start != fileSize:
        a, b = getRequest()
        part = str(a) + '-' + str(b)
        request = 'GET /%s HTTP/1.1\r\nHost:%s\r\nConnection: keep-alive\r\nRange: bytes=%s\r\n\r\n' % (file, host, part)
        #print(threadName, part)
        s.send(request.encode())
        header = ''
        d = ''
        while True:
            msg = s.recv(1)
            if len(msg) < 1:
                s.close()
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.connect((host, port))
                s.send(request.encode())
                header = ''
                continue
            header += str(msg)[2:-1]
            if '\\r\\n\\r\\n' in header:
                break

        for i in range(chunkSize+1):
            msg = s.recv(1)
            d += msg.decode()
        data.put((a, d))
        kt2 = timeit.default_timer()
        l.append((b, kt2-start_time))
    s.close()
    loglist.append(l)

threads = []

for i in range(numTCP):
    l = []
    t = threading.Thread(target=sendRequest, args=(l, "Thread-" + str(i),))
    t.start()
    threads.append(t)

for t in threads:
    t.join()

end_time = timeit.default_timer()
time_taken = (end_time - start_time)

if len(sys.argv) > 6:
	MD5Sum = hashlib.md5()
	while not data.empty():
		s = data.get()
		MD5Sum.update(s[1].encode())
	print('MD5Sum = {}'.format(MD5Sum.hexdigest()))
	exit()

print("Downloading in File: {}".format(output_file))
f = open(output_file, 'w')
MD5Sum = hashlib.md5()

while not data.empty():
    s = data.get()
    MD5Sum.update(s[1].encode())
    print(s[1], end='', file=f)

f.close()

print('MD5Sum = {}'.format(MD5Sum.hexdigest()))
print('Time to Download = {}'.format(time_taken))
print()

plt.figure()
for i in range(len(loglist)):
    l1 = loglist[i]
    timek = []
    bytek = []
    lb = 'Connection-' + str(i)
    for j in range(len(l1)):
        timek.append(l1[j][1])
        bytek.append(l1[j][0])
    plt.plot(timek, bytek, label=lb)

plt.legend()
plt.title('TCP Connection Log')
plt.ylabel('Bytes Downloaded')
plt.xlabel('Time(s)')
c = 'log' + sys.argv[5] + '.png'
plt.savefig(c)
plt.show()
plt.close()


""" Plot Functions """
import matplotlib.pyplot as plt

def plot_matches(genuine_match, disguise_match, simulated_match, threshold):
    """ Plot Match Point percentages """

    plt.figure(figsize = (8, 8))
    match = [genuine_match, disguise_match, simulated_match]
    plt.hist(match, label=['Genuine', 'Disguise', 'Simulated'])
    plt.legend()
    plt.title('Matched Points')
    plt.savefig('../Plots/PMatchpoints' + str(threshold) + '.jpg')
    plt.show()

def plot_EER(theta_range, far, frr, threshold):
    """ Plot EER vs Theta """

    plt.figure(figsize = (8, 8))
    plt.plot(theta_range, far, color = 'red')
    plt.plot(theta_range, frr, color = 'blue')
    plt.title('EER vs Theta')
    plt.legend(['FAR', 'FRR'])
    plt.savefig('../Plots/Error' + str(threshold) + '.jpg')
    plt.show()