import feedparser
import requests
import sqlite3
import time
from datetime import datetime, timezone, timedelta

webHookUrl = ['https://discord.com/api/webhooks/...AccountidA',
              'https://discord.com/api/webhooks/...AccountidB']

PTTRssUrl = ['https://www.ptt.cc/atom/PC_Shopping.xml',
             'https://www.ptt.cc/atom/HardwareSale.xml']

def req_webhook(webHookUrl: str, DATA: dict, URL: str):
    try:
        time.sleep(5)
        SendToDC = requests.post(
            webHookUrl, json=DATA)
        if 200 <= SendToDC.status_code < 300:
            print(
                f"Send Webhook {str(URL)}\nStatus:{SendToDC.status_code}")
        else:
            print(
                f"Send Webhook {str(URL)}\nStatus:{SendToDC.status_code}, response:\n{SendToDC.json()}")
    except Exception as e:
        print(e)

def main(Webhookurl: str, RssUrl: str):
    con = sqlite3.connect('rss.db')
    cur = con.cursor()
    cur.execute(
        '''CREATE TABLE IF NOT EXISTS Stuff(url TEXT PRIMARY KEY NOT NULL)''')
    data = feedparser.parse(RssUrl)
    Provider = data.feed.title
    for i in data.entries:
        Title = i.title
        hyperLink = i.link
        description = i.summary
        Author = i.author

        exist = con.execute(
            "SELECT * FROM Stuff WHERE url = ?", ([hyperLink])).fetchone()
        if exist is None:
            # print(shorTcode)
            cur.execute(
                "INSERT INTO Stuff VALUES(?)", [hyperLink])
            con.commit()
            time_now = datetime.now(timezone(timedelta(hours=+8))).isoformat()
            embed = {
                "description":  description[5:len(description)-7].replace("*", "\*"),
                "title": Title,
                "timestamp": time_now,
                "color": 0x546e7a,
                "author": {
                    "name": Provider+" ( "+Author+" )",
                },
                "footer": {
                    "text": "Powered by vincent-chang-rightfighter/DiscordRSS-Improve",
                    "icon_url": "https://raw.githubusercontent.com/vincent-chang-rightfighter/DiscordWebhook-InstagramUrl/main/icon.png"
                },
            }
            data = {"content": f"{Title}\n{hyperLink}",
                    "embeds": [embed], }
            req_webhook(Webhookurl, data, hyperLink)

        else:
            print("Yep exists")
    con.close()

if __name__ == "__main__":
    # main()
    for i in range(len(PTTRssUrl)):
        main(webHookUrl[i], PTTRssUrl[i])


import pygame
import time
import random
from constant import *
from background import Background
from hand import Hand
from hand_tracking import HandTracking
from penguin import Penguin
from bomb import Bomb
import cv2
import ui

class Game:
    def __init__(self, surface):
        self.surface = surface
        self.background = Background()
        self.capture = cv2.VideoCapture(0) # Load camera using the default camera (0)
        self.sounds = {}
        self.sounds["grab"] = pygame.mixer.Sound(f"Assets/Sounds/grab.mp3")
        self.sounds["grab"].set_volume(SOUNDS_VOLUME)
        self.sounds["explosion"] = pygame.mixer.Sound(f"Assets/Sounds/explosion.wav")
        self.sounds["explosion"].set_volume(SOUNDS_VOLUME-0.6)
        self.sounds["click"] = pygame.mixer.Sound(f"Assets/Sounds/click.mp3")
        self.sounds["click"].set_volume(SOUNDS_VOLUME)

    # Reset all the needed variables when a new game starts
    def reset(self):
        self.hand_tracking = HandTracking()
        self.hand = Hand()
        self.objects = []
        self.objects_spawn_timer = 0
        self.score = 0
        self.game_start_time = time.time()

    def spawn_objects(self):
        t = time.time() #Gets the current time in seconds
        if t > self.objects_spawn_timer:
            self.objects_spawn_timer = t + PENGUIN_SPAWN_TIME #A new penguin will be spawned every 1 second

            # Increase the probability that the object will be a bomb over time
            # nb = number of bombs
            nb = (GAME_DURATION-self.time_left)/GAME_DURATION * 100 / 2  #Increase from 0 to 50 during all the game (linear)

            # Add randomness and unpredictability to the game
            if random.randint(0, 100) < nb:
                self.objects.append(Bomb())
            else:
                self.objects.append(Penguin())

            # Spawn additional bomb and penguin after the half of the game to increase difficulty
            if self.time_left < GAME_DURATION/2:
                self.objects.append(Bomb())

            if self.time_left < GAME_DURATION/2:
                self.objects.append(Penguin())

    # Keep track of the remaining time in the game
    def update_game_time(self):
        self.time_left = max(round(GAME_DURATION - (time.time() - self.game_start_time), 1), 0) #The result is 1 decimal place, with the maximum value between the result and 0

    # Capture single frame from the camera
    def load_camera(self):
        _, self.frame = self.capture.read() #method to read the next frame from the camera
        # '_' used to unpack tuple returned by the method, containing boolean whether frame was read successfully
        # frame assigned to the self.frame

    # Track player's hand in real-time
    def set_hand_position(self):
        self.frame = self.hand_tracking.scan_hands(self.frame) #Scanning the hands in the frame
        (x, y) = self.hand_tracking.get_hand_position()
        self.hand.rect.center = (x, y) #Sets the hand object position on the screen to match the position of the hand in the webcam frame
                                        # thus hand object positioned on the screen at the same location as hand in the webcam frame

    # Drawing elements of the game on the screen
    def draw(self):
        self.background.draw(self.surface) #Draw the background
        for object in self.objects: #Draw the objects
            object.draw(self.surface)
        self.hand.draw(self.surface) #Draw the hand
        ui.draw_text(self.surface, f"Score : {self.score}", (5, 5), COLORS["score"], font=FONTS["medium"],
                    shadow=True, shadow_color=(255,255,255)) #Draw the score
        timer_text_color = (160, 40, 0) if self.time_left < 5 else COLORS["timer"] #Change the text color remaining time less than 5s
        ui.draw_text(self.surface, f"Time left : {self.time_left}", (SCREEN_WIDTH//2, 5),  timer_text_color, font=FONTS["medium"],
                    shadow=True, shadow_color=(255,255,255)) #Draw the remaining time

    # Update the game's state
    def update(self):
        self.load_camera()
        self.set_hand_position()
        self.update_game_time()
        self.draw()

        if self.time_left > 0:
            self.spawn_objects()
            (x, y) = self.hand_tracking.get_hand_position()
            self.hand.rect.center = (x, y)
            self.hand.left_click = self.hand_tracking.hand_closed #Sets the mouse left click to hand closing
            # print("Hand closed", self.hand.left_click)
            if self.hand.left_click: #If hand is closed
                self.hand.image = self.hand.smaller_image.copy() #Use the hand closed image that has been scaled smaller
            else:
                self.hand.image = self.hand.original_image.copy() #Use the hand opened image
            self.score = self.hand.catch_objects(self.objects, self.score, self.sounds)
            for object in self.objects:
                object.move() #Updates its position on the screen

        else: # When the game is over
            if ui.button(self.surface, 400, "BACK", click_sound=self.sounds["click"]):
                return "menu"

        cv2.imshow("Frame", self.frame) #Shows webcam frame on the screen
        cv2.waitKey(1) #Waits for 1ms before the next frame to create illusion of a live video stream


#!/bin/env python3

import os
from datetime import datetime
import csv
import config

#os.environ[""]

def get_date():
    date = datetime.now()
    #print("date: ",date)
    return date

def get_user(number):
    # From the phone number, function will return user line with no,name,number,position
    player = "unfound"
    with open('player.csv') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            if number == row['number']:
                player_row = row
                return player_row
                break

    if player == "unfound":
        print("No player found, exiting")

    elif player != "unfound":
        print("Player found, %s." %(player_row['name']))

    print(player_row['name'])
    return player_row

def get_next_game():
    date = get_date()
    next_game = "unfound"
    with open('games.csv') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:

            row_date = datetime.strptime(row['date'],'%Y-%m-%d')
            if row_date > date:
                next_game = row

                return next_game
                break

        if next_game == "unfound":
            print("No next game found, exiting")

        elif next_game != "unfound":
            print("Game found, next one on: %s" %(next_game))

        print(next_game)
    return next_game

def get_raw_msg():
    return user_number, raw_msg

def get_function(raw_msg):
    if raw_msg == 'abscent':
        function = "abscent"

    elif raw_msg == 'present':
        function = 'present'
    else:
        function = 'unknown'

    return function

def analyse_function(function):
    date = get_date()
    next_game = get_next_game()
    msg = ""
    if function == 'unknown':
        msg = "Commande non comprise, utiliser present ou abscent pour indiquer votre presende a la prochaine partie."

    elif function == 'abscent' or 'present':
        msg = "Merci, je vous met %s pour la prochaine partie du %s, a l'arena: %s" % (function, next_game['date'], next_game['arena'])
    # print(msg)
    return msg

def send_msg(msg,):
    return

def get_number():
    return

def nearest(items, pivot):
    return min(items, key=lambda x: abs(x - pivot))

def pushover(msg, user):
    # Your Account Sid and Auth Token from twilio.com/console

    account_sid = os.environ["TWILIO_ACCOUNT_SID"]
    auth_token = os.environ["TWILIO_AUTH_TOKEN"]

    numbers = config.NUMBERS
    client = Client(account_sid, auth_token)

    for number in numbers:
        message = client.api.account.messages.create(
                            body=msg,
                            from_=config.FROM_NUMBER,
                            to=number)
        print(message.sid)
        sleep(2)

if __name__ == "__main__":
    msg = analyse_function('abscent')
    print(msg)


import json
import numpy as np
from setuptools.command.easy_install import easy_install

class SatcomScanConfig:
    def __init__(self,
                 ifov_radians=np.deg2rad(15),
                 game_divisor=(6 * 60),
                 lookang_degree=20,
                 tl=[4.472, 102.00],
                 tr=[4.472, 105.00],
                 bl=[-1, 102.00],
                 br=[-1, 105.00],
                 extra=30,
                 gmat_exec="",
                 gmat_root_dir="",
                 gmat_io__dir="",
                 gmat_startup_template="",
                 gmat_script_template=""):
        self.ifov_radians = ifov_radians
        self.game_divisor = game_divisor
        self.lookang_degree = lookang_degree
        self.tl = tl
        self.tr = tr
        self.bl = bl
        self.br = br
        self.extra = extra
        self.gmat_exec = gmat_exec
        self.gmat_root_dir = gmat_root_dir
        self.gmat_io__dir = gmat_io__dir
        self.gmat_startup_template = gmat_startup_template
        self.gmat_script_template = gmat_script_template

    def __repr__(self):
        return str(vars(self))

# ----------------------------------------------------------

class JsonEncoder(json.JSONEncoder):
    def default(self, obj):
        result = None
        if isinstance(obj, SatcomScanConfig):
            result = {
                "__class__": "satcom_scan_configuration",
                "ifov_radians": obj.ifov_radians,
                "game_divisor": obj.game_divisor,
                "lookang_degree": obj.lookang_degree,
                "tl": obj.tl,
                "tr": obj.tr,
                "bl": obj.bl,
                "br": obj.br,
                "extra": obj.extra,
                "gmat_exec": obj.gmat_exec,
                "gmat_root_dir": obj.gmat_root_dir,
                "gmat_io__dir": obj.gmat_io__dir,
                "gmat_startup_template": obj.gmat_startup_template,
                "gmat_script_template": obj.gmat_script_template
            }
        else:
            print(type(obj))
            result = super().default(obj)

        return result

class JsonDecoder(json.JSONDecoder):
    def __init__(self, *args, **kwargs):
        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)

    def object_hook(self, obj):
        result = None
        if '__class__' not in obj:
            result = obj
        else:
            class_type = obj['__class__']
            if class_type == 'satcom_scan_configuration':
                result = self.parse_satcom_scan_configuration(obj)
            else:
                print("Unsupported class type")

        return result

    @staticmethod
    def parse_satcom_scan_configuration(obj):
        return SatcomScanConfig(
            obj['ifov_radians'],
            obj['game_divisor'],
            obj['lookang_degree'],
            obj['tl'],
            obj['tr'],
            obj['bl'],
            obj['br'],
            obj['extra'],
            obj['gmat_exec'],
            obj['gmat_root_dir'],
            obj['gmat_io__dir'],
            obj['gmat_startup_template'],
            obj['gmat_script_template']
        )


from collections import defaultdict

class Solution:
    def isPrintable(self, targetGrid: List[List[int]]) -> bool:
        """
        1. find color range
        - color2Range={colorID: (left, right, top, bottom)}
        2. find color order
        - build graph, top color as parent, others as children
        3. check cycle
        """
        color2Range = {}
        M = len(targetGrid)
        N = len(targetGrid[0])

        # define range
        for ii in range(M):
            for jj in range(N):
                color = targetGrid[ii][jj]
                if color not in color2Range:
                    color2Range[color] = (N, 0, M, 0)
                color2Range[color] = (min(color2Range[color][0], jj),
                                      max(color2Range[color][1], jj),
                                      min(color2Range[color][2], ii),
                                      max(color2Range[color][3], ii)
                                      )

        # define color order via building graph
        g = defaultdict(set)
        for ii in range(M):
            for jj in range(N):

                for color in range(1, 61):
                    if color not in color2Range:
                        continue
                    if jj < color2Range[color][0] or  \
                       jj > color2Range[color][1] or  \
                       ii < color2Range[color][2] or  \
                       ii > color2Range[color][3]:
                        continue

                    if color != targetGrid[ii][jj]:
                        g[targetGrid[ii][jj]].add(color)

        visited = {}

        def has_cycle_dfs(curr):
            """
            return True if has cycle
                False otherwise
            :param curr:
            :return:
            """
            if curr in visited and visited[curr] == 1:
                return False
            if curr in visited and visited[curr] == 2:
                return True  # has cycle

            visited[curr] = 2
            for child in g[curr]:
                if has_cycle_dfs(child):
                    return True
            visited[curr] = 1

            return False

        for color in range(1, 61):
            if color not in color2Range:
                continue
            if has_cycle_dfs(color):
                return False

        return True


import pyperclip
from textual.app import App, ComposeResult
from textual.containers import Vertical
from textual.widgets import Button, Header, Input, Static

class MainMenu(App):
    def compose(self) -> ComposeResult:
        yield Header(show_clock=True)
        yield Static("Select an option:", classes="center", id="toptext")
        yield Static(
            "Download info - Downloads info from Twitter in JSON form.",
            classes="center",
        )
        yield Static(
            "Download media from info - Looks at the URLs in the JSON files and then downloads their media.",
            classes="center",
        )
        yield Vertical(
            Button("Download info", id="info", variant="primary", classes="button"),
            Button(
                "Download media from info",
                id="download",
                variant="error",
                classes="button",
            ),
            classes="center",
        )

    def on_button_pressed(self, event: Button.Pressed) -> None:
        self.exit(event.button.id)

class ChooseOption(App):
    def compose(self) -> ComposeResult:
        yield Header(show_clock=True)
        yield Static("Select an option:", classes="center", id="toptext")
        yield Vertical(
            Button(
                "Tweets and retweets", name="1", variant="primary", classes="button"
            ),
            Button("Likes", name="2", variant="secondary", classes="button"),
            Button("Bookmarks", name="3", variant="primary", classes="button"),
            Button("Following", name="4", variant="secondary", classes="button"),
            classes="center",
        )

    def on_button_pressed(self, event: Button.Pressed) -> None:
        self.exit(int(event.button.name))

class InputCode(App):
    def action_copy_text(self):
        pyperclip.copy(self.authorize_url)
        self.bell()

    def compose(self) -> ComposeResult:
        yield Header(show_clock=True)
        yield Static(
            f"Open this site (click to copy):\n[@click='copy_text()']{self.authorize_url}[/]",
            classes="center",
            id="toptext",
        )
        yield Static(
            "Input the code that you received here:", classes="center", id="lilpad"
        )
        yield Input(placeholder="Paste it here", id="input")
        yield Vertical(
            Button("PASTE", variant="secondary", classes="button", id="paste"),
            Button("CONFIRM", variant="primary", classes="button"),
            classes="center",
        )

    def on_button_pressed(self, event: Button.Pressed) -> None:
        if event.button.id == "paste":
            self.query_one(Input).value = pyperclip.paste()
        else:
            self.exit(self.query_one(Input).value)


import shutil
import sys
import json
from dpla_local_map import dpla_local_map

def rec_gen(source_file):
    """
    :param source_file: JSON file of SSDN records
    :return: Generator or records in source_file
    """
    with open(source_file + ".bak") as f:
        recs = json.load(f)
        for rec in recs:
            yield rec

def sub_gen(rec):
    """
    :param rec: JSON record
    :return: Generator of subjects in rec
    """
    try:
        for sub in rec['sourceResource']['subject']:
            yield sub
    except KeyError:
        pass

# create a backup of input file
shutil.move(sys.argv[1], sys.argv[1] + ".bak")
out = open(sys.argv[1], 'a', encoding='utf8', newline='\n')
for rec in rec_gen(sys.argv[1]):
    for sub in sub_gen(rec):
        '''
        check existing subjects against mapped terms
        and make sure supplied subject isn't already
        in record
        '''
        if sub['name'] in dpla_local_map.keys() and dpla_local_map[sub['name']][0][0] not in [term['name'] for term in
                                                                                rec['sourceResource'][
                                                                                'subject']]:
            if len(dpla_local_map[sub['name']]) > 1:
                for item in dpla_local_map[sub['name']]:
                    rec['sourceResource']['subject'].append({'name': item[0], "@id": item[1]})
                break
            else:
                rec['sourceResource']['subject'].append(
                    {'name': dpla_local_map[sub['name']][0][0], "@id": dpla_local_map[sub['name']][0][1]})
                break
    out.write(json.dumps(rec) + '\n')


from datetime import datetime
import matplotlib.pyplot as plt
import numpy as np
import os.path
import pandas as pd
import sqlalchemy as sa

from . import model

def main(cli, args):
    db = cli.connect_db(args.db)

    format = args.format
    if not format:
        _, ext = os.path.splitext(args.output_file)
        if ext == '.csv':
            format = 'csv'
        elif ext == '.xlsx':
            format = 'excel'
        else:
            cli.abort('could not guess file format from extension')

    if format == 'csv':
        format_is_text = True
    elif format == 'excel':
        format_is_text = False
    else:
        cli.abort('unrecognized file format')

    Tweet = model.Tweet
    User = model.User

    as_txt = lambda col: sa.cast(col, sa.Text).label(col.name)
    q = (
        db.query(
            as_txt(Tweet.id),
            Tweet.created_at,
            as_txt(Tweet.user_id),
            User.nick,
            Tweet.user_description,
            Tweet.user_verified,
            Tweet.user_followers_count,
            Tweet.user_friends_count,
            Tweet.user_listed_count,
            Tweet.user_statuses_count,
            Tweet.user_favorites_count,
            Tweet.user_created_at,
            as_txt(Tweet.in_reply_to_tweet_id),
            as_txt(Tweet.in_reply_to_user_id),
            as_txt(Tweet.quoted_tweet_id),
            as_txt(Tweet.rt_tweet_id),
            Tweet.updated_at,
            Tweet.favorite_count,
            Tweet.retweet_count,
            Tweet.reply_count,
            Tweet.quote_count,
            Tweet.lang,
            Tweet.text,
        )
        .join(
            model.User,
            model.User.id == model.Tweet.user_id,
        )
        .order_by(model.Tweet.created_at.asc())
    )

    df = model.query_to_pandas(q)

    with cli.output_file(args.output_file, text=format_is_text) as fp:
        if format == 'csv':
            df.to_csv(fp, index=False)
        elif format == 'excel':
            with pd.ExcelWriter(
                fp,
                engine='xlsxwriter',
                options=dict(
                    strings_to_numbers=False,
                ),
            ) as writer:
                df.to_excel(writer, index=False)

def main_plot(cli, args):
    db = cli.connect_db(args.db)

    date_col = sa.func.date(model.Tweet.created_at).label('date')
    tweets = (
        db.query(
            date_col,
            sa.func.count().label('count'),
        )
        .filter(model.Tweet.created_at.between(
            datetime(2019, 3, 20),
            datetime(2019, 5, 10),
        ))
        # .filter(model.Tweet.in_reply_to_tweet_id.is_(None))
        .filter(model.Tweet.rt_tweet_id.is_(None))
        .filter(sa.func.lower(model.Tweet.text).like('%#saam%'))
        .group_by(date_col)
        .order_by(date_col.asc())
        .all()
    )
    dates = [t.date for t in tweets]
    counts = [t.count for t in tweets]

    fig, ax = plt.subplots()
    x = np.arange(len(dates))
    plt.bar(x, counts)
    plt.xticks(x, dates)
    with cli.output_file(args.output_file, text=False) as fp:
        plt.savefig(fp, format='png')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created on 2017

@author: li.taojun
'''

import requests
from uopweixin.util.configurl import userActiviesUrl
from uopweixin.util.jsonTransform import transUopHttpHears
# from com.uop.weixin.user.center.address.json.memberAddJonsParseport parseMemberDefalutAddJSON
# from com.uop.weixin.user.activity.raffle.json.signUpActivitiesJsonParse import transUserSignupActivitiesHttpJson
from uopweixin.activity.raffle.userSignActivitiesCpr import checkSignUpActivitiesResultFormat
from uopweixin.center.activities.ParseCenterActivitiesJson import parseActivitiesStatus,parseActivitiesIdFromJson,parseMyActivitiesIDFromJson
import json
class PersonalActiviesService(object):
    '''
    classdocs
    '''
    #userSignupActivitiesurl = signUpOpenActUrl
    memberId = None
    openid = None
    def __init__(self, memberId,openid):
        '''
        Constructor li.taojun
        '''
        self.memberId = memberId
        self.openid = openid
        self.userActiviesUrl = userActiviesUrl

    #获取我的活动列表
    def userActivies(self):
        jsonheart = transUopHttpHears(self.memberId,self.openid)
        actjson = {"memberId":self.memberId,"page":1,"limit":10}
        activityrspjson = requests.post(url=self.userActiviesUrl,json = actjson,headers = jsonheart,verify=False)
        return activityrspjson

    #根据活动ID获取我的活动中信息
    def getMyActivitiesByID(self,activitiesId = "",activityrspjson=""):
#       actjson = self.userActivies()
        myactivities = parseMyActivitiesIDFromJson(activityrspjson,activitiesId)
        return myactivities

    #获取活动状态
    def getMyActivitiesStatus(self,activitiesId):
        myactivities = self.getMyActivitiesByID(activitiesId)
        actstatus = parseActivitiesStatus(myactivities)
        return actstatus

    #得到活动ID列表
    def getActivitiesidList(self):
        actjson = self.userActivies()
        actidls = parseActivitiesIdFromJson(response = json.loads(actjson.text))
        return actidls

    #检查活动ID是否存在
    def checkActivitiesExsit(self,activiId = ""):
        sign = False
        actidls = self.getActivitiesidList()
        if activiId in actidls:
            sign = True
        return sign

    def setUserActiviesUrl(self,url):
        self.userActiviesUrl = url

if __name__ == '__main__':
    a = PersonalActiviesService('997da560-6de2-4056-8614-e7cd95dd967b','ovQBPxGwi5RUfDoZDc-xep7EraEI')
    a.setUserActiviesUrl('https://dev-uop-api.opg.cn/activities-service/raffleResult/getMemberRaffleReCords')
    acils = a.getActivitiesidList()
    print(acils)


from collections import defaultdict

from sklearn import clone
from sklearn.svm import LinearSVC

from binary_tree_regressor import BinaryTreeRegressor

class RegressionQuantifier(object):
    def __init__(self, pipeline=None):
        if pipeline is None:
            self._pipeline = BinaryTreeRegressor(base_estimator=LinearSVC(C=100.0))
        else:
            self._pipeline = pipeline

    def fit(self, X, y, groups):
        self._true_global_prevalences = defaultdict(float)
        self._values = list(set(y))
        self._values.sort()
        for rate in self._values:
            self._true_global_prevalences[rate] = y.count(rate) / len(y)
        self._estimated_global_prevalences = defaultdict(float)
        for group in set(groups):
            group_X = [atext for atext, agroup in zip(X, groups) if agroup == group]
            notgroup_X = [atext for atext, agroup in zip(X, groups) if agroup != group]
            notgroup_y = [alabel for alabel, agroup in zip(y, groups) if agroup != group]
            pipclone = clone(self._pipeline)
            model = pipclone.fit(notgroup_X, notgroup_y)
            predictions = model.predict(group_X)

            for rate in self._values:
                self._estimated_global_prevalences[rate] += predictions.count(rate)

        for rate in self._values:
            self._estimated_global_prevalences[rate] /= len(y)

        self._model = self._pipeline.fit(X, y)

    def predict(self, X, groups):
        predictions = self._model.predict(X)
        quantifications = dict()
        test_global_prevalences = defaultdict(float)
        for rate in self._values:
            test_global_prevalences[rate] = predictions.count(rate) / len(X)
        for group in set(groups):
            group_predictions = [prediction for prediction, agroup in zip(predictions, groups) if agroup == group]
            simple_prevanlences = list()
            corrected_prevalences = list()
            test_corrected_prevalences = list()
            for rate in self._values:
                prevalence = group_predictions.count(rate) / len(group_predictions)
                simple_prevanlences.append(prevalence)
                if self._estimated_global_prevalences[rate] != 0:
                    corrected_prevalences.append(
                        prevalence * self._true_global_prevalences[rate] / self._estimated_global_prevalences[rate])
                else:
                    corrected_prevalences.append(prevalence)
                if test_global_prevalences[rate] != 0:
                    test_corrected_prevalences.append(
                        prevalence * self._true_global_prevalences[rate] / test_global_prevalences[rate])
                else:
                    test_corrected_prevalences.append(prevalence)

            cumulative = sum(corrected_prevalences)
            corrected_prevalences = [corrected_prevalence / cumulative for corrected_prevalence in
                                     corrected_prevalences]
            cumulative = sum(test_corrected_prevalences)
            test_corrected_prevalences = [test_corrected_prevalence / cumulative for test_corrected_prevalence in
                                          test_corrected_prevalences]

            quantifications[group] = (simple_prevanlences, corrected_prevalences, test_corrected_prevalences)
        return quantifications


import networkx as nx

def groupSum(s, t, group):
	assert type(group) is tuple, "group is not appropriate"
	for n in group:
		assert type(n) is int and n > 1, "a group entry is not appropriate"
	rank = len(group)
	assert type(s) is tuple and len(s) == rank and type(t) is tuple and len(t) == rank, "a group element is not a tuple of the correct length"
	for i in range(rank):
		assert type(s[i]) is int and 0 <= s[i] < group[i] and type(t[i]) is int and 0 <= t[i] < group[i], "a group element has an entry which is not appropriate"
	return tuple([(s[i]+t[i])%group[i] for i in range(rank)])

class AbelianCayley:

	def __init__(self, group, gens): # group is a tuple of positive integers (residues), group is a list of tuples of the same length where each is a proper residue. will generate a multigraph, so duplicates (up to sign) get counted twice
		assert type(group) is tuple, "group is not appropriate"
		for n in group:
			assert type(n) is int and n > 1, "a group modulus is not appropriate"
		self.rank = len(group)
		assert type(gens) is list or type(gens) is int, "gens is not appropriate"
		for s in gens:
			assert type(s) is tuple and len(s) == len(group), "a generator is not a tuple of the correct length"
			for i in range(self.rank):
				assert type(s[i]) is int and 0 <= s[i] < group[i], "a generator has an entry which is not appropriate"
		self.size = 1
		for n in group:
			self.size *= n
		V = [[n] for n in range(group[0])]
		for i in range(1, self.rank):
			W = []
			for t in V:
				for m in range(group[i]):
					W.append(t + [m])
			V = W
		V = [tuple(t) for t in V]
		self.deg = 2*len(gens)
		self.graph = nx.MultiGraph()
		for v in V:
			vString = str(v)
			for s in gens:
				w = groupSum(v, s, group)
				wString = str(w)
				self.graph.add_edge(vString, wString)

# G = AbelianCayley((2,5), [(1,1)])
# print(G.graph.number_of_edges())


import pandas as pd
import numpy as np

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.externals import joblib

import xgboost as xgb
import lightgbm as lgb

train_file = '../input/train.csv'
test_file = '../input/test.csv'
train = pd.read_csv(train_file)
test = pd.read_csv(test_file)
test_ID = test['ID']
y_train = train['target']
y_train = np.log1p(y_train)
train.drop("ID", axis = 1, inplace = True)
train.drop("target", axis = 1, inplace = True)
test.drop("ID", axis = 1, inplace = True)
NUM_OF_DECIMALS = 32
train = train.round(NUM_OF_DECIMALS)
test = test.round(NUM_OF_DECIMALS)
train_zeros = pd.DataFrame({'Percent_zero':((train.values)==0).mean(axis=0),
                           'Column' : train.columns})

high_vol_columns = train_zeros['Column'][train_zeros['Percent_zero'] < 0.70].values
low_vol_columns = train_zeros['Column'][train_zeros['Percent_zero'] >= 0.70].values

train = train.replace({0:np.nan})
test = test.replace({0:np.nan})
cluster_sets = {"low":low_vol_columns, "high":high_vol_columns}
for cluster_key in cluster_sets:
    for df in [train,test]:
        df["count_not0_"+cluster_key] = df[cluster_sets[cluster_key]].count(axis=1)
        df["sum_"+cluster_key] = df[cluster_sets[cluster_key]].sum(axis=1)
        df["var_"+cluster_key] = df[cluster_sets[cluster_key]].var(axis=1)
        df["median_"+cluster_key] = df[cluster_sets[cluster_key]].median(axis=1)
        df["mean_"+cluster_key] = df[cluster_sets[cluster_key]].mean(axis=1)
        df["std_"+cluster_key] = df[cluster_sets[cluster_key]].std(axis=1)
        df["max_"+cluster_key] = df[cluster_sets[cluster_key]].max(axis=1)
        df["min_"+cluster_key] = df[cluster_sets[cluster_key]].min(axis=1)
        df["skew_"+cluster_key] = df[cluster_sets[cluster_key]].skew(axis=1)
        df["kurtosis_"+cluster_key] = df[cluster_sets[cluster_key]].kurtosis(axis=1)

train_more_simplified = train.drop(high_vol_columns,axis=1).drop(low_vol_columns,axis=1)
test_more_simplified = test.drop(high_vol_columns,axis=1).drop(low_vol_columns,axis=1)
train_more_simplified.head()
def run_xgb(train_X, train_y, val_X, val_y, test_X):
    params = {'objective': 'reg:linear',
          'eval_metric': 'rmse',
          'eta': 0.001,
          'max_depth': 6,
          'subsample': 0.6,
          'colsample_bytree': 0.6,
          'alpha':0.001,
          'random_state': 42,
          'silent': True}
    print("Load matrices")
    tr_data = xgb.DMatrix(train_X, train_y)
    va_data = xgb.DMatrix(val_X, val_y)

    print("Set watchlist")
    watchlist = [(tr_data, 'train'), (va_data, 'valid')]

    print("Train model")
    model_xgb = xgb.train(params, tr_data, 20000, watchlist, maximize=False, early_stopping_rounds = 100, verbose_eval=100)

    dtest = xgb.DMatrix(test_X)
    xgb_pred_y = np.expm1(model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit))

    return xgb_pred_y, model_xgb
dev_X, val_X, dev_y, val_y = train_test_split(train_more_simplified, y_train, test_size = 0.2, random_state = 40)
pred_test_xgb, model_xgb = run_xgb(dev_X, dev_y, val_X, val_y, test_more_simplified)
print("Finished!")
sub = pd.read_csv('../input/sample_submission.csv')
sub["target"] = pred_test_xgb
sub.to_csv('sub_XGB_Aggregate_v2.csv', index=False)


import tkinter as tk
import ttkbootstrap as ttk
from ttkbootstrap.constants import *
from ttkbootstrap.tooltip import ToolTip
'''设置窗口大小和名称的类,不受窗口类型的影响,使用方法为win_geometry(窗口名,宽度,高度),
此类不进行继承,经过处理的窗口按原类调用'''

class win_initialize:
    def __init__(self, master: ttk.Window, inputwidth, inputheight, title='untitled'):
        self.master = master
        self.width = inputwidth
        self.height = inputheight
        self.titile = title
        self.master.title(self.titile)
        screenwidth = self.master.winfo_screenwidth()
        screenheight = self.master.winfo_screenheight()
        width = self.width
        height = self.height
        x = int((screenwidth-width)/2)
        y = int((screenheight-height)/2)
        if self.width >= int(13*screenwidth/14):
            x = 0
        if self.height >= int(12*screenheight/14):
            y = 0
        # if self.height-screenheight <= int(screenheight/12) and self.width == screenwidth:
        #     try:
        #         self.master.state('zoomed')
        #     except:
        #         self.master.attributes('-zoomed', True)
        #     return None
        # if width>=screenwidth or height>=screenheight:
        #     try:
        #         self.master.state('zoomed')
        #     except:
        #         self.master.attributes('-zoomed', True)
        #     return None
        self.master.geometry('{}x{}+{}+{}'.format(width, height, x, y))

# --------------------------------------------------------------------------------------------------
    # 使用实例化Treeview的方法对该类进行实例化，
    # 必要传入的参数有columnid：一个列index的列表；
    # showname，一个与columnid对应的显示列名的列表
    # columnwidth，列宽度；其余参数按treeview类传入
    #!!!!!只能接受数据库cursor反馈的数据

class table_view(ttk.Treeview):
    def __init__(self, parent, columnid, showname,  data, columnwidth=100, selectmode=BROWSE, readonly=0, *args, **kwargs):
        ttk.Treeview.__init__(self, parent, column=columnid,
                              show='headings', selectmode=BROWSE, *args, **kwargs)
        TreeviewStyle = ttk.Style()
        TreeviewStyle.configure('Treeview', rowheight=40)
        self.parent = parent
        self.columnid = columnid
        self.showname = showname
        self.columnwidth = columnwidth
        self.database_rows = data
        self.selectmode = selectmode
        self.readonly = readonly
        self.amended_array = []
        self.delete_array = []
        self.tag_configure('amended', background='#009688', foreground='white')
        self.tag_configure('deleted', background='#F44336', foreground='white')
        self.tag_configure('outdated', background='#FF5722',
                           foreground='white')
        for i in range(len(columnid)):
            self.column(self.columnid[i],
                        width=self.columnwidth, anchor='center')
            self.heading(self.columnid[i], text=self.showname[i])
        self.bind('<Double-1>', self.entry_deploy)
    # 双击触发entry

    def entry_deploy(self, *args):
        def entry_destroy(*args):
            TreeviewEntry.destroy()
            return False
    # 不对entry中的值做任何处理，直接摧毁
        # def srollbar_entry_destroy(event):
        #     if event.x >= self.winfo_width() or event.y >= self.winfo_height() or event.x <= 0 or event.y <= 0:
        #         try:
        #             TreeviewEntry.get()
        #         except:
        #             pass
        #         else:
        #             value_deliver()
        #     pass
        # 检测到有人点击滚轮条，直接删除entry
        # 相关动作以在table_viewS中内置，该方法弃用

        def value_deliver():
            if value_list[value_id] == TreeviewEntry.get():
                TreeviewEntry.destroy()
                self.reload()
                return False
            value_list[value_id] = TreeviewEntry.get()
            # templist = (value_list[0], value_list[value_id], value_id)   #老式值返回方案，已弃用
            TreeviewEntry.destroy()
            temp = []
            for row in self.delete_array:
                if value_list[0] == row[0]:
                    temp.append(row)
                    break
            for item in temp:
                self.delete_array.remove(row)
            temp = []
            for row in self.amended_array:
                if value_list[0] == row[0]:
                    temp.append(row)
                    break
            for row in temp:
                self.amended_array.remove(row)
            self.amended_array.append(value_list)
            self.reload()
            return True

            # 修改行所有的值会被作为一个列表添加到self.amended_array中
            # 在老式返回方案中
            # deliver方法向self.amended_array末尾添加一个含三个元素的元组
            # 元组构成（所在行第一列当作主键的元素值，修改后的值，被修改元素的索引）
        if(args[0] == 'd' or self.readonly == 1):
            for widget in self.winfo_children():
                if isinstance(widget, ttk.Entry):
                    widget.destroy()
            return True
        box = self.bbox(self.selection())
        TreeviewEntry = ttk.Entry(
            self, justify='center', width=25)
        column_len = len(self.columnid)
        value_list = list(self.item(self.selection(), option='values'))
        if len(box) < 3:
            return False
        bwidth = int(box[2]/column_len)
        entry_x = box[0]
        value_id = 0
        # 点击y轴是否在列范围内，且此时Treeview应为单选模式
        # ----------------------------------------------------------------------------------------------------------------------------

        if (args[0].y in range(box[1], box[1]+box[3])) and self.selectmode == BROWSE:
            for i in range(column_len):  # 默认第一列为不可更改的primary key
                if int(args[0].x) in range(entry_x+(i+1)*bwidth, entry_x+(i+2)*bwidth):
                    entry_x += (i+1)*bwidth  # if直接从第二列判定鼠标坐标是否在范围内
                    value_id += (i+1)  # 确定所在列index，改index为所在列的值列表的index
                    if value_id >= len(self.columnid):
                        return False
                    TreeviewEntry.place(
                        x=entry_x, y=box[1], width=bwidth, height=box[3], anchor=NW)
                    TreeviewEntry.insert(-1, value_list[value_id])
                    self.see(self.selection())
                    TreeviewEntry.focus_set()
                    TreeviewEntry.bind(
                        '<FocusOut>', lambda event: value_deliver())
                    self.bind('<Configure>', lambda event: entry_destroy())
                    #self.parent.bind('<1>', srollbar_entry_destroy)
                    #self.bind('<Leave>', srollbar_entry_destroy)
                    TreeviewEntry.bind(
                        '<MouseWheel>', lambda event: entry_destroy())
                    self.bind('<MouseWheel>', lambda event:  entry_destroy())
                    self.parent.bind(
                        '<MouseWheel>', lambda event:  entry_destroy())
                    TreeviewEntry.bind(
                        '<Return>', lambda event: value_deliver())
                    TreeviewEntry.bind(
                        '<Escape>', lambda event: entry_destroy())
        return 0
        # 进行监视的三个动作分别为：entry失焦：直接摧毁entry中止输入；
        # sheetview大小改变：直接摧毁entry中止输入，此举是防止窗口大小改变后entry在错误位置
        # 回车键被按下：执行值传递

    def clear(self):
        childrens = self.get_children()
        for i in childrens:
            self.delete(i)
        return True
        # 清空sheetview上的所有内容

    def deliver(self):
        temp = self.amended_array
        self.amended_array = []
        return temp
        # 将值传出后重置实例内部存储的值

    def delete_passing(self):
        temp = []
        for row in self.delete_array:
            temp.append(row[0])
        self.delete_array = []
        return temp

    def get_amended(self):
        return self.amended_array
        # 传出值，但不重置实例存储数据

    def get_deleted(self):
        return self.delete_array

    def load(self):
        insert_flag = 0
        for row in self.database_rows:
            for amended_row in self.amended_array:
                if row[0] == amended_row[0]:
                    self.insert('', 'end', values=amended_row, tags='amended')
                    insert_flag = 1
                    break
            for deleted_row in self.delete_array:
                if row[0] == deleted_row[0]:
                    self.insert('', 'end', values=deleted_row, tags='deleted')
                    insert_flag = 1
                    break
            if insert_flag == 0:
                self.insert('', 'end', values=row)
            insert_flag = 0
        return True

    def entry_state(self, state):
        if state == 0:
            self.config(selectmode=EXTENDED)
            self.selectmode = EXTENDED
            return True
        if state == 1:
            self.config(selectmode=BROWSE)
            self.selectmode = BROWSE
            return True
        return False

    def reload(self, data=0):
        if data != 0:
            self.database_rows = data
        self.clear()
        self.load()
        return True

    def delete_row(self):
        delete_table_index = self.selection()
        delete_index_dict = {}
        for row in self.delete_array:
            delete_index_dict[row[0]] = row
        for item in delete_table_index:
            if(self.item(item, option='values'))[0] not in delete_index_dict:
                self.delete_array.append(
                    list(self.item(item, option='values')))
            else:
                if (list(self.item(item, option='values'))) in self.database_rows:
                    self.delete_array.remove(
                        delete_index_dict[(self.item(item, option='values'))[0]])
                if (list(self.item(item, option='values'))) not in self.database_rows:
                    self.amended_array.append(
                        list(self.item(item, option='values')))
                    self.delete_array.remove(
                        delete_index_dict[(self.item(item, option='values'))[0]])
        self.reload()
        temp = []
        for row in self.amended_array:
            for delete_row in self.delete_array:
                if row[0] == delete_row[0]:
                    temp.append(row)
                    break
        for row in temp:
            self.amended_array.remove(row)
        temp = []
        self.reload()
        return True

    def selected_value(self):
        temp = []
        for index in self.selection():
            temp.append(self.item(index, option='values'))
        return temp

class table_viewS(ttk.Frame):
    def __init__(self, parent, columnid, showname,  data, columnwidth=100, selectmode=BROWSE, readonly=0, *args, **kwargs):
        ttk.Frame.__init__(self, parent)
        # 变量转接
        self.parent = parent
        self.columnid = columnid
        self.showname = showname
        self.columwidth = columnwidth
        self.data = data
        self.selectmode = selectmode
        self.readonly = readonly
        self.args = args
        self.kargs = kwargs

        # 内部变量
        self.search_result = []
        self.delete_array = []
        self.multiselect_var = ttk.IntVar(value=1)

        # 内部方法
        def search():
            data_array = self.data
            for row in data_array:
                for item in row:
                    if search_entry.get().upper() in str(item).upper():
                        self.search_result.append(row)
                        break
            self.temp_load(self.search_result)
            self.search_result = []
            return True

        def reset():
            self.reload()
            return True

        def multi_select():
            if self.multiselect_var.get() == 0:
                self.InnerTableview.entry_state(self.multiselect_var.get())
            if self.multiselect_var.get() == 1:
                self.InnerTableview.entry_state(self.multiselect_var.get())
                temp_id_array = self.InnerTableview.selection()
                for item in temp_id_array:
                    self.InnerTableview.selection_remove(item)
            return True

        # def delete():
        #     pass

        # 组件定义
        self.InnerTableview = table_view(
            self, self.columnid, self.showname,  self.data, self.columwidth, selectmode=self.selectmode, readonly=self.readonly)
        search_entry = ttk.Entry(
            self, width=40, validate='key', validatecommand=search)
        search_entry.bind('<Return>', lambda event: search)
        search_label = ttk.Label(self, text='  搜索:', justify=RIGHT)
        scrollbar = ttk.Scrollbar(
            self, command=self.InnerTableview.yview)

        def scroll_command(*args):
            self.InnerTableview.entry_deploy('d')
            scrollbar.set(first=args[0], last=args[1])
        self.InnerTableview.config(yscrollcommand=scroll_command)
        search_button = ttk.Button(self, text='搜索', command=search)
        ToolTip(search_button, text="点击搜索")
        reset_button = ttk.Button(self, text='刷新', command=reset)
        ToolTip(reset_button, text="退出搜索恢复到默认视野")
        multiselect_button = ttk.Checkbutton(
            self, text='多选模式', variable=self.multiselect_var, command=multi_select, onvalue=0, offvalue=1, bootstyle="warning-square-toggle")
        ToolTip(multiselect_button,
                text='''开启多选模式，按住Ctrl键点击待删除行，开启后无法触发修改，关闭后恢复。如果想取消修改
                ，双击需取消删除行的任一可修改项后，按Enter键将行状态修改为待修改即可。''', wraplength=90)
        delete_button = ttk.Button(
            self, text='删除', command=self.delete_row, bootstyle='warning')
        if self.readonly == 1:
            delete_button.config(state='disable')
            multiselect_button.config(state='disable')
        if readonly != 1:
            ToolTip(delete_button, text="将目标标记为删除，点击应用修改后生效", wraplength=10)
        else:
            ToolTip(delete_button, text="查询模式下不可进行删除操作！", wraplength=10)
        search_label.grid(column=0, row=0, padx=(10, 0),
                          pady=5,  sticky=(W, S, N, E))
        search_entry.grid(column=1, row=0, columnspan=3,
                          padx=(0, 5), pady=5, sticky=(W, S, N, E))
        search_button.grid(column=4, row=0, padx=5,
                           pady=5, sticky=(W, S, N, E))
        reset_button.grid(column=5, row=0, padx=5, pady=5, sticky=(W, S, N, E))
        multiselect_button.grid(column=6, row=0, padx=5,
                                pady=5, sticky=(W, S, N, E))
        delete_button.grid(column=7, row=0, padx=5,
                           pady=5, sticky=(W, S, N, E))
        self.InnerTableview.grid(
            column=0, row=1, columnspan=8, rowspan=6, sticky=(W, S, N, E))
        scrollbar.grid(column=8, row=1, rowspan=6, sticky=(W, S, N, E))
        for i in range(6):
            self.columnconfigure(i, weight=1)
        for i in range(1, 7):
            self.rowconfigure(i, weight=1)

# 传递table_view的方法
    def clear(self):
        t = self.InnerTableview.clear()
        return t

    def deliver(self):
        t = self.InnerTableview.deliver()
        return t

    def delete_passing(self):
        t = self.InnerTableview.delete_passing()
        return t

    def get_amended(self):
        t = self.InnerTableview.get_amended()
        return t

    def get_deleted(self):
        t = self.InnerTableview.get_deleted()
        return t

    def load(self):
        t = self.InnerTableview.load()
        return t

    def reload(self, data=0):
        if data != 0:
            self.data = data
        else:
            data = self.data
        t = self.InnerTableview.reload(data)
        return t

    def temp_load(self, data=0):
        t = self.InnerTableview.reload(data)
        return t

    def entry_state(self, state):
        t = self.InnerTableview(state)
        return t

    def delete_row(self):
        t = self.InnerTableview.delete_row()
        return t

    def selected_value(self):
        t=self.InnerTableview.selected_value()
        return t


import logging
from collections import defaultdict
from collections.abc import Iterable
from typing import Any

import numpy as np
import pandas as pd

from hetdesrun.adapters.exceptions import (
    AdapterClientWiringInvalidError,
    AdapterHandlingException,
)
from hetdesrun.adapters.generic_rest.external_types import ExternalType
from hetdesrun.adapters.generic_rest.load_framelike import load_framelike_data
from hetdesrun.models.data_selection import FilteredSource

logger = logging.getLogger(__name__)

async def load_ts_data_from_adapter(
    filtered_sources: list[FilteredSource],
    filter_params: Iterable[tuple[str, Any]],
    adapter_key: str,
) -> pd.DataFrame:
    """Load data from generic rest adapter timeseries endpoint

    filtered_sources are expected to all have the exact same type which must be a
    timeseries(...) variant.

    Data is returned as a DataFrame with a timestamp column,
    a "value" column with automatically inferred dtype
    and a timeseriesId column with dtype str.

    This uses request with stream=True in combination with the raw response as file-like together
    with Pandas read_json class method to get some minor performance out of the streaming.

    It therefore currently isn't async.
    """

    df = await load_framelike_data(
        filtered_sources=filtered_sources,
        additional_params=list(filter_params),
        adapter_key=adapter_key,
        endpoint="timeseries",
    )

    if "timeseriesId" in df.columns:
        df["timeseriesId"] = df["timeseriesId"].astype("string")

    return df

def extract_one_channel_series_from_loaded_data(
    df: pd.DataFrame, ts_id: str
) -> pd.Series:
    try:
        extracted_df = df[df["timeseriesId"] == ts_id].copy()
        extracted_df.index = extracted_df["timestamp"]
        extracted_series = extracted_df["value"].sort_index()
        extracted_series.attrs = df.attrs.get(ts_id, {})
        logger.debug(
            "extracted attributes %s for series with id %s",
            extracted_series.attrs,
            ts_id,
        )
    except KeyError as e:
        msg = (
            f"Missing keys in received timeseries records. Got columns {str(df.columns)}"
            f" with dataframe of shape {str(df.shape)}:\n{str(df)}"
        )
        logger.info(msg)
        raise AdapterHandlingException(msg) from e

    extracted_series.name = ts_id

    return extracted_series

async def load_grouped_timeseries_data_together(
    data_to_load: dict[str, FilteredSource], adapter_key: str
) -> dict[str, pd.Series]:
    """Reorganize query information by timestamp pairs and load timeseries data

    Generic Rest Adapter allows to query for multiple timeseries in one request but then only with
    one timestamp filter pair and same (requested) value type for all those timeseries.

    This function expects data refs of the timeseries type,
    groups them together if they have same filter timestamp pairs and same value type,
    loads each such group in one request
    and returns all results gathered.
    """
    loaded_data = {}

    # group by occuring timestamp pairs
    group_by_filters_and_external_type: dict[
        tuple[frozenset[tuple[Any, Any]], ExternalType],
        dict[str, FilteredSource],
    ] = defaultdict(dict)

    for filtered_source in data_to_load.values():
        if (
            not isinstance(filtered_source.filters.get("timestampFrom", None), str)
        ) or (not isinstance(filtered_source.filters.get("timestampTo", None), str)):
            raise AdapterClientWiringInvalidError(
                "Timeseries data with no to/from filters."
            )

    for key, filtered_source in data_to_load.items():
        filtered_source.filters["from"] = filtered_source.filters.pop("timestampFrom")
        filtered_source.filters["to"] = filtered_source.filters.pop("timestampTo")
        group_by_filters_and_external_type[
            (
                frozenset(filtered_source.filters.items()),
                ExternalType(filtered_source.type),
            )
        ][key] = filtered_source

    # load each group together:
    for group_tuple, grouped_source_dict in group_by_filters_and_external_type.items():
        loaded_ts_data_from_adapter = await load_ts_data_from_adapter(
            list(grouped_source_dict.values()),
            group_tuple[0],
            adapter_key=adapter_key,
        )

        loaded_data.update(
            {
                key: extract_one_channel_series_from_loaded_data(
                    loaded_ts_data_from_adapter,
                    filtered_source.ref_id,  # type: ignore
                )
                for key, filtered_source in grouped_source_dict.items()
            }
        )

        try:
            received_ids = loaded_ts_data_from_adapter["timeseriesId"].unique()
        except KeyError as e:
            msg = (
                f"Missing keys in received timeseries records."
                f" Got columns {str(loaded_ts_data_from_adapter.columns)}"
                f" with dataframe of shape {str(loaded_ts_data_from_adapter.shape)}:\n"
                f"{str(loaded_ts_data_from_adapter)}"
            )
            logger.info(msg)
            raise AdapterHandlingException(msg) from e

        queried_ids = [fs.ref_id for fs in grouped_source_dict.values()]

        if not np.isin(received_ids, np.array(queried_ids)).all():
            msg = (
                f"Found timeseries ids in received data that were not queried."
                f" Received timeseriesId unique values were:\n{str(received_ids.tolist())}"
                f" \nQueried ids were:\n{str(queried_ids)}."
                "\nThis unassignable data will be discarded. This indicates an error in the adapter"
                f" implementation of the adapter {str(adapter_key)}!"
            )
            logger.warning(msg)

    return loaded_data


#!/usr/bin/python

__author__     = "Michael Lienhardt"
__copyright__  = "Copyright 2019-2020, Michael Lienhardt"
__license__    = "GPL3"
__version__    = "1"
__maintainer__ = "Michael Lienhardt"
__email__      = "michael.lienhardt@onera.fr"
__status__     = "Prototype"

# Graph loading and generation
import random
import sys
import argparse
import sys

# Parsing
import pydot
import lrparsing

# Solving
import z3

#
# CONSTRAINT SYNTAX AND TRANSLATION IN Z3
#

KW_AND = 'and'
KW_NOT = 'not'
KW_XOR = 'xor'
KW_OR  = 'or'
KW_IMPLIES = '=>'
KW_IFF = '<=>'
KW_TRUE = 'True'
KW_FALSE = 'False'

class T(lrparsing.TokenRegistry):
  AND     = lrparsing.Keyword(KW_AND)
  NOT     = lrparsing.Keyword(KW_NOT)
  XOR     = lrparsing.Keyword(KW_XOR)
  OR      = lrparsing.Keyword(KW_OR)
  IMPLIES = lrparsing.Token(KW_IMPLIES)
  IFF     = lrparsing.Token(KW_IFF)
  TRUE    = lrparsing.Keyword(KW_TRUE)
  FALSE   = lrparsing.Keyword(KW_FALSE)
  # special symbols
  LPAREN     = lrparsing.Token('(')
  RPAREN     = lrparsing.Token(')')

class C(lrparsing.Grammar):
  c_base = lrparsing.Token(re=r'[0-9a-zA-Z_]+') | (T.LPAREN + lrparsing.Ref('c_iff') + T.RPAREN)
  c_not = T.NOT*(0,1) + c_base
  c_and = c_not + (T.AND + c_not)*()
  c_xor = c_and + (T.XOR + c_and)*()
  c_or = c_xor + (T.OR + c_xor)*()
  c_implies = c_or | (c_or + T.IMPLIES + c_or)
  c_iff = c_implies | (c_implies + T.IFF + c_implies)
  START = c_iff

lrparsing.compile_grammar(C)

class c_translator(object):
  __slots__ = '_features'
  def __init__(self):
    self._features = {}

  def c_translate_base(self, parse_tree):
    #print("        c_base: {}".format(parse_tree))
    if(parse_tree[1][1] == '('): return self.c_translate_iff(parse_tree[2])
    elif(parse_tree[1][1] == KW_TRUE): return True
    elif(parse_tree[1][1] == KW_FALSE): return False
    else:
      name = parse_tree[1][1]
      res = self._features.get(name)
      if(res is None):
        res = z3.Bool(name)
        self._features[name] = res
      return res
  def c_translate_not(self, parse_tree):
    if(parse_tree[1][1] != KW_NOT): res = self.c_translate_base(parse_tree[1])
    else: res = z3.Not(self.c_translate_base(parse_tree[2]))
    #print("     c_not: {}".format(spc_debug(res)))
    return res
  def c_translate_and(self, parse_tree):
    subs = tuple(self.c_translate_not(el) for el in parse_tree if((len(el) > 0) and (el[0].name == "c_not")))
    #print("    c_and: {}".format([spc_debug(el) for el in subs]))
    if(len(subs) == 1): return subs[0]
    else: return z3.And(*subs)
  def c_translate_xor(self, parse_tree):
    subs = tuple(self.c_translate_and(el) for el in parse_tree if((len(el) > 0) and (el[0].name == "c_and")))
    #print("   c_xor: {}".format([spc_debug(el) for el in subs]))
    if(len(subs) == 1): return subs[0]
    else: return z3.Xor(*subs)
  def c_translate_or(self, parse_tree):
    subs = tuple(self.c_translate_xor(el) for el in parse_tree if((len(el) > 0) and (el[0].name == "c_xor")))
    #print("  c_or: {}".format([spc_debug(el) for el in subs]))
    if(len(subs) == 1): return subs[0]
    else: return z3.Or(*subs)
  def c_translate_implies(self, parse_tree):
    subs = tuple(self.c_translate_or(el) for el in parse_tree if((len(el) > 0) and (el[0].name == "c_or")))
    #print(" c_implies: {}".format([spc_debug(el) for el in subs]))
    if(len(subs) == 1): return subs[0]
    else: return z3.Implies(subs[0], subs[1])
  def c_translate_iff(self, parse_tree):
    subs = tuple(self.c_translate_implies(el) for el in parse_tree if((len(el) > 0) and (el[0].name == "c_implies")))
    #print("c_iff: {}".format([spc_debug(el) for el in subs]))
    if(len(subs) == 1): return subs[0]
    else: return subs[0] == subs[1]

  def c_translate(self, s): return self.c_translate_iff(C.parse(s)[1])

#
# CORE IMPLEMENTATION OF FTS
#

z3_id = 0
def fresh_var():
  global z3_id
  res = z3_id
  z3_id = z3_id + 1
  return z3.Bool(str(res))

class State(object):
  """Simple implementation of a state class"""
  __slots__ = '_id', '_in', '_out', '_z3_var', '_is_hidden_deadlock'
  def __init__(self, id):
    self._id = id
    self._in = set()
    self._out = set()
    self._z3_var = fresh_var()
    self._is_hidden_deadlock = None
  def __str__(self): return str(self._id)

class Transition(object):
  """Generic edge class specification"""
  __slots__ = '_in', '_out', '_label', '_constraint', '_z3_var', '_is_dead', '_is_false_optional'
  def __init__(self, _in, _out, label, constraint):
    self._in = _in
    self._out = _out
    self._label = label
    self._constraint = constraint
    self._z3_var = fresh_var()
    self._is_dead = None
    self._is_false_optional = None
  def __str__(self): return f"({str(self._in)}, {str(self._out)}, {self._label})"

class FTS(c_translator):
  __slots__ = '_name', '_fm', '_states', '_transitions', '_initial', '_set_hidden_deadlock', '_set_dead', '_set_false_optional'

  def __init__(self, name, fm):
    super().__init__()
    self._name = name
    self._fm = self.c_translate(fm[1:-1] if(fm is not None) else KW_TRUE)                # the feature model of the FTS
    self._states = {}
    self._transitions = []
    self._initial = None         # the initial state of the FTS
    self._set_hidden_deadlock = None # the list of hidden deadlocks in the FTS
    self._set_dead = None # the list of dead transitions in the FTS
    self._set_false_optional = None

  def state(self, id):
    """this method adds the state $id to the FTS"""
    res = self._states.get(id)
    if(res is None):
      res = State(id)
      self._states[id] = res
    return res

  def transition(self, _in, _out, label, constraint):
    """this method adds the transition between the states $_in and $_out, with the label $label and the constraint $constraint to the FTS"""
    _in = self.state(_in)
    _out = self.state(_out)
    transition = Transition(_in, _out, label, self.c_translate(constraint))
    _in._out.add(transition)
    _out._in.add(transition)
    self._transitions.append(transition)

  # this method sets the initial state of the FTS
  def initial_state(self, id):
    self._initial = self.state(id)
    return self._initial

  def report(self):
    print(f"{self._name}: {'not live' if self._set_hidden_deadlock else 'live'}")
    print(f"  HIDDEN DEADLOCKS = {tuple(str(n) for n in self._set_hidden_deadlock)}")
    if(self._set_false_optional is None):
      print("  FALSE OPTIONAL   = NOT ANALYSED")
    else: print(f"  FALSE OPTIONAL   = {tuple(str(e) for e in self._set_false_optional)}")
    if(self._set_dead is None):
      print("  DEAD TRANSITIONS = NOT ANALYSED")
    else: print(f"  DEAD TRANSITIONS = {tuple(str(e) for e in self._set_dead)}")

#
# FTS LOADING AND TRANSLATION
#

def load_dot(f):
  g = pydot.graph_from_dot_data(f.read())[0]

  if(g.get_type() != "digraph"):
    print("ERROR: the input graph is not a digraph (it is a {} instead)".format(g.get_type()))
    return

  g_attributes = g.get_attributes()
  fm = g_attributes.get('FM')
  name = g_attributes.get('name')
  name = name[1:-1] if(name is not None) else file_name

  res = FTS(name, fm)
  for node in g.get_nodes():
    node_name = node.get_name()
    if(node_name not in ('node', 'FeatureModel')):
      res.state(node_name)
      if(node.get_attributes().get('initial', False)): res.initial_state(node_name)

  for edge in g.get_edges():
    entry = edge.get_source()
    exit = edge.get_destination()
    label, constraint = edge.get_attributes().get('label', "|True")[1:-1].split('|')
    #print(action, formula)
    res.transition(entry, exit, label, constraint)

  return res

###############################################################################
# SAT ANALYSIS
###############################################################################

def one_exit(n):
  out = n._out
  if(len(out) > 1):
    return tuple(z3.Implies(e1._z3_var, z3.Not(z3.Or(*tuple(e2._z3_var for e2 in out if e2 is not e1)))) for e1 in out)
  else: return ()

def no_exit(n):
  return tuple(z3.Not(e._z3_var) for e in n._out)

def z3_translator(fts):
    # z3_states = the constraint when each node is accessible, or within a loop
    z3_states = (fts._initial._z3_var,) #(fts._initial._z3_var == True,)
    z3_states = z3_states + tuple(z3.Implies(n._z3_var, z3.Or(*tuple(z3.And(e._z3_var, e._constraint, e._in._z3_var) for e in n._in))) for n in fts._states.values() if n != fts._initial)
    # forbid loops
    no_loop_partial = []
    for n in fts._states.values(): no_loop_partial.extend(one_exit(n))
    return (fts._fm, *z3_states, *no_loop_partial)

def z3_analyse_full(fts):
  print("setting up solver ...", flush=True)
  solver = z3.Solver()
  constraint_list = z3_translator(fts)
  solver.add(*constraint_list)
  print(" ... done", flush=True)

  for i,n in enumerate(fts._states.values()):
    print(f"managing node {str(i)}: {str(n)} ...")
    if(n._out):
      constraint_annex = (n._z3_var, *no_exit(n)) + tuple(z3.Not(e._constraint) for e in n._out)
      n._is_hidden_deadlock = (solver.check(*constraint_annex,) == z3.sat)
    else: n._is_hidden_deadlock = False

  for i,t in enumerate(fts._transitions):
    print(f"managing transition {str(i)}: {str(t)} ...")
    t_in_useful = (t._in._z3_var, *no_exit(t._in))
    #n._is_hidden_deadlock = (solver.check(*constraint_annex, *tuple(z3.Not(e._constraint) for e in n._out)) == z3.sat)
    if(t._constraint is True):
      t._is_dead = not (solver.check(*t_in_useful) == z3.sat)
      t._is_false_optional = False
    elif(t._constraint is False):
      t._is_dead = True
      t._is_false_optional = False
    else:
      t._is_dead = not (solver.check(*t_in_useful, t._constraint) == z3.sat)
      if(t._is_dead): t._is_false_optional = False
      else: t._is_false_optional = not (solver.check(*t_in_useful, z3.Not(t._constraint)) == z3.sat)

  # Collect all information
  fts._set_hidden_deadlock = tuple( n for n in fts._states.values() if n._is_hidden_deadlock )
  fts._set_false_optional = tuple( e for e in fts._transitions if e._is_false_optional )
  fts._set_dead = tuple( e for e in fts._transitions if e._is_dead )

def z3_analyse_hdead(fts):
  print("setting up solver ...", flush=True)
  solver = z3.Solver()
  constraint_list = z3_translator(fts)
  solver.add(*constraint_list)
  print(" ... done", flush=True)

  for i,n in enumerate(fts._states.values()):
    print(f"managing node {str(i)}: {str(n)} ...")
    if(n._out):
      constraint_annex = (n._z3_var, *no_exit(n)) + tuple(z3.Not(e._constraint) for e in n._out)
      n._is_hidden_deadlock = (solver.check(*constraint_annex,) == z3.sat)
    else: n._is_hidden_deadlock = False

  # Collect all information
  fts._set_hidden_deadlock = tuple( n for n in fts._states.values() if n._is_hidden_deadlock )

def z3_analyse_quick(fts):
  print("setting up solver ...", flush=True)
  solver = z3.Solver()
  constraint_list = z3_translator(fts)
  solver.add(*constraint_list)
  print(" ... done", flush=True)

  for i,n in enumerate(fts._states.values()):
    print(f"managing node {str(i)}: {str(n)} ...")
    if(n._out):
      constraint_exit = tuple(z3.Not(e._constraint) for e in n._out)
      if(solver.check(*constraint_exit) != z3.sat):
        n._is_hidden_deadlock = False
      else:
        constraint_annex = (n._z3_var, *no_exit(n)) + constraint_exit
        n._is_hidden_deadlock = (solver.check(*constraint_annex) == z3.sat)
    else: n._is_hidden_deadlock = False

  # Collect all information
  fts._set_hidden_deadlock = tuple( n for n in fts._states.values() if n._is_hidden_deadlock )

def z3_analyse_alt(fts):
  print("setting up solver ...", flush=True)
  solver = z3.Solver()
  constraint_list = z3_translator(fts)
  solver.add(*constraint_list)
  print(" ... done", flush=True)

  #solver_optim = z3.Solver(ctx=z3.Context())
  #solver_optim.add(fts._fm)
  #print(tuple(f"node {str(n)} => {n._z3_var}" for n in fts._states.values()) + tuple(f"edge {str(e)} => {e._z3_var}" for e in fts._transitions))
  #print(constraint_list)
  #exit()
  #print(solver)
  for i,n in enumerate(fts._states.values()):
    print(f"managing node {str(i)}: {str(n)} ...")
    if(n._out):
      constraint_annex = (n._z3_var, *no_exit(n))
      n._is_hidden_deadlock = (solver.check(*tuple(z3.Not(e._constraint) for e in n._out)) == z3.sat)
      if(n._is_hidden_deadlock):
        n._is_hidden_deadlock = (solver.check(*constraint_annex, *tuple(z3.Not(e._constraint) for e in n._out)) == z3.sat)
      #n._is_hidden_deadlock = (solver.check(*constraint_annex, *tuple(z3.Not(e._constraint) for e in n._out)) == z3.sat)
      for e in n._out:
        if(e._constraint is True):
          e._is_dead = not (solver.check(*constraint_annex) == z3.sat)
          e._is_false_optional = False
        else:
          e._is_dead = not (solver.check(*constraint_annex, e._constraint) == z3.sat)
          if(e._is_dead): e._is_false_optional = False
          else: e._is_false_optional = not (solver.check(*constraint_annex, z3.Not(e._constraint)) == z3.sat)
      #solver.pop()
    else: n._is_hidden_deadlock = False
    print(f" ... done: hidden={n._is_hidden_deadlock}")
    for e in n._out:
      print(f"     edge {e}: false_opt={e._is_false_optional}, dead={e._is_dead}", flush=True)

  # Collect all information
  fts._set_hidden_deadlock = tuple( n for n in fts._states.values() if n._is_hidden_deadlock )
  fts._set_false_optional = tuple( e for e in fts._transitions if e._is_false_optional )
  fts._set_dead = tuple( e for e in fts._transitions if e._is_dead )

###############################################################################
# ADDITIONAL ANALYSIS FOR TESTING
###############################################################################

def check_always_accessible(graph, state_name):
  n_focus = graph._states[state_name]

  constraint_core = (n_focus._z3_var, *z3_translator(graph), *no_exit(n_focus)) # vincolo per un path verso s3
  to_hide = tuple(e._z3_var for e in graph._transitions) + tuple(n._z3_var for n in graph._states.values() if state_name != n_focus) # nascondo tutte le transizione e i stati diversi da s3
  solver = z3.Solver()
  solver.add(z3.Not( z3.Exists(to_hide, z3.And(*constraint_core)) )) # check se tautologia

  if(solver.check() == z3.sat):
    print(f"There exists a configuration in which {state_name} is not accessible:")
    model = solver.model()
    print(f"  {[ f'{f} => {model.eval(v)}' for f,v in graph._features.items()]}")
  else: print("OK")  # è una tautologia

def clean_path(graph, path, end):
  n = graph._initial
  res = []
  while(n != end):
    for e in n._out:
      if(e in path):
        res.append(e)
        n = e._out
        break
  return res

def compute_paths(graph, state_name):
  n_focus = graph._states[state_name]

  constraint_core = (n_focus._z3_var, *z3_translator(graph), *no_exit(n_focus))
  solver = z3.Solver()
  solver.add(constraint_core)
  while(solver.check() == z3.sat):
    model = solver.model()
    solution = {e for e in graph._transitions if z3.is_true(model.eval(e._z3_var))}
    solution = clean_path(graph, solution, n_focus)
    product = {f for f,v in graph._features.items() if z3.is_true(model.eval(v))} # this is just a possible product for this path, not all of them
    print(f"found path [product={product}]: { [str(e) for e in solution] }")
    solver.add(z3.Not(z3.And(*tuple(e._z3_var for e in solution)))) # next, search another path

#
# RANDOM ACYCLIC FTS GENERATION
#

def random_fts(nb_node, percent_edge):
  nb_rank = random.randint(0, nb_node - 1)
  nodes = { i: random.randint(0,nb_rank) for i in range(nb_node) }
  full_edges = { i: [j for j in range(nb_node) if nodes[i] < nodes[j]] for i in range(nb_node) }
  edges = { i: (random.choices(full_edges[i], k=((len(full_edges[i])*percent_edge)//100)) if(full_edges[i]) else []) for i in range(nb_node)}

  res = FTS("random", "True")
  for i, ns in edges.items():
    for j in ns:
      res.transition(f"S{i}", f"S{j}", "no_action", "True")
  res.initial_state("S0")
  return res

###############################################################################
# MAIN PROGRAM
###############################################################################

def set_default_subparser(parser, name):
  commands = { cmd for x in parser._subparsers._actions if(isinstance(x, argparse._SubParsersAction)) for cmd in x._name_parser_map.keys() }
  commands.update(('-h', '--help'))
  if(len(sys.argv) > 1):
    option = sys.argv[1]
    if(option in commands): return
  sys.argv.insert(1, name)

self_program_name = None
def main_manage_cmd_options():
  parser = argparse.ArgumentParser()
  subparsers = parser.add_subparsers()

  main_parser = subparsers.add_parser('main', add_help=False)
  main_parser.set_defaults(cmd='main')
  main_parser.add_argument('-g', '--generate', nargs=2, default=None)
  analysis_option = main_parser.add_mutually_exclusive_group()
  main_parser.add_argument('file', nargs='?', default=None)
  analysis_option.add_argument('--full', action="store_true")
  analysis_option.add_argument('--hdead', action="store_true")
  analysis_option.add_argument('--quick', action="store_true")
  analysis_option.add_argument('--alt', action="store_true")

  acc_parser = subparsers.add_parser('acc', add_help=False)
  acc_parser.set_defaults(cmd='acc')
  acc_parser.add_argument('state')
  acc_parser.add_argument('file', nargs='?', default=None)

  path_parser = subparsers.add_parser('path', add_help=False)
  path_parser.set_defaults(cmd='path')
  path_parser.add_argument('state')
  path_parser.add_argument('file', nargs='?', default=None)

  set_default_subparser(parser, 'main')
  #
  global self_program_name
  self_program_name = parser.prog

  args = parser.parse_args()

  if(args.cmd == "main"):
    if((args.generate is not None) and (args.file is not None)):
      print("ERROR: options -g and -f are incompatible")
      exit(-1)
    elif(args.generate is not None):
      fts = random_fts(int(args.generate[0]), int(args.generate[1]))
    elif(args.file is not None):
      fts = None
      with open(args.file, 'r') as f:
        fts = load_dot(f)
    else: fts = load_dot(sys.stdin)
    func_analyse = z3_analyse_full
    if args.hdead: func_analyse = z3_analyse_hdead
    elif args.quick: func_analyse = z3_analyse_quick
    elif args.alt: func_analyse = z3_analyse_alt
    func_display = lambda fts: fts.report()
    return func_analyse, func_display, fts
  elif(args.cmd == "acc"):
    if(args.file is not None):
      with open(args.file, 'r') as f:
        fts = load_dot(f)
    else: fts = load_dot(sys.stdin)
    state = args.state
    func_analyse = lambda fts: check_always_accessible(fts, state)
    func_display = None
    return func_analyse, func_display, fts
  elif(args.cmd == "path"):
    if(args.file is not None):
      with open(args.file, 'r') as f:
        fts = load_dot(f)
    else: fts = load_dot(sys.stdin)
    state = args.state
    func_analyse = lambda fts: compute_paths(fts, state)
    func_display = None
    return func_analyse, func_display, fts

#if(__name__ == "__main__"):
#  fts_implem, nb_node, pct_edge = main_manage_cmd_options_random()
#  fts = random_fts(fts_implem, nb_node, pct_edge)
#  print(fts.dot("random FTS"))

if(__name__ == "__main__"):
  func_analyse, func_display, fts = main_manage_cmd_options()
  #fts_debug(name, fts)
  func_analyse(fts)
  if(func_display): func_display(fts)


import numpy as np
from galpy import potential,df
from galpy.orbit import Orbit
from astropy import units as u

kpc = u.kpc
kms = u.km/u.s
deg = u.deg
Gyr = u.Gyr

def orbit_sampling(N_samples,rmin=0.0,rmax=200):
    pot  = potential.NFWPotential(a=19.0*kpc)
    distr_funct = df.isotropicNFWdf(pot=pot,ro=8.0*kpc,vo=220*kms,rmax=rmax*kpc)
    samples = distr_funct.sample(n=N_samples,return_orbit=True,rmin=rmin*kpc)
    return samples

def orbit_coordinates(samples,T_Gyr,nframes,ctype='Cartesian'):
    N_samples = len(samples)
    ts = np.linspace(0.0,T_Gyr*u.Gyr,nframes)
    Coords = np.zeros(shape=(nframes,samples.dim(),N_samples))

    for i in range(N_samples):
        R   = samples[i].R()
        z   = samples[i].z()
        vR  = samples[i].vR()
        vT  = samples[i].vT()
        vz  = samples[i].vz()
        phi = samples[i].phi()*180/np.pi
        o = Orbit(vxvv=[R*kpc,vR*kms,vT*kms,z*kpc,vz*kms,phi*deg])
        o.integrate(ts,potential.MWPotential2014)
        if ctype == 'Cartesian':
            Coords[:,:,i] = np.column_stack((o.x(ts),o.y(ts),o.z(ts)))
        elif ctype == 'Cylindrical':
            Coords[:,:,i] = np.column_stack((o.R(ts),o.phi(ts),o.z(ts)))
        else:
            print("Type not valid!")
    return Coords

def get_profile(nsamples,bins=500,rmin=0.0,rmax=200):
    '''

    Reproduces the density profile from the sampled miniclusters

    Use: r,rho = get_profile(nsamples)

    Input:
        - nsamples: the sample array

    Output:
       - r: radial coordinates in kpc
       - rho: density profiles in a.u.
    '''
    N_samples = int(nsamples)
    pot  = potential.NFWPotential(a=19.0*kpc)
    distr_funct = df.isotropicNFWdf(pot=pot,ro=8.0*kpc,vo=220*kms,rmax=rmax*kpc)
    samples = distr_funct.sample(n=N_samples,return_orbit=True,rmin=rmin*kpc)
    R = samples.R()
    dP,rb = np.histogram(np.log10(R),bins=bins,range=[0,np.log10(200)])
    rb = 10**rb
    rc = (rb[0:-1]+rb[1:])/2
    dr = rb[1:]-rb[0:-1]
    rho = (1/(4*np.pi*rc**2))*(dP/dr)
    rho = rho/rho[np.argmin(np.abs(rc-19.0))]
    return rc, rho

def disk_encounters(orbits,maxrad=0):
    '''
    Computes the number of disk encounters (crossing z=0) for each orbit, discards encounters with radii larger
    than maxrad
    Input:
        - Orbits coordinates from orbit_coordinates function with shape (nframes,3,nsamples)
        - Maximum radius to consider the disk encounter in kpc
    Output:
        - List with number of encounters for each orbit, shape (nsamples)
    '''
    enc_li = []
    for i in range(orbits.shape[-1]):
        z_path = orbits[:,:,i][:,2]
        R_path  = np.sqrt(z_path**2+(orbits[:,:,i][:,1])**2+(orbits[:,:,i][:,0])**2)
        enc_array = np.abs(np.diff(np.sign(z_path)))
        if maxrad > 0:
            mask = np.argwhere(R_path <= maxrad)
            enc_array = enc_array[mask-1] # minus 1
        enc  = int(enc_array.sum()/2)
        enc_li.append(enc)
    return np.array(enc_li)


import os
import re
import json
import hashlib
from wasabi import msg
from thefuzz import fuzz
import xml.etree.ElementTree as et

global user_answers
user_answers = []

def key_to_md5(key):
    """
    convert bibtex-key to shortened md5-sum to avoid special characters
    in the keys.

    :param key: bibtex-key of an article
    :return: converted key, first six characters of the key md5-sum
    """
    m = hashlib.md5()
    m.update(key.encode("utf-8"))
    hd = m.hexdigest()
    shorthd = hd[:6]
    if shorthd.isdigit():
        return shorthd + "a"
    return shorthd

def find_urls(string):
    """
    Check if the given string contains an url,
    used to find the pdfs of all articles.

    :param string: link to the pdf of an article
    :return: url if one is found
    """
    url = re.findall(
        r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
        string,
    )
    return url

def find_author(author_json):
    """
    detect correct surnames of all authors out of the author-bibtex-field
    :param author_json: author-field of an article from json export
    :return: list of all surnames
    """
    if "," not in author_json:
        pattern = re.compile(r"(?P<name>[A-Za-z\-]+)(?: +and +| +AND +|$)")
    else:
        pattern = re.compile(r"(?:^|and +|AND +)(?P<name>[A-Za-z\- ]+)")
    authors = pattern.findall(author_json)
    return authors

def find_doi(input):
    """
    checks if the doi of an article is syntactically correct or empty
    :param input: doi of an article
    :return: correct doi if found, otherwise None
    """
    if input is None:
        return None
    regex = r"(\d\d\.\d+\/\S+)"
    regex = re.compile(regex)
    m = regex.search(input)
    if m is not None:
        return m.group()
    return None

def find_matching_authors(artauthors, otherauthors):
    """
    calculates the total number of authors and the number of shared authors
    taking into account first authors
    :param artauthors: authors of first examined article
    :param otherauthors: authors of second article
    :return: two paramenters to calculate the score of the two articles
    """
    counter = 0
    number = len(artauthors) + len(otherauthors)
    if len(artauthors) == 0 or len(otherauthors) == 0:
        return counter, number
    for author in artauthors:
        if author == artauthors[0]:
            if author == otherauthors[0]:
                counter += 5
                number -= 1
            elif (
                next(
                    (
                        a
                        for a in otherauthors
                        if fuzz.ratio(author.upper(), a.upper()) >= 95
                    ),
                    None,
                )
                is not None
            ):
                counter += 3
                number -= 1
        elif author == otherauthors[0]:
            counter += 3
            number -= 1
        elif (
            next(
                (
                    a
                    for a in otherauthors
                    if fuzz.ratio(author.upper(), a.upper()) >= 95
                ),
                None,
            )
            is not None
        ):
            counter += 1
            number -= 1
    return counter, number

def citation_matching(
    doi_art,
    doi_ref,
    title_art,
    title_ref,
    author_art,
    authors_ref,
    without_interactive_queries,
):
    """
    Checks if a reference and an article of the citation graph match,
    if yes a citation is found.

    :param doi_art: doi of the article
    :param doi_ref: doi of the reference
    :param title_art: article title
    :param title_ref: reference title
    :param author_art: article authors
    :param authors_ref: reference authors
    :param without_interactive_queries: true for interactive mode
    :return: True iff a match is found
    """

    def ask_user(title_art, title_ref, author_art, authors_ref):

        global user_answers

        if without_interactive_queries:
            return False
        else:

            # Check if user already answered this question before
            question = (title_art, title_ref, author_art, authors_ref)
            for user_answer in user_answers:
                if user_answer["question"] == question:
                    return user_answer["answer"]

            # Prompt a question to the user and get his answer
            msg.warn("Unsure whether the entries belong together or not:")
            print(
                "Article:  Title = "
                + title_art
                + "\n\t  Authors = "
                + str(author_art).strip("[]")
                + "\nReference: Title = "
                + title_ref
                + "\n\t Authors = "
                + str(authors_ref).strip("[]")
            )

            msg.text(
                "Do both entries belong to the same article?",
                color="grey",
            )
            msg.text(
                "Please enter 'y' or 'n'...",
                color="grey",
            )
            if input() == "y":
                user_answers.append({"question": question, "answer": True})
                return True
            else:
                user_answers.append({"question": question, "answer": False})
                return False

    doi_art = find_doi(doi_art)
    doi_ref = find_doi(doi_ref)

    if doi_art == doi_ref and doi_art is not None and doi_ref is not None:
        # That was easy, DOIs match.
        return True

    elif type(title_art) == str and type(title_ref) == str:

        # Normalize titles
        clean_title_art = title_art.replace("{", "").replace("}", "").upper()
        clean_title_ref = title_ref.replace("{", "").replace("}", "").upper()

        # Get similarity between titles
        lev = fuzz.ratio(clean_title_art, clean_title_ref)
        lev_partial = fuzz.partial_ratio(clean_title_art, clean_title_ref)

        if lev > 90 or (lev_partial > 95 and lev > 70):
            # Titles can be considered matching.
            counter, number = find_matching_authors(author_art, authors_ref)

            if counter >= 2:
                # Authors can be considered matching.
                return True
            else:
                # Authors seem not to match. Ask the user for help!
                return ask_user(
                    clean_title_art, clean_title_ref, author_art, authors_ref
                )

        elif lev_partial > 90 and lev > 60:
            # Titles seem not to match. Ask the user for help!
            return ask_user(
                clean_title_art, clean_title_ref, author_art, authors_ref
            )

        else:
            # Consider it certain that the titles do not match.
            return False
    else:
        return False

def build_graph_model(
    json_bib_file,
    tei,
    graph_dir,
    graph_filename,
    original_bibtex_keys,
    without_interactive_queries,
):
    """
    Output looks like this:
        {
        "years": [2017, 2016, 2019, 2019, 2020, 2019,...],
        "year_arts": {
            "2010": ["Hoffmann et al (2010)"],
            "2011": ["Weng et al (2011)", "Tu et al (2011)"]
            ...
        },
        "articles": [
            {
            "title": "Bootstrapping for {Numerical} {Open} {IE}",
            "author": ["Saha", "Pal"],
            "key": "Saha et al (2017)",
            "year": "2017"
            },
            {
            "title": "Numerical Relation Extraction with Minimal Supervision",
            "author": ["Madaan", "Mittal"],
            "key": "Madaan and Mittal (2016)",
            "year": "2016"
            }
            ...
        ],
        "edges": [
            {
            "from": "Saha et al (2017)",
            "to": "Hoffmann et al (2010)"
            },
            {
            "from": "Saha et al (2017)",
            "to": "Madaan and Mittal (2016)"
            },
            ...
        ]
    }"""
    msg.divider("Build citation graph model from references in PDFs")

    with open(json_bib_file, "r") as file:  # encoding='utf8'
        bib = json.load(file)

    articles = bib["final selection articles"]

    if not (original_bibtex_keys):
        for article in articles:
            article["bibtex_key"] = key_to_md5(article["bibtex_key"])

    # Init graph
    graph = {}

    # Add years (e.g., "years": [2017, 2016, 2019, 2019, 2020, 2019,...])
    years = []
    for article in articles:
        if article["year"] is not None:
            years.append(int(article["year"]))

    graph["years"] = years

    # Add year_arts
    graph["year_arts"] = {}
    for year in range(min(years), max(years) + 1):
        this_year_arts = []
        for article in articles:
            if int(article["year"]) == year:
                this_year_arts.append(article["bibtex_key"])
        graph["year_arts"][year] = this_year_arts

    # Add articles
    graph["articles"] = []
    for article in articles:
        authors = find_author(article["author"])
        article_dict = {
            "title": article["title"],
            "author": authors,
            "key": article["bibtex_key"],
            "year": article["year"],
        }
        graph["articles"].append(article_dict)

    # Add edges
    graph["edges"] = []
    namespace = "{http://www.tei-c.org/ns/1.0}"

    for article in articles:

        # Check if field for file references is used
        if article["file"] is None:
            continue

        # Get filename of TEI corresponding to PDF
        xmlName = os.path.basename(article["file"])[:-4]
        tei_file = os.path.join(tei, "{}.tei.xml".format(xmlName))

        # Check if TEI file exists
        if not os.path.isfile(tei_file):
            msg.fail("tei-file not found for " + article["title"])
            continue

        # Check if TEI content is empty
        with open(tei_file, "r", encoding="utf8") as f:
            tei_content = f.read()

        if tei_content in [
            "[NO_BLOCKS] PDF parsing resulted in empty content",
            "[BAD_INPUT_DATA] PDF to XML conversion failed with error code: 1",
        ]:
            msg.fail("tei-file is empty for", article["title"])
            continue

        print("")
        msg.text("Processing " + tei_file, color="blue")

        # Loop over all references that Grobid found
        xml = et.parse(tei_file)
        for ref in xml.findall(".//{}biblStruct".format(namespace)):

            # Get title of referenced paper
            ref_title = ref.find(".//{}title".format(namespace)).text
            ref_title = ref_title  # if ref_title is not None else 'Ohne Titel'

            # Get DOI of referenced paper
            if (
                ref.find('.//{}idno[@type="doi"]'.format(namespace))
                is not None
            ):
                ref_doi = ref.find(
                    './/{}idno[@type="doi"]'.format(namespace)
                ).text
            else:
                ref_doi = None

            # Get authors of referenced paper
            ref_authors = []
            for ref_author in ref.findall(".//{}surname".format(namespace)):
                ref_authors.append(ref_author.text)

            # Check if the referenced paper is included in the
            # given set of papers. If so, add an edge between
            # the current paper and the referenced paper.
            for art in articles:
                art_authors = find_author(art.get("author"))
                art_doi = art.get("doi")

                if art["title"] is article["title"]:
                    # An article cannot cite itself
                    continue

                elif citation_matching(
                    art_doi,
                    ref_doi,
                    art["title"],
                    ref_title,
                    art_authors,
                    ref_authors,
                    without_interactive_queries,
                ):
                    # This article matches the references, therefore add an
                    # edge between this article and the article referencing
                    # this article.
                    edge = {
                        "from": article["bibtex_key"],
                        "to": art["bibtex_key"],
                    }
                    graph["edges"].append(edge)
                    break

    with open(os.path.join(graph_dir, graph_filename), "w") as jf:
        json.dump(graph, jf, indent=2)


"""
Config subroutine for YiQi. Manages configuration files in order for
other parts of the program to call back to and get user data.
"""

import json
import os
import shutil

configPath = 'config/keys.json'
emptyConfigPath = 'config/keyBlanks.json'

def init():
    """
    Initializes the config file. If a file exists, loads the config
    file into the configs variable for access later and sets the
    file to global. If it doesn't exist or the file cannot be
    decoded, the file is destroyed and recreated.
    """

    global configs
    try:
        configFile = open(configPath)
    except FileNotFoundError:
        dest = shutil.copyfile(configPath, emptyConfigPath)
        configFile = open(dest)
    finally:
        try:
            configs = json.load(configFile)
        # If there is an issue, reset the config file
        except json.decoder.JSONDecodeError:
            configFile.close()
            os.remove(configPath)
            init()
        configFile.close()

def exit():
    configFile = open(configPath, 'w')
    json.dump(configs, configFile)
    configFile.close()

def main():
    """
    The main function of Config handles the initialization of the
    entire class to ensure that the data is clear and usable.
    """
    init()


from django import template

register = template.Library()

@register.simple_tag
def formfield(field, id, label):
    return """
<li>
    %(errors)s
    <label for="%(id)s">
        %(label)s
    </label>
    %(field)s
</li>
    """ % {
        'errors': field.errors,
        'id': id,
        'label': label,
        'field': field,
    }


import torch
import torch.nn as nn
from .base import BaseModel
from torch.utils.data import DataLoader, TensorDataset
from uncertaintylearning.utils import get_dropout_uncertainty_estimate

class MCDropout(BaseModel):
    def __init__(self, train_X, train_Y,
                 network,
                 optimizer,
                 batch_size=16,
                 device=torch.device("cpu")):
        super().__init__()
        self.train_X = train_X
        self.train_Y = train_Y

        self.input_dim = train_X.size(1)
        self.output_dim = train_Y.size(1)

        self.epoch = 0
        self.loss_fn = nn.MSELoss()

        self.f_predictor = network
        self.device = device

        self.f_optimizer = optimizer

        self.actual_batch_size = min(batch_size, len(self.train_X) // 2)

    def fit(self, epochs=100):
        """
        Update a,f,e predictors with acquired batch
        """

        self.train()
        data = TensorDataset(self.train_X, self.train_Y)

        loader = DataLoader(data, shuffle=True, batch_size=self.actual_batch_size)
        for epoch in range(epochs):
            for batch_id, (xi, yi) in enumerate(loader):
                xi = xi.to(self.device)
                yi = yi.to(self.device)
                self.f_optimizer.zero_grad()
                y_hat = self.f_predictor(xi)
                f_loss = self.loss_fn(y_hat, yi)
                f_loss.backward()
                self.f_optimizer.step()

            self.epoch += 1

        return {
            'f': f_loss.detach().item(),
        }

    def _uncertainty(self, x, num_samples=100):
        """
        Computes uncertainty for input sample and
        returns total uncertainty estimate.
        """
        _, var = get_dropout_uncertainty_estimate(self.f_predictor, x, num_samples=num_samples)
        var = var ** 2
        return var

    def get_prediction_with_uncertainty(self, x):
        out = super().get_prediction_with_uncertainty(x)
        if out is None:
            self.eval()
            mean = self.f_predictor(x)
            self.train()
            var = self._uncertainty(x)
            return mean, var
        return out


import logging
from exchange.binance import BinanceClient
from data_collector import collect_all
import backtester
import datetime
from utils import TF_EQUIV
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

formatter = logging.Formatter("%(asctime)s %(levelname)s :: %(message)s")

stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
stream_handler.setLevel(logging.INFO)
file_handler = logging.FileHandler("info.log")
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.DEBUG)

logger.addHandler(file_handler)
logger.addHandler(stream_handler)

# logger.info("this a info log")

# only this file run then only run otherwise if imported in other then dont run

#1. asking from what you want to perfrom ( add data of 1 min, and later can change to any other form, or backtest, optimize)

#2 . asking for which client you want to fetch data from

#3. What currency pair your want to pull

#4 if backtest choose strategy, like 1.moving average 2. vwap startegy , timeframe (Reseampled timeframe using numpy library)

if __name__ == "__main__":
    mode = input("choose  the program mode (data/ backtest / optimize) : ").lower()
    client = BinanceClient()
    while True:
        exchange = input("Choose a exchange : ")
        if exchange == "binance":
            break

    while True:
        symbol = input("Choose a symbol : ").upper()
        if symbol in client.symbols:
            break

    if mode == "data":
        collect_all(client, exchange, symbol)

    elif mode == "backtest":

        #strategy testing

        available_strategies =['obv', 'ichimoku']
        while True:
            strategy = input(f"Chose a strategy: ({', '.join(available_strategies)}) : ").lower()

            if strategy in available_strategies:
                break
        #timeframe
        while True:
            tf = input(f"Chose a timeframe: ({', '.join(TF_EQUIV.keys())}): ").lower()

            if tf in TF_EQUIV.keys():
                break
         #from_time
        while True:
            from_time = input("backtest from (yyyy-mm-dd or Press Enter): ")

            if from_time == "":
                from_time = 0
                break
            try:
                from_time = int(datetime.datetime.strptime(from_time,"%y-%m-%d").timestamp()*1000)
                break
            except ValueError:
                continue

        # to Time
        while True:
            to_time = input("backtest to (yyyy-mm-dd or Press Enter) : ")

            if to_time == "":
                to_time = int(datetime.datetime.now().timestamp()*1000)
                break
            try:
                to_time = int(datetime.datetime.strptime(to_time, "%y-%m-%d").timestamp() * 1000)
                break
            except ValueError:
                continue

        backtester.run(exchange,symbol,strategy, tf, from_time,to_time)