import pos
import utils

class Sentence:
	'''A structure containing all words by one speaker between eol charachters ... . ! ?

	a sentence contains word objects which also link to the chunk object
	(chunk object reflect the ortogrpahic annotation structure)
	'''

	def __init__(self,words,sentence_number = 0):
		'''Make sentence object from a list of word object that form a sentence'''
		self.words = words
		self.nwords = len(words)
		self.sentence_number = sentence_number
		self.st = self.words[0].st
		self.et = self.words[-1].et
		try: self.duration = round(self.et - self.st,3)
		except: self.duration = -0.999
		self.find_chunk_numbers()
		self.check_sentence()
		self.npos_ok = False
		self.overlap = False
		self.overlap_unknown = True
		utils.make_attributes_available(self,'w',self.words)
		self.set_info()

	def __str__(self):
		a = ['sentence:\t'+self.string_words()]
		a.append('register:\t'+str(self.register))
		a.append('corpus:\t\t'+str(self.corpus))
		a.append('fid:\t\t'+str(self.fid))
		a.append('sid:\t\t'+str(self.sid))
		a.append('nwords:\t\t'+str(self.nwords))
		a.append('start_time:\t'+str(self.st))
		a.append('end_time:\t'+str(self.et))
		a.append('duration:\t'+str(self.duration))
		a.append('nchunks:\t'+str(self.n_chunks))
		a.append('chunk_numbers:\t'+' '.join(map(str,self.chunk_numbers)))
		a.append('sentence ok:\t'+str(self.ok))
		a.append('npos_ok:\t'+str(self.npos_ok))
		return '\n'.join(a)

	def __repr__(self):
		return self.string_words().ljust(150) + ' dur: ' + str(self.duration) + '\tregister: ' + self.register + '\tok: ' + str(self.ok)

	def set_info(self):
		self.sid = self.words[0].sid
		self.fid = self.words[0].fid
		self.register = self.words[0].register
		self.corpus = self.words[0].corpus

	def print_words(self):
		'''Print the info of each word'''
		print(self)
		print('-'*50)
		for w in self.words:
			print(w)
			print('-'*50)

	def string_words(self):
		'''Create an ascii sentence out of all words in the sentence'''
		output = []
		for w in self.words:
			output.append( w.word )
		return ' '.join(output)

	def string_utf8_words(self):
		'''Create an utf8 sentence out of all words in the sentence'''
		output = []
		for w in self.words:
			output.append( w.word_utf8_nocode )
		return ' '.join(output)

	def string_utf8_words_no_diacritics(self, fix_apostrophe = True):
		'''Create an utf8 sentence out of all words in the sentence'''
		output = []
		for w in self.words:
			output.append( w.word_utf8_nocode_nodia(fix_apostrophe) )
		return ' '.join(output)

	def find_chunk_numbers(self):
		'''Create a list of all chunk numbers of the words in the sentence
		(this does not mean all words of these chunks are in this sentence)
		'''
		self.chunk_numbers = []
		for w in self.words:
			if w.chunk_number not in self.chunk_numbers:
				self.chunk_numbers.append(w.chunk_number)
		self.n_chunks = len(self.chunk_numbers)

	def check_sentence(self):
		'''Check whether last word is an eol word (this should be the case)'''
		self.ok = True
		for i,w in enumerate(self.words):
			if w.eol == True and i != (len(self.words) - 1):
				self.ok = False

	def add_pos_to_words(self,pos_tags):
		'''Add POS tag info to each word

		Uses the FROG POS output and assumes that the output has the identical
		sentence structure as the object (number of words and order)
		'''
		if self.nwords == len(pos_tags):
			self.npos_ok = True
			for wi,pos_tag in enumerate(pos_tags):
				w = self.words[wi]
				w.pos = pos.Pos(pos_tag,self.sentence_number)
				if w.word_utf8_nocode == w.pos.token: w.pos_ok = True
				else: w.pos_ok = False

	def check_overlap(self):
		'''check whether the sentence overlaps in time with a sentence of other speaker.'''
		self.overlap = False
		for w in self.words:
			if w.overlap:
				self.overlap = True
		self.overlap_unknown = False

	def set_samplenumbers(self):
		'''set sample time based on the start and end sample of first and last word respectively.'''
		self.st_sample = self.words[0].st_sample
		self.et_sample = self.words[0].et_sample


import cv2
import os
import numpy as np

def create_video_from_images(image_folder, output_video_path, fps=24, frame_width=1920, frame_height=1080,
                             video_length=None, codec='none'):
    images = sorted([img for img in os.listdir(image_folder) if img.endswith((".jpg", ".jpeg", ".png", ".JPG"))])

    if video_length is not None:
        fps = len(images) / video_length

    # Získáme rozměry prvního obrázku (předpokládáme, že všechny mají stejné rozměry)
    image_height, image_width = cv2.imread(os.path.join(image_folder, images[0]), 0).shape[:2]

    if output_video_path.lower().endswith(".mp4"):
        if codec == 'none':
            codec = 'H265'
        elif codec not in ('H264', 'X264', 'H265', 'VP90', 'mp4v', 'DIVX', 'XVID', 'FMP4', 'avc1'):
            print("Nepodporovaný codec pro mp4.")
            return
    elif output_video_path.lower().endswith(".avi"):
        if codec == 'none':
            codec = 'DIVX'
        elif codec not in ('DIVX', 'XVID', 'MJPG', 'WMV1', 'WMV2', 'mpg1', 'I420', 'IYUV', 'H264'):
            print("Nepodporovaný codec pro mp4.")
            return
    else:
        print("Nepodporovaný nebo nezadaný formát.")
        return

    # Nastavení kodeku a vytvoření objektu pro video zápis
    #                                   VideoWriter objekt s nekomprimovaným kodekem (VYUY)
    # fourcc = cv2.VideoWriter_fourcc('Y', 'U', 'Y', 'V') # - nefunkční
    fourcc = cv2.VideoWriter_fourcc(*codec)
    out = cv2.VideoWriter(output_video_path, fourcc, fps, (frame_width, frame_height), isColor=True)

    # Výpočet poměru šířky a výšky načteného obrázku
    image_ratio = image_width / image_height

    # Výpočet poměru šířky a výšky požadovaného pole
    desired_ratio = frame_width / frame_height

    # Rozhodnutí o změně velikosti obrázku
    if image_ratio > desired_ratio:
        new_width = frame_width
        new_height = int(frame_width / image_ratio)
    else:
        new_width = int(frame_height * image_ratio)
        new_height = frame_height

    # Umístění obrázku do pole, aby zabíralo maximální plochu
    x_offset = (frame_width - new_width) // 2
    y_offset = (frame_height - new_height) // 2

    for image_name in images:
        image = cv2.imread(os.path.join(image_folder, image_name), 1)

        # Změna velikosti obrázku
        resized_image = cv2.resize(image, (new_width, new_height))

        # Vytvoření prázdného pole
        output_image = np.zeros((frame_height, frame_width, 3), dtype=np.uint8)

        # Umístěte fotografii na střed obrazu
        output_image[y_offset:y_offset + new_height, x_offset:x_offset + new_width] = resized_image
        out.write(output_image)

    # Uzavření objektu pro video zápis
    out.release()

    print("hotovo")

def play_video(video_path):
    cap = cv2.VideoCapture(video_path)
    close = False

    while True:
        while cap.isOpened():
            ret, frame = cap.read()
            if not ret:
                break

            cv2.namedWindow('Video', cv2.WINDOW_NORMAL)
            cv2.resizeWindow('Video', 1920 // 2, 1080 // 2)
            cv2.imshow('Video', frame)

            if cv2.waitKey(25) & 0xFF == 27:  # Klávesa "Esc" pro ukončení
                close = True
                break

        # Posuneme časový ukazatel zpět na začátek videa
        cap.set(cv2.CAP_PROP_POS_FRAMES, 0)

        if close:  # Klávesa "Esc" pro ukončení
            break

    cap.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    image_path = "photos"
    output_path = "output_video.mp4"

    create_video_from_images(image_path, output_path, video_length=60, codec='mp4v',
                             frame_width=3840, frame_height=2160)

    if os.path.exists(output_path):
        play_video(output_path)
    else:
        create_video_from_images(image_path, output_path)
        play_video(output_path)

    """print("\nChcete video smazat?")
    while True:
        ans = input("\tZadejte Y / N: ")
        if ans == "Y":
            os.remove(output_video_path)
            break
        elif ans == "N":
            break
        else:
            print("Zadejte platnou odpověď.")"""


from django.shortcuts import render
import json
import requests

def index(request):
    return render(request, 'index.html')

def weather(request):
    if request.method == 'POST':
        # collect data from index and instantiate to city
        city = request.POST['city']

        response = requests.get('https://api.openweathermap.org/data/2.5/weather?q='+city+'&appid={api_key}')
        # converting the response to json data
        json_data = response.json()

        #creating a dictionary with the json_data for reference in the templates
        data = {
            "country" : str(json_data['name']),
            "country_code" : str(json_data['sys']['country']),
            "coordinate": str(json_data['coord']['lon']) + ' ' + str(json_data['coord']['lat']),
            "temp": str(json_data['main']['temp']) + 'k',
            "pressure": str(json_data['main']['pressure']),
            "humidity":str(json_data['main']['humidity']),
        }

    context = {
        'city': city, 'data': data
    }
    return render(request, 'weather.html', context)


from datetime import datetime, timedelta
from subscription_handler import SubscriptionHandler
from customer import Customer, NotEnoughCredit, CustomerInDebt
from subscription import Subscription
from display import AlertMessage

class Subject:
    def __init__(self):
        self._observers = {}

    def register(self, observer_key, observer):
        if observer not in self._observers:
            self._observers[observer_key] = observer

    def detach(self, observer_key):
        try:
            del self._observers[observer_key]
        except ValueError:
            pass

    def notify(self, data = None):
        for _, observer in self._observers.items():
            observer.update(data)

class SubscriptionSystem(Subject):
    """General class to manage subscriptions"""
    def __init__(self, subscription_interval=timedelta(minutes=3)):
        Subject.__init__(self)
        self.subscription_interval = subscription_interval
        self.subscriptions = {}
        self._time = datetime.now()
        self.customers = {}

    def run(self):
        while True:
            try:
                command = input()
                self.handle_command(command)
                self.pass_time()

            except NotEnoughCredit as e:
                AlertMessage(e.message())
            except KeyError:
                AlertMessage(f"Key(customer_id/subscription_title) not found")

            # Test:
            print(f"--> System Time: {self._time}")

    def handle_command(self, command):
        if command == "":
            return

        tokens = command.split(" ")
        invalid_command = False

        if tokens[0] == "add":
            if tokens[1] == "sub":
                title, price = tokens[2], int(tokens[3])
                self.add_subscription(title, price)

            elif tokens[1] == "customer":
                customer_id, username , credit = tokens[2], tokens[3], int(tokens[4])
                self.add_customer(customer_id, username, credit)

            elif tokens[1] == "credit":
                customer_id, credit_amount = tokens[2], int(tokens[3])
                self.customers[customer_id].add_credit(credit_amount)

            else:
                invalid_command = True

        elif tokens[0] == "subscribe":
            customer_id, title = tokens[1], tokens[2]
            self.subscribe_customer(customer_id, title)

        elif tokens[0] == "deact":
            customer_id, title = tokens[1], tokens[2]
            self.deactivate_subscription(customer_id, title)

        elif tokens[0] == "react":
            customer_id, title = tokens[1], tokens[2]
            self.activate_subscription(customer_id, title, self._time)

        elif tokens[0] == "pay":
            customer_id = tokens[1]
            self.customers[customer_id].pay_debts()

        elif tokens[0] == "ls":
            customer_id = tokens[2]
            if tokens[1] == "subs":
                self.list_customer_subs(customer_id)
            elif tokens[1] == "invoices":
                self.customers[customer_id].display_invoices()
            else:
                invalid_command = True

        elif tokens[0] == "report":
            customer_id = tokens[1]
            self.customers[customer_id].display_report()

        elif tokens[0] == "help":
            try:
                self.prompt_help(' '.join(tokens[1:]))
                return
            except IndexError:
                self.prompt_help()
                return

        if invalid_command:
            self.prompt_help()

    def add_subscription(self, title, price):
        subscription = Subscription(title, price)
        self.subscriptions[title] = subscription

    def add_customer(self, customer_id, username, credit):
        customer = Customer(int(customer_id), username, credit)
        self.customers[customer_id] = customer

    def subscribe_customer(self, customer_id, title):
        handler_key = customer_id + title
        if handler_key in self._observers.keys():
            AlertMessage("Already subscribed!")
        else:
            try:
                handler = SubscriptionHandler(
                    self.customers[customer_id], self.subscriptions[title],
                    self.subscription_interval, self._time
                )

                Subject.register(self, handler_key, handler)
            except CustomerInDebt as e:
                AlertMessage(e.message())

    def deactivate_subscription(self, customer_id, title):
        handler_key = customer_id + title
        self._observers[handler_key].deactivate()

    def activate_subscription(self, customer_id, title, time):
        handler_key = customer_id + title
        self._observers[handler_key].activate(time)

    def list_customer_subs(self, customer_id):
        for key, sub in self._observers.items():
            if key.startswith(customer_id):
                sub.display()

    def pass_time(self):
        self._time += timedelta(minutes=1)
        self.notify(self._time)

    def prompt_help(self, parent=""):
        commands = ["add", "subscribe", "deact", "react", "pay", "ls", "report"]

        print()
        print("-------------------------------------------------------------------")
        if parent == "add":
            print(f"Usage: {parent} [sub/customer/credit] [ARG...]")
            print()
            print("Add items to the subscription system")
        elif parent == "add sub":
            print(f"Usage: {parent} title price")
            print()
            print("Add subscription to the system")
        elif parent == "add customer":
            print(f"Usage: {parent} id username credit")
            print()
            print("Add customer to the system")
        elif parent == "add credit":
            print(f"Usage: {parent} customer_id amount")
            print()
            print("Add credit to customer")
        elif parent == "subscribe":
            print(f"Usage: {parent} customer_id subscription_title")
            print()
            print("Subscribe customer to a subscription")
        elif parent in ["deact", "react"]:
            print(f"Usage: deact/react customer_id subscription_title")
            print()
            print("Deactivate/activate a subscription")
        elif parent == "pay":
            print(f"Usage: {parent} customer_id")
            print()
            print("Pay user's debts")
        elif parent == "ls":
            print(f"Usage: {parent} [subs/invoices] customer_id")
            print()
            print("List customer's [subscriptions/invoices]")
        elif parent == "report":
            print(f"Usage: {parent} customer_id")
            print()
            print("Generate report of customer's invoices and credit spent in the system")
        else:
            print("Usage: COMMAND customer_id [ARGS...]")
            print("Commands:")
            for  cmd in commands:
                print()
                print("  " + cmd)

        print()
        print("For additional detail on each command try `help COMMAND [SUBCOMMAND]`")
        print("-------------------------------------------------------------------")
        print()


from setuptools import find_packages,setup
from typing import List

HYPEN_E_DOT = '-e .'

def get_req(a:str)->List[str]:
    requirements=[]
    with open(a) as file_obj:
        reuirements=file_obj.readlines()
        [i.replace("/n","") for i in requirements]

        if HYPEN_E_DOT in requirements:
            requirements.remove(HYPEN_E_DOT)
    return requirements

setup(name='MLproject',
versio='0.0.1',
author='chandu',
packages=find_packages(),
install_requires=get_req('requirements.txt')
)


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split, GridSearchCV, KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import OneHotEncoder
from sklearn.linear_model import Lasso
from sklearn.ensemble import RandomForestRegressor, StackingRegressor
from xgboost import XGBRegressor
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import MaxAbsScaler

# Load the data
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# EDA
plt.figure(figsize=(12, 6))
sns.histplot(train['SalePrice'], kde=True)
plt.title('Sale Price Distribution')
plt.show()

plt.figure(figsize=(12, 6))
sns.scatterplot(x=train['GrLivArea'], y=train['SalePrice'])
plt.title('Sale Price vs. GrLivArea')
plt.show()

correlation_matrix = train.corr()
plt.figure(figsize=(12, 12))
sns.heatmap(correlation_matrix, cmap='coolwarm', annot=False)
plt.title('Correlation Matrix')
plt.show()

# Identify categorical and numerical columns
categorical_columns = train.select_dtypes(include=['object']).columns
numerical_columns = train.select_dtypes(exclude=['object']).drop(['Id', 'SalePrice'], axis=1).columns

# Preprocessing
preprocessor = ColumnTransformer(
    transformers=[
        ('num', SimpleImputer(strategy='mean'), numerical_columns),
        ('cat', Pipeline(steps=[
            ('impute', SimpleImputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))]), categorical_columns)])

# Prepare the data
X_train = train.drop(['Id', 'SalePrice'], axis=1)
y_train = train['SalePrice']
X_test = test.drop(['Id'], axis=1)

# Preprocess the data
X_train = preprocessor.fit_transform(X_train)
X_test = preprocessor.transform(X_test)

# Feature scaling
scaler = MaxAbsScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Define base models for stacking
base_models = [
    ('random_forest', RandomForestRegressor(n_jobs=-1, random_state=42)),
    ('xgb_regressor', XGBRegressor(n_jobs=-1, random_state=42)),
    ('lasso', Lasso(max_iter=10000, random_state=42))
]

# Set up stacking regressor
stacking_regressor = StackingRegressor(estimators=base_models, final_estimator=Lasso(max_iter=10000, random_state=42), n_jobs=-1)

# Perform KFold cross-validation
kf = KFold(n_splits=5, random_state=42, shuffle=True)
rmse_scores = np.sqrt(-cross_val_score(stacking_regressor, X_train, y_train, scoring='neg_mean_squared_error', cv=kf))

# Print mean and standard deviation of RMSE scores
print(f"KFold Cross-Validation RMSE scores: {rmse_scores}")
print(f"Mean RMSE score: {rmse_scores.mean():.2f}")
print(f"Standard deviation of RMSE scores: {rmse_scores.std():.2f}")

# Train the stacking model
stacking_regressor.fit(X_train, y_train)

# Generate predictions
predictions = stacking_regressor.predict(X_test)

# Create the submission file
submission = pd.DataFrame({'Id': test['Id'], 'SalePrice': predictions})
submission.to_csv('submission.csv', index=False)

print("Submission file created.")


import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
from window2 import Ventana
from gi.repository import GdkPixbuf
from random import randrange

class Celda(Gtk.EventBox):
    name = None
    image = Gtk.Image()
    imageM = Gtk.Image()
    jugador = None
    maquina = None
    resultado = None

    def __init__(self, name, image):
        super().__init__()
        self.name = name
        self.image = image
        box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=4)
        box.pack_start(image, True, True, 0)
        self.add(box)
        self.connect("button-release-event", self.on_click)

    def on_click(self, widget, event):
        self.seleccionJugador()
        self.seleccionMaquina()
        self.getImagePlayer()
        self.getImageMachine()

        self.result()

        win = Ventana(self.image, self.imageM,self.resultado)
        win.show_all()
        Gtk.main()

    def seleccionJugador(self):
        if self.name=="piedra":
            self.jugador=1
        if self.name=="papel":
            self.jugador=2
        if self.name=="tijera":
            self.jugador=3

    def seleccionMaquina(self):
        maquina = randrange(1, 4)
        self.maquina=maquina

    def getImagePlayer(self):
        img = Gtk.Image()
        pixbuf = None
        if self.name == "piedra":
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale("data/piedra.jfif", 200, 200, False)
        elif self.name == "papel":
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale("data/papel.jfif", 200, 200, False)
        elif self.name == "tijera":
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale("data/tijera.jfif", 200, 200, False)
        img.set_from_pixbuf(pixbuf)
        self.image = img

    def getImageMachine(self):
        img = Gtk.Image()
        pixbuf = None
        if self.maquina == 1:
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale("data/piedra.jfif", 200, 200, False)
        elif self.maquina == 2:
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale("data/papel.jfif", 200, 200, False)
        elif self.maquina == 3:
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale("data/tijera.jfif", 200, 200, False)
        img.set_from_pixbuf(pixbuf)
        self.imageM=img

    def result(self):
        result = None
        if self.jugador==self.maquina:
            result= "Habeis seleccionado lo mismo, habeis empatado!"
        if self.jugador==1 and self.maquina==2:
            result= "Has seleccionado piedra, la máquina ha seleccionado papel, has perdido!"
        if self.jugador==1 and self.maquina==3:
            result = "Has seleccionado piedra, la máquina ha seleccionado tijeras, has ganado!"
        if self.jugador==2 and self.maquina==1:
            result ="Has seleccionado papel, la máquina ha seleccionado piedra, has ganado!"
        if self.jugador==2 and self.maquina==3:
            result = "Has seleccionado papel, la máquina ha seleccionado tijeras, has perdido!"
        if self.jugador==3 and self.maquina==1:
            result = "Has seleccionado tijeras, la máquina ha seleccionado piedra, has perdido!"
        if self.jugador==3 and self.maquina==2:
            result = "Has seleccionado tijeras, la máquina ha seleccionado papel, has ganado!"

        self.resultado=result


#
from typing import Optional, Dict, Tuple, Any, Union
import logging
import numpy as np
import gym
import torch
from torch import nn
from torch import Tensor
from tianshou.data import Batch

import lib.networks.encoders as encoders
import lib.networks.decoders as decoders
from lib.networks.encoders import GRAPH_ENCODERS
from lib.networks.aggregator import Aggregator, DummyAggregator
from lib.networks.decoders.iqn import IQN
from lib.networks.formats import Obs, GraphObs, Emb
from lib.utils import count_parameters

logger = logging.getLogger(__name__)

class Model(nn.Module):
    """
    Model wrapper providing unified interface
    for different encoder/decoder architectures.

    Args:
        problem: current problem type
        observation_space: env observation space
        action_space: env action space
        policy_type: used policy type to determine model output format
                    (default 'DQN'; 'IQN' for implicit quantile network)
        decoder_type: type of decoder
        encoder_args: additional arguments for encoder creation
        decoder_args: additional arguments for decoder creation
        aggregator_args: additional arguments for aggregator creation
        embedding_dim: general embedding dimension of model
        device: device of model (CPU or GPU)
        env_modes: mode cfg of the env
    """
    def __init__(self,
                 problem: str,
                 observation_space: gym.spaces.Dict,
                 action_space: Union[gym.spaces.Discrete, gym.spaces.MultiDiscrete],
                 policy_type: str = "DQN",
                 decoder_type: Union[str, nn.Module] = "FFDecoder",
                 encoder_args: Optional[Dict] = None,
                 decoder_args: Optional[Dict] = None,
                 aggregator_args: Optional[Dict] = None,
                 iqn_args: Optional[Dict] = None,
                 embedding_dim: int = 128,
                 device: Union[str, int, torch.device] = "cpu",
                 env_modes: Dict = None,
                 **kwargs):
        super(Model, self).__init__()

        self.problem = problem
        self._observation_space = observation_space
        self._action_space = action_space
        self.decoder_type = decoder_type
        self.encoder_args = encoder_args if encoder_args is not None else {}
        self.decoder_args = decoder_args if decoder_args is not None else {}
        self.aggregator_args = aggregator_args if aggregator_args is not None else {}
        self.iqn_args = iqn_args if iqn_args is not None else {}
        self.embedding_dim = embedding_dim
        self._device = torch.device(device)
        self.policy_type = policy_type.upper()
        #
        self._idx_inc = None
        self._static_nbh_edge_buffer = None
        self._ref_bs = None

        # get dims from obs and act space
        _, f = self._observation_space.spaces['node_features'].shape
        self.node_feature_dim = f
        self.node_max_dim = None
        self.action_output_dim = self._action_space.n

        # initialize encoder model
        self.encoder = Encoder(
            problem=self.problem,
            embedding_dim=embedding_dim,
            node_feature_dim=self.node_feature_dim,
            edge_feature_dim=1,
            **self.encoder_args, **kwargs)

        # initialize aggregator
        if self.aggregator_args:
            self.aggregator = Aggregator(
                observation_space=self._observation_space,
                action_space=self._action_space,
                emb_dim=embedding_dim,
                env_modes=env_modes,
                **self.aggregator_args)
        else:
            self.aggregator = DummyAggregator()

        # initialize decoder network
        decoder_cl = getattr(decoders, decoder_type) if isinstance(decoder_type, str) else decoder_type
        # jit decoder model
        self.decoder = torch.jit.script(decoder_cl(
            input_dim=self.embedding_dim,
            output_dim=self.action_output_dim,
            **self.decoder_args, **kwargs
        ))

        self.iqn = IQN(
            input_dim=self.action_output_dim,
            output_dim=self.action_output_dim,
            **self.iqn_args
        ) if self.policy_type == "IQN" else None

        self.reset_parameters()
        self.to(device=self._device)

    def __repr__(self):
        super_repr = super().__repr__()     # get torch module str repr
        n_enc_p = count_parameters(self.encoder)
        n_agg_p = count_parameters(self.aggregator)
        n_dec_p = count_parameters(self.decoder)
        n_iqn_p = count_parameters(self.iqn) if self.iqn is not None else 0
        add_repr = f"\n-----------------------------------" \
                   f"\nNum Parameters: " \
                   f"\n  (encoder): {n_enc_p} " \
                   f"\n  (aggregator): {n_agg_p} " \
                   f"\n  (decoder): {n_dec_p + n_iqn_p} " \
                   f"\n  total: {n_enc_p+n_agg_p+n_dec_p+n_iqn_p}"
        return super_repr + add_repr

    def reset_parameters(self):
        """Reset model parameters."""
        self.encoder.reset_parameters()
        self.aggregator.reset_parameters()

    def forward(self,
                obs: Batch,
                state: Optional[np.ndarray] = None,
                info: Optional[Dict] = None,
                **kwargs) -> Tuple[Union[Tensor, Tuple], Any]:
        """
        Meta forward pass wrapper doing required data batch collation and conversion.

        Args:
            obs: ts.batch of observations
            state: optional RNN state
            info: additional env info dict

        Returns:
            logits: logits over action space dimension.
                    depending on policy can also be a tuple, e.g.
                    (mu: Tensor, sigma: Tensor) for Gaussian PPO policy or
                    (logits: Tensor, taus: Tensor) for IQN
            state: optional RNN state

        """
        # convert to torch tensors and push to device
        obs.to_torch(device=self._device)
        bs = obs.shape[0]
        recompute = True
        sample_size = kwargs.pop("sample_size", 0)
        self.node_max_dim = obs.node_features.size(1)

        # collate and format batch
        if self.encoder.uses_graph:
            # convert edges into running idx format expected by torch_geometric
            nbh_e = obs.get('nbh_edges')
            nbh_w = obs.get('nbh_weights')

            # during inference we can keep static components (e.g. node neighborhood)
            # fixed and do not need to recompute them every step
            if (
                self.training   # during training
                or len(info.shape) == 0     # on init
                or bs != self._ref_bs       # bs change
                or (len(info.shape) > 0 and np.any(info['step'] == 0))  # start of new episode
            ):
                self._ref_bs = bs
                self._idx_inc = torch.arange(0, self.node_max_dim * bs, self.node_max_dim,
                                             device=self._device)[:, None, None]
                if nbh_e is not None:
                    self._static_nbh_edge_buffer = self._to_edge_batch(nbh_e, self._idx_inc, self.node_max_dim)
            else:
                recompute = False

            cur_sol_e = self._to_edge_batch(obs['current_sol'], self._idx_inc, self.node_max_dim)
            best_sol_e = self._to_edge_batch(obs['best_sol'], self._idx_inc, self.node_max_dim)

            if self.problem.upper() == "JSSP":
                prep_obs = GraphObs(
                    batch_size=bs,
                    batch_idx=None,     # type: ignore
                    node_features=obs['node_features'].view(-1, self.node_feature_dim),
                    current_sol=None,   # type: ignore
                    current_sol_edges=cur_sol_e,  # type: ignore
                    current_sol_weights=obs['current_sol_w'].view(-1),
                    best_sol=None,  # type: ignore
                    best_sol_edges=best_sol_e,  # type: ignore
                    best_sol_weights=obs['best_sol_w'].view(-1),
                    nbh_edges=self._static_nbh_edge_buffer,  # type: ignore
                    nbh_weights=nbh_w.view(-1) if nbh_w is not None else None,
                    current_sol_seqs=obs['current_sol_seq'],
                    best_sol_seqs=obs['best_sol_seq'],
                    meta_features=obs['meta_features'],
                )
            else:
                if not hasattr(obs, "batch_idx"):
                    raise RuntimeError(f"Specified encoder needs observations from RPGraphVecEnv.")
                prep_obs = GraphObs(
                    batch_size=bs,
                    batch_idx=obs['batch_idx'].view(-1).to(torch.long),
                    node_features=obs['node_features'].view(-1, self.node_feature_dim),
                    current_sol=obs['current_sol'],
                    current_sol_edges=cur_sol_e,  # type: ignore
                    current_sol_weights=self._compute_tour_edge_weights(obs['coords'], cur_sol_e),
                    best_sol=obs['best_sol'],
                    best_sol_edges=best_sol_e,  # type: ignore
                    best_sol_weights=self._compute_tour_edge_weights(obs['coords'], best_sol_e),
                    nbh_edges=self._static_nbh_edge_buffer,  # type: ignore
                    nbh_weights=nbh_w.view(-1) if nbh_w is not None else None,
                    current_sol_seqs=obs['current_sol_seq'],
                    best_sol_seqs=obs['best_sol_seq'],
                    meta_features=obs['meta_features'],
                )
        else:
            assert self.problem.upper() != "JSSP"
            prep_obs = Obs(
                node_features=obs['node_features'],
                current_sol=obs['current_sol'],
                best_sol=obs['best_sol'],
                current_sol_seqs=obs['current_sol_seq'],
                best_sol_seqs=obs['best_sol_seq'],
                meta_features=obs['meta_features'],
            )

        logits, state = self._dqn_forward(bs, prep_obs, state, recompute=recompute, **kwargs)
        if self.policy_type == "IQN":
            # obs batch object does not need mask since logits are already masked, i.e. set to -inf where infeasible
            logits = self.iqn(logits, sample_size, **kwargs)    # returns logits tuple!

        return logits, state

    def _dqn_forward(self,
                     bs: int,
                     prep_obs: Union[Obs, GraphObs],
                     state: Optional[np.ndarray] = None,
                     **kwargs) -> Tuple[Union[Tensor, Tuple], Any]:
        """Inner forward calling encoder, aggregator and decoder models."""
        # encode
        emb = self.encoder(prep_obs, **kwargs)
        # aggregate embeddings
        emb = self.aggregator(
            prep_obs, emb, dims=(bs, self.node_max_dim, self.node_feature_dim)
        )
        # decode
        return self.decoder(emb, state)

    @staticmethod
    @torch.jit.script
    def _to_edge_batch(x: Tensor, idx_inc: Tensor, node_max_dim: int) -> Tensor:
        """Convert a batch of single instance edges to a batch of edges with running idx."""
        bs, e, n = x.size()
        assert e == 2 and x.max() < node_max_dim
        return (x + idx_inc).permute(1, 0, 2).reshape(2, -1)

    @staticmethod
    @torch.jit.script
    def _compute_tour_edge_weights(coords: Tensor, tour_edges: Tensor):
        """Calculate edge weights as euclidean distance between coords."""
        # calc edge weights (Euclidean distance)
        idx_coords = coords.view(-1, 2)[tour_edges]
        return torch.norm(idx_coords[0] - idx_coords[1], p=2, dim=-1)

    def get_optimizer(self, lr: float = 1e-3, **kwargs):
        """Return torch optimizer instance for network."""
        return torch.optim.Adam(self.parameters(), lr=lr, **kwargs)

class Encoder(nn.Module):
    """
    Encoder model wrapper to enable different encoding levels
    between nodes, edges, tours and graph.
    """

    def __init__(self,
                 problem: str,
                 embedding_dim: int,
                 node_feature_dim: int,
                 edge_feature_dim: Optional[int] = None,
                 node_encoder_type: Optional[str] = None,
                 node_encoder_cfg: Optional[Dict] = None,
                 sol_encoder_type: Optional[str] = None,
                 sol_encoder_cfg: Optional[Dict] = None,
                 sub_graph_encoder_type: Optional[str] = None,
                 sub_graph_encoder_cfg: Optional[Dict] = None,
                 graph_encoder_type: Optional[str] = None,
                 graph_encoder_cfg: Optional[Dict] = None,
                 ):
        super(Encoder, self).__init__()

        self.problem = problem
        self.embedding_dim = embedding_dim
        self.node_feature_dim = node_feature_dim
        self.edge_feature_dim = edge_feature_dim
        self.node_encoder_type = node_encoder_type
        self.node_encoder_cfg = node_encoder_cfg if node_encoder_cfg is not None else {}
        self.sol_encoder_type = sol_encoder_type
        self.sol_encoder_cfg = sol_encoder_cfg if sol_encoder_cfg is not None else {}
        self.sub_graph_encoder_type = sub_graph_encoder_type
        self.sub_graph_encoder_cfg = sub_graph_encoder_cfg if sub_graph_encoder_cfg is not None else {}
        self.graph_encoder_type = graph_encoder_type
        self.graph_encoder_cfg = graph_encoder_cfg if graph_encoder_cfg is not None else {}

        self.node_encoder = None
        self.sol_encoder = None
        self.sub_graph_encoder = None
        self.graph_encoder = None
        self.uses_graph = False

        if self.node_encoder_type is not None:
            self.node_encoder = self._init_encoder(
                encoder_type=self.node_encoder_type,
                input_dim=node_feature_dim,
                output_dim=embedding_dim,
                edge_feature_dim=edge_feature_dim,
                **self.node_encoder_cfg
            )

        # group pooling layer
        if self.sol_encoder_type is not None:
            self.sol_encoder = self._init_encoder(
                encoder_type=self.sol_encoder_type,
                input_dim=embedding_dim,
                output_dim=embedding_dim,
                problem=self.problem,
                **self.sol_encoder_cfg
            )

        if self.sub_graph_encoder_type is not None:
            self.sub_graph_encoder = self._init_encoder(
                encoder_type=self.sub_graph_encoder_type,
                input_dim=embedding_dim,
                output_dim=embedding_dim,
                **self.sub_graph_encoder_cfg
            )

        if self.graph_encoder_type is not None:
            self.graph_encoder = self._init_encoder(
                encoder_type=self.graph_encoder_type,
                input_dim=embedding_dim,
                output_dim=embedding_dim,
                **self.graph_encoder_cfg
            )

    def _init_encoder(self,
                      encoder_type: str,
                      input_dim: int,
                      output_dim: int,
                      **kwargs) -> nn.Module:
        self.uses_graph = self.uses_graph or (encoder_type in GRAPH_ENCODERS)
        cl = getattr(encoders, encoder_type) if isinstance(encoder_type, str) else encoder_type
        return cl(
            input_dim=input_dim,
            output_dim=output_dim,
            **kwargs
        )

    def reset_parameters(self):
        if self.node_encoder is not None:
            self.node_encoder.reset_parameters()
        if self.sol_encoder is not None:
            self.sol_encoder.reset_parameters()
        if self.sub_graph_encoder is not None:
            self.sub_graph_encoder.reset_parameters()
        if self.graph_encoder is not None:
            self.graph_encoder.reset_parameters()

    def forward(self, obs: Union[Obs, GraphObs], **kwargs) -> Emb:
        """
        Args:
            obs: batched observation tuple

        Returns:
            emb: tuple of created embeddings
        """
        emb = Emb()
        if self.node_encoder is not None:
            emb = self.node_encoder(obs, emb, **kwargs)
        if self.sol_encoder is not None:
            emb = self.sol_encoder(obs, emb, **kwargs)
        if self.sub_graph_encoder is not None:
            emb = self.sub_graph_encoder(obs, emb, **kwargs)
        if self.graph_encoder is not None:
            emb = self.graph_encoder(obs, emb, **kwargs)
        return emb

# ============= #
# ### TEST #### #
# ============= #
def _test(
    problem: str = "TSP",
    bs: int = 3,
    size: int = 10,
    n: int = 20,
    seed: int = 1,
    n_steps: int = 20,
    mode='ACCEPT_LS',
    mode_args={'ls_op': 'TWO_OPT'},
    graph: bool = False,
    **kwargs
):
    import tianshou as ts
    from lib.env import VecEnv, RPGraphVecEnv

    env_kwargs = {
        'num_steps': n_steps,
        'sampling_args': {'sample_size': size, 'graph_size': n},
        'mode': mode,
        'mode_args': mode_args,
        'debug': False,
    }
    if graph:
        env_cl = RPGraphVecEnv
        train_envs = env_cl(num_envs=bs, problem=problem, env_kwargs=env_kwargs, create_nbh_graph=True)
        test_envs = env_cl(num_envs=bs, problem=problem, env_kwargs=env_kwargs, create_nbh_graph=True)
    else:
        env_cl = VecEnv
        train_envs = env_cl(num_envs=bs, problem=problem, env_kwargs=env_kwargs)
        test_envs = env_cl(num_envs=bs, problem=problem, env_kwargs=env_kwargs)

    train_envs.seed(seed)
    test_envs.seed(seed)

    net = Model(observation_space=train_envs.observation_space, action_space=train_envs.action_space)
    print(net)

    optim = net.get_optimizer()
    policy = ts.policy.DQNPolicy(net, optim, discount_factor=0.99, estimation_step=3, target_update_freq=256)

    train_collector = ts.data.Collector(
        policy, train_envs, ts.data.PrioritizedVectorReplayBuffer(20000, bs), exploration_noise=True
    )
    test_collector = ts.data.Collector(
        policy, test_envs, exploration_noise=False
    )

    result = ts.trainer.offpolicy_trainer(
        policy, train_collector, test_collector,
        max_epoch=2,
        step_per_epoch=200,
        step_per_collect=100,
        update_per_step=0.1,
        episode_per_test=100,
        batch_size=64,
        train_fn=lambda epoch, env_step: policy.set_eps(0.1),
        test_fn=lambda epoch, env_step: policy.set_eps(0.05),
    )
    print(result)


import os
import sys
import itertools as it

from xml.etree import ElementTree as ET
import datetime

from collections.abc import Sequence
from collections import defaultdict

from tlgraphsum.utils import iter_dirs, iter_files, fst
from calendar import monthrange

import pickle
from collections import namedtuple

HARDCODED_PROBLEMATIC_DOCS = {'crisis-libya': {'768', '4746', '156', '981', '434', '1001', '5456', '1086', '825'}, 'tl17-libya': {'308', '257', '3', '212', '341', '309'}, 'tl17-haiti': {'820'}, 'tl17-bpoil': {'124', '364', '489', '2131', '1908', '1965'}, 'tl17-h1n1': {'519'}, 'tl17-finan': {'2124'}, 'crisis-egypt': {'2342', '506', '1808', '3312', '4831', '4983', '5047', '1268', '4933', '1569', '3110', '2510', '2190', '608', '385', '4633'}, 'crisis-yemen': {'3664', '592', '2787'}, 'tl17-syria': {'328', '283', '293'}, 'tl17-mj': {'176'}}

def load_corpus(fname, filter_blacklist=True):
    with open(fname, "rb") as f:
        corpus = pickle.load(f)
    corpus_id, _ = os.path.basename(fname).split(".", 1)
    if filter_blacklist:
        print("Filtering...")
        corpus.per_date_documents = filter_corpus_by_blacklist(corpus.per_date_documents, HARDCODED_PROBLEMATIC_DOCS.get(corpus_id, []))

    corpus.real_name = fname
    corpus.real_basename = os.path.basename(fname)
    if filter_blacklist:
        base_path, corpus_name = os.path.split(fname)
        name, _ = corpus_name.rsplit(".", 1)
        fname = os.path.join(base_path, name + "+filtered" + ".pkl")
    corpus.name = fname
    corpus.basename = os.path.basename(fname)

    return corpus

def filter_corpus_by_blacklist(per_date_documents, blacklist):
    new_per_date_documents = {}
    for date, docs in per_date_documents.items():
        new_per_date_documents[date] = [doc for doc in docs if doc.docid not in blacklist]
    return new_per_date_documents

class TokenListView(Sequence):
    def __init__(self, container, attributes):
        self.container = container
        self.attributes = attributes

    def __getitem__(self, idx):
        tok = self.container.tokens[idx]

        if isinstance(idx, slice):
            return [tuple([getattr(t, arg) for arg in self.attributes]) for t in tok]
        else:
            return tuple([getattr(tok, arg) for arg in self.attributes])

    def __len__(self):
        return len(self.container.tokens)

    def __repr__(self):
        content = ", ".join(map(lambda x: repr(x), self))
        return "TokenListView([{}])".format(content)

class TokenAttrListView(Sequence):
    def __init__(self, container, attr):
        self.container = container
        self.attr = attr

    def __getitem__(self, idx):
        tok = self.container.tokens[idx]

        if isinstance(idx, slice):
            return [getattr(t, self.attr) for t in tok]
        else:
            return getattr(tok, self.attr)

    def __len__(self):
        return len(self.container.tokens)

class TimelineCorpus:
    def __init__(self, per_date_documents, blacklist=set()):
        super(TimelineCorpus, self).__init__()
        self.per_date_documents = dict()

        for date, docs in per_date_documents.items():
            self.per_date_documents[date] = [doc for doc in docs if doc.docid not in blacklist]

    def __iter__(self):
        return (doc for _, docs in sorted(self.per_date_documents.items()) for doc in docs)

    @property
    def num_documents(self):
        return sum(map(lambda l: len(l), self.per_date_documents.values()))

    def iter_dates(self):
        return iter(self.per_date_documents.keys())

    def docs_for_date(self, date):
        return self.per_date_documents.get(date, [])

    def as_docs_attr_sequence(self, arg):
        return [doc.as_token_attr_sequence(arg) for doc in self]

    @property
    def plaintext_docs(self):
        plaintext_docs = []
        for doc in self:
            plaintext_docs.append(doc.plaintext)

        return plaintext_docs

    @property
    def sentences(self):
        return [sent for doc in self for sent in doc]

    @property
    def sorted_sentences(self):
        return [sent for doc in sorted(self, key=lambda d: d.name) for sent in doc]

    def __len__(self):
        return sum(map(len, self.per_date_documents.values()))

    def docs_between_dates(self, first_date, last_date):
        relevant_docs = []

        for date, docs in self.per_date_documents.items():
            if date >= first_date and date <= last_date:
                relevant_docs.extend(docs)

        return relevant_docs

    def docs_at_dates(self, dates):
        relevant_docs = []

        for date, docs in self.per_date_documents.items():
            if date in dates:
                relevant_docs.extend(docs)

        return relevant_docs

class Document:
    def __init__(self, new_sentences, name=None):
        self.name = name
        self.sentences = []
        self._tok_list = None

        for sent in new_sentences:
            self.add_sentence(sent)

    def __iter__(self):
        return iter(self.sentences)

    def __len__(self):
        return len(self.sentences)

    @property
    def docid(self):
        did, _ = os.path.basename(self.name).split(".", 1)

        return did

    @property
    def plaintext(self):
        return " ".join(self.as_token_attr_sequence("form"))

    @property
    def tokens(self):
        if self._tok_list is None:
            self._tok_list = list(it.chain(*map(lambda s: s.tokens, self.sentences)))
        return self._tok_list

    @property
    def basename(self):
        return self.name.split("/")[-1].split(".")[0]

    def add_sentence(self, sentence):
        sentence.idx = len(self.sentences)
        self.sentences.append(sentence)
        sentence.document = self

    def as_token_tuple_sequence(self, *args):
        return TokenListView(self, args)

    def as_token_attr_sequence(self, arg):
        return TokenAttrListView(self, arg)

    def as_sentence_attr_sequence(self, arg):
        return [sent.as_token_attr_sequence(arg) for sent in self.sentences]

class Sentence:
    def __init__(self, tokens,
                 dependency_tree=None, idx=None,
                 document=None, zero_based_indexing=True):
        self.tokens = tokens
        self.idx = idx
        self.dependency_tree = dependency_tree
        self.document = document
        self.zero_based_indexing = zero_based_indexing

        for token in self.tokens:
            token.sentence = self

    @property
    def plaintext(self):
        return self.as_tokenized_string()

    def as_tokenized_string(self):
        return " ".join(map(lambda t: t.form, self.tokens))

    def as_tokenized_string_with_attribute(self, attr):
        toks = []
        for t in self.tokens:
            str_repr = "/".join((t.form, getattr(t, attr)))
            toks.append(str_repr)
        return " ".join(toks)

    def __iter__(self):
        return iter(self.tokens)

    def __getitem__(self, idx):
        if not self.zero_based_indexing:
            idx -= 1
        return self.tokens[idx]

    def __len__(self):
        return len(self.tokens)

    def as_token_tuple_sequence(self, *args):
        return TokenListView(self, args)

    def as_token_attr_sequence(self, arg):
        return TokenAttrListView(self, arg)

class Token:
    def __init__(self,
                 form=None,
                 lemma=None,
                 pos=None,
                 coarse_pos=None,
                 idx=None,
                 coref_chain=None,
                 timex=None):
        self.form = form
        self.lemma = lemma
        self.pos = pos
        self.coarse_pos = coarse_pos
        self.idx = idx
        self.sentence = None
        self.coref_chain = coref_chain
        self.timex = timex

    def __repr__(self):
        return "Token(idx = {!r}, {!r}, sentence_idx = {!r})".format(
        self.idx, self.form, self.sentence.idx if self.sentence is not None else None)

    @property
    def form_lowercase(self):
        return self.form.lower()

class StanfordXMLReader:
    def __init__(self):
        pass

    def run(self, filename):
        tree = ET.parse(filename)
        root = tree.getroot()

        sents = []

        for xml_sent in root.find("document").find("sentences").iter("sentence"):
            sent = self.process_sentence(xml_sent)
            sents.append(sent)

        self.process_coref(root.find("document").find("coreference"), sents)

        return Document(sents, filename)

    def process_coref(self, all_coref_xml, sents):
        if all_coref_xml is None:
            return

        for coref_xml in all_coref_xml.iter("coreference"):
            coref_chain = []
            for mention in coref_xml.iter("mention"):
                sent_idx = int(mention.find("sentence").text) - 1
                head_idx = int(mention.find("head").text) - 1

                tok = sents[sent_idx][head_idx]

                tok.coref_chain = coref_chain

    def process_sentence(self, xml_sent):
        tokens = []
        token_idx_map = {}
        for token in xml_sent.find("tokens").iter("token"):
            pos = token.find("POS").text

            if pos.startswith("N"):
                coarse_pos = "NOUN"
            elif pos.startswith("V"):
                coarse_pos = "VERB"
            elif pos.startswith("A"):
                coarse_pos = "ADJ"
            else:
                coarse_pos = None

            tid = token.attrib["id"]
            token = Token(
                idx=int(tid) - 1,
                form=token.find("word").text,
                lemma=token.find("lemma").text,
                pos=pos,
                coarse_pos=coarse_pos)

            tokens.append(token)
            token_idx_map[tid] = token

        dep_tree = None
        for deps in xml_sent.iter("dependencies"):
            if deps.attrib["type"] == "enhanced-plus-plus-dependencies":
                dep_tree = self.process_dependencies(deps, tokens)
                break

        return Sentence(tokens, zero_based_indexing=False, dependency_tree=dep_tree)

    def process_dependencies(self, xml_deps, tokens):
        tree = DependencyTree()
        node_idx_map = {}
        for tok in tokens:
            tree.add_token(tok)

        for dep in xml_deps.iter("dep"):
            # CoreNLP uses 1 based indexing
            gov_id = int(dep.find("governor").attrib["idx"]) - 1
            dep_id = int(dep.find("dependent").attrib["idx"]) - 1
            extra = dep.attrib.get("extra", False)
            if extra:
                continue
            edge_type = dep.attrib["type"]

            if edge_type != "root":
                gov_tok = tokens[gov_id]
                dep_tok = tokens[dep_id]
                tree.add_edge(gov_tok, dep_tok, edge_type)

        return tree

class DUC2005Reader:
    def __init__(self, basedir):
        self.basedir = basedir

    def read_document_cluster(self, cluster_id):
        doc_dir = self._find_cluster_doc_dir(cluster_id)

        all_docs = []

        for doc_fname in os.listdir(doc_dir):
            full_path = os.path.join(doc_dir, doc_fname)
            doc = self.read_document(full_path)

            all_docs.append(doc)

        return cluster_id, all_docs

    @property
    def docs_path(self):
        return os.path.join(
            self.basedir,
            "duc2005_docs"
        )

    def _find_cluster_doc_dir(self, cluster_id):
        for path in os.listdir(self.docs_path):
            if path.startswith("d" + str(cluster_id)):
                return os.path.join(self.docs_path, path)

        return None

    def read_all_document_clusters(self):
        for doc_dirname in os.listdir(self.docs_path):
            num = doc_dirname[1:-1]
            yield self.read_document_cluster(num)

    def read_document(self, doc_path):
        tree = ET.parse(doc_path)
        paragraphs = tree.getroot().findall(".//P")
        if len(paragraphs) == 0:
            text = tree.getroot().find("TEXT")
            if text is not None:
                paragraphs = [text]
        return os.path.basename(doc_path), "\n".join([p.text for p in paragraphs])

TimeEX = namedtuple("TimeEX", "tid type_ value tokens")

class TimeMLReader:
    def __init__(self, preserve_linebreaks=False):
        self.preserve_linebreaks = preserve_linebreaks

    def run(self, fname):
        etree = ET.parse(fname)
        root = etree.getroot()
        if not self.preserve_linebreaks:
            tokens = [(tok, None) for tok in root.text.split()]
        else:
            tokens = []
            lines = root.text.split("\n")
            for line in lines:
                tokens.extend([(tok, None) for tok in line.split(" ")])
                tokens.append(("\n", None))

        for elem in root:
            time_ex = TimeEX(
                elem.attrib["tid"],
                elem.attrib["type"],
                elem.attrib["value"],
                elem.text.split()
            )
            for tok in elem.text.split():
                tokens.append((tok, time_ex))
            if not self.preserve_linebreaks:
                for tok in elem.tail.split():
                    tokens.append((tok, None))
            else:
                lines = elem.tail.split("\n")
                for line in lines:
                    tokens.extend([(tok, None) for tok in line.split(" ")])
                    tokens.append(("\n", None))

        return tokens

def compute_month_week_range(year, month):
    try:
        _, start_week, _ = datetime.date(year, month, 1).isocalendar()
        _, month_len = monthrange(2011, 2)
        _, end_week, _ = datetime.date(year, month, month_len).isocalendar()
    except ValueError:
        pass
    else:
        return range(start_week, end_week + 1)

    return set()

def compute_week_from_day(year, month, day):
    try:
        _, week, _ = datetime.date(year, month, day).isocalendar()
    except ValueError:
        pass
    else:
        return week

    return None

class DateTag:
    YEAR = 1
    MONTH = 2
    DAY = 3
    WEEK = 4

    @staticmethod
    def from_timex(expr):
        parts = expr.split("-")

        if len(parts) == 1:
            if not parts[0].isdigit():
                raise ValueError("Expression {!r} is not a date tag".format(expr))

            return DateTag(DateTag.YEAR, int(parts[0]))

        if len(parts) == 3:
            return DateTag(
                DateTag.DAY,
                int(parts[0]),
                int(parts[1]),
                int(parts[2]))

        if len(parts) == 2:
            if parts[1].startswith("W"):
                return DateTag(DateTag.WEEK, int(parts[0]), int(parts[1][1:]))
            else:
                return DateTag(DateTag.MONTH, int(parts[0]), int(parts[1]))

        raise ValueError("Expression {!r} is not a date tag".format(expr))

    def __init__(self, dtype, *args):
        self.dtype = dtype

        self.year = None
        self.month = None
        self.week = None
        self.day = None

        if dtype == DateTag.YEAR:
            self.year = args[0]
        elif dtype == DateTag.MONTH:
            self.year = args[0]
            self.month = args[1]
        elif dtype == DateTag.DAY:
            self.year = args[0]
            self.month = args[1]
            self.day = args[2]
        elif dtype == DateTag.WEEK:
            self.year = args[0]
            self.week = args[1]

    def __contains__(self, other):
        if self.dtype == other.dtype:
            return self == other
        elif self.dtype == DateTag.DAY and other.dtype in (DateTag.YEAR, DateTag.MONTH, DateTag.WEEK):
            return False
        elif self.dtype == DateTag.WEEK and other.dtype in (DateTag.YEAR, DateTag.MONTH):
            return False
        elif self.dtype == DateTag.MONTH and other.dtype == DateTag.YEAR:
            return False

        if other.dtype == DateTag.MONTH:
            return self.month == other.month and self.year == other.year
        elif other.dtype == DateTag.WEEK:
            if self.dtype == DateTag.YEAR:
                return other.year == self.year
            elif self.dtype == DateTag.MONTH:
                return other.week in compute_month_week_range(self.year, self.month)
            assert False
        elif other.dtype == DateTag.DAY:
            if self.dtype == DateTag.MONTH:
                return self.year == other.year and self.month == other.month
            elif self.dtype == DateTag.WEEK:
                return self.week == compute_week_from_day(other.year, other.month, other.day)
            elif self.dtype == DateTag.YEAR:
                return self.year == other.year

        assert False

    def __eq__(self, other):
        return self.dtype == other.dtype and self.year == other.year and self.month == other.month and self.day == other.day and self.week == other.week

    def __hash__(self):
        return hash((self.dtype, self.year, self.month, self.week, self.day))

    def __repr__(self):
        if self.dtype != DateTag.WEEK:
            return "DateTag({}, {}, {}, {})".format(self.dtype, self.year, self.month, self.day)
        else:
            return "DateTag({}, {}, {})".format(self.dtype, self.year, self.week)

    def __str__(self):
        if self.dtype != DateTag.WEEK:
            return "{}-{}-{}".format(self.year, self.month, self.day)
        else:
            return "{}-W{}".format(self.year, self.week)

    @property
    def datetime(self):
        return datetime.datetime(self.year, self.month, self.day)

class DatedSentenceReader:
    def __init__(self):
        self.stanford_reader = StanfordXMLReader()
        self.time_ml_reader = TimeMLReader()

    def read(self, stanford_path, timeml_path, dct):
        doc = self.stanford_reader.run(stanford_path)

        timeml_tokens = self.time_ml_reader.run(timeml_path)

        doc_tok_iter = (tok for sent in doc for tok in sent)

        for sent in doc.sentences:
            sent.time_expressions = []

        doc_tok_iter = (tok for sent in doc for tok in sent)

        for timetok, doc_tok in zip(timeml_tokens, doc_tok_iter):
            (time_tok, timex) = timetok
            assert time_tok == doc_tok.form, "{} != {}".format(time_tok, doc_tok.form)

            if timex is not None:
                doc_tok.timex = timex

                if len(doc_tok.sentence.time_expressions) == 0 or doc_tok.sentence.time_expressions[-1] != timex:
                    doc_tok.sentence.time_expressions.append(timex)

        doc.dct_tag = DateTag(DateTag.DAY, dct.year, dct.month, dct.day)
        doc.all_date_tags = set()
        for sent in doc.sentences:
            available_timeexs = filter(lambda tx: tx.type_ == "DATE", sent.time_expressions)

            possible_exact_dates = []
            all_date_tags = set()
            for timeex in available_timeexs:
                try:
                    tag = DateTag.from_timex(timeex.value)
                except ValueError:
                    continue
                all_date_tags.add(tag)
                doc.all_date_tags.add(tag)
                if tag.dtype == DateTag.DAY:
                    possible_exact_dates.append(tag)

            sent.all_date_tags = all_date_tags
            sent.exact_date_references = possible_exact_dates

            if len(possible_exact_dates) > 0:
                # TODO: Find better heuristic
                pred_date = possible_exact_dates[0]
                try:
                    sent.predicted_date = datetime.date(pred_date.year, pred_date.month, pred_date.day)
                except ValueError:
                    sent.predicted_date = dct
            else:
                sent.predicted_date = dct

        return doc

class DependencyTree:
    def __init__(self):
        self.token_heads = {}
#        self.nodes = []
#        self.roots = []

    def add_token(self, token):
        self.token_heads[token] = (None, None)

    def add_edge(self, head_token, token, edge_type):
        self.token_heads[token] = (head_token, edge_type)

    def __str__(self):
        return "\n".join(map(lambda n: str(n), self.roots))

    def as_head_idx_sequence(self):
        seq = []
        for token, (head_token, edge_type) in self.token_heads.items():
            if head_token is None:
                seq.append((token.idx, -1))
            else:
                seq.append((token.idx, head_token.idx))

        seq.sort()
        return [head_idx for _, head_idx in seq]

    def extract_svo_tuples(self):
        possible_tuples = defaultdict(dict)

        for token, (head, label) in self.token_heads.items():
            if label == "dobj":
                possible_tuples[head]["dobj"] = token
            elif label == "nsubj":
                possible_tuples[head]["subj"] = token

        tuples = []
        for root, args in possible_tuples.items():
            if args.get("subj") is not None:
                tuples.append(
                    (root, args.get("subj"), args.get("dobj"))
                )
        return tuples

class DatedTimelineCorpusReader:
    def __init__(self, parsed_suffix=".tokenized", timeml_suffix=".tokenized.timeml"):
        self.reader = DatedSentenceReader()

        self.parsed_suffix = parsed_suffix
        self.timeml_suffix = timeml_suffix

    def run(self, document_dir, timeml_dir, ):
        date_dict = defaultdict(list)
        for date_dir in iter_dirs(document_dir):
            print("Reading", date_dir)
            dir_date = datetime.datetime.strptime(os.path.basename(date_dir), "%Y-%m-%d").date()

            for doc_fname in iter_files(date_dir, self.parsed_suffix):
                fname = os.path.basename(doc_fname)
                prefix = fname[:-len(self.parsed_suffix)]
                timeml_fname = os.path.join(timeml_dir, os.path.basename(date_dir), prefix + self.timeml_suffix)
                sentences = self.reader.read(doc_fname, timeml_fname, dir_date)
                date_dict[dir_date].append(sentences)

        return TimelineCorpus(date_dict)

if __name__ == "__main__":
    # java -cp "*" -Xmx2g edu.stanford.nlp.pipeline.StanfordCoreNLP -annotators tokenize,ssplit,pos,lemma -file input.txt
    reader = TimeMLReader(True)
    outdir = sys.argv[2]
    for crisis_dir in iter_dirs(sys.argv[1]):
        crisis_outdir = os.path.join(outdir, os.path.basename(crisis_dir))
        os.mkdir(crisis_outdir)

        doc_dir = os.path.join(crisis_dir, "articles")
        for date_dir in iter_dirs(doc_dir):
            date_out_dir = os.path.join(crisis_outdir, os.path.basename(date_dir))
            os.mkdir(date_out_dir)
            for fname in iter_files(date_dir, "timeml"):
                basename = os.path.basename(fname)[:-7]
                toks = reader.run(fname)

                with open(date_out_dir + "/" + basename + ".tokenized", "w") as f_out:
                    f_out.write(" ".join(map(fst, toks)))


from PIL import Image
import os
from tqdm import tqdm
import json
import time
import shutil

def sort_images_by_resolution(input_path, output_root, max_retries=10, retry_delay=0.1):
    if not os.path.exists(output_root):
        os.makedirs(output_root)

    processed_images = set()
    checkpoint_file = os.path.join(output_root, 'checkpoint.json')

    if os.path.exists(checkpoint_file):
        with open(checkpoint_file, 'r') as checkpoint:
            processed_images = set(json.load(checkpoint))

    file_list = os.listdir(input_path)
    total_images = len(file_list)
    remaining_images = total_images - len(processed_images)

    images_to_delete = []

    for filename in tqdm(file_list, desc="Sorting images", total=total_images, initial=total_images - remaining_images):
        if filename in processed_images:
            continue

        input_file = os.path.join(input_path, filename)

        if os.path.isfile(input_file):
            retries = 0
            success = False

            while retries < max_retries:
                try:
                    img = Image.open(input_file)
                    width, height = img.size

                    resolution_dir = os.path.join(output_root, f"{width}x{height}")
                    if not os.path.exists(resolution_dir):
                        os.makedirs(resolution_dir)

                    output_file = os.path.join(resolution_dir, filename)
                    shutil.copy2(input_file, output_file)  # Copy the file

                    images_to_delete.append(input_file)  # Mark for deletion

                    processed_images.add(filename)
                    remaining_images -= 1

                    # Update checkpoint
                    with open(checkpoint_file, 'w') as checkpoint:
                        json.dump(list(processed_images), checkpoint)

                    success = True
                    break

                except Exception as e:
                    print(f"Error processing {filename}: {str(e)}")
                    retries += 1
                    time.sleep(retry_delay)

            if not success:
                print(f"Failed to process {filename} after {max_retries} retries.")

    # Delete the original files
    for file_to_delete in images_to_delete:
        try:
            os.remove(file_to_delete)
        except Exception as e:
            print(f"Error deleting {file_to_delete}: {str(e)}")

    # Remove the checkpoint file
    if os.path.exists(checkpoint_file):
        os.remove(checkpoint_file)

    print("Image sorting completed.")

if __name__ == "__main__":
    input_directory = "D:\My Work - Personal\Genevieve\Preprocessed"
    output_directory = "D:\My Work - Personal\Genevieve\Preprocessed\Sorted"

    sort_images_by_resolution(input_directory, output_directory)


import os
import string
import time
from datetime import datetime
from pathlib import Path

import jwt
from dotenv import load_dotenv
from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from starlette.staticfiles import StaticFiles
from starlette import responses

# Create the app
from WebApplication.Controllers import router

app = FastAPI(
    title='Open Chat',
    version='1.0.0'
)

# Add origins in this array to allow them only
origins = [
    "http://localhost",
    "http://localhost:4200",
]

# Add middlewares, origin also comes as middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Mount specific folders
app.mount("/static", StaticFiles(directory="static"), name="static")

# Enable routing
app.include_router(router)

public_routs = [
    '/api/signup',
    '/api/login',
    '/docs',
    '/redoc',
    '/openapi.json',
    '/favicon.ico',
    '/senders'
]

@app.middleware("http")
async def validate_jwt(request: Request, call_next):
    try:
        start_time = time.time()
        application_root_path = str(Path(__file__).parent.parent)
        if load_dotenv(dotenv_path=application_root_path + '/.env'):
            secret_key: string = os.getenv('SECRET_KEY')
            security_algo: string = os.getenv("SECURITY_ALGORITHM")
            if secret_key is None or security_algo is None:
                raise HTTPException(
                    status_code=401,
                    detail='Unable to parse authentication token'
                )
            current_route: string = request.url.path

            # No auth required if it's public url
            if current_route in public_routs:
                response = await call_next(request)
                process_time = time.time() - start_time
                response.headers["X-Process-Time"] = str(process_time)
                return response
            else:
                token: str = request.headers.get('Authorization')
                if token:
                    token = token.split()[1].replace('"','')
                    decoded: dict = jwt.decode(jwt=token, key=secret_key, algorithms=security_algo)
                    if datetime.fromtimestamp(decoded['exp']) >= datetime.now() or current_route in public_routs:
                        response = await call_next(request)
                        process_time = time.time() - start_time
                        response.headers["X-Process-Time"] = str(process_time)
                        return response
                else:
                    return responses.JSONResponse(
                        status_code=401,
                        content='Unauthorized Access'
                    )
        return responses.JSONResponse(
            status_code=401,
            content='Unauthorized Access 2'
        )
    except Exception as e:
        return responses.JSONResponse(
            status_code=500,
            content=str(e)
        )

# @app.middleware('ws')
# async def validate_jwt_for_socket(request: Request, call_next):
#     print(request)
#     print(call_next)
#     return


#!/usr/bin/env python3.9
# -*- coding: utf-8 -*-
"""
Programme ou module pour ...

Créé le Fri Nov 26 15:36:57 2021

@author: ejetzer
"""

import datetime

import sqlalchemy as sqla

from sqlalchemy import Column, MetaData, Table
from sqlalchemy.orm import declarative_base

from ..outils.database.dtypes import get_type, column

metadata = MetaData()

def colonnes_communes():
    return (column('index', int, primary_key=True),)

cols = colonnes_communes() + (column('Technicien', str),
                              column('Payeur', str),
                              column('Date', datetime.datetime),
                              column('Description des travaux effectués', str),
                              column('Demandeur', str),
                              column('Heures', float),
                              column('Atelier', bool),
                              column('Précision si pour département', str),
                              column('Autres', str))
cols_admin = cols + (column('Facturé', bool),)

heures = Table('heures', metadata, *cols)
#admin = Table('admin', metadata, *cols_admin)


import argparse
import os
import random
import sys

import django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')

django.setup()
from datacenter.models import Schoolkid, Lesson, Commendation, Chastisement, Mark

def fetch_schoolkid(child_name):
    child = Schoolkid.objects.get(full_name__contains=child_name)
    return child

def fix_marks(child):
    negative_marks = Mark.objects.filter(schoolkid=child, points__lte=3)
    for mark in negative_marks:
        target_mark = Mark.objects.get(pk=mark.pk)
        target_mark.points = 5
        target_mark.save()

def create_commendation(child, subject_title):
    commendations = [
        'Молодец!',
        'Отлично!',
        'Хорошо!',
        'Гораздо лучше, чем я ожидал!',
        'Ты меня приятно удивил!'
    ]
    year = child.year_of_study
    letter = child.group_letter
    lessons = Lesson.objects.filter(year_of_study=year, group_letter=letter)
    target_lessons = lessons.filter(subject__title=subject_title)
    last_lesson = target_lessons.order_by('-date').first()
    if last_lesson:
        Commendation.objects.create(
            text=random.choice(commendations),
            created=last_lesson.date,
            schoolkid=child,
            subject=last_lesson.subject,
            teacher=last_lesson.teacher
        )
    else:
        raise Lesson.DoesNotExist

def remove_chastisements(child):
    chastisement = Chastisement.objects.filter(schoolkid=child)
    chastisement.delete()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-ln', '--last_name', help='Фамилия ученика', default='Фролов')
    parser.add_argument('-fn', '--first_name', help='Имя ученика', default='Иван')
    parser.add_argument('-p', '--patronymic', help='Отчетсво ученика', default='')
    parser.add_argument('-s', '--subject', help='Название предмета', default='Математика')
    args = parser.parse_args()
    kid_full_name = f'{args.last_name} {args.first_name} {args.patronymic}'
    try:
        schoolkid = fetch_schoolkid(kid_full_name)
    except Schoolkid.MultipleObjectsReturned:
        print('Слишком много учеников с указанным именем. Пожалуйста уточните имя.')
        sys.exit()
    except Schoolkid.DoesNotExist:
        print('Ученик с таким именем отсутствует. Пожалуйста уточните имя.')
        sys.exit()
    print('Начинаю исправлять оценки')
    fix_marks(schoolkid)
    print('Оценки исправлены')
    print('Начинаю удалять замечания')
    remove_chastisements(schoolkid)
    print('Замечания удалены')
    print('Начинаю добавлять похвалу')
    try:
        create_commendation(schoolkid, args.subject)
    except Lesson.DoesNotExist:
        print('Неправильно введено название предмета. Пожалуйста выберети правильный предмет из расписания.')
        sys.exit()
    print('Похвала добавлена')


from argparse import ArgumentParser, REMAINDER

class MbtParams:

    def __init__(self, config, context, prog_name="mbt"):
        self.config = config
        self.context = context
        self.parser = ArgumentParser(prog=prog_name)
        self.requires_topic = False
        self.requires_series = False
        self.requires_variant = False
        self.requires_installation = False

    def add_topic_arg(self):
        if self.requires_topic:
            return
        self.requires_topic = True
        self.parser.add_argument("-t", "--topic",
                                 required=(self.context.topic is None),
                                 help="Description of the changes, part of"
                                 " the feature branch name",
                                 default=self.context.topic
                                 )

    def add_series_arg(self):
        if self.requires_series:
            return
        self.requires_series = True
        self.parser.add_argument("-s", "--series",

                                 required=(self.context.series is None),
                                 help="Release series to work on",
                                 choices=self.config.series,
                                 default=self.context.series
                                 )

    def add_variant_arg(self):
        if self.requires_variant:
            return
        self.requires_variant = True
        self.parser.add_argument("-v", "--variant",
                                 required=(self.context.variant is None),
                                 help="Build variant to use",
                                 choices=self.config.build_configs.keys(),
                                 default=self.context.variant
                                 )

    def add_installation_arg(self):
        if self.requires_installation:
            return
        self.requires_installation = True

        self.add_topic_arg()
        self.add_series_arg()
        self.add_variant_arg()

        self.parser.add_argument("-i", "--installation",
                                 required=(self.context.installation is None),
                                 help="Tag of the installation",
                                 default=self.context.installation
                                 )

    def add_remote_arg(self):
        self.parser.add_argument("-r", "--remote",
                                 required=True,
                                 help="Remote to be used",
                                 choices=self.config.remotes.keys()
                                 )

    def add_remaining_args(self):
        self.parser.add_argument('remaining_args', nargs=REMAINDER)

    def add_boolean_arg(self, name):
        self.parser.add_argument("--"+name, action="store_true")

    def add_int_arg(self, name, default=None):
        self.parser.add_argument("--"+name, type=int, default=default)

    def parse(self, args):
        self.results = self.parser.parse_args(args)
        # Work around an argparse limitation:
        # The first remaining arg can't be an optional, so in this caes,
        # -- has to be specified first
        if "remaining_args" in self.results:
            rargs = self.results.remaining_args
            if len(rargs) and rargs[0] == "--":
                self.results.remaining_args.pop(0)
        return self.results


# test_AddFollow
import json
import logging

import allure
import pytest

from Business.Urlpath.UrlPath_userlogin import UrlPath_add_follow
from Common.login import login
from Common.show_sql import showsql
from Common.sign import get_sign

from Common.requests_library import Requests

from Common.tools.read_write_yaml import yamltoken

from glo import JSON, HTTP

# @pytest.mark.skip(reason="调试中 ")
@allure.feature('主页信息-关注用户')
class TestAddFollow():
    @classmethod
    def setup_class(cls) -> None:
        cls.session = Requests().get_session()
        login()

    def tearDown(self) -> None:
        Requests(self.session).close_session()

    # @pytest.mark.skip(reason="调试中 ")
    def test_AddFollow(self):
        # 关注用户url
        url_add_follow = HTTP + UrlPath_add_follow
        # 拼装headers参数
        header = {}
        header.update(JSON)
        headers = {}
        headers.update(header)
        token = {"token": yamltoken()}
        headers.update(token)  # 将token更新到headers
        # print(headers)
        # 从数据库获取用户id
        userId = showsql(
            '192.168.1.237', 'root', '123456', "user_account",
            "select user_id from t_user_account where `zr_no`= '10000039';"
        )

        paylo = {
            "userId": list(list(userId)[0])[0]
        }
        sign1 = {"sign": get_sign(paylo)}  # 把参数签名后通过sign1传出来
        payload1 = {}
        payload1.update(paylo)
        payload1.update(sign1)

        payload = json.dumps(dict(payload1))

        r = Requests(self.session).post(
            url=url_add_follow, headers=headers, data=payload, title="关注用户"
        )
        j = r.json()
        # print(j)
        assert r.status_code == 200
        try:
            assert j.get("code") == "000000"
            assert j.get("msg") == "ok"
            if j.get("data") == True:
                logging.info("是关注用户")

            else:
                logging.info("不是关注用户")
        except:
            raise AssertionError(
                f"\n请求地址：{url_add_follow}"
                f"\nbody参数：{payload}"
                f"\n请求头部参数：{headers}"
                f"\n返回数据结果：{j}"
            )

        else:
            raise ValueError(f"{j}")


from django.http import HttpResponse
from django.shortcuts import render

def index(request):
    return render(request, 'index.html')

def analyze(request):

    djtext = request.GET.get('text', 'default')

    removepunc = request.GET.get('removepunc', 'off')
    fullcaps = request.GET.get('fullcaps', 'off')
    newlineremover = request.GET.get('newlineremover', 'off')
    extraspaceremover = request.GET.get('extraspaceremover', 'off')

    # Check which checkbox is on
    if removepunc == "on":
        punctuations = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
        analyzed = ""
        for char in djtext:
            if char not in punctuations:
                analyzed = analyzed + char
        djtext = analyzed

    if(fullcaps == "on"):
       djtext = djtext.upper()

    if (extraspaceremover == "on"):
        djtext = djtext.replace(" ", "")

    if (newlineremover == "on"):
        analyzed = ""
        for char in djtext:
            if char != "\n":
                analyzed = analyzed + char
        djtext = analyzed

    params = {'purpose': 'New Data', 'analyzed_text': djtext}
    return render(request, 'analyze.html', params)


import numpy as np
import pandas as pd
import requests
import xlsxwriter
import math
import requests_cache
import tabulate
import json

from io import StringIO
from tabulate import tabulate

from _secrets import IEX_CLOUD_API_TOKEN

print(f'token: {IEX_CLOUD_API_TOKEN}')

session = requests_cache.CachedSession('http_cache', backend='filesystem')
print(session.cache.cache_dir)

base_url = 'https://cloud.iexapis.com'

my_columns = ['Ticker', 'Stock Price', 'Market Capitalization', 'Number of Shares to Buy']
df = pd.DataFrame(columns=my_columns)

stocks = pd.read_csv('starter_files/sp_500_stocks.csv')

step = 100
all_symbols = stocks['Ticker']

for i in range(0, len(all_symbols), step):
    symbols = all_symbols[i:i + step]
    symbols_text = ','.join(symbols)
    url = f'{base_url}/stable/stock/market/batch/?symbols={symbols_text}&types=quote&token={IEX_CLOUD_API_TOKEN}'
    response = session.get(url)

    if response.status_code == 404:
        continue

    data = response.json()

    for symbol in data:
        item = data[symbol]['quote']
        # print(json.dumps(item, indent=2))

        price = item['latestPrice']
        market_cap = item['marketCap']
        new_row = pd.DataFrame([[symbol, price, market_cap, 'N/A']], columns=my_columns)
        df = pd.concat([df, new_row], ignore_index=True)

# portfolio_size = input('enter the value of your portfolio')
# print(portfolio_size)
#
# try:
#     val = float(portfolio_size)
#     print(val)
# except ValueError:
#     print('please enter a number')

print(tabulate(df, headers='keys', tablefmt='psql'))
portfolio_size = 1000000
position_size = portfolio_size / len(df.index)

for i in range(0, len(df.index)):
    price = df.loc[i, 'Stock Price']
    df.loc[i, 'Number of Shares to Buy'] = math.floor(position_size / price)

print(tabulate(df, headers='keys', tablefmt='psql'))

writer = pd.ExcelWriter('foo.xlsx', engine='xlsxwriter')
df.to_excel(writer, "foo", index=False)
bg_color = '#0a0a23'
font_color = '#ffffff'

string_format = writer.book.add_format({
    'font_color': font_color,
    'bg_color': bg_color,
    'border': 1
})
dollar_format = writer.book.add_format({
    'num_format': '$0.00',
    'font_color': font_color,
    'bg_color': bg_color,
    'border': 1
})
integer_format = writer.book.add_format({
    'num_format': 0,
    'font_color': font_color,
    'bg_color': bg_color,
    'border': 1
})

col_formats = {
    'A': ['Ticker', string_format],
    'B': ['Stock Price', dollar_format],
    'C': ['Market Capitalization', dollar_format],
    'D': ['Number of Shares to Buy', integer_format],
}

for col in col_formats.keys():
    f = col_formats[col][1]
    title = col_formats[col][0]
    writer.sheets['foo'].set_column(f'{col}:{col}', 18, f)
    writer.sheets['foo'].write(f'{col}1', title, string_format)

writer.save()

algorithmic-trading-python-tutorial/algo trading course - project 1.py


# Divide and Conquer, Sorting and Searching, and Randomized Algorithms
# https://www.coursera.org/learn/algorithms-divide-conquer

from random import randrange, randint
import operator as op
from functools import reduce
from math import log, ceil, floor
import sys
from utils import swap

# week 2
def count_inversions(arr):
	"""
	Returns the number of inversions in an array
		Parameters:
			arr (int[]): An array of integers

		Returns:
			(int): Count of inversions
	"""
	global count
	count = 0
	# implement using merge sort, which easily allows to count inversions during the merge phase
	def merge_sort(arr):
		# exit early if array is empty or there is only one element
		if len(arr) < 2:
			return arr

		mid = floor(len(arr) / 2)
		left = merge_sort(arr[0:mid])
		right = merge_sort(arr[mid:])
		return merge(left, right)

	# helper function to merge arrays and count split inversions
	def merge(a1, a2):
		global count
		res = []
		i = 0
		j = 0
		while True:
			# there are no more elements to look at - exit
			if len(a1) == i and len(a2) == j:
				break

			# array1 was fully merged but one element in array2 remains
			# push the remaining element into the result array
			# and advance to the next iteration of the loop
			if len(a1) == i and len(a2) != j:
				res.append(a2[j])
				j += 1
				continue

			# array2 was fully merged but one element in array1 remains
			# push the remaining element into the result array
			# and advance to the next iteration of the loop
			if len(a1) != i and len(a2) == j:
				res.append(a1[i])
				i += 1
				continue

			# if the element in the second array is larger, this is a split inversion for all remaining elements of a1
			# add the remaining length of a1 to count
			if a1[i] > a2[j]:
				res.append(a2[j])
				j += 1
				count += len(a1) - i
			# if the element in the first array is larger or equal, there is no split inversion - simply advance
			else:
				res.append(a1[i])
				i += 1

		return res

	merge_sort(arr)
	return count

def strassen_mult(X, Y):
	"""
	Returns the multiplication of two matrices using Strassen's method
	Currently only works for even matrices due to partitioning, this can be circumvented using padding
		Parameters:
			X (int[][]): A 2D array of integers
			Y (int[][]): A 2D array of integers

		Returns:
			(int[][]): A 2D array of integers
	"""

	# helper function to split a matrix into 4 sub-matrices
	def split_matrix(M):
		if len(M) == 1 and len(M[0]) == 1:
			return M
		n = len(M)
		half_size = int(n / 2)
		A = empty_matrix(half_size, half_size)
		B = empty_matrix(half_size, half_size )
		C = empty_matrix(half_size, half_size)
		D = empty_matrix(half_size, half_size)

		for i in range(half_size):
			for j in range(half_size):
				A[i][j] = M[i][j]
				B[i][j] = M[i][j + half_size]
				C[i][j] = M[i + half_size][j]
				D[i][j] = M[i + half_size][j + half_size]
		return (A, B, C, D)

	# helper function to create an empty matrix of dimensions x by y
	def empty_matrix(x, y):
		M = [[]] * x
		for i in range(x):
			M[i] = [0] * y
		return M

	def add_matrices(A, B):
		n = len(A)
		M = empty_matrix(n, n)
		for i in range(n):
			for j in range(n):
				M[i][j] = A[i][j] + B[i][j]
		return M

	def subtract_matrices(A, B):
		n = len(A)
		M = empty_matrix(n, n)
		for i in range(n):
			for j in range(n):
				M[i][j] = A[i][j] - B[i][j]
		return M

	n = len(X)
	if n == 1:
		return [[X[0][0] * Y[0][0]]]
	else:
		A, B, C, D = split_matrix(X)
		E, F, G, H = split_matrix(Y)
		# p1 = A(F - H)
		P1 = strassen_mult(A, subtract_matrices(F, H))
		# p2 = (A + B)H
		P2 = strassen_mult(add_matrices(A, B), H)
		# p3 = (C + D)E
		P3 = strassen_mult(add_matrices(C, D), E)
		# p4 = D(G - E)
		P4 = strassen_mult(D, subtract_matrices(G, E))
		# p5 = (A + D)(E + H)
		P5 = strassen_mult(add_matrices(A, D), add_matrices(E, H))
		# p6 = (B - D)(G + H)
		P6 = strassen_mult(subtract_matrices(B, D), add_matrices(G, H))
		# p7 = (A - C)(E + F)
		P7 = strassen_mult(subtract_matrices(A, C), add_matrices(E, F))

		# top left quadrant of the final matrix = P5 + P4 - P2 + P6
		Q1 = add_matrices(add_matrices(P5, P4), subtract_matrices(P6, P2))
		# top right quadrant = P1 + P2
		Q2 = add_matrices(P1, P2)
		# bottom left quadrant = P3 + P4
		Q3 = add_matrices(P3, P4)
		# bottom right quadrant = P1 + P5 - P3 - P7
		Q4 = subtract_matrices(subtract_matrices(add_matrices(P1, P5), P3), P7)

		m = int(n / 2)
		Z = empty_matrix(n, n)
		for i in range(m):
			for j in range(m):
				Z[i][j] = Q1[i][j]
				Z[i][j + m] = Q2[i][j]
				Z[i + m][j] = Q3[i][j]
				Z[i + m][j + m] = Q4[i][j]
		return Z

# week 3
def quick_sort(arr):
	"""
	Sorts an array using quick sort
		Parameters:
			arr (int[]): An array of integers

		Returns:
			(int[]): A sorted array of integers
	"""
	if len(arr) < 2:
		return arr

	def pivot(a):
		pivot_idx = 0
		pivot = a[pivot_idx]

		for i in range(1, len(a)):
			if pivot > a[i]:
				pivot_idx += 1
				swap(a, i , pivot_idx)

		swap(a, 0, pivot_idx)
		return pivot_idx

	pivot_idx = pivot(arr)
	leftArr = arr[0:pivot_idx + 1]
	rightArr = arr[pivot_idx + 1:]
	return quick_sort(leftArr) + quick_sort(rightArr)

# week 4
def min_cut(graph):
	"""
	Calculates the minimal cut of a graph using Karger's randomized contraction algorithm
		Parameters:
			graph (Graph): A graph to be examined

		Returns:
			(dict): A dictionary containing the remaining vertices which represent the minimal cut
	"""
	while len(graph.get_vertices()) > 2:
		v1, v2 = graph.get_random_edge()
		graph.contract_edge(v1, v2)
	return graph.get_vertices()

def min_cut_len(graph, increase_success_rate = False):
	"""
	Calculates the amount of the edges intersecting the minimal cut of a graph
		Parameters:
			graph (Graph): A graph to be examined

		Returns:
			(int): The amount of edges intersecting the minimal cut of a graph
	"""

	# calculate n choose r
	# https://stackoverflow.com/questions/4941753/is-there-a-math-ncr-function-in-python
	def ncr(n, r):
		r = min(r, n - r)
		numer = reduce(op.mul, range(n, n - r, -1), 1)
		denom = reduce(op.mul, range(1, r + 1), 1)
		return numer // denom

	# calculate the amount of attempts that should be executed to increase the probability of a correct result roughly to 1/n
	# where n is the amount of vertices
	# https://en.wikipedia.org/wiki/Karger%27s_algorithm#Success_probability_of_the_contraction_algorithm
	# this is optional as it would make the algorithm too slow for larger graphs
	n = len(graph.get_vertices())
	attempts = ceil(ncr(n, 2) * log(n)) if increase_success_rate else n
	minimum = sys.maxsize
	for _ in range(attempts):
		g = graph.deep_copy()
		result = min_cut(g)
		key = list(result.keys())[0]
		minimum = min(minimum, len(result[key]))
	return minimum

def randomized_selection(arr, k):
	"""
	Finds the kth smallest element in an array of integers
		Parameters:
			arr (int[]): An array of integers
			k (int): the position of the element to look for

		Returns:
			(int): The kth smallest element of the input array
	"""
	assert k > 0, "Second argument must be greater than 0"

	n = len(arr)
	if n == 0:
		return None
	if n == 1:
		return arr[0]

	# generate pivot index at random
	pivot_idx = randrange(0, n)
	pivot = arr[pivot_idx]
	# move pivot to the very end of array
	swap(arr, pivot_idx, n - 1)

	# i represents the tail of the "left side" of the partitioned array (everything that is less than the pivot)
	i = 0
	# j represents the index of the current item being looked at
	j = 0
	while j < n:
		# if at the end of the loop, swap pivot into its rightful place and exit the loop
		if j == n - 1:
			swap(arr, i, j)
			break
		# if the current item is less than the pivot, move it to the left
		if arr[j] < pivot:
			swap(arr, i, j)
			i += 1
			j += 1
		# otherwise simply advance the loop
		else:
			j += 1

	# translate k into computer's idea of index, since we're looking for the kth smallest item, we're looking for (k-1)th index
	search_idx = k - 1
	# if the kth smallest element happens to be the one at ith position, return it
	if i == search_idx:
		return arr[search_idx]
	# if we overshot the element, recurse on the left side of the array
	elif i > search_idx:
		return randomized_selection(arr[0:i + 1], k)
	# if we undershot, recurse on the right side
	else:
		return randomized_selection(arr[i:n], k - i)