from PySide6.QtWidgets import QPushButton, QWidget, QLabel, QMessageBox, QApplication
from PySide6.QtGui import QCloseEvent

class Gra(QWidget):
    def __init__(self):
        super(Gra, self).__init__()
        self.setFixedSize(640, 420)
        self.setWindowTitle("Nowa gra")

class Interfejs(QWidget):
    def __init__(self):
        super().__init__()
        self.setFixedSize(640, 420)
        self.setWindowTitle("Statki - NPG 2022")
        self.a = None
        self.run()
    def run(self):

        self.button_start = QPushButton(self)
        self.button_start.move(200, 180)
        self.button_start.resize(150, 30)
        self.button_start.setText("Rozpocznij grę")
        self.button_start.clicked.connect(self.Okno)

        self.button_quit = QPushButton(self)
        self.button_quit.move(200, 220)
        self.button_quit.resize(150, 30)
        self.button_quit.setText("Wyjście")
        self.button_quit.clicked.connect(QApplication.instance().quit)

        self.show()

    def Okno(self, checked):
        if self.a is None:
            self.a = Gra()
        self.a.show()

    def closeEvent(self, event: QCloseEvent):
        zamkniecie = QMessageBox.question(self, "Zamykanie aplikacji", "Czy chcesz zamknac aplikacje?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
        if zamkniecie == QMessageBox.StandardButton.Yes:
            event.accept()
        else:
            event.ignore()

app = QApplication([])
interfejs = Interfejs()
app.exec()


# Link: https://codeforces.com/problemset/problem/1730/A
#Q: Planets

# Time: O(N)
# Space: O(N)

from collections import Counter

t = int(input())

for _ in range(t):
    n, c = map(int, input().split())
    planets = list(map(int, input().split()))

    counter = Counter(planets)
    output = 0

    for key in counter:
        if counter[key] <= c:
            output += counter[key]
        else:
            output += c

    print(output)


"""generator that yields Superpositions and the functions that support it

A Superposition tracks the relationship between a given Figure and all
the valid Numerals. It does this by counting the differences between
the provided Figure and actual Figure for each valid Numeral.
The generator yields this relationship as a dictionary:
  superposition[difference_count] = numeral_set
  {difference_count: set of valid numerals with that many differences}
"""

from parse import settings
from parse.validators import Validate

def superpositions_from_figures(figures):
    "generator that consumes Figures and yields SuperPositions"
    for figure in figures:
        _validate_figure(figure)
        yield _superposition_from_figure(figure)

def _validate_figure(figure):
    "confirm figure type, length, and composition or raise ValueError"
    Validate.type(basestring, figure, 'Figure')
    Validate.length(settings.strokes_per_figure, figure, 'Figure')
    Validate.composition(settings.valid_strokes, figure, 'Figure')

def _superposition_from_figure(figure):
    "return Superposition represented by Figure"
    d = {}
    for valid_figure, numeral in settings.figures.items():
        difference_count = _count_differences(figure, valid_figure)
        d.setdefault(difference_count, set()).add(numeral)
    return d

def _count_differences(figure_a, figure_b):
    "return count of differing Strokes between two Figures"
    return sum(a != b for a, b in zip(figure_a, figure_b))


import pytest

from fizzbuzz.web.api import app

@pytest.fixture
def client():
    app.config["TESTING"] = True
    with app.test_client() as client:
        yield client

@pytest.mark.parametrize(
    "input,expected",
    [
        (3, b"fizz"),
        (5, b"buzz"),
        (15, b"fizzbuzz"),
    ],
)
def test_fizzbuzz(client, input, expected):
    actual = client.get(f"/fizzbuzz/{input}")

    assert expected == actual.data


# Neural Network
import numpy as np
import utils

class Network(object):
    """ Simple neural network class to take an arbitrary number of layers.
    -------
    Args: layers; list of layer objects
          loss; the loss function to use for this neural net
          optimizer; the optimizer to use for this neural net
    """
    def __init__(self, layers, loss, optimizer):
        self.layers = layers
        self.loss_func = loss
        self.optimizer = optimizer
        self.depth = len(self.layers)
        self.reg_funcs = [lay.reg for lay in layers]

    def forward(self,x):
        """ This method calculates the result of the network. Here we just loop
        through the layers calculating the output and passing it to the next layer
        We also calculate any regularization associated with a layer and add that
        to a collector to be added later to the loss
        -------
        Args: x; numpy array of dimension nxm where n in the number of points and
                    m is the number of features
        -------
        Returns: numpy array of shape denoted by the final layer
        """
        self.reg_fwd = []
        for i,(layer,rf) in enumerate(zip(self.layers, self.reg_funcs)):
            x = layer(x)
            if not layer.act:
                self.reg_fwd.append(rf.f(layer.params['W'], layer.reg_param))
        return x

    def loss(self,x,y):
        """ This method calls forward internally and calculates the loss of the network
        ------
        Args: x; numpy array of dimension nxm where n in the number of points and
                    m is the number of features
              y; numpy array of size nxc where n is the number of points and c is
                    the number of classes - one in the case of regression.
        ------
        Returns: float; the loss of the network
        """
        x = self.forward(x)
        return self.loss_func(x,y)+np.sum(self.reg_fwd)

    def backward(self):
        """ This method back propogates the gradient from the loss and updates the
        values of the parameters.
        ------
        Args: None
        ------
        Returns: the gradient of the most-backward layer
        ------
        NOTE: You must call the loss function before you call this function  or this will not work.
        """
        self.params,self.grads = [],[]
        grad = self.loss_func.backward()
        for layer in reversed(self.layers):
            grad = layer.backward(grad)
            if not layer.act:
                self.params.extend([layer.params['W'],layer.params['b']])
                self.grads.extend([layer.grads['dW'],layer.grads['db']])
        return grad

    def predict(self,x):
        """ This just takes the argmax to get the prediction for a classification problem. """
        return np.argmax(self.forward(x),axis=1)

    def train(self,x,y,epochs=100,batch_size=64,val_sets =(),one_hot=True,verbose=False):
        """ trains the network for the specified number of epochs.
        -------
        Args: x; numpy array of dimension nxm where n in the number of points and
                    m is the number of features
              y; numpy array of size nxc wher n is the number of points and c is the
                    number of classes (classes=1 for regression)
              epochs; int, the number of times the network runs through the dataset passed to it.
              batch_size; int, the size of the batch to run through the network (usually a power of 2)
              val_sels; tuple with the x and y similar to the input for validation accuracy (defaults to empty)
              one_hot; bool, if true the y values are alrady one hot encoded.
              verbose; bool, just what it sounds like
        -------
        Returns: tuples with the losses and training accuracy it each batch, or tuple with those two
                    and the validation accuracy if validation sets passed in the args
        """
        losses,val_acc,tr_acc = [],[],[]
        for e in range(epochs):
            batch_sampler = utils.Batcher(x,y,batch_size)
            for batch_x, batch_y in batch_sampler:

                loss_val = self.loss(batch_x, batch_y)
                self.backward()
                self.optimizer.step(self)

            if verbose:
                if e % int(epochs/10)==0:
                    print(f'loss at epoch {e}: {loss_val}')
            if len(val_sets)>0:
                val_acc.append(utils.accuracy(self, val_sets[0], val_sets[1], convert=one_hot))
            losses.append(loss_val)
            tr_acc.append(utils.accuracy(self, batch_x, batch_y, convert=one_hot))
        if len(val_sets)>0:
            return losses, tr_acc, val_acc
        return losses, tr_acc


import os
import fnmatch
import glob
import xlsxwriter
from xlsxwriter.utility import xl_rowcol_to_cell
import xlrd
import pandas as pd
import pyDOE as doe
import numpy as np
import random as rnd

def compute_grid(nsamples_x, nsamples_y, var_LB, var_UB, NVARS):
    '''
    var_LB/var_UB: vector that contains the min/max values of its parameter
    e.g. for initial example:
    xmin = [Tmin = 100, Rac_ratio = 0.8, Micro_Pow = 150] and
    xmax = [Tmax = 200, Rac_ratio = 1.8, MIcro_Pow = 250]
    '''
	# number of variables
    c_1 = np.linspace(var_LB[0], var_UB[0], nsamples_x)
    c_2 = np.linspace(var_LB[1], var_UB[1], nsamples_y)

    # print (np.shape(M))
    M = np.array(np.meshgrid([c_1], [c_2])).T

    nsamples = nsamples_x * nsamples_y
    M = M.reshape(nsamples,NVARS)
    # print("M matrix in mamin, compute grid = /n", M )
    # print("M size: ", np.shape(M))
    return (M)

def compute_LHS(num_samples, var_LB, var_UB):
	'''
	var_LB/var_UB: vector that contains the min/max values of its parameter
	e.g. for initial example:
	xmin = [Tmin = 100, Rac_ratio = 0.8, Micro_Pow = 150] and
	xmax = [Tmax = 200, Rac_ratio = 1.8, MIcro_Pow = 250]
	'''
	# number of variables
	dim = len(var_LB)
	lhs_set = doe.lhs(dim, num_samples)
	# Convert output to parameter space
	for j in range(dim):
		lhs_set[:,j] = var_LB[j] + lhs_set[:,j]*(var_UB[j] - var_LB[j])
	# print(np.shape(lhs_set))

	return lhs_set

myPath= r'/Users/va00173/Desktop/plate_optimiser/sycofinder-master/ramp_crystallisation_tool/MDL_screens_database'
code_name = 'MD1-40*'

for file in os.listdir(myPath):
	if fnmatch.fnmatch(file, code_name):
		print ("The file you called is: \n", file)
		file_found = file
newpath = os.path.join(myPath, file_found)

xls = pd.ExcelFile(newpath)
df1 = pd.read_excel(xls)
searchedValue = "B1"

df_searchedValue = df1[df1["Well #"] == searchedValue]
df_new = df1.set_index("Well #", drop = False)
df_hit_well = df_new.loc[[searchedValue]]
# df_hit_well_variables = series_hit_well_variables.to_frame()
# print(type(df_hit_well))

# print ("Conditions where we get a hit: \n", df_hit_well)
df_hit_values = df_hit_well.dropna(axis='columns')
print("df_hit_values", df_hit_values)

# exctract indexes for each component of the file
concentrations = df_hit_values.filter(like='Conc').columns

print(concentrations, type(concentrations))
ph =  df_hit_values.filter(like='pH').columns
units = df_hit_values.filter(like='Units').columns
salts = df_hit_values.filter(like='Salt').columns
buff =  df_hit_values.filter(like='Buffer').columns
precip =  df_hit_values.filter(like='Precipitant').columns

var = np.array(len(concentrations))
print("var", var)
var = df_hit_values[concentrations].to_numpy()
reagent = np.array(len(salts)+len(buff)+len(precip))
# reagent = [df_hit_values[salts].to_numpy(), df_hit_values[buff].to_numpy(), df_hit_values[precip].to_numpy()]
print("reagents: \n", reagent)
print("reagent type", type(reagent))
var = var.T
print(len(var))
num_var = 2
range_LB = ([var[0] - 0.01, var[1] - 0.1])
range_UB = ([var[0] + 0.1, var[1] + 0.1])
nsamples_x = 8
nsamples_y = 12
num_samples = nsamples_x * nsamples_y

pp = compute_grid(nsamples_x, nsamples_y, range_LB, range_UB, num_var)
print(pp)

kk = compute_LHS(num_samples, range_LB, range_UB)


from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import pandas as pd
import praw
from dotenv import dotenv_values
from datetime import datetime
from helpers import util
from datetime import datetime

from sqlalchemy import create_engine

from airflow import DAG
from airflow.providers.postgres.hooks.postgres import PostgresHook
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.operators.python import PythonOperator
from airflow.operators.bash import BashOperator

def get_subreddit_list():
    config = dotenv_values("./api_keys/.env")
    reddit = praw.Reddit(
        client_id=config["REDDIT_CLIENT_ID"],
        client_secret=config["REDDIT_CLIENT_SECRET"],
        user_agent=config["REDDIT_USER_AGENT"],
    )
    wikipage = reddit.subreddit("Music").wiki["musicsubreddits"]
    soup = BeautifulSoup(wikipage.content_html, "html.parser")
    wiki_list = soup.find("div", {"class": "md wiki"})
    bullets = wiki_list.find_all("li")
    all_a = [i.a for i in bullets if i.a is not None]
    subreddits = pd.DataFrame([a["href"] for a in all_a if a["href"].startswith("/r/")])
    subreddits.columns = ["subreddit"]
    subreddits["name"] = subreddits.apply(lambda x: x["subreddit"][3:], axis=1)
    subreddits = subreddits.drop_duplicates()
    util.create_dir("output")
    subreddits.to_csv("output/subreddits.csv", index=False)

def export_subreddit_list():
    config = dotenv_values("./api_keys/.env")

    df = pd.read_csv("output/subreddits.csv")

    engine = create_engine(config["COCKROACH_ALCHEMY"])

    df.to_sql(
        "subreddits",
        engine,
        if_exists="replace",
        index=False,
    )

default_args = {
    "owner": "scott",
    "retries": 0,
    "retry_delay": timedelta(seconds=2),
    "schedule_interval": "@weekly",
}

with DAG(
    dag_id="subreddit_list",
    default_args=default_args,
    start_date=datetime(2022, 12, 29),
    schedule_interval="0 2 * * 0",
    catchup=False,
) as dag:
    t1 = PythonOperator(
        task_id="read_subreddit_list", python_callable=get_subreddit_list
    )

    t2 = PostgresOperator(
        task_id="create_table",
        postgres_conn_id="postgres_cockroach",
        sql="sql/create_table.sql",
    )

    t3 = PostgresOperator(
        task_id="clear_table",
        postgres_conn_id="postgres_cockroach",
        sql="sql/clear_table.sql",
    )

    t4 = PythonOperator(
        task_id="export_to_postgres", python_callable=export_subreddit_list
    )

    t5 = BashOperator(
        task_id="clean_directory",
        bash_command="rm ${AIRFLOW_HOME}/output/subreddits.csv",
    )

    t1 >> t2 >> t3 >> t4 >> t5


import torch
import torch.nn as nn
import math

from utils.BiGRU import GRU, BiGRU
from .Knowledge_graph import KnowledgeGraph

class TransferNet(nn.Module):
    def __init__(self, args, dim_word, dim_hidden, vocab):
        super().__init__()
        self.args = args
        self.vocab = vocab
        self.kg = KnowledgeGraph(args, vocab)
        num_words = len(vocab['word2id'])
        num_entities = len(vocab['entity2id'])
        num_relations = len(vocab['relation2id'])
        self.num_steps = args.num_steps
        self.aux_hop = args.aux_hop

        self.question_encoder = BiGRU(dim_word, dim_hidden, num_layers=1, dropout=0.2)

        self.word_embeddings = nn.Embedding(num_words, dim_word)
        self.word_dropout = nn.Dropout(0.2)
        self.step_encoders = []
        for i in range(self.num_steps):
            m = nn.Sequential(
                nn.Linear(dim_hidden, dim_hidden),
                nn.Tanh()
            )
            self.step_encoders.append(m)
            self.add_module('step_encoders_{}'.format(i), m)
        self.rel_classifier = nn.Linear(dim_hidden, num_relations)
        # self.q_classifier = nn.Linear(dim_hidden, num_entities)

        self.hop_selector = nn.Linear(dim_hidden, self.num_steps)

    def follow(self, e, r):
        x = torch.sparse.mm(self.kg.Msubj, e.t()) * torch.sparse.mm(self.kg.Mrel, r.t())
        return torch.sparse.mm(self.kg.Mobj.t(), x).t() # [bsz, Esize]

    def forward(self, questions, e_s, answers=None, hop=None):
        question_lens = questions.size(1) - questions.eq(0).long().sum(dim=1) # 0 means <PAD>
        q_word_emb = self.word_dropout(self.word_embeddings(questions)) # [bsz, max_q, dim_hidden]
        q_word_h, q_embeddings, q_hn = self.question_encoder(q_word_emb, question_lens) # [bsz, max_q, dim_h], [bsz, dim_h], [num_layers, bsz, dim_h]

        device = q_word_h.device
        bsz = q_word_h.size(0)
        dim_h = q_word_h.size(-1)
        last_e = e_s
        word_attns = []
        rel_probs = []
        ent_probs = []
        for t in range(self.num_steps):
            cq_t = self.step_encoders[t](q_embeddings) # [bsz, dim_h]
            q_logits = torch.sum(cq_t.unsqueeze(1) * q_word_h, dim=2) # [bsz, max_q]
            q_dist = torch.softmax(q_logits, 1).unsqueeze(1) # [bsz, 1, max_q]
            word_attns.append(q_dist.squeeze(1))
            ctx_h = (q_dist @ q_word_h).squeeze(1) # [bsz, dim_h]
            rel_dist = torch.softmax(self.rel_classifier(ctx_h), 1) # [bsz, num_relations]
            rel_probs.append(rel_dist)

            last_e = self.follow(last_e, rel_dist)

            # reshape >1 scores to 1 in a differentiable way
            m = last_e.gt(1).float()
            z = (m * last_e + (1-m)).detach()
            last_e = last_e / z

            # Specifically for MetaQA: reshape cycle entities to 0, because A-r->B-r_inv->A is not allowed
            if t > 0:
                prev_rel = torch.argmax(rel_probs[-2], dim=1)
                curr_rel = torch.argmax(rel_probs[-1], dim=1)
                prev_prev_ent_prob = ent_probs[-2] if len(ent_probs)>=2 else e_s
                # in our vocabulary, indices of inverse relations are adjacent. e.g., director:0, director_inv:1
                m = torch.zeros((bsz,1)).to(device)
                m[(torch.abs(prev_rel-curr_rel)==1) & (torch.remainder(torch.min(prev_rel,curr_rel),2)==0)] = 1
                ent_m = m.float() * prev_prev_ent_prob.gt(0.9).float()
                last_e = (1-ent_m) * last_e

            ent_probs.append(last_e)

        hop_res = torch.stack(ent_probs, dim=1) # [bsz, num_hop, num_ent]
        hop_logit = self.hop_selector(q_embeddings)
        hop_attn = torch.softmax(hop_logit, dim=1) # [bsz, num_hop]
        last_e = torch.sum(hop_res * hop_attn.unsqueeze(2), dim=1) # [bsz, num_ent]

        # Specifically for MetaQA: for 2-hop questions, topic entity is excluded from answer
        m = hop_attn.argmax(dim=1).eq(1).float().unsqueeze(1) * e_s
        last_e = (1-m) * last_e

        # question mask, incorporate language bias
        # q_mask = torch.sigmoid(self.q_classifier(q_embeddings))
        # last_e = last_e * q_mask

        if not self.training:
            return {
                'e_score': last_e,
                'word_attns': word_attns,
                'rel_probs': rel_probs,
                'ent_probs': ent_probs
            }
        else:
            # Distance loss
            weight = answers * 9 + 1
            loss_score = torch.mean(weight * torch.pow(last_e - answers, 2))
            loss = {'loss_score': loss_score}

            if self.aux_hop:
                loss_hop = nn.CrossEntropyLoss()(hop_logit, hop-1)
                loss['loss_hop'] = 0.01 * loss_hop

            return loss


import numpy as np
import cv2
import cv2.aruco as aruco

face_cascade = cv2.CascadeClassifier('Haarcascades/haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier('Haarcascadeshaarcascade_eye.xml')

cap = cv2.VideoCapture(0);

while(True):
	ret, frame = cap.read()
	gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

	aruco_dict = aruco.Dictionary_get(aruco.DICT_6X6_250)
	parameters = aruco.DetectorParameters_create()
	corners, ids, rejectedImgPoints = aruco.detectMarkers(gray, aruco_dict, parameters=parameters)
	frame_markers =  aruco.drawDetectedMarkers(frame.copy(), corners, ids)

	cv2.imshow('img',frame)
	if cv2.waitKey(1) & 0xFF == ord('q'):
		break
cv2.destroyAllWindows()


import base64
import json
from datetime import datetime, timedelta, timezone

import numpy as np

from bytewax.dataflow import Dataflow
from bytewax.inputs import ManualInputConfig, distribute
from bytewax.window import EventClockConfig, TumblingWindowConfig
from bytewax.execution import run_main
from bytewax.outputs import StdOutputConfig

from websocket import create_connection
from ticker_pb2 import Ticker

## input
ticker_list = ['AMZN', 'MSFT']

def yf_input(worker_tickers, state):
        ws = create_connection("wss://streamer.finance.yahoo.com/")
        ws.send(json.dumps({"subscribe": worker_tickers}))
        while True:
            yield state, ws.recv()

def input_builder(worker_index, worker_count, resume_state):
    state = resume_state or None
    worker_tickers = list(distribute(ticker_list, worker_index, worker_count))
    print({"subscribing to": worker_tickers})
    return yf_input(worker_tickers, state)

flow = Dataflow()
flow.input("input", ManualInputConfig(input_builder))

# Protobuf deserialization
def deserialize(message):
    '''Use the imported Ticker class to deserialize
    the protobuf message

    returns: ticker id and ticker object
    '''
    ticker_ = Ticker()
    message_bytes = base64.b64decode(message)
    ticker_.ParseFromString(message_bytes)
    return ticker_.id, ticker_

flow.map(deserialize)

def build_array():
    return np.empty((0,3))

# This is the accumulator function, and outputs a numpy array of time and price
def acc_values(np_array, ticker):
    return np.insert(np_array, 0, np.array((ticker.time, ticker.price, ticker.dayVolume)), 0)

# This function instructs the event clock on how to retrieve the
# event's datetime from the input.
# Note that the datetime MUST be UTC. If the datetime is using a different
# representation, we would have to convert it here.
def get_event_time(ticker):
    return datetime.utcfromtimestamp(ticker.time/1000).replace(tzinfo=timezone.utc)

# Configure the `fold_window` operator to use the event time.
cc = EventClockConfig(get_event_time, wait_for_system_duration=timedelta(seconds=10))

# And a 5 seconds tumbling window, that starts at the beginning of the minute
start_at = datetime.now(timezone.utc)
start_at = start_at - timedelta(
    seconds=start_at.second, microseconds=start_at.microsecond
)
wc = TumblingWindowConfig(start_at=start_at, length=timedelta(seconds=60))
flow.fold_window("1_min", cc, wc, build_array, acc_values)
flow.inspect(print)

def calculate_features(ticker__data):
    ticker, data = ticker__data
    return (ticker, {"time":data[-1][0], "min":np.amin(data[:,1]), "max":np.amax(data[:,1]), "first_price":data[:,1][-1], "last_price":data[:,1][0], "volume":data[:,2][0] - data[:,2][-1]})

flow.map(calculate_features)

flow.capture(StdOutputConfig())

if __name__ == "__main__":
    run_main(flow)


# -*- encoding: utf-8 -*-
"""Logica del recurso Emails."""
from flask import jsonify, make_response, request
from flask_restful import Api, Resource

from .. import blueprint_emails
from ...lib.errors import error_409, error_410, error_422, error_500
from .models import Email, EmailSchema

schema = EmailSchema()
api = Api(blueprint_emails)

class EmailList(Resource):

    def get(self):
        try:
            query_set = Email.query.all()
            res = schema.dump(query_set, many=True).data
            return res, 200
        except Exception as e:
            return error_500()

class EmailDetail(Resource):

    def get(self, id):
        try:
            query_set = Email.query.get(id)
            if query_set is None:
                return error_410()
            else:
                res = schema.dump(query_set).data
                return res, 200
        except Exception as e:
            return error_500()

    def put(self, id):
        """Actualiza un Ususrio."""
        json_data = request.get_json()
        if not json_data:
            err = {"datos": ["Información insuficientes."]}
            return error_422(err)
        data, errors = schema.load(json_data)
        if errors:
            return error_409(errors)
        try:
            email = Email.query.get(id)
            if email is None:
                return error_410()
            else:
                sub = data['email']
                setattr(email, 'email', sub)
                email.update()
                res = schema.dump(email).data
                return res, 200
                # return self.get(id)
        except Exception as e:
            return e

    def delete(self, id):
        """Elimina un usuario."""
        try:
            email = Email.query.get(id)
            if email is None:
                return error_410()
            else:
                email.delete(email)
                return None, 204
        except Exception as e:
            return error_500()

api.add_resource(EmailList, '/emails')
api.add_resource(EmailDetail, '/emails/<int:id>')


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch.utils.data as data
import pycocotools.coco as coco
import numpy as np
import torch
import json
import cv2
import os
import math
from utils.image import flip, color_aug
from utils.image import get_affine_transform, affine_transform
from utils.image import gaussian_radius, draw_umich_gaussian, draw_msra_gaussian
import pycocotools.coco as coco

import random

from scipy.spatial.transform import Rotation

class FishDataset(data.Dataset):
  def _coco_box_to_bbox(self, box):
    bbox = np.array([box[0], box[1], box[0] + box[2], box[1] + box[3]],
                    dtype=np.float32)
    return bbox

  def __getitem__(self, index):
    img_id = self.images[index]
    img_info = self.coco.loadImgs(ids=[img_id])[0]
    img_path = os.path.join(self.img_dir, img_info['file_name'])

    ann_ids = self.coco.getAnnIds(imgIds=[img_id])
    img_annos = self.coco.loadAnns(ids=ann_ids)
    boxes = [self._coco_box_to_bbox(ann['bbox']) for ann in img_annos]

    # ===========shape============
    hm = np.zeros((self.opt.num_classes, self.opt.output_h, self.opt.output_w), dtype=np.float32)
    reg = np.zeros((self.max_objs, 2), dtype=np.float32)

    dep = np.zeros((self.max_objs, 1), dtype=np.float32)
    dim = np.zeros((self.max_objs, 3), dtype=np.float32)
    rot = np.zeros((self.max_objs, 2), dtype=np.float32)

    # wh = np.zeros((self.max_objs, 2), dtype=np.float32)

    # reid = np.zeros((self.max_objs, self.opt.max_id), dtype=np.float32)

    reg_mask = np.zeros((self.max_objs), dtype=np.uint8)
    ind = np.zeros((self.max_objs), dtype=np.int64)
    # ===========shape============

    # process img
    img = cv2.imread(img_path)
    img = self._preprocess_input(img)

    cts = []
    for k in range(len(boxes)):
      ann = img_annos[k]
      dim_avg = np.array([0.1457903,0.06654171,0.3345097])

      bbox = self._preprocess_bbox(boxes[k])

      cls_id = int(self.cat_ids[1])

      h, w = bbox[3] - bbox[1], bbox[2] - bbox[0]
      h = h*1.6
      w = w*1.6
      # only taking hm where h and w >0
      if h > 0 and w > 0:

        # HEATMAP AND CENTER
        cx_3d = ann['cx']
        cy_3d = ann['cy']

        cx_2d = bbox[0] + w/2
        cy_2d = bbox[1] + h/2

        ct = np.array(
          [cx_2d,cy_2d], dtype=np.float32)
        ct_int = ct.astype(np.int32)

        cts.append(ct_int)
        radius = gaussian_radius((h, w))
        radius = max(1, int(radius))

        draw_msra_gaussian(hm[0], ct, radius)

        angle_max = (2*np.pi)
        alphaX = np.degrees((ann['alphax'] + angle_max) % angle_max)
        alphaY = np.degrees((ann['alphay'] + angle_max) % angle_max)

        # angle = Rotation.from_euler('xyz', [alphaX, alphaY, 0], degrees=True)
        # angle = angle.as_quat()
        angle = np.array([alphaX,alphaY])

        dep[k] = ann['depth']
        dim[k] = ann['dim'] - dim_avg
        rot[k] = angle

        # reid[k][ann['fish_id']+1] = 1

        # wh[k] = [w,h]

        ind[k] = ct_int[1] * self.opt.output_w + ct_int[0]
        reg[k] = ct - ct_int

        reg_mask[k] = 1

    ret = {'input': img,
          'hm': hm,
          'reg' : reg,
          'reg_mask': reg_mask,
          'ind': ind,
          'dep': dep,
          'dim': dim,
          'rot':rot,
          # 'wh' :wh,
          # 'reid':reid,
          }

    return ret

  def _aug(self,img):
    kerner_size = [5,7,9]
    k = kerner_size[random.randrange(3)]

    img = cv2.GaussianBlur(img, (k, k), 0)
    return img

  def _preprocess_input(self,img):

    # aug = random.choice([True, False])

    # if aug:
    #   img = self._aug(img)

    inp = (img.astype(np.float32) / 255.)
    inp = (inp - self.mean) / self.std
    inp = inp.transpose(2, 0, 1)

    return inp

  def _preprocess_bbox(self,bbox):
    bbox = np.array(bbox)
    bbox = bbox / self.opt.down_ratio

    return bbox


import requests
from bs4 import BeautifulSoup
import pandas as pd
from time import sleep
import numpy

#----------------------------Scap whole page
what_input = input("\n\t\tEnter job title, keywords or company :")
what_input.replace(" ", "+")        # replace space with + to match indeed's input format
where_input = input("\n\t\tEnter location : ")

URL="https://ca.indeed.com/jobs?q="+what_input+"+&l="+where_input+"&start="

title = []
company =[]
location = []
difficulty = []
date = []
remote = []
salary = []
rating = []

proxy = {           # use this website to check proxy "http://free-proxy.cz/en/proxylist/country/CA/https/ping/level1"
    "http":"http://144.217.101.245:3129",
    "https":"http://144.217.101.245:3129"
}   # use proxy to bypass block from indeed

for pg in range(0,3):
    URL_page = URL + str(pg*10)
    # page = requests.get(URL_page,headers={'User-Agent': 'Mozilla/5.0'},proxies=proxy)
    page = requests.get(URL_page)

    soup = BeautifulSoup(page.text, 'html.parser')
    #soup = BeautifulSoup(page.content, 'html.parser')

    data = soup.find_all("div",{"data-tn-component":"organicJob"})
    #data = soup.find_all("div",{"class":"jobsearch-SerpJobCard"})
    #data = soup.select("div.jobsearch-SerpJobCard")

    for d in list(data):
        try:
            # title.append(d.find("h2",{"class":"title"}).get_text().replace('\n\n','').replace('\nnew',''))
            title.append(d.find("a",{"data-tn-element":"jobTitle"}).get_text().replace('\n',''))
        except:
            title.append(None)

        try:
            company.append(d.find("span",{"class":"company"}).get_text().replace('\n',''))
            # company.append(d.find("span",{"class":"company"}).get_text())
        except:
            company.append(None)

        try:
            location.append(d.find("span",{"class":"location"}).get_text())
        except:
            location.append(None)

        try:
            difficulty.append(d.find("span",{"class":"iaLabel iaIconActive"}).get_text())
        except:
            difficulty.append(None)

        try:
            date.append(d.find("span",{"class":"date"}).get_text())
        except:
            date.append(None)

        try:
            salary.append(d.find("span",{"class":"salaryText"}).get_text().replace('\n',''))
            # salary.append(d.find("span",{"class":"salaryText"}).get_text())
        except:
            salary.append(None)

        try:
            remote.append(d.find("span",{"class":"remote"}).get_text())
        except:
            remote.append(None)

        try:
            rating.append(d.find("span",{"class":"ratingsDisplay"}).get_text().replace('\n',''))
            # rating.append(d.find("span",{"class":"ratingsDisplay"}).get_text())
        except:
            rating.append(None)

    delays = [3, 0.5, 1]
    delay = numpy.random.choice(delays)
    sleep(delay)

save_data = pd.DataFrame({
    "Title": title,
    "Company": company,
    "Location": location,
    "Salary": salary,
    "Date": date,
    "Rating": rating,
    "Difficulty": difficulty,
    "Remote": remote
})

save_data.fillna("---", inplace = True)         # replace empty cell to ---
# (from pandas import ExcelWriter)
# create excel writer object
#writer = ExcelWriter('./Data/Indeed_Data.xlsx')    # pylint: disable=abstract-class-instantiated
# write dataframe to excel
#save_data.to_excel(writer,"henry",index = False, header=True)
# save the excel
#writer.save()

#save data to csv file
save_data.to_csv('./Data/indeed_data_original.csv', index = False, header=True)
save_data.to_csv('./Data/indeed_data_modified.csv', index = False, header=True, sep='|')         #replace '|' as delimter (instead of ',')

#----------------------------------------------------------#
'''     # Testing
has_salary=[]
for s in save_data["Salary"]:
    try:
        if len(s) == 0:
            has_salary.append(False)
        else:
            has_salary.append(True)
    except:
        has_salary.append(False)
n = 0
save_salary = []
for s in has_salary:
    if s:
        # print(save_data.iloc[n])        #loc, iloc, ix
        save_salary.append(save_data.iloc[n])        # table format
    n+=1

for s in save_salary:
    print(s)
    print("\n")
'''