from bs4 import BeautifulSoup
import requests
import spotipy
from spotipy.oauth2 import SpotifyOAuth
import os

SPOTIPY_CLIENT_ID = os.environ.get("SPOTIPY_CLIENT_ID")
SPOTIPY_CLIENT_SECRET = os.environ.get("SPOTIPY_CLIENT_SECRET")
USERNAME = os.environ.get("USERNAME")
USER_ID = os.environ.get("USER_ID")

SPOTIPY_REDIRECT_URI = "http://example.com"
SCOPE = "playlist-modify-public"

year = input("Which year do you want to travel to? Type the date in this format YYYY-MM-DD: ")
URL = f"https://www.billboard.com/charts/hot-100/{year}"
billboard = requests.get(url=URL)

soup = BeautifulSoup(billboard.text, "html.parser")

songs = soup.find_all("span", class_="chart-element__information__song text--truncate color--primary")

playlist = [song.text for song in songs]

sp = spotipy.Spotify(auth_manager=SpotifyOAuth(scope=SCOPE, redirect_uri=SPOTIPY_REDIRECT_URI))
playlist_url = sp.user_playlist_create(user=USERNAME, name=f"{year} Nostalgia Songs", description=f"{year} in Songs")
playlist_id = playlist_url["id"]

results = [sp.search(q=f"{song_name}", type="track") for song_name in playlist]

tracks = [x["tracks"]["items"][0]["external_urls"]["spotify"] for x in results]

sp.playlist_add_items(playlist_id=playlist_id, items=tracks)
print("Successfully Done Boss")


import json
import os
from googletrans import Translator

from services.analyze_tweets.tweet_text import get_tweet_text
from services.analyze_tweets.spacy_matcher import filter_tweet
from services.analyze_tweets.topic_modelling import topic_modelling
from services.analyze_tweets.define_topics import define_topics
from services.analyze_tweets.sentiment_analysis import classify_sentiment

def save_files_full(file_path, data, users):
    new_data = []
    with open(file_path, 'w') as f:
        for tweet in data:
            new_tweet = {'lang': tweet['lang'],
                         'id_str': tweet['id_str'],
                         'text_original': get_tweet_text(tweet)}  # not cleaned yet

            new_user = {'lang': tweet['user']['lang'],
                        'id': tweet['user']['id_str'],
                        'name': tweet['user']['name'],
                        'screen_name': tweet['user']['screen_name'],
                        'location_original': tweet['user']['location'],
                        'description_original': tweet['user']['description']}

            new_tweet['user'] = new_user
            new_data.append(new_tweet)

            f.write(json.dumps(new_tweet) + '\n')
    return new_data

def save_complete_processed_data(file_path, data):
    TRANSLATOR = Translator()

    with open('data/test_translated.json', 'w') as f:
        new_data = []
        for new_tweet in data:
            if new_tweet['lang'] != 'en':

                text_translated = TRANSLATOR.translate(new_tweet['text_original'], dest='en')
                new_tweet['text_english'] = text_translated.text
                new_tweet['text_lang_detected'] = text_translated.src

            else:
                new_tweet['text_english'] = new_tweet['text_original']
                new_tweet['text_lang_detected'] = 'en'

            new_user = new_tweet['user']

            if new_user['lang'] != 'en':

                location_translated = TRANSLATOR.translate(new_user['location_original'], dest='en')

                new_user['location_english'] = location_translated.text
                new_user['location_lang_detected'] = location_translated.src

            else:
                new_user['location_english'] = new_user['location_original']
                new_user['location_lang_detected'] = 'en'
                new_user['description_english'] = new_user['description_original']
                new_user['description_lang_detected'] = 'en'

            new_user = new_tweet['user']

            if new_user['lang'] != 'en':
                description_translated = TRANSLATOR.translate(new_user['description_original'], dest='en')
                new_user['description_english'] = description_translated.text
                new_user['description_lang_detected'] = description_translated.src

            new_data.append(new_tweet)
            f.write(json.dumps(new_tweet) + '\n')
        return new_data

def sentiment_analysis(tweet_data):
    n, p, neutral = 0, 0, 0
    for tweet in tweet_data:
        sentiment_result, confidence_probabilities = classify_sentiment(text=tweet)
        r = max(confidence_probabilities)
        print(r)
        if r == 'negative':
            n += 1
        elif r == 'positive':
            p += 1
        else:
            neutral += 1
    return n, p, neutral

def save_data_full(file_path, data):
    users = [d['user'] for d in data]
    return save_files_full(file_path, data, users)

def mySource():
    # data = download_tweets(0)
    # return {2,3}, [2,3,1]
    data = []

    current_dir = os.path.dirname(__file__)
    file_path = os.path.join(current_dir, "data", "test.json")
    with open(file_path, "r") as json_file:
        for line in json_file:
            data.append((json.loads(line)))

    data = filter_tweet(data)  # output is a list of dict

    # this file will contain all data fields of interest
    filtered_data_full_tweets = save_data_full('data\\full_0.json', data)

    # this file will contain tweets only which have been translated
    data = save_complete_processed_data('data\\data.json', filtered_data_full_tweets)

    print(data)

    tweet_data = []
    for text in data:
        tweet_data.append(text["text_original"])

    keywords = topic_modelling(tweet_data)

    topics = define_topics(keywords)

    sentiment_analysis_result = sentiment_analysis(tweet_data)

    # location_data = save_extracted_location_data('data\\location.json', data)

    # age_data = save_extracted_age('data\\age.json', location_data)

    return topics, sentiment_analysis_result


import util
from sklearn import tree
from sklearn.metrics import accuracy_score
from features import create_features_and_split
from sklearn import ensemble
from sklearn import linear_model
from sklearn.ensemble import GradientBoostingClassifier

def get_feats_labels(df):
    feats = df.drop('device_class', axis=1)
    labels = df['device_class']
    return feats, labels

def get_data():
    df = util.load_data_to_dataframe('dataset/train_split_orig.json')
    train, val = create_features_and_split(df)
    train_feats, train_labels = get_feats_labels(train)
    val_feats, val_labels = get_feats_labels(val)
    return train_feats, train_labels, val_feats, val_labels

def main():
    train_feats, train_labels, val_feats, val_labels = get_data()

    gb = GradientBoostingClassifier(verbose=True, n_estimators=100, learning_rate = 0.05, max_features=20, max_depth = 20, random_state = 0)
    gb.fit(train_feats, train_labels)

    val_predict = gb.predict(val_feats)
    print(accuracy_score(val_labels, val_predict))

main()


import gym

from flatland.envs.malfunction_generators import malfunction_from_params
from flatland.envs.rail_env import RailEnv
from flatland.envs.rail_generators import sparse_rail_generator
from flatland.envs.schedule_generators import sparse_schedule_generator
from flatland.utils.rendertools import RenderTool

from src.common.deadlocks import DeadlocksDetector
from src.common.observation import NormalizeObservations
from src.common.wrappers import RewardsWrapper, StatsWrapper

class FlatlandRailEnv:
    """
    Flatland environment to deal with wrappers.
    """
    def __init__(self,
                 train_params,
                 env_params,
                 observation,
                 custom_observations,
                 reward_wrapper,
                 stats_wrapper):

        self.env = FlatlandGymEnv(self._launch(env_params, observation),
                                   custom_observations,
                                   env_params,
                                   render=train_params.render)

        self.state_size = self.env.state_size

        if reward_wrapper:
            self.env = RewardsWrapper(self.env,
                                       env_params.invalid_action_penalty,
                                       env_params.stop_penalty,
                                       env_params.deadlock_penalty,
                                       env_params.shortest_path_penalty_coefficient,
                                       env_params.done_bonus,
                                      env_params.uniform_reward)
        if stats_wrapper:
            self.env = StatsWrapper(self.env,
                                     env_params)

    def _launch(self, env_params, observation):
        return RailEnv(
            width=env_params.x_dim,
            height=env_params.y_dim,
            rail_generator=sparse_rail_generator(
                max_num_cities=env_params.n_cities,
                grid_mode=False,
                max_rails_between_cities=env_params.max_rails_between_cities,
                max_rails_in_city=env_params.max_rails_in_city,
                seed=env_params.seed
            ),
            schedule_generator=sparse_schedule_generator(env_params.speed_profiles),
            number_of_agents=env_params.n_agents,
            malfunction_generator_and_process_data=malfunction_from_params(env_params.malfunction_parameters),
            obs_builder_object=observation,
            random_seed=env_params.seed
        )

    def reset(self):
        return self.env.reset()

    def step(self, action_dict):
        return self.env.step(action_dict)

    def get_rail_env(self):
        return self.env.rail_env

class FlatlandGymEnv(gym.Env):
    """
    gym.Env wrapper of the Flatland environment providing deadlocks and observation normalization.
    """
    def __init__(self,
                 rail_env,
                 custom_observations,
                 env_params,
                 render=False,
                 regenerate_rail_on_reset=True,
                 regenerate_schedule_on_reset=True):

        self._regenerate_rail_on_reset = regenerate_rail_on_reset
        self._regenerate_schedule_on_reset = regenerate_schedule_on_reset
        self.rail_env = rail_env
        self.deadlocks_detector = DeadlocksDetector()

        self.observation_normalizer = NormalizeObservations(self.rail_env.obs_builder.observation_dim,
                                                            env_params.observation_tree_depth,
                                                            custom_observations,
                                                            self.rail_env.width,
                                                            self.rail_env.height,
                                                            env_params.observation_radius)

        self.state_size = self.observation_normalizer.state_size

        self.render = render
        self.env_renderer = None

    def reset(self):
        obs, info = self.rail_env.reset(regenerate_rail=self._regenerate_rail_on_reset,
                                        regenerate_schedule=self._regenerate_schedule_on_reset)
        # Reset rendering
        if self.render:
            self.env_renderer = RenderTool(self.rail_env, gl="PGL")
            self.env_renderer.set_new_rail()

        # Reset custom observations
        self.observation_normalizer.reset_custom_obs(self.rail_env)

        # Compute deadlocks
        self.deadlocks_detector.reset(self.rail_env.get_num_agents())
        info["deadlocks"] = {}

        for agent in range(self.rail_env.get_num_agents()):
            info["deadlocks"][agent] = self.deadlocks_detector.deadlocks[agent]

        # Normalization
        for agent in obs:
            if obs[agent] is not None:
                obs[agent] = self.observation_normalizer.normalize_observation(obs[agent], self.rail_env,
                                                                               agent, info["deadlocks"][agent])

        return obs, info

    def step(self, action_dict):
        """
        Normalize observations by default, update deadlocks and step.

        :param action_dict:
        :return:
        """
        obs, rewards, dones, info = self.rail_env.step(action_dict)

        # Compute deadlocks
        deadlocks = self.deadlocks_detector.step(self.rail_env)
        info["deadlocks"] = {}
        for agent in range(self.rail_env.get_num_agents()):
            info["deadlocks"][agent] = deadlocks[agent]

        # Normalization
        for agent in obs:
            if obs[agent] is not None:
                obs[agent] = self.observation_normalizer.normalize_observation(obs[agent], self.rail_env,
                                                                               agent, info["deadlocks"][agent])

        return obs, rewards, dones, info

    def show_render(self):
        """
        Open rendering window.

        :return:
        """
        if self.render:
            return self.env_renderer.render_env(
                show=True,
                frames=False,
                show_observations=False,
                show_predictions=False)

    def close(self):
        """
        Close rendering window.
        :return:
        """
        if self.render:
            return self.env_renderer.close_window()


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

"""
Applying ANN
--> ANN  86.8%
--> Logistic Regression 88.5%
--> SVM  88.5%
--> KNN  90.1%

"""

# In[ ]:

# Importing modules

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
get_ipython().run_line_magic('matplotlib', 'inline')

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# In[ ]:

import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LeakyReLU,PReLU,ELU
from keras.layers import Dropout
from sklearn.metrics import accuracy_score

# In[ ]:

df = pd.read_csv('../input/heart-attack-analysis-prediction-dataset/heart.csv')
df

# In[ ]:

dfo = pd.read_csv('../input/heart-attack-analysis-prediction-dataset/o2Saturation.csv')
#dfo

# In[ ]:

df.isnull().sum()

# # **EDA and Plotting**

# In[ ]:

sns.scatterplot(x='age',y='chol',data=df)

# In[ ]:

sns.barplot(x='cp',y='age',data=df)

# In[ ]:

sns.scatterplot(x='trtbps',y='chol',data=df)

# In[ ]:

sns.scatterplot(x='trtbps',y='thalachh',data=df)

# # **Train Test Split and Scaling**

# In[ ]:

x=df.drop(['output'],axis=1)
y=df['output']

# In[ ]:

xtrain,xtest,ytrain,ytest = train_test_split(x,y,test_size=0.2,random_state=42)

# In[ ]:

sc=StandardScaler()
xtrain=sc.fit_transform(xtrain)
xtest=sc.fit_transform(xtest)

# # **DL ANN**

# In[ ]:

clf = Sequential()

# In[ ]:

clf.add(Dense(input_dim=13,units=13,kernel_initializer='he_uniform',activation='relu'))
clf.add(Dropout(0.3))
clf.add(Dense(units=13,kernel_initializer='he_uniform',activation='relu'))
clf.add(Dropout(0.3))
clf.add(Dense(units=13,kernel_initializer='he_uniform',activation='relu'))
clf.add(Dropout(0.3))
clf.add(Dense(units=1,kernel_initializer='glorot_uniform',activation='sigmoid'))
clf.compile(optimizer='Adamax',loss='binary_crossentropy',metrics=['accuracy'])

# In[ ]:

model = clf.fit(xtrain,ytrain,validation_split=0.2,epochs=100,batch_size=10)

# In[ ]:

yp=clf.predict(xtest)
yp=yp>0.5

# In[ ]:

score=accuracy_score(yp,ytest)
score

# In[ ]:

print(model.history.keys())

# In[ ]:

# In[ ]:

plt.figure(figsize=(15, 10))
plt.plot(model.history['loss'])
plt.plot(model.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# In[ ]:

plt.figure(figsize=(15, 10))
plt.plot(model.history['accuracy'])
plt.plot(model.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('acc')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# # **Logistic Regression , SVM , KNN**

# In[ ]:

lr=LogisticRegression()
lr.fit(xtrain,ytrain)
lr.score(xtest,ytest)

# In[ ]:

sv=SVC(C=10,gamma=0.2,kernel='linear')
sv.fit(xtrain,ytrain)
sv.score(xtest,ytest)

# In[ ]:

from sklearn.neighbors import KNeighborsClassifier

# In[ ]:

kn = KNeighborsClassifier(n_neighbors=5)
kn.fit(xtrain,ytrain)
kn.score(xtest,ytest)

# In[ ]:

# In[ ]:

# In[ ]:


import math
import numpy as np
import curses
alpha_values=np.linspace(0,2*math.pi,45)
beta_values=np.linspace(0,2*math.pi,200)
aux1=np.linspace(0,math.pi/4,50)
aux1rev=aux1[::-1]
xtheta_value=[*aux1,*aux1rev]
ytheta_value=np.linspace(0,math.pi,100)
ztheta_value=0
Rinx=8
Riny=4
Rinz=4
Rtx=20
Rty=9

zbuffer=np.full((40,60),-100)
shadsign=['.',',','-','~',':',';','=','!','*','#','$','@']

e=math.cos(ztheta_value)
e1=math.sin(ztheta_value)
curses.initscr()
win=curses.newwin(60,60,0,0)
curses.curs_set(False)
for k in range(100):
    win.clear()
    c=math.cos(xtheta_value[k])
    c1=math.sin(xtheta_value[k])
    d=math.cos(ytheta_value[k])
    d1=math.sin(ytheta_value[k])
    for i in range(200):
        b=math.cos(beta_values[i])
        b1=math.sin(beta_values[i])
        for j in range(45):
            a=math.cos(alpha_values[j])
            a1=math.sin(alpha_values[j])
            dx=round((Rtx+Rinx*a)*b*d*e+(Rty+Riny*a)*b1*(-c*e1+e*d1*c1)+Rinz*a1*(e*c*d1+e1*c1))+29
            dy=round((Rtx+Rinx*a)*b*d*e1+Rinz*a1*(d*e1*d1-e*c1)+(Rty+Riny*a)*b1*(e*c+e1*d1*c1))+13
            dz=round(-(Rtx+Rinx*a)*b*d1+e*(Rinz*c*a1+(Rty+Riny*a)*b1*c1))
            shadcomp=abs(round(11*(-a*b*d1+d*(c*a1+a*b1*c1))))
            if dz > zbuffer[dy,dx]:
                zbuffer[dy,dx]=dz
                win.addstr(dy,dx,shadsign[shadcomp])
            else:
                pass
    win.refresh()
    zbuffer=np.full((40,60),-100)
    win.move(0,0)
curses.endwin()


import main1 as basic
import kivy
kivy.require('1.0.6')
from kivy.app import App
from kivy.properties import NumericProperty,ReferenceListProperty,ObjectProperty,StringProperty
from kivy.uix.widget import Widget
from kivy.vector import Vector
from kivy.uix.gridlayout import GridLayout
from kivy.uix.label import Label
from kivy.clock import Clock
from kivy.uix.button import Button
from kivy.uix.textinput import TextInput
from random import randint
#
class SearchScreen(Widget):
    b=[0,0,0,0,0]
    text_input=TextInput(multiline=False)
    text_output=TextInput()
    def put_bar(self):
        self.text=''
    def update(self,dt):
        ##print(str(self.text_input.text),self.text)
        if(str(self.text_input.text)==self.text+'\n'):
            self.text_input.text=''
            basic.insert1(self.text)
            self.text=''
            self.text_output.text+=(self.b[0].text+' ')
        elif(str(self.text_input.text)==self.text+' '):
            if(basic.in_list(0,self.b[0].text)):
                basic.insert1(self.b[0].text)
                self.text_output.text+=(self.b[0].text+' ')
            elif(self.b[1].text!=''):
                basic.insert1(self.b[1].text)
                self.text_output.text+=(self.b[1].text+' ')
            else:
                basic.insert1(self.b[0].text)
                self.text_output.text+=(self.b[0].text+' ')
            self.text_input.text=''
        elif(str(self.text_input.text)!=self.text and str(self.text_input.text!='')):
            self.text=self.text_input.text
            self.search_result=basic.search1(self.text)
            self.print_list=list(self.search_result.keys()).copy()
            #print(self.search_result)
            if(len(self.print_list)>0):
                n=len(self.print_list)
                for i in range (n-1):
                    ##print(self.#print_list)
                    for j in range(0,n-i-1):
                        if(self.search_result[self.print_list[j]]<self.search_result[self.print_list[j+1]]):
                    #        #print(self.search_result[self.#print_list[j]],self.search_result[self.#print_list[j+1]])
                    #        #print(self.#print_list[j],self.#print_list[j+1])
                            temp=self.print_list[j]
                            self.print_list[j]=self.print_list[j+1]
                            self.print_list[j+1]=temp
                    #        #print(self.search_result[self.#print_list[j]],self.search_result[self.#print_list[j+1]])
                    #        #print(self.#print_list[j],self.#print_list[j+1])
            self.send_list=[]
            if(len(self.print_list)>4):
                i=0
                while(len(self.send_list)!=5):
                    self.send_list.append(self.print_list[i])
                    i+=1
            else:
                self.send_list=self.print_list.copy()
            if(self.text) in self.send_list:
                self.send_list.pop(self.send_list.index(self.text))
                self.send_list.insert(0,self.text)
            else:
                self.send_list.insert(0,self.text)
            self.display_words(self.send_list)

    def display_words(self,word_list):
        try:
            for i in range(5):
                self.remove_widget(self.b[i])
        except:
            pass
        #print(word_list)
        if(len(word_list)<5):
            while(len(word_list)!=5):
                word_list.append('')
        for i in range (5):
            self.b[i]=Button(text=word_list[i],x=(self.x+(self.width*(i)/5)),center_y=(self.center_y+(self.height/2)-50),size=(self.width/5,self.height/10),font_size='12sp')
            self.b[i].bind(on_press=self.b_press)
            self.add_widget(self.b[i])
    def b_press(self,obj):
        basic.insert1(obj.text)
class SearchApp(App,Widget):
    def build(self):
        game=SearchScreen()
        game.put_bar()
        Clock.schedule_interval(game.update,1.0/60.0)
        return(game)

if __name__=="__main__":
    SearchApp().run()


import numpy as np
import pandas as pd
import h5py
import scipy.io
np.random.seed(1337) # for reproducibility
import torch.nn.functional as F
import torch
import torch.nn as nn
from sklearn.metrics import roc_auc_score

from models import Deep_SEA, CNN_BiLSTM_Attension

save_to_dir = 'output/result.csv'
print ('loading data')

try:
    print(len(trainmat),'Already dataset here')
except:
    trainmat = h5py.File('data/train.mat')
    validmat = scipy.io.loadmat('data/valid.mat')
    testmat = scipy.io.loadmat('data/test.mat')

    X_train = np.transpose(np.array(trainmat['trainxdata']),axes=(2,0,1))
    y_train = np.array(trainmat['traindata']).T
    X_test = np.transpose(testmat['testxdata'],axes=(0,2,1))
    y_test = testmat['testdata']

torch.cuda.set_device(4)
device = torch.device('cuda')
model = CNN_BiLSTM_Attension(conv_dr=0.2 , lstm_dr=0.3,fc_dr=0.5)
model = Deep_SEA(conv1_ch=320,conv2_ch=480,conv3_ch=960,drop_rate1 =0.5, drop_rate2 = 0.2, drop_rate3 = 0.2)
model.to(device)

data_size = len(X_train)
batch_division_size = 10
batch_size = data_size//batch_division_size
mini_batch_size = 16
max_epoch = 100
l1 = 1e-08
l2 = 5e-07
wd = 8e-7
learning_rate = 1e-2
cnn_optimizer = torch.optim.Adam([{'params': model.parameters(), 'lr':l2}],lr= learning_rate ,weight_decay =wd)
loss_fn = torch.nn.BCELoss()
test_auc_list = []

for epoch in range(max_epoch):
    losses= 0
    for batch in range(batch_division_size):
        X_tr = X_train[batch_size*(batch):batch_size*(batch+1)]
        y_tr = y_train[batch_size*(batch):batch_size*(batch+1)]

        trainDataset = torch.utils.data.TensorDataset(torch.FloatTensor(X_tr), torch.FloatTensor(y_tr.astype(int)))
        trainLoader = torch.utils.data.DataLoader(dataset = trainDataset, batch_size= mini_batch_size, shuffle=False, num_workers= 0)

        model.train()
        for i, (x, target) in enumerate(trainLoader):
            y = target.float().to(device)
            x = x.to(device)
            pred_y = model(x)
            pred_y = pred_y.to(device)
            loss = loss_fn(pred_y,y)
            regularization_loss = 0
            for param in model.parameters():
                regularization_loss += torch.sum(abs(param))

            losses += loss.item()+l1*regularization_loss
            cnn_optimizer.zero_grad()
            loss.backward()
            cnn_optimizer.step()

        print("batch:",batch, "  losses:", losses/(data_size*(batch+1)))

    with torch.no_grad():
        temp_AUC_list = []
        for i in range(len(X_test)//100):
            x_ts = X_test[100*(i):100*(i+1)]
            y_ts = y_test[100*(i):100*(i+1)]
            x_ts  = torch.FloatTensor(x_ts)
            y_ts  = torch.FloatTensor(y_ts)
            x_ts = x_ts.to(device)

            model.eval()
            pred_y = model(x_ts)
            pred_y = pred_y.cpu()
            test_AUC = roc_auc_score(y_ts.view(-1,1).detach().numpy(),pred_y.view(-1,1).detach().numpy())
            temp_AUC_list.append(test_AUC)

        temp_mean_auc = torch.mean(torch.Tensor(temp_AUC_list))
        test_auc_list.append(temp_mean_auc)
        print("test_AUC: ",temp_mean_auc)
        temp_df = pd.DataFrame(data=test_auc_list)
        temp_df.to_csv(save_to_dir,sep='\t',index=None)

    print("epoch: ",epoch, "  avg_losses: ", losses/data_size)


import argparse
import os
import numpy as np
import os
import sys
current_dir = os.getcwd()
sys.path.append(current_dir)

import torch
import transformers
from tqdm import tqdm

from bias_bench.dataset import load_sentence_debias_data
from bias_bench.debias import DensRay

from bias_bench.model import models
from bias_bench.util import generate_experiment_id

thisdir = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(
    description="Computes the bias subspace for Densray."
)
parser.add_argument(
    "--persistent_dir",
    action="store",
    type=str,
    default=os.path.realpath(os.path.join(thisdir, "..")),
    help="Directory where all persistent data will be stored.",
)
parser.add_argument(
    "--model",
    action="store",
    type=str,
    default="BertModel",
    choices=["BertModel", "AlbertModel", "RobertaModel", "GPT2Model"],
    help="Model (e.g., BertModel) to compute the SentenceDebias subspace for. "
    "Typically, these correspond to a HuggingFace class.",
)
parser.add_argument(
    "--model_name_or_path",
    action="store",
    type=str,
    default="bert-base-uncased",
    choices=["bert-base-uncased",'bert-base-multilingual-uncased', 'bert-base-multilingual-cased', "albert-base-v2", "roberta-base", "gpt2"],
    help="HuggingFace model name or path (e.g., bert-base-uncased). Checkpoint from which a "
    "model is instantiated.",
)
parser.add_argument(
    "--bias_type",
    action="store",
    type=str,
    choices=["gender", "religion", "race"],
    required=True,
    help="The type of bias to compute the bias subspace for.",
)
parser.add_argument(
    "--lang_debias",
    action="store",
    type=str,
    default='en',
    choices=['en','nl','de','fr','pl','ru','ca'],
    help="Batch size to use while encoding.",
)
parser.add_argument(
    "--batch_size",
    action="store",
    type=int,
    default=32,
    help="Batch size to use while encoding.",
)

if __name__ == "__main__":
    args = parser.parse_args()
    print('Densray')
    experiment_id = generate_experiment_id(
        name="densray",
        model=args.model,
        model_name_or_path=args.model_name_or_path,
        bias_type=args.bias_type,
        lang_debias=args.lang_debias,
    )

    print("Computing bias direction densray:")
    print(f" - persistent_dir: {args.persistent_dir}")
    print(f" - model_name_or_path: {args.model_name_or_path}")
    print(f" - model: {args.model}")
    print(f" - bias_type: {args.bias_type}")
    print(f" - batch_size: {args.batch_size}")
    print(f" - language debias : {args.lang_debias}")

    # Get the data to compute the SentenceDebias bias subspace.
    data = load_sentence_debias_data(
        persistent_dir=args.persistent_dir, bias_type=args.bias_type, lang_debias=args.lang_debias
    ) #female_example

    # Load model and tokenizer.
    model = getattr(models, args.model)(args.model_name_or_path)
    #for name, param in model.named_parameters():
    #    print(name)

    model.eval()
    tokenizer = transformers.AutoTokenizer.from_pretrained(args.model_name_or_path)
    #tokenize the examples (data['female_example] en data['male_example'])
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model.to(device)
    all_embeddings_male = []
    all_embeddings_female = []

    n_batches = len(data) // args.batch_size
    print(n_batches)
    for i in tqdm(range(n_batches), desc="Encoding gender examples"):
        offset = args.batch_size * i

        inputs_male = tokenizer(
            [example["male_example"] for example in data[offset : offset + args.batch_size]],
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=128,
        )

        inputs_female = tokenizer(
            [
                example["female_example"]
                for example in data[offset : offset + args.batch_size]
            ],
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=128,
        )

        male_input_ids = inputs_male["input_ids"].to(device)
        female_input_ids = inputs_female["input_ids"].to(device)

        male_attention_mask = inputs_male["attention_mask"].to(device)
        female_attention_mask = inputs_female["attention_mask"].to(device)
        with torch.no_grad():
            # Compute average representation from last layer.
            # embedding_male.shape == (batch_size, 128, 768).
            embedding_male = model(
                input_ids=male_input_ids, attention_mask=male_attention_mask
            )["last_hidden_state"]
            embedding_male *= male_attention_mask.unsqueeze(-1)
            embedding_male = embedding_male.sum(dim=1)
            embedding_male /= male_attention_mask.sum(dim=1, keepdims=True)

            embedding_female = model(
                input_ids=female_input_ids, attention_mask=female_attention_mask
            )["last_hidden_state"]
            embedding_female *= female_attention_mask.unsqueeze(-1)
            embedding_female = embedding_female.sum(dim=1)
            embedding_female /= female_attention_mask.sum(dim=1, keepdims=True)

        embedding_male /= torch.norm(embedding_male, dim=-1, keepdim=True)
        embedding_female /= torch.norm(embedding_female, dim=-1, keepdim=True)

        all_embeddings_male.append(embedding_male.cpu().numpy())
        all_embeddings_female.append(embedding_female.cpu().numpy())

    # all_embeddings_male.shape == (num_examples, dim).
    all_embeddings_male = np.concatenate(all_embeddings_male, axis=0)
    all_embeddings_female = np.concatenate(all_embeddings_female, axis=0)
    #L, R = analogy.get_embeddings_from_cropus(corpus, layer)
    # compute densray
    print('now we will compute the bias dimensions')
    densray = DensRay(torch.from_numpy(all_embeddings_male),torch.from_numpy(all_embeddings_female))
    densray.fit()

    print(
        f"Saving computed eigenvector to: {args.persistent_dir}/results/densray/{experiment_id}.pt."
    )
    os.makedirs(f"{args.persistent_dir}/results/densray", exist_ok=True)
    torch.save(
        densray.eigvecs, f"{args.persistent_dir}/results/densray{experiment_id}.pt"
    )
    torch.save(densray.mean, f"{args.persistent_dir}/results/densray{experiment_id}_mean.pt")
    torch.save(densray.std, f"{args.persistent_dir}/results/densray{experiment_id}_std.pt")


from odoo import fields, models, _, api
from odoo.exceptions import UserError, ValidationError
import logging
_logger = logging.getLogger(__name__)

class AccountCheckOperation(models.Model):
    _inherit = 'account.check.operation'

    operation = fields.Selection(selection_add=[
        ('draft', 'Draft'),
        ('used', 'In use'),
        ('negotiated', 'Negotiate'),
    ])
    lot_operation = fields.Char(
        string='Lot operation'
    )

class AccountCheck(models.Model):
    _inherit = 'account.check'

    name = fields.Char(
        required=False,
        readonly=True,
        copy=False,
        states={'draft': [('readonly', False)]},
        index=True,
    )
    state = fields.Selection(selection_add=[
        ('draft', 'Draft'),
        ('used', 'In use'),
        ('negotiated', 'Negotiate'),
    ])
    deposited_date = fields.Date(
        string='Deposited date'
    )
    deposited_journal_id = fields.Many2one(
        'account.journal',
        string='Deposited Journal'
    )
    deposited_bank_id = fields.Many2one(
        'res.bank',
        string='Deposited Bank'
    )
    notes = fields.Text(
        string='Note'
    )
    lot_operation = fields.Char(
        string='Lot operation',
        copy=True,
        compute='_compute_lot_operation',
    )
    first_operation_id = fields.Many2one(
        'account.check.operation',
        compute='_compute_first_partner',
        string='First operation',
        readonly=True,
        store=True,
        compute_sudo=True
    )
    note = fields.Char(string='Note')
    not_endorsable = fields.Boolean(
        string='Not Endorsable',
        readonly=True,
        states={'draft': [('readonly', False)],
                'holding': [('readonly', False)]}
    )
    first_partner_id = fields.Many2one(
        'res.partner',
        compute='_compute_first_partner',
        string='First operation partner',
        readonly=True,
        store=False,
        compute_sudo=True
    )

    @api.depends('operation_ids', 'operation_ids.partner_id')
    def _compute_first_partner(self):
        for rec in self:
            if not rec.operation_ids:
                rec.first_partner_id = False
                rec.first_operation_id = False
                continue
            rec.first_partner_id = rec.operation_ids[-1].partner_id
            rec.first_operation_id = rec.operation_ids[-1].id

    @api.depends(
        'operation_ids.operation',
        'operation_ids.lot_operation',
    )
    def _compute_lot_operation(self):
        for rec in self:
            if rec.operation_ids.sorted():
                rec.lot_operation = rec.operation_ids.sorted()[0].lot_operation
            else:
                rec.lot_operation = False

    @api.onchange('number')
    def onchange_number(self):
        for rec in self:
            rec.name = self._get_name_from_number(rec.number)

    @api.onchange('journal_id')
    def onchange_journal_id(self):
        for rec in self:
            if rec.journal_id and rec.type == 'issue_check':
                rec.bank_id = rec.journal_id.bank_id

    @api.onchange('amount', 'currency_id')
    def onchange_amount(self):
        for rec in self:
            if rec.amount and rec.currency_id and rec.currency_id != rec.company_currency_id:
                rec.amount_company_currency = rec.currency_id._convert(
                    rec.amount, rec.company_id.currency_id,
                    rec.company_id, rec.issue_date)
            else:
                rec.amount_company_currency = rec.amount

    def _add_operation(
            self, operation, origin, partner=None, date=False, lot_operation=False):
        for rec in self:
            if not self._context.get('no_check'):
                rec._check_state_change(operation)
                # agregamos validacion de fechas
                date = date or fields.Datetime.now()
                if rec.operation_ids and rec.operation_ids[0].date > date:
                    raise ValidationError(_(
                        'The date of a new check operation can not be minor than '
                        'last operation date.\n'
                        '* Check Id: %s\n'
                        '* Check Number: %s\n'
                        '* Operation: %s\n'
                        '* Operation Date: %s\n'
                        '* Last Operation Date: %s') % (
                        rec.id, rec.name, operation, date,
                        rec.operation_ids[0].date))
            vals = {
                'operation': operation,
                'date': date,
                'check_id': rec.id,
                'origin': origin and '%s,%i' % (origin._name, origin.id) or False,
                'partner_id': partner and partner.id or False,
                'lot_operation': lot_operation,
            }
            rec.operation_ids.create(vals)

    def used(self):
        if all([x.state in ['draft'] and x.type == 'issue_check' for x in self]):
            if self._context.get('lot_operation', False):
                name = _('Lot Checks "%s" used') % (self._context.get('lot_operation'))
            else:
                name = _('Check "%s" used') % (self[0].name)
            view_id = self.env.ref('account.view_move_form').id
            journal = self[0].journal_id
            partner_id = self._context.get('partner', False)
            vals = self.get_used_values(journal, name, partner_id, self._context.get('lot_operation'))
            action_context = vals
            return {
                'name': name,
                'view_type': 'form',
                'view_mode': 'form',
                'res_model': 'account.move',
                'view_id': view_id,
                'type': 'ir.actions.act_window',
                'context': action_context,
            }

    def get_used_values(self, journal, name, partner_id, lot_operation):
        amount = sum(self.mapped('amount'))
        credit_account = journal.default_debit_account_id
        credit_line_vals = {
            'name': name,
            'account_id': credit_account.id,
            'debit': 0.0,
            'credit': amount,
            'exclude_from_invoice_tab': True
        }
        if partner_id:
            credit_line_vals['partner_id'] = partner_id.id
        return {
            'default_ref': name,
            'default_type': 'entry',
            'default_auto_post': True,
            'default_journal_id': journal.id,
            'default_partner_id': partner_id and partner_id.id or False,
            'default_date': self._context.get('action_date'),
            'default_used_check_ids': [(6, 0, self.ids)],
            'default_lot_operation': lot_operation,
            'default_line_ids': [(0, 0, credit_line_vals)]
        }

    def negotiated(self):
        if all([x.state in ['draft', 'holding'] for x in self]):
            action_date = self._context.get('action_date')
            partner = self._context.get('partner', False) or self.company_id.partner_id
            lot_operation = self._context.get('lot_operation')
            self._add_operation('negotiated', False, partner, action_date, lot_operation)

    def selled(self):
        if all([x.state in ['negotiated'] for x in self]):
            action_date = self._context.get('action_date')
            partner = self._context.get('partner', False)
            journal = self._context.get('journal', False)
            expense_account = self._context.get('expense_account', False)
            debit_note = self._context.get('debit_note', False)
            expense_amount = self._context.get('expense_amount', 0.0)
            if not self[0].company_id._get_check_account('selled'):
                raise UserError(_(
                    'You must set the checking account sold in the accounting settings.'))
            if journal and expense_account:
                if self._context.get('lot_operation', False):
                    name = _('Lot Checks "%s" selled') % (self._context.get('lot_operation'))
                else:
                    name = _('Check "%s" selled') % (self[0].name)
                amount_total = sum(x.amount for x in self)
                vals = self[0].get_move_values(name,
                    debit_note, journal, expense_account, expense_amount, amount_total)

                action_date = self._context.get('action_date')
                lot_operation = self._context.get('lot_operation', False)
                vals['date'] = action_date
                move = self.env['account.move'].create(vals)
                move.post()
                self._add_operation('selled', move, partner, action_date, lot_operation)
                if debit_note and expense_amount > 0.0:
                    tax_ids = self._context.get('tax_ids')
                    return self[0].action_create_debit_note('selled', 'supplier', partner,
                                                            expense_account, expense_amount, tax_ids)

    def deposited(self):
        if all([x.state in ['draft', 'holding'] for x in self]) or \
                (all([x.state in ['rejected'] for x in self]) and self._context.get('no_check_op', False)):
            journal = self._context.get('journal', False)
            if journal:
                payment_ids = self.env['account.payment']
                if self[0].type == 'third_check':
                    payment_ids += self.create_payment_deposited(journal, self[0].type)
                else:
                    for rec in self:
                        payment_ids += rec.create_payment_deposited(journal, rec.type)
                action = self.env.ref('account_payment_group.action_account_payments_transfer').read()[0]
                action['name'] = _('Deposited Checks')
                action['domain'] = [('id', 'in', payment_ids.ids)]
                return action

    def create_payment_deposited(self, journal, type):
        vals = self[0].get_payment_values(self[0].journal_id)
        ctx = dict(self._context)
        if type == 'third_check':
            vals['check_ids'] = [(4, check.id, None) for check in self]
            payment_method_code_id = self.env['account.payment.method'].search(
                                                [('code', '=', 'delivered_third_check')])
            if payment_method_code_id:
                vals['payment_method_id'] = payment_method_code_id.id
        else:
            # vals['check_id'] = self.id
            vals['check_ids'] = [(4, self.id, None)]
            vals['check_number'] = self.number
            vals['checkbook_id'] = self.checkbook_id.id
            vals['check_name'] = self.name
            payment_method_code_id = self.env['account.payment.method'].search(
                [('code', '=', 'issue_check')])
            if payment_method_code_id:
                vals['payment_method_id'] = payment_method_code_id.id
        vals['communication'] = self._context.get('communication', False)
        vals['payment_type'] = 'transfer'
        vals['destination_journal_id'] = journal.id,
        payment = self.env['account.payment'].with_context(ctx).create(vals)
        payment.post()
        return payment

    def get_move_values(self, name, debit_note, journal, expense_account, expense_amount, amount_total=0.0):
        if self.type == 'third_check':
            credit_account = self[0].journal_id.default_credit_account_id
        else:
            credit_account = self[0].company_id._get_check_account('selled')
        debit_account = journal.default_debit_account_id
        debit = amount_total
        amount_move = amount_total
        line_ids = []
        debit_line_vals1 = False
        if not debit_note:
            debit = amount_move - expense_amount
            debit_line_vals1 = {
                'name': name,
                'account_id': expense_account.id,
                'debit': expense_amount,
                'exclude_from_invoice_tab': True
                # 'amount_currency': self.amount_currency,
                # 'currency_id': self.currency_id.id,
            }
        debit_line_vals = {
            'name': name,
            'account_id': debit_account.id,
            'debit': debit,
            'exclude_from_invoice_tab': True
            # 'amount_currency': self.amount_currency,
            # 'currency_id': self.currency_id.id,
        }
        line_ids.append((0, False, debit_line_vals))
        if debit_line_vals1:
            line_ids.append((0, False, debit_line_vals1))
        credit_line_vals = {
            'name': name,
            'account_id': credit_account.id,
            'credit': amount_move,
            'exclude_from_invoice_tab': True
            # 'amount_currency': self.amount_currency,
            # 'currency_id': self.currency_id.id,
        }
        line_ids.append((0, False, credit_line_vals))
        return {
            'ref': name,
            'journal_id': journal.id,
            'date': fields.Date.today(),
            'line_ids': line_ids,
            'type': 'entry',
        }

    def _check_state_change(self, operation):
        """
        We only check state change from _add_operation because we want to
        leave the user the possibility of making anything from interface.
        Necesitamos este chequeo para evitar, por ejemplo, que un cheque se
        agregue dos veces en un pago y luego al confirmar se entregue dos veces
        On operation_from_state_map dictionary:
        * key is 'to state'
        * value is 'from states'
        """
        self.ensure_one()
        # if we do it from _add_operation only, not from a contraint of before
        # computing the value, we can just read it
        old_state = self.state
        operation_from_state_map = {
            # 'draft': [False],
            'holding': [
                'draft', 'deposited', 'changed', 'delivered', 'transfered'],
            'delivered': ['holding'],
            'deposited': ['holding', 'rejected', 'draft'],
            'changed': ['holding'],
            'handed': ['draft'],
            'transfered': ['holding'],
            'withdrawed': ['draft'],
            'rejected': ['delivered', 'deposited', 'changed', 'handed'],
            'debited': ['handed', 'selled'],
            'returned': ['handed', 'holding'],
            'used': ['draft'],
            'negotiated': ['draft', 'holding'],
            'selled': ['negotiated'],
            'cancel': ['draft'],
            'reclaimed': ['rejected'],
        }
        from_states = operation_from_state_map.get(operation)
        if not from_states:
            raise ValidationError(_(
                'Operation %s not implemented for checks!') % operation)
        if old_state not in from_states:
            raise ValidationError(_(
                'You can not "%s" a check from state "%s"!\n'
                'Check nbr (id): %s (%s)') % (
                    self.operation_ids._fields['operation'].convert_to_export(
                        operation, self),
                    self._fields['state'].convert_to_export(old_state, self),
                    self.name,
                    self.id))

    def handed_reconcile(self, move):
        """
        Funcion que por ahora solo intenta conciliar cheques propios entregados
        cuando se hace un debito o cuando el proveedor lo rechaza
        """

        self.ensure_one()
        if self.state == 'selled':
            debit_account = self.company_id._get_check_account('selled')
            operation = self._get_operation('selled')
        else:
            debit_account = self.company_id._get_check_account('deferred')
            operation = self._get_operation("handed")

        # conciliamos
        if debit_account.reconcile:
            if operation.origin._name == 'account.payment':
                move_lines = operation.origin.move_line_ids
            elif operation.origin._name == 'account.move':
                move_lines = operation.origin.line_ids
            move_lines |= move.line_ids
            move_lines = move_lines.filtered(
                lambda x: x.account_id == debit_account)
            if len(move_lines) != 2:
                raise ValidationError(_(
                    'We have found more or less than two journal items to '
                    'reconcile with check debit.\n'
                    '*Journal items: %s') % move_lines.ids)
            move_lines.reconcile()

    def _get_related_invoice(self):
        self.ensure_one()
        related_inv = False
        for move in self.operation_ids.filtered(lambda o: o.operation == 'holding').sorted(lambda o: o.id):
            if move.origin and getattr(move.origin, 'invoice_ids') and move.origin.invoice_ids:
                related_inv = move.origin.invoice_ids.filtered(lambda x: x.is_invoice())
            elif move.origin and getattr(move.origin, 'reconciled_invoice_ids') and move.origin.reconciled_invoice_ids:
                related_inv = move.origin.reconciled_invoice_ids.filtered(
                    lambda x: x.is_invoice() and x.l10n_latam_document_type_id)
        if related_inv:
            return related_inv[0]
        else:
            return related_inv

    def action_create_debit_note(
            self, operation, partner_type, partner, account, amount=0.0, tax_ids=False):
        self.ensure_one()
        action_date = self._context.get('action_date')

        if partner_type == 'supplier':
            invoice_type = 'in_invoice'
            journal_type = 'purchase'
        else:
            invoice_type = 'out_invoice'
            journal_type = 'sale'

        journal = self.env['account.journal'].search([
            ('company_id', '=', self.company_id.id),
            ('type', '=', journal_type),
        ], limit=1)

        related_inv = self._get_related_invoice()

        # si pedimos rejected o reclamo, devolvemos mensaje de rechazo y cuenta
        # de rechazo
        if operation in ['rejected', 'reclaimed']:
            name = 'Rechazo cheque "%s"' % (self.name)
        # si pedimos la de holding es una devolucion
        elif operation == 'returned':
            name = 'Devolución cheque "%s"' % (self.name)
        elif operation == 'selled':
            name = 'Venta de cheque "%s"' % (self.name)
        else:
            raise ValidationError(_(
                'Debit note for operation %s not implemented!' % (
                    operation)))
        price_unit = self.amount
        if amount > 0.0:
            price_unit = amount

        inv_line_vals = {
            # 'product_id': self.product_id.id,
            'name': name,
            'account_id': account.id,
            'price_unit': price_unit,
            # 'invoice_id': invoice.id,
        }
        if tax_ids:
            inv_line_vals['tax_ids'] = [(6, 0, tax_ids.ids)]

        inv_vals = {
            # this is the reference that goes on account.move.line of debt line
            # 'name': name,
            # this is the reference that goes on account.move
            'rejected_check_id': self.id,
            'ref': name,
            'invoice_date': action_date,
            'invoice_origin': _('Check nbr (id): %s (%s)') % (self.name, self.id),
            'journal_id': journal.id,
            # this is done on muticompany fix
            # 'company_id': journal.company_id.id,
            'partner_id': partner.id,
            'type': invoice_type,
            'debit_origin_id': related_inv and related_inv.id or False,
            'invoice_line_ids': [(0, 0, inv_line_vals)],
        }
        if self.currency_id:
            inv_vals['currency_id'] = self.currency_id.id
        # we send internal_type for compatibility with account_document
        invoice = self.env['account.move'].with_context(
            company_id=journal.company_id.id, force_company=journal.company_id.id,
            internal_type='debit_note').create(inv_vals)
        if not operation == 'selled':
            self._add_operation(operation, invoice, partner, date=action_date)

        return {
            'name': name,
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'account.move',
            'res_id': invoice.id,
            'type': 'ir.actions.act_window',
        }

    def name_get(self):
        result = []
        for record in self:
            name = record.name or ''
            if not record.name:
                name = record.number and str(record.number) or ''
            if record.bank_id:
                name += ' / ' + record.bank_id.name
            if record.owner_name:
                name += ' / ' + record.owner_name
            result.append((record.id, name))
        return result

    def _get_name_from_number(self, number):
        padding = 8
        if len(str(number)) > padding:
            padding = len(str(number))
        return ('%%0%sd' % padding % number)

    def _get_number(self, checkbook_id, number=False):
        if checkbook_id and not checkbook_id.numerate_on_printing:
            sequence = checkbook_id.sequence_id
            if number:
                sequence.sudo().write(
                    {
                        'number_next_actual': number
                    })
            else:
                number = checkbook_id.next_number
            sequence.next_by_id()
        return number

    @api.model
    def create(self, vals):
        checkbook = self.env['account.checkbook']
        if vals.get('checkbook_id') and not vals.get('number', False):
            checkbook_id = checkbook.browse(vals.get('checkbook_id'))
            vals['number'] = self._get_number(checkbook_id)
        if (not vals.get('name', False) or vals['name'] == '00000000')and vals['number']:
            vals['name'] = self._get_name_from_number(vals['number'])
        return super(AccountCheck, self).create(vals)

    def _update_check_cancel(self, payment):
        for rec in self:
            if rec.state in ['holding'] and rec.type == 'third_check' and payment.payment_type in ['inbound',
                                                                                'transfer']:
                operation = rec._get_init_operation('holding', True)
                if operation and operation.origin and operation.origin._name == 'account.payment':
                    rec.with_context(no_check=True)._add_operation('draft', payment, operation.partner_id, date=operation.date)
                if payment.payment_method_code == 'delivered_third_check' and payment.payment_type == 'transfer':
                    inbound_method = (payment.destination_journal_id.inbound_payment_method_ids)
                    if len(inbound_method) == 1 and (
                            inbound_method.code == 'received_third_check'):
                        rec.journal_id = operation.origin.journal_id.id

            elif rec.state in ['changed', 'deposited'] and rec.type == 'third_check' and payment.payment_type in ['transfer']:
                operation = rec._get_operation(rec.state, False)
                if operation and operation.origin and operation.origin._name == 'account.payment':
                    operation = rec._get_init_operation('holding', True)
                    payment_link = operation.origin if operation.origin._name == 'account.payment' else payment
                    rec.with_context(no_check=True)._add_operation('holding', payment_link, operation.partner_id, date=operation.date)

            elif rec.state in ['delivered'] and rec.type == 'third_check' and payment.payment_type == 'outbound':
                operation = rec._get_operation('delivered', True)
                if operation and operation.origin and operation.origin._name == 'account.payment':
                    operation_holding = rec._get_init_operation('holding', True)
                    payment_link = operation_holding.origin if operation_holding.origin and operation_holding.origin._name == 'account.payment' else False
                    rec.with_context(no_check=True)._add_operation('holding', payment_link, operation_holding.partner_id, date=operation.date)

            elif rec.state in ['handed'] and rec.type == 'issue_check':
                operation = rec._get_operation('handed', True)
                if operation and operation.origin and operation.origin._name == 'account.payment':
                    rec.with_context(no_check=True)._add_operation('draft', payment, operation.partner_id, date=operation.date)

            elif rec.state in ['deposited'] and rec.type == 'issue_check' and payment.payment_type == 'transfer':
                operation = rec._get_operation('deposited', False)
                if operation and operation.origin and operation.origin._name == 'account.payment':
                    rec.with_context(no_check=True)._add_operation('draft', payment, operation.partner_id, date=operation.date)

            elif rec.state in ['withdrawed'] and rec.type == 'issue_check' and payment.payment_type == 'transfer':
                operation = rec._get_operation('withdrawed', False)
                if operation and operation.origin and operation.origin._name == 'account.payment':
                    rec.with_context(no_check=True)._add_operation('draft', payment, operation.partner_id, date=operation.date)

            elif rec.state in ['rejected'] and payment.payment_type == 'transfer' and self._context.get('no_check_op', False):
                return
            # elif rec.state in ['transfered'] and payment.payment_type == 'transfer':
            #     operation = self.operation_ids.search([('check_id', '=', self.id)], limit=2)
            #     if operation and operation[0].origin and operation[0].origin._name == 'account.payment':
            #         rec.with_context(no_check=True)._add_operation(operation[1].operation, operation[1].origin, operation.partner_id,
            #                               date=operation.date)
            else:
                raise UserError(
                    _(
                        'You cannot cancel a payment with a %s check because you are in another state unrelated to the payment, this check operation must be changed.') % rec.number)

    def _get_init_operation(self, operation, partner_required=False):
        self.ensure_one()
        op = self.operation_ids.search([
            ('check_id', '=', self.id), ('operation', '=', operation)], order='date,id asc',
            limit=1)
        if partner_required:
            if not op.partner_id:
                raise ValidationError((
                    'The %s (id %s) operation has no partner linked.'
                    'You will need to do it manually.') % (operation, op.id))
        return op

    def _get_operation(self, operation, partner_required=False):
        self.ensure_one()
        op = self.operation_ids.search([
            ('check_id', '=', self.id), ('operation', '=', operation)],
            limit=1)
        if partner_required:
            if not op.partner_id:
                raise ValidationError((
                                          'The %s (id %s) operation has no partner linked.'
                                          'You will need to do it manually.') % (operation, op.id))
        return op

    def bank_debit(self):
        self.ensure_one()
        if self.state in ['handed']:
            payment_values = self.get_payment_values(self.journal_id)
            payment = self.env['account.payment'].with_context(
                default_name=_('Check "%s" debit') % (self.name),
                force_account_id=self.company_id._get_check_account(
                    'deferred').id,
            ).create(payment_values)
            self.post_payment_check(payment)
            self.handed_reconcile(payment.move_line_ids.mapped('move_id'))
            self._add_operation('debited', payment, date=payment.payment_date)

    def bank_debit(self):
        for rec in self:
            if rec.state in ['handed', 'selled'] and rec.type == 'issue_check':
                if rec.state == 'selled':
                    vals = rec.get_bank_vals(
                        'bank_debit_selled', rec.journal_id)
                else:
                    vals = rec.get_bank_vals(
                        'bank_debit', rec.journal_id)
                action_date = self._context.get('action_date')
                vals['date'] = action_date
                move = self.env['account.move'].create(vals)
                rec.handed_reconcile(move)
                move.post()
                rec._add_operation('debited', move, date=action_date)

    def get_bank_vals(self, action, journal):
        self.ensure_one()
        if action == 'bank_debit':
            credit_account = journal.default_debit_account_id
            debit_account = self.company_id._get_check_account('deferred')
            name = _('Check "%s" debit') % (self.name)
        elif action == 'bank_debit_selled':
            credit_account = journal.default_debit_account_id
            debit_account = self.company_id._get_check_account('selled')
            name = _('Check "%s" debit') % (self.name)
        # elif action == 'bank_reject':
        #     credit_account = journal.default_debit_account_id
        #     debit_account = self.company_id._get_check_account('rejected')
        #     name = _('Check "%s" rejection') % (self.name)
        else:
            raise ValidationError(_(
                'Action %s not implemented for checks!') % action)
        debit_line_vals = {
            'name': name,
            'account_id': debit_account.id,
            'debit': self.amount,
            'amount_currency': self.amount_company_currency * -1 if self.currency_id != self.company_currency_id else 0.0,
            'currency_id': self.currency_id.id if self.currency_id != self.company_currency_id else False,
        }
        credit_line_vals = {
            'name': name,
            'account_id': credit_account.id,
            'credit': self.amount,
            'amount_currency': self.amount_company_currency * -1 if self.currency_id != self.company_currency_id else 0.0,
            'currency_id': self.currency_id.id if self.currency_id != self.company_currency_id else False,
        }
        return {
            'ref': name,
            'journal_id': journal.id,
            'date': fields.Date.today(),
            'line_ids': [
                (0, False, debit_line_vals),
                (0, False, credit_line_vals)],
        }

    def re_deposited(self):
        self.ensure_one()
        if self.state in ['rejected']:
            operation = self._get_operation(self.state)
            op_deposited = self.operation_ids.search([
                ('check_id', '=', self.id), ('id', '<', operation.id)],
                limit=1)
            action_date = self._context.get('action_date')
            if action_date < operation.date:
                raise ValidationError(_(
                    'The re-deposit date must be greater than the banks rejection date.'))
            journal = operation.origin.journal_id
            if op_deposited.operation == 'deposited':
                if operation.origin and operation.origin._name == 'account.payment':
                    payment_vals = self.get_payment_values(journal)
                    payment_vals['payment_type'] = 'inbound'
                    payment = self.env['account.payment'].with_context(
                        default_name=_('Check "%s" re-deposited') % (self.name),
                        force_account_id=self.company_id._get_check_account(
                            'rejected').id,
                    ).create(payment_vals)
                    self.post_payment_check(payment)
                    self._add_operation('deposited', payment, date=payment.payment_date)
                #         and \
                #         op_deposited.origin._name == 'account.payment':
                #     operation.origin.with_context(no_check_op=True).cancel()
                #     # move_account_line = op_deposited.origin.move_line_ids
                #     # move_account_line.remove_move_reconcile()
                #     op_deposited.origin.with_context(no_check_op=True).cancel()
                #     self.with_context(no_check_op=True, journal=journal).deposited()
                # elif operation.origin and operation.origin._name == 'account.payment' and \
                #         op_deposited.origin._name == 'account.move':
                #     operation.origin.button_cancel()
                #     op_deposited.origin.button_cancel()
                #     self.with_context(no_check_op=True, journal=journal).deposited()
                else:
                    raise ValidationError(_(
                        'This operation is only allowed if the rejection of the check was from the bank.'))
            else:
                raise ValidationError(_(
                    'This operation is only allowed if the rejection of the check was from the bank.'))
        else:
            raise ValidationError(_(
                'This operation is only allowed if the rejection of the check was from the bank.'))


import sys
from PyQt5 import QtWidgets, QtGui, QtWidgets
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *

# Global Variables for stack index
LOGIN_REGISTER = 0
HOME = 1

#comment

# To change between Pages - self.parent().setCurrentIndex(HOME)
#          Changes depending on what you want to switch to ^
class LoginRegisterWidget(QtWidgets.QWidget):  # Normal LoginPage
    def __init__(self, parent):
        QtWidgets.QWidget.__init__(self, parent)
        self.setup()

    def setup(self):
        # Our Layout
        self.box_layout = QtWidgets.QVBoxLayout()
        # Login Button
        self.login_button = QtWidgets.QPushButton("Login", self)
        self.login_button.clicked.connect(self.login_click)

        # Register Button
        self.register_button = QtWidgets.QPushButton("Register", self)
        self.register_button.clicked.connect(self.register_click)

        # Adding buttons to layout
        self.box_layout.addWidget(self.login_button)
        self.box_layout.addWidget(self.register_button)

        self.setLayout(self.box_layout)

    def login_click(self):
        self.login_dialog = QDialog()
        self.login_dialog.setWindowTitle("Login")
        self.login_dialog.exec_()

    def register_click(self):
        self.register_dialog = QDialog()
        self.register_dialog.setWindowTitle("Register")
        self.register_dialog.exec_()

class HomeWidget(QtWidgets.QWidget): # HomeWidget for joining, creating match
    def __init__(self, parent):
        QtWidgets.QWidget.__init__(self, parent)
        self.setup()

    def setup(self):
        # Our Layout
        self.box_layout = QtWidgets.QVBoxLayout()
        # Create Match Button
        self.create_button = QtWidgets.QPushButton("Create Match", self)
        self.create_button.clicked.connect(self.create_click)

        # Join Match Button
        self.join_button = QtWidgets.QPushButton("Join Match", self)
        self.join_button.clicked.connect(self.join_click)

        # Current Match Button
        self.current_button = QtWidgets.QPushButton("Current Match", self)
        self.current_button.clicked.connect(self.current_click)

        # Adding buttons to layout
        self.box_layout.addWidget(self.create_button)
        self.box_layout.addWidget(self.join_button)
        self.box_layout.addWidget(self.current_button)

        self.setLayout(self.box_layout)

    def create_click(self):
        self.garbage = 0

    def join_click(self):
        self.garbage = 0

    def current_click(self):
        self.garbage = 0

# INDEXES FOR STACK
# 0 - LoginRegisterWidget
# 1 - HomeWidget

class MainWindow(QtWidgets.QMainWindow):  # Main Window
    def __init__(self):
        QtWidgets.QMainWindow.__init__(self)
        self.setup()

    def setup(self):
        self.setGeometry(0, 0, 700, 700)
        self.setFixedSize(700, 700)  # eliminating resizing
        self.setWindowTitle("Fantasy Football")

        self.PageStack = QStackedWidget(self)

        # Creating our pages for the stack
        self.main_page = LoginRegisterWidget(self)
        self.home_page = HomeWidget(self)

        # Adding to stack; Pages indexed in order of addition
        self.PageStack.addWidget(self.main_page) # Index 0
        self.PageStack.addWidget(self.home_page) # Index 1

        self.setCentralWidget(self.PageStack)
        self.show()

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    main_window = MainWindow()
    main_window.show()
    app.exec_()


"""EHF_ACORNSAT.py calculates heatwave stats from ACORNSAT station data
"""
import glob
import numpy as np
import pandas as pd
import datetime as dt
import os
import compute_EHFheatwaves_station as comp
import netCDF4

def dstr_to_dt(string):
    """Convert a string do a datetime object with the format %Y%m%d
    """
    return dt.datetime.strptime(string, '%Y%m%d')

def load_acornsat(fnamelist):
    """Load the acornsat data as a pandas dataframe object.
    """
    station_no = 1
    dataframe = pd.read_table('acorn.sat.maxT.002012.daily.txt',
            sep=' ',
            skipinitialspace=True,
            skiprows=1,
            index_col=0,
            header=None,
            names=['Dates',station_no],
            na_values=99999.9,
            parse_dates=True,
            date_parser=dstr_to_dt)
    dates = dataframe.index
    data = pd.DataFrame(index=dates)
    for file_name in fnamelist:
        station_no = file_name[15:21]
        dataframe = pd.read_table(file_name,
                sep=' ',
                skipinitialspace=True,
                skiprows=1,
                index_col=0,
                header=None,
                names=['Dates',station_no],
                na_values=99999.9,
                parse_dates=True,
                date_parser=dstr_to_dt)
        data[station_no] = dataframe
    return data

def do_detrend(data):
    '''do_detrend detrends station data using an OLS linear
    regression while ignoring missing values.

    Arguments
    data -- station data with time on the 0 axis and station on the 1 axis

    Returs
    data -- detrended data
    '''
    from numpy import array, arange, nan, ones, empty, where, isnan
    from scipy.stats import linregress
    for station in range(data.shape[1]):
        # Select the station to detrend
        a_series = data[:,station]
        # Create an empty array and independant variable
        result = ones(len(a_series))*nan
        x = arange(len(a_series))
        # Locate nans and remove from the series and independant variable
        nan_index = isnan(a_series)
        a_series = a_series[where(nan_index==False)]
        x = x[where(nan_index==False)]
        # Detrend
        slope, intcpt, r, p, err = linregress(x, a_series)
        trend = x*slope + intcpt
        result[nan_index==False] = a_series - trend
        # Return series to data matrix
        data[:,station] = result
    return data

if __name__ == '__main__':
    syear = 1911
    eyear = 2014

    # Define the file names.
    os.chdir('/srv/ccrc/data35/z5032520/ACORNSAT/')
    file_list = sorted(glob.glob('acorn.sat.maxT.??????.daily.txt'))
    thres_file = 'EHF_heatwaves_ACORNSAT_1961_1990_summer.nc'
    #thres_file=None

    # Load the files.
    tmax = load_acornsat(file_list)

    # Select the analysis period
    dates = tmax.index
    tmax = tmax[(dates.year>=syear)&(dates.year<=eyear)]
    dates = tmax.index

    # Find the latitude and longitudes for each station
    latslons = np.genfromtxt('BoM_STN_latlon.txt')
    station_ids = tmax.columns.values
    lats = np.zeros(len(station_ids))
    lons = np.zeros(len(station_ids))
    for i, station in enumerate(station_ids):
        station = float(station)
        lats[i] = latslons[np.where(latslons[:,0]==station)[0],1]
        lons[i] = latslons[np.where(latslons[:,0]==station)[0],2]

    # Convert to a regular numpy array for compute_EHF
    tmax = np.array(tmax)
    # Detrend
    tmax = do_detrend(tmax)

    # Repeat for tmin
    file_list = sorted(glob.glob('acorn.sat.minT.??????.daily.txt'))
    tmin = load_acornsat(file_list)
    dates = tmin.index
    tmin = tmin[(dates.year>=syear)&(dates.year<=eyear)]
    dates = tmin.index
    tmin = np.array(tmin)
    tmin = do_detrend(tmin)
#    import numpy as np
#    np.save('tminfile', tmin)
#    np.save('tmaxfile', tmax)
    # Caclulate heatwave stats
    HWA, HWM, HWF, HWN, HWD, HWT, pct = comp.compute_EHF(tmax,
            tmin,
            dates=dates,
            modified=True,
            thres_file=None,
            season='summer',
            bsyear=1961,
            beyear=1990)

    # Save to a netCDF file.
    years = range(syear,eyear+1,1)
    season = 'summer'
    ofile_name = 'EHF_heatwaves_ACORNSAT_%s_%s_%s.nc' %(syear, eyear, season)
    outfile = netCDF4.Dataset(ofile_name, mode='w')

    # Create dimentions
    outfile.createDimension('time',len(HWA[:,0]))
    outfile.createDimension('day',365)
    outfile.createDimension('station', len(HWA[0,:]))

    # Create variables
    otime = outfile.createVariable("time", 'i', dimensions=('time'))
    setattr(otime, "long_name", "Time")
    setattr(otime, "units", "year")
    setattr(otime, "standard_name", "time")

    oday = outfile.createVariable("day", 'i', dimensions=('day'))
    setattr(oday, "long_name", "Day of Year")
    setattr(oday, "units", "days since 0000-01-01")

    olat = outfile.createVariable("lat",'f', dimensions=('station'))
    setattr(olat, "long_name", "Latitude")
    setattr(olat, "units", "degrees_north")
    setattr(olat, "standard_name", "latitude")

    olon = outfile.createVariable("lon",'f',dimensions=('station'))
    setattr(olon, "long_name", "Longitude")
    setattr(olon, "units", "degrees_east")
    setattr(olon, "standard_name", "longitude")

    ostation = outfile.createVariable("station", 'f', dimensions=('station'))
    setattr(ostation, "long_name", "Station ID")

    oHWA = outfile.createVariable("HWA", 'f', dimensions=(['time','station']))
    setattr(oHWA, "long_name"," Heat Wave Amplitude")
    setattr(oHWA, "units", "C2")
    setattr(oHWA, "description", "Peak of the hottest heatwave")

    oHWM = outfile.createVariable("HWM", 'f', dimensions=(['time','station']))
    setattr(oHWM, "long_name", "Heat Wave Magnitude")
    setattr(oHWM, "units", "C2")
    setattr(oHWM, "description", "Average of heatwave magnitude")

    oHWF = outfile.createVariable("HWF", 'f', dimensions=(['time','station']))
    setattr(oHWF, "long_name", "Heat Wave Frequency")
    setattr(oHWF, "units", "days")
    setattr(oHWF, "description", "Number of heat wave days")

    oHWN = outfile.createVariable("HWN", 'f', dimensions=(['time','station']))
    setattr(oHWN, "long_name", "Heat Wave Number")
    setattr(oHWN, "units", "heatwaves")
    setattr(oHWN, "description", "Number of heat waves")

    oHWD = outfile.createVariable("HWD", 'f', dimensions=(['time','station']))
    setattr(oHWD, "long_name", "Heat Wave Duration")
    setattr(oHWD, "units", "days")
    setattr(oHWD, "description", "Duration of the longest heat wave")

    oHWT = outfile.createVariable("HWT", 'f', dimensions=(['time','station']))
    setattr(oHWT, "long_name","Heat Wave Timing")
    if season == 'summer':
        setattr(oHWT, "description",
                "First heat wave day of the year from 1st of Nov")
    elif season == 'yearly':
        setattr(oHWT, "description",
                "First heat wave day of the year from 1st of Jul")
    setattr(oHWT, "units", "day")

    opct = outfile.createVariable("PRCTILE90", 'f', dimensions=(['day','station']))
    setattr(opct, "long_name", "90th Percentile")
    setattr(opct, "units", "C")
    setattr(opct, "description", "90th percentile used as threshold")

    ostids = outfile.createVariable("stationid", 'i', 'station')
    setattr(opct, "long_name", "Station ID")
    setattr(opct, "cf_role", "timeseries_id")

    # Transfer data
    otime[:] = years
    oday[:] = range(1,366,1)
    olat[:] = lats
    olon[:] = lons
    ostation[:] = station_ids
    oHWA[:] = HWA
    oHWM[:] = HWM
    oHWF[:] = HWF
    oHWN[:] = HWN
    oHWD[:] = HWD
    oHWT[:] = HWT
    opct[:] = pct
    ostids[:] = station_ids

    # Global attributes
    setattr(outfile, "author", "Tammas Loughran")
    setattr(outfile, "contact", "t.loughran@student.unsw.edu.au")
    setattr(outfile, "featureType", "timeSeries")
    setattr(outfile, "date", dt.datetime.today().strftime('%Y-%m-%d'))
    setattr(outfile, "script", "EHF_ACORNSAT.py")

    outfile.close()


from itertools import combinations
class Solution(object):
    def numIdenticalPairs(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        pairs = 0
        dic = {}
        for i in range(len(nums)):
            if nums[i] in dic:
                (dic[nums[i]]).append(i)
            else:
                dic[nums[i]]=[i]
        for _,value in dic.items():
            pairs+=len(list(combinations(value,2)))
        return pairs


"""
This module tests the interaction of Kafka with Zookeeper with authentication enabled
"""
import logging
import uuid
import pytest

import shakedown

import sdk_auth
import sdk_cmd
import sdk_hosts
import sdk_install
import sdk_marathon
import sdk_repository
import sdk_utils

from tests import auth
from tests import config
from tests import test_utils

log = logging.getLogger(__name__)

@pytest.fixture(scope='module', autouse=True)
def kafka_principals():
    fqdn = "{service_name}.{host_suffix}".format(service_name=config.SERVICE_NAME,
                                                 host_suffix=sdk_hosts.AUTOIP_HOST_SUFFIX)

    brokers = [
        "kafka-0-broker",
        "kafka-1-broker",
        "kafka-2-broker",
    ]

    principals = []
    for b in brokers:
        principals.append("kafka/{instance}.{domain}@{realm}".format(
            instance=b,
            domain=fqdn,
            realm=sdk_auth.REALM))

    principals.append("client@{realm}".format(realm=sdk_auth.REALM))

    yield principals

def get_node_principals():
    """Get a list of zookeeper principals for the agent nodes in the cluster"""
    principals = []

    agent_ips = shakedown.get_private_agents()
    agent_dashed_ips = list(map(
        lambda ip: "ip-{dashed_ip}".format(dashed_ip="-".join(ip.split("."))), agent_ips))
    for b in agent_dashed_ips:
        principals.append("zookeeper/{instance}.{domain}@{realm}".format(
            instance=b,
            # TODO(elezar) we need to infer the region too
            domain="us-west-2.compute.internal",
            realm=sdk_auth.REALM))

    return principals

@pytest.fixture(scope='module', autouse=True)
def zookeeper_principals():
    zk_fqdn = "{service_name}.{host_suffix}".format(service_name="kafka-zookeeper",
                                                    host_suffix=sdk_hosts.AUTOIP_HOST_SUFFIX)

    zk_ensemble = [
        "zookeeper-0-server",
        "zookeeper-1-server",
        "zookeeper-2-server",
    ]

    principals = []
    for b in zk_ensemble:
        principals.append("zookeeper/{instance}.{domain}@{realm}".format(
            instance=b,
            domain=zk_fqdn,
            realm=sdk_auth.REALM))

    principals.extend(get_node_principals())
    yield principals

@pytest.fixture(scope='module', autouse=True)
def kerberos(configure_security, kafka_principals, zookeeper_principals):
    try:
        principals = []
        principals.extend(kafka_principals)
        principals.extend(zookeeper_principals)

        kerberos_env = sdk_auth.KerberosEnvironment()
        kerberos_env.add_principals(principals)
        kerberos_env.finalize()

        yield kerberos_env

    finally:
        kerberos_env.cleanup()

@pytest.fixture(scope='module')
def zookeeper_server(kerberos):
    service_kerberos_options = {
        "service": {
            "name": "kafka-zookeeper",
            "security": {
                "kerberos": {
                    "enabled": True,
                    "kdc_host_name": kerberos.get_host(),
                    "kdc_host_port": int(kerberos.get_port()),
                    "keytab_secret": kerberos.get_keytab_path(),
                }
            }
        }
    }

    # TODO: Remove once kafka-zookeeper is available in the universe
    zookeeper_stub = "https://infinity-artifacts.s3.amazonaws.com/permanent/kafka-zookeeper/20171128-113715-h2qHdoIXKgAEgPOy/stub-universe-kafka-zookeeper.json"
    stub_urls = sdk_repository.add_stub_universe_urls([zookeeper_stub, ])

    try:
        sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
        sdk_install.install(
            "beta-kafka-zookeeper",
            "kafka-zookeeper",
            6,
            additional_options=service_kerberos_options,
            timeout_seconds=30 * 60)

        yield {**service_kerberos_options, **{"package_name": "beta-kafka-zookeeper"}}

    finally:
        sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
        sdk_repository.remove_universe_repos(stub_urls)

@pytest.fixture(scope='module', autouse=True)
def kafka_server(kerberos, zookeeper_server):

    # Get the zookeeper DNS values
    zookeeper_dns = sdk_cmd.svc_cli(zookeeper_server["package_name"],
                                    zookeeper_server["service"]["name"],
                                    "endpoint clientport", json=True)["dns"]

    service_kerberos_options = {
        "service": {
            "name": config.SERVICE_NAME,
            "security": {
                "kerberos": {
                    "enabled": True,
                    "enabled_for_zookeeper": True,
                    "kdc_host_name": kerberos.get_host(),
                    "kdc_host_port": int(kerberos.get_port()),
                    "keytab_secret": kerberos.get_keytab_path(),
                }
            }
        },
        "kafka": {
            "kafka_zookeeper_uri": ",".join(zookeeper_dns)
        }
    }

    sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
    try:
        sdk_install.install(
            config.PACKAGE_NAME,
            config.SERVICE_NAME,
            config.DEFAULT_BROKER_COUNT,
            additional_options=service_kerberos_options,
            timeout_seconds=30 * 60)

        yield {**service_kerberos_options, **{"package_name": config.PACKAGE_NAME}}
    finally:
        sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)

@pytest.fixture(scope='module', autouse=True)
def kafka_client(kerberos, kafka_server):

    brokers = sdk_cmd.svc_cli(
        kafka_server["package_name"],
        kafka_server["service"]["name"],
        "endpoint broker", json=True)["dns"]

    try:
        client_id = "kafka-client"
        client = {
            "id": client_id,
            "mem": 512,
            "user": "nobody",
            "container": {
                "type": "MESOS",
                "docker": {
                    "image": "elezar/kafka-client:latest",
                    "forcePullImage": True
                },
                "volumes": [
                    {
                        "containerPath": "/tmp/kafkaconfig/kafka-client.keytab",
                        "secret": "kafka_keytab"
                    }
                ]
            },
            "secrets": {
                "kafka_keytab": {
                    "source": kerberos.get_keytab_path(),

                }
            },
            "networks": [
                {
                    "mode": "host"
                }
            ],
            "env": {
                "JVM_MaxHeapSize": "512",
                "KAFKA_CLIENT_MODE": "test",
                "KAFKA_TOPIC": "securetest",
                "KAFKA_BROKER_LIST": ",".join(brokers)
            }
        }

        sdk_marathon.install_app(client)
        yield {**client, **{"brokers": list(map(lambda x: x.split(':')[0], brokers))}}

    finally:
        sdk_marathon.destroy_app(client_id)

@pytest.mark.dcos_min_version('1.10')
@sdk_utils.dcos_ee_only
@pytest.mark.sanity
def test_client_can_read_and_write(kafka_client, kafka_server):
    client_id = kafka_client["id"]

    auth.wait_for_brokers(kafka_client["id"], kafka_client["brokers"])

    topic_name = "authn.test"
    sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"],
                    "topic create {}".format(topic_name),
                    json=True)

    test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name)

    message = str(uuid.uuid4())

    assert auth.write_to_topic("client", client_id, topic_name, message)

    assert message in auth.read_from_topic("client", client_id, topic_name, 1)


from tkinter import *
from tkinter import filedialog
import pandas as pd
import matplotlib.pyplot as plt
import astropy.io.ascii as ascii
import astropy.table as table
import xlsxwriter
from scipy.signal import savgol_filter

# This function opern window to open several excel files and return a list of names of the files
def get_files():
    root = Tk()
    root.withdraw()
    root.filename = filedialog.askopenfilenames(initialdir = "/",title = "Select file",filetypes = (("asci files","*.asc"),("all files","*.*")))
    root.destroy()
    return root.filename

frame = get_files()

lista = []
for j in range(1, len(frame)+1):
    lista.append('df_%s' % j)

print(lista)

dic = {}
for i, j in zip(frame, lista):
    a = ascii.read(i, data_start=54) #54, 23
    dic[j] = pd.DataFrame(a.as_array())

#df = *dataframe with identifiers for rows and dates for columns*
#for date in list_of_dates:
#    df['New Column Name'] = *a new column with a new date to be added to the df*

def add_columns(dic,lista_nombres):
    df_0 = dic["df_1"]["col1"]
    n = 0
    for i in lista:
        c = dic[i]['col2']
        s_c = savgol_filter(c, 51, 3)
        s_c = pd.DataFrame(s_c, columns= ["df_"+ str(n)])
        df = pd.concat([df_0, s_c], axis=1)
        df_0 = df
        n += 1
    return df_0

dataframe = add_columns(dic,lista)

plt.plot(dataframe["col1"], dataframe.iloc[:,2:-1])
plt.show()

#This code plot df.iloc[:,0] and df.iloc[:,1] to every df_i
#df.iloc[:,0] is the wavelength and df.iloc[:,1] is the property
for i in lista:
    d = dic[i]
    plt.plot(d['col1'], d['col2'])
    plt.xlabel('Wavelength')
    plt.ylabel('Fluorescence')
    plt.xlim(332,600)

#archivo = str(input("Ingrese el nombre del archivo: "))
#archivo_0 = archivo + ".xlsx"

#this function open windows to save the file in xlsx format
def save_file():
    root = Tk()
    root.withdraw()
    root.filename = filedialog.asksaveasfilename(initialdir = "/",title = "Select file",filetypes = (("Excel files","*.xlsx"),("all files","*.*")))
    root.destroy()
    return root.filename

archivo_1 = save_file() + ".xlsx"
print(archivo_1)

with pd.ExcelWriter(archivo_1, engine='xlsxwriter') as writer:
    dataframe.to_excel(writer, sheet_name="datos_titulacion")


from flask import Flask
from flask import render_template
from flask import abort, redirect, url_for
import subprocess

app = Flask(__name__)
home = '/home/cloudy/Github/automation-sop-ajk-using-ansible/'

@app.route('/')
def hello_world():
    return render_template('index.html')

@app.route('/start')
def start():
    command = "ansible-playbook -i %shosts %sservices.yml --tags 'start'" % ( home, home )
    print(command)
    subprocess.call(command, shell=True)
    return redirect('/')

@app.route('/restart')
def restart():
    command = "ansible-playbook -i %shosts %sservices.yml --tags 'restart'" % ( home, home )
    print(command)
    subprocess.call(command, shell=True)
    return redirect('/')

@app.route('/stop')
def stop():
    command = "ansible-playbook -i %shosts %sservices.yml --tags 'stop'" % ( home, home )
    print(command)
    subprocess.call(command, shell=True)
    return redirect('/')


import requests
import collections
import datetime
import matplotlib.pyplot as plt
import matplotlib.dates

params = {
    'action': 'query',
    'format': 'json',
    'prop':   'revisions',
    'rvlimit': 500,
    'rvstart':'2022-01-26T23:59:59Z',
    'rvend':  '2016-01-01T00:00:00Z',
    'rvprop': 'timestamp|user',
    'rvexcludeuser': 'Cyberbot_I', # Can only exclude one user. This bot posted 500 edits in 1 day
    'titles': 'Russo-Ukrainian War'
}
banned_users = ['Cyberbot_I', 'Aleksandr Grigoryev', 'RenatUK', '178.35.34.32']

timestamp_datetimes = []

while True:
    response = requests.get('https://en.wikipedia.org/w/api.php', params=params).json()
    timestamps = [rev['timestamp'] for rev in filter(lambda e: e['user'] not in banned_users, response['query']['pages']['42085878']['revisions'])]
    timestamp_datetimes.extend(map(lambda s: datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ'), timestamps))
    if 'continue' not in response:
        print('Reached final targeted revision')
        break
    else:
        if 'rvstart' in params:
            del params['rvstart']
        params['rvstartid'] = response['continue']['rvcontinue'].split('|')[1]

date_counts = collections.Counter(map(lambda stamp: stamp.date(), timestamp_datetimes))

dates = list(date_counts.keys())
values = list(date_counts.values())

print("no. of days =", dates[1] - dates[-1] + datetime.timedelta(days=1))

days_rolling_avg = 15
rolling_avg = {}
diff = datetime.timedelta(days=days_rolling_avg//2)
base_day = dates[0] - diff
while base_day >= dates[-1] + diff:
    date_range = [base_day + diff - datetime.timedelta(days=x) for x in range(days_rolling_avg)]
    date_values = list(map(lambda date: date_counts[date], date_range))
    rolling_avg[base_day] = sum(date_values)/len(date_values)
    base_day -= datetime.timedelta(days=1)

fig = plt.figure(figsize=(10.00, 6.50))
plt.style.use('fivethirtyeight')

ax1 = fig.add_subplot(111)
ax1.grid(True)
ax1.set_ylim([0.0, 18])
ax1.yaxis.set_major_locator(plt.MultipleLocator(base=2))
ax1.bar(matplotlib.dates.date2num(dates), values, color='tab:blue', width=4.0, alpha=0.20, label='Day-by-day')
for tl in ax1.get_yticklabels():
    tl.set_color('tab:blue')

ax2 = ax1.twinx()
ax2.grid(False)
ax2.set_ylim([0.0, 9.0])
ax2.yaxis.set_major_locator(plt.MultipleLocator(base=1.0))
ax2.plot_date(matplotlib.dates.date2num(list(rolling_avg.keys())), list(rolling_avg.values()), 'r-', linewidth=0.9, label=f'{days_rolling_avg}-day rolling average')
for tl in ax2.get_yticklabels():
    tl.set_color('r')

ax1.legend(loc='upper left')
ax2.legend(loc='upper right')
plt.title(label='Edits to Wikipedia article $\it{Russo-Ukrainian}$ $\it{War}$', pad=35)
fig.savefig('figure.png', bbox_inches='tight', pad_inches=0.45)
plt.show()


"""
DAG utility class
"""
from datetime import (datetime, timedelta,)

from helpers.configs import (
    DAG_EMAIL, DAG_EMAIL_ON_FAILURE, DAG_EMAIL_ON_RETRY,
    DAG_NO_OF_RETRIES, DAG_OWNER, DAG_RETRY_DELAY_IN_MINS,
)

class DagUtility:
    def __init__(self):
        pass

    @staticmethod
    def get_dag_default_args():
        return {
            'owner': DAG_OWNER,
            'depends_on_past': False,
            'start_date': datetime(2019, 10, 21),
            'email': [DAG_EMAIL],
            'email_on_failure': DAG_EMAIL_ON_FAILURE,
            'email_on_retry': DAG_EMAIL_ON_RETRY,
            'catchup_by_default': False,
            'retries': DAG_NO_OF_RETRIES,
            'retry_delay': timedelta(minutes=DAG_RETRY_DELAY_IN_MINS),
        }


from flask import Flask, render_template, request
# from flask_lt import run_with_lt
# from a2weboriginal import Query
from a2web import Query
import os
import time

Q = Query()
app = Flask(__name__)
# run_with_lt(app)
last_time = {}

barGraphExists = os.path.exists('./global_bargraph.png')
wordCloudExists = os.path.exists('./global_word_cloud.png')

if not (barGraphExists) and not (wordCloudExists):
    # Q.global_visualization(Q.topics_to_doc, Q.cleaned_docs)
    pass

@app.route('/', methods=['GET', 'POST'])
def home():
    if request.method == 'GET':
        result = None
        query = None
        model = None
        length = None
        visualdata = None

    if request.method == 'POST':
        client_ip = request.remote_addr
        current_time = int(time.time())
        time_difference = current_time - last_time.get(client_ip, 0)
        if time_difference < 5:
            return 'Form submission rate exceeded.'
        else:
            last_time[client_ip] = current_time
            data = request.form
            query = data.get("query")
            model = data.get("model")
            print(query, model)
            result = Q.search(query, model)
            length = len(result["result"])
            visualdata = result["query_visualizations"]
            if model == "jdrs":
                model = "JDRS"
    return render_template('search.html', result=result, query=query, model=model, length=length, visualdata=visualdata)

@app.route('/test', methods=['GET'])
def test():
    return render_template('test1.html')

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=int(
        os.environ.get("FLASK_RUN_PORT", "80")), debug=True)


import hashlib
from datetime import datetime
from flask import Flask
from flask import request
from copy import deepcopy
import json
import requests
node = Flask(__name__)

class Block:
    def __init__(self, index, timestamp, data, previous_hash):
        self.index = index
        self.timestamp = timestamp
        self.data = data
        self.previous_hash = previous_hash
        self.hash = self.hash_block()

    def hash_block(self):
        sha = hashlib.sha256()
        input_str = str(self.index) +\
                    str(self.timestamp) +\
                    str(self.data) +\
                    str(self.previous_hash)
        input_bytes = bytes(input_str, 'utf-8')
        sha.update(input_bytes)
        return sha.hexdigest()

def create_genesis_block():
    # Manually construct a block with
    # index zero and arbitrary previous hash
    return Block(0, datetime.now(),
                 {"proof-of-work": 9,
                  "transactions": None},
                 "0")

# A completely random address of the owner of this node
miner_address = "q3nf394hjg-random-miner-address-34nf3i4nflkn3oi"

blockchain = []
blockchain.append(create_genesis_block())
this_nodes_transactions = []
peer_nodes = []

mining = True

@node.route("/txion", methods=["POST"])
def transactions():
    new_txion = request.get_json()
    this_nodes_transactions.append(new_txion)
    print("New transaction")
    print("FROM: {}".format(new_txion["from"]))
    print("TO: {}".format(new_txion["to"]))
    print("AMOUNT: {}\n".format(new_txion["amount"]))
    return "Transaction submission successful\n"

@node.route("/blocks", methods=["GET"])
def get_blocks():
    chain_to_send = deepcopy(blockchain)
    for i in range(len(chain_to_send)):
        block = chain_to_send[i]
        block_index = str(block.index)
        block_timestamp = str(block.timestamp)
        block_data = str(block.data)
        block_hash = block.hash
        chain_to_send = {
            "index": block_index,
            "timestamp": block_timestamp,
            "data": block_data,
            "hash": block_hash
        }
    chain_to_send = json.dumps(chain_to_send)
    return chain_to_send

def find_new_chains():
    other_chains = []
    for node_url in peer_nodes:
        block = requests.get(node_url + "/blocks").content
        block = json.loads(block)
        other_chains.append(block)
    return other_chains

def consensus():
    global blockchain
    other_chains = find_new_chains()
    longest_chain = blockchain
    for chain in other_chains:
        if len(longest_chain) < len(chain):
            longest_chain = chain
    blockchain = longest_chain

def proof_of_word(last_proof):
    incrementor = last_proof + 1
    while not (incrementor % 9 == 0 and incrementor % last_proof == 0):
        incrementor += 1
    return incrementor

@node.route('/mine', methods=["Get"])
def mine():
    last_block = blockchain[-1]
    last_proof = last_block.data["proof-of-work"]
    proof = proof_of_word(last_proof)
    this_nodes_transactions.append(
        {"from": "network", "to": miner_address, "amount": 1}
    )

    new_block_data = {
        "proof-of-work": proof,
        "transactions": list(this_nodes_transactions)
    }
    new_block_index = last_block.index + 1
    new_block_timestamp = datetime.now()
    last_block_hash = last_block.hash

    this_nodes_transactions[:] = []

    mined_block = Block(
        new_block_index,
        new_block_timestamp,
        new_block_data,
        last_block_hash
    )

    blockchain.append(mined_block)

    return json.dumps({
        "index": new_block_index,
        "timestamp": str(new_block_timestamp),
        "data": new_block_data,
        "hash": last_block_hash,
    }) + "\n"

if __name__ == "__main__":
    node.run()


import itertools

def combinationSum3(k: int, n: int) -> List[List[int]]:
    nums = [1, 2, 3, 4, 5, 6, 7, 8, 9]
    combos = itertools.combinations(nums, k)
    result = []
    for c in combos:
        if sum(c) == n:
            result.append(c)

    return result

def combinationSum3Bactrack(k, n):
    result = []
    def backtrack(remain, combo, next_start):
        if remain == 0 and len(combo) == k:
            result.append(list(combo))
            return
        elif remain < 0 or len(combo) == k:
            return
            # no match was found
        for i in range(next_start, 9):
            combo.append(i+1)
            backtrack(remain-i-1, combo, i+1)
            combo.pop()
    backtrack(n, [], 0)
    return result


import requests
from bs4 import BeautifulSoup
query = input ("search for ... ")
url= "https://www.bing.com/search?q=" + query
print(url)
data= requests.get(url)
print(data)
soup= BeautifulSoup(data.content,"html.parser")
ResponseData=soup.find_all("li",{'class':'b_algo'})
#print(ResponseData[0].text)
for result in ResponseData :
	print(result.text)
	print(" ")