import pandas as pd
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.lm.preprocessing import padded_everygram_pipeline, flatten
from nltk.lm import MLE
from nltk import bigrams
import os
import re
import math

#import nltk # only need this for the data download
#nltk.download("stopwords") #Only need to run this once
#nltk.download('punkt') #Only need to run this once

# get current directory path.
current_dir_path = os.getcwd()

# Define the Input files
train_file = current_dir_path+r'\archive\olid-training-v1.0.tsv'
test_file_a = current_dir_path+r'\archive\testset-levela.tsv'
labels_a = current_dir_path+r'\archive\labels-levela.csv'
offensive_train_file = current_dir_path+r'\offensive_train_file.tsv'
not_offensive_train_file = current_dir_path+r'\not_offensive_train_file.tsv'

#Function to write the dataframe out to a tsv file
def output_tsv_file(dataframe_in,file_path,file_name):
    dataframe_in.to_csv(file_path+'\\'+file_name+'.tsv',sep="\t",index=False)

def create_offensive_train_file():
    df = pd.read_csv(train_file, delimiter = '\t' )
    working_DF = df[df['subtask_a'] == 'OFF']
    output_tsv_file(working_DF,current_dir_path,'offensive_train_file')
    print("Your offensive data train file was saved as {}".format(current_dir_path+'\\offensive_train_file.tsv'))

def create_not_offensive_train_file():
    df = pd.read_csv(train_file, delimiter = '\t' )
    working_DF = df[df['subtask_a'] == 'NOT']
    output_tsv_file(working_DF,current_dir_path,'not_offensive_train_file')
    print("Your non-offensive data train file was saved as {}".format(current_dir_path+'\\not_offensive_train_file.tsv'))

def merge_test_files():
    # get the test file and create a pandas dataframe
    df = pd.read_csv(test_file_a, delimiter='\t')

    #get the full labels dataset
    labels_df_full = pd.read_csv(labels_a,names = ['id','subtask_a'],header = None)

    #merge the test dataset and the labels dataset in order to filter the test subsets(offensive, not offensive)
    test_DF = pd.merge(df,labels_df_full,on ='id')

    return test_DF

# Methods to handle creation of training files containing
# non-offensive and offensive data separately.
# Note: If the code LM.py is executed again, do not forget to close the files,
# otherwise you will get a permission error.
create_not_offensive_train_file()
create_offensive_train_file()

# `clean_data` method to handle preprocessing of the data.
# Removal of punctuations, stopwords.
# Takes the Tweet string as input, performs lower, and
# splits it on any character that is not in the list.
def clean_data(input_string):
    # Remove commonly used words i.e., stopwords.
    stop_words = set(stopwords.words("english"))
    stop_words.add('vs.')
    stop_words.add('url')
    stop_words.add('re')
    stop_words.add('user')

    # Keep stopwords like `i`, `not`, `user` which adds more sense in the data.
    stop_words.remove('i')
    stop_words.remove('not')

    cleaned_data = [x.strip(" '") for x in re.split(r"[^a-z'.]",input_string.lower())if len(x) > 0 and x not in stop_words]
    #cleaned_data leaves the end of sentence period attached to the word.
    cleaned_data_string = " ".join(cleaned_data)
    cleaned_data = word_tokenize(cleaned_data_string)
    return cleaned_data

# `train_LM` method, to train LM model using training dataset.
# This method train a Maximum Likelihood Estimator(MLE) for bigrams.
# Returns following three LM models based on the type of input training file:
#   a. LM_full -> trained on all the olid training data.
#   b. LM_not  -> trained on all the non-offensive (NOT) training data
#   c. LM_off  -> trained on all the offensive (OFF) training data
#------------------------------------------------------------------------
# @parameter - `path_to_train_file`: path of file which can be used for
#               training LM model, input file could be olid(train_file),
#               non-offensive(offensive_train_file), or,
#               offensive(not_offensive_train_file) training data.
#------------------------------------------------------------------------
def train_LM(path_to_train_file):

    #get the file and create a pandas dataframe
    #--Note--# a dataframe is easier to manipulate compared to multiple "for" loops
    df = pd.read_csv(path_to_train_file, delimiter = '\t' )

    # Training data preprocessing using `clean_data` method.
    df['bigrams'] = df['tweet'].apply(lambda x: clean_data(x))

    # Create a list of tweets to create the model
    tweet_list = df['bigrams'].tolist()

    # Create the train ngram and vocab string for the model
    train, vocab = padded_everygram_pipeline(2,tweet_list)

    # Create the language model
    lm = MLE(2)

    # Fit the model
    lm.fit(train,vocab)
    return lm

#`test_LM` method test every three trained LM models against each test dataset.
# For each instance in the test file, it assigns a MLE score for each test text.
# The function then generates an output file as the same format as the test file
# and adds a new column with the MLE score.
# For each test file, calculate avergae score against each LM models.
#------------------------------------------------------------------------
# @parameters -
#   `path_to_test_file`: path of file which can be used for
#                        testing trained LM model, input file could be olid(test_file),
#                        non-offensive(offensive_test_file), or,
#                        offensive(not_offensive_test_file) test data.
#   `LM_model`: For each type of LM model(i.e., `LM_full`, `LM_not`, and, `LM_off`), this method test on
#                   a. the full test set
#                   b. the not offensive subset,
#                   c. the offensive subset
#------------------------------------------------------------------------
def test_LM(path_to_test_file,LM_model):

    # get the test file and create a pandas dataframe
    df = pd.read_csv(path_to_test_file, delimiter='\t')

    # get the full labels dataset
    labels_df_full = pd.read_csv(labels_a,names = ['id','subtask_a'],header = None)

    # merge the test dataset and the labels dataset in order to
    # filter the test subsets(offensive, not offensive).
    test_DF = pd.merge(df,labels_df_full,on ='id')

    # Test data preprocessing using `clean_data` method.
    # Split the tweet into words list the same as we did the train dataset.
    # Takes the tweet column and applies the normalization and tokenization
    # returns a new column "clean_data".
    test_DF['clean_data'] = test_DF['tweet'].apply(lambda x: list(flatten(clean_data(x))))

    # Create the "bigrams" column from "clean_data" column.
    # Takes the formatted list and turns it into bigrams to feed the model.
    test_DF['bigrams'] = test_DF['clean_data'].apply(lambda x: list(bigrams(x)))

    # Full test set MLE Score.
    # Finds the MLE scores of each bigram in the tweet.
    test_DF['MLE_data'] = test_DF['bigrams'].apply(lambda x: [LM_model.score(y[1],y[0].split()) for y in x] )

    # Smoothing: Substitute a 1 for any score of 0
    test_DF['MLE_data_filtered'] = test_DF['MLE_data'].apply(lambda x: [y if y != 0 else 1 for y in x] )

    # computes the score of the text by taking product of the individual bigram scores(Multiply).
    test_DF['MLE_score'] = test_DF['MLE_data_filtered'].apply(lambda x: math.prod(x))

    # save score of each test data against each model in a csv file at this location.
    score_file_dir = current_dir_path+r'\LM_testData_score'

    # output the full test data file.
    full_df_out = test_DF[['id','tweet','MLE_score']]
    output_tsv_file(full_df_out,score_file_dir,'full_test_data_output')

    # calculate average score for full.
    print("-------------------------")
    print('Average score - full_TestData: ', full_df_out['MLE_score'].mean())
    print("Your full data test results file was saved as {}".format(score_file_dir+'\\full_test_data_output.tsv'))
    print("-------------------------")

    # filter the Not offensive test data and output to file.
    not_df = test_DF[test_DF['subtask_a'] == 'NOT']
    not_df_out = not_df[['id','tweet','MLE_score']]
    output_tsv_file(not_df_out,score_file_dir,'not_off_test_output')

    # calculate average score for non-offensive.
    print('Average score - not_off_TestData: ', not_df_out['MLE_score'].mean())
    print("Your non-offensive data test results file was saved as {}".format(score_file_dir+'\\not_off_test_output.tsv'))
    print("-------------------------")

    # filter the Offensive output test data and output to file.
    off_df = test_DF[test_DF['subtask_a'] == 'OFF']
    off_df_out = off_df[['id','tweet','MLE_score']]
    output_tsv_file(off_df_out,score_file_dir,'off_test_data_output')

    # calculate avergae score for offensive.
    print('Average score - off_TestData: ', off_df_out['MLE_score'].mean())
    print("Your offensive data test results file was saved as {}".format(score_file_dir+'\\off_test_data_output.tsv'))
    print("-------------------------")

# For ease of testing,
# I have included these functions too
# beside the two required functions(i.e., `train_LM`, `test_LM`).
# run this function after you load the file.
def run():
   lm_model = train_LM(train_file)
   test_LM(test_file_a,lm_model)
print('done')


import torch.utils.data as data
import os
import torch
import platform
import numpy as np
sysstr = platform.system()
import nibabel as nib
from matplotlib import pyplot as plt
from skimage import transform
from PIL import Image
import random
import cv2

class Dataset_covid(data.Dataset):
    def __init__(self, ct_paths,lung_path, mask_paths,phase, num_split,crop_size,dataset):
        super(Dataset_covid, self).__init__()
        self.lung_path=lung_path
        self.ct_paths = ct_paths
        self.mask_paths = mask_paths
        self.phase=phase
        self.num_split=num_split
        self.crop_size = crop_size
        self.crop_stride =crop_size
        self.dataset = dataset

    def __len__(self):
        return len(self.ct_paths)
    def __getitem__(self, index):

        ct_nii = nib.load(self.ct_paths[index])
        ct_nii = np.int16(ct_nii.dataobj)  # (512,512,200)
        ct_nii[(ct_nii > 200)] = 200
        ct_nii[(ct_nii < -1400)] = -1400
        ct_max = np.max(ct_nii, axis=(0, 1))
        ct_min = np.min(ct_nii, axis=(0, 1))
        scans = ((ct_nii - (ct_min)) / (ct_max - ct_min) * 1.0)  # (512,512,200)
        scans = scans[..., ::-1]
        if self.ct_paths[index][11:14]=='cap' and int(self.ct_paths[index][24:-7])>=214:
            scans = np.rot90(scans,k=-1)
        if self.ct_paths[index][11:14] == 'nor' and int(self.ct_paths[index][27:-7]) >= 235:
            scans = np.rot90(scans,k=-1)
        ct_dim = scans.shape  # [512,512,200]
        scan_start = 0
        scan_end =ct_dim[2]-1

        gt_nii = nib.load(self.mask_paths[index])

        seg_gt = np.uint8(gt_nii.dataobj)  # (512,512,200)
        assert seg_gt.shape == scans.shape
        seg_gt = np.fliplr(seg_gt)
        seg_gt[seg_gt > 0] = 1

        lung_mask=np.load(self.lung_path[index])# (512,512,200)
        lung_mask[lung_mask > 0] = 1
        lung_mask = lung_mask[:, :, ::-1]
        lung_mask = np.rot90(lung_mask)
        lung_mask = lung_mask[ :,::-1, :]
        lung_mask =lung_mask[::-1,:,:]

        scans=scans*lung_mask
        slice_width = (scan_end - scan_start) / self.num_split  # 2.8
        crop_num = [np.int16(scan_start + slice_width * i) for i in range(self.num_split)]
        ct_scans = scans[..., crop_num]
        ct_scans = ct_scans.transpose((2, 0, 1))

        ct_scans =np.array(ct_scans)
        ct_scans =np.expand_dims(ct_scans, axis=0)
        ct_scans = np.float32(ct_scans / 1.0)#  [1,num_split,512,512]

        if self.phase=='Train' or self.phase=='Val' :
            gt_crop_count = 0
            count = 1
            Random = random.uniform(0, 1)
            if Random >= 0.4:
                while gt_crop_count < np.floor(
                        self.crop_size[0] * self.crop_size[1] * self.crop_size[2] * 0.01) and (count < 50):
                    crop_center_x = int(random.randint(100 + self.crop_size[0] / 2, 412 - self.crop_size[0] / 2))
                    crop_center_y = int(random.randint(100 + self.crop_size[1] / 2, 412 - self.crop_size[1] / 2))
                    if ct_dim[2] < 100:
                        crop_center_z = int(
                            random.randint(self.crop_size[2] / 2, ct_dim[2] - self.crop_size[2] / 2))
                    else:
                        crop_center_z = int(
                            random.randint(30 + self.crop_size[2] / 2, ct_dim[2] - 30 - self.crop_size[2] / 2))

                    gt_crop = seg_gt[
                              int(crop_center_x - self.crop_size[0] / 2):int(crop_center_x + self.crop_size[0] / 2),
                              int(crop_center_y - self.crop_size[1] / 2):int(crop_center_y + self.crop_size[1] / 2),
                              int(crop_center_z - self.crop_size[2] / 2):int(crop_center_z + self.crop_size[2] / 2)
                              ]
                    count += 1
                    gt_crop_count = np.sum(gt_crop)

            else:
                crop_center_x = int(random.randint(0 + self.crop_size[0] / 2, 512 - self.crop_size[0] / 2))
                crop_center_y = int(random.randint(0 + self.crop_size[1] / 2, 512 - self.crop_size[1] / 2))
                if ct_dim[2] < 100:
                    crop_center_z = int(
                        random.randint(self.crop_size[2] / 2, ct_dim[2] - self.crop_size[2] / 2))
                else:
                    crop_center_z = int(
                        random.randint(0 + self.crop_size[2] / 2, ct_dim[2] - 0 - self.crop_size[2] / 2))

                gt_crop = seg_gt[
                          int(crop_center_x - self.crop_size[0] / 2):int(crop_center_x + self.crop_size[0] / 2),
                          int(crop_center_y - self.crop_size[1] / 2):int(crop_center_y + self.crop_size[1] / 2),
                          int(crop_center_z - self.crop_size[2] / 2):int(crop_center_z + self.crop_size[2] / 2)
                          ]
            ct_crop = scans[
                      int(crop_center_x - self.crop_size[0] / 2):int(crop_center_x + self.crop_size[0] / 2),
                      int(crop_center_y - self.crop_size[1] / 2):int(crop_center_y + self.crop_size[1] / 2),
                      int(crop_center_z - self.crop_size[2] / 2):int(crop_center_z + self.crop_size[2] / 2)
                      ]
            lung_crop = lung_mask[
                        int(crop_center_x - self.crop_size[0] / 2):int(crop_center_x + self.crop_size[0] / 2),
                        int(crop_center_y - self.crop_size[1] / 2):int(crop_center_y + self.crop_size[1] / 2),
                        int(crop_center_z - self.crop_size[2] / 2):int(crop_center_z + self.crop_size[2] / 2)
                        ]
            ct_crop = np.expand_dims(ct_crop, axis=0)
            ct_crop = np.float32(ct_crop / 1.0)  # (1,128,128,64)
            gt_crop = np.expand_dims(gt_crop, axis=0)
            gt_crop = np.concatenate(((1 - gt_crop), gt_crop), axis=0)
            gt_crop = np.float32(gt_crop)  # (1,128,128,64)
            lung_crop = np.expand_dims(lung_crop, axis=0)
            lung_crop = np.concatenate(((1 - lung_crop), lung_crop), axis=0)
            lung_crop = np.float32(lung_crop / 1.0)  # (1,128,128,64)
            ct_crop = ct_crop.transpose((0, 3, 1, 2))
            gt_crop = gt_crop.transpose((0, 3, 1, 2))
            lung_crop = lung_crop.transpose((0, 3, 1, 2))
            return  torch.from_numpy(ct_scans),torch.from_numpy(ct_crop),torch.from_numpy(gt_crop),torch.from_numpy(lung_crop),[crop_center_x,crop_center_y,crop_center_z,ct_dim[2]]

        elif self.phase=='Test':
            ct_crops = []

            scans_pad = np.zeros(shape=(scans.shape[0], scans.shape[1], int(self.crop_size[2] + self.crop_stride[2] * np.ceil((ct_dim[2] - self.crop_size[2]) / self.crop_stride[2]) - ct_dim[2])), dtype=np.uint8)
            gt_pad = np.zeros(shape=(scans.shape[0], scans.shape[1], int(
                self.crop_size[2] + self.crop_stride[2] * np.ceil(
                    (ct_dim[2] - self.crop_size[2]) / self.crop_stride[2]) - ct_dim[2])), dtype=np.uint8)
            scans = np.concatenate((scans, scans_pad), axis=2)
            seg_gt = np.concatenate((seg_gt, gt_pad), axis=2)

            for h in range(int(np.ceil((ct_dim[2] - self.crop_size[2]) / self.crop_stride[2]) + 1)):
                for j in range(int((ct_dim[1] - self.crop_size[1]) / self.crop_stride[1]) + 1):
                    for i in range(int((ct_dim[0] - self.crop_size[0]) / self.crop_stride[0]) + 1):
                        crop_center_x = self.crop_size[0] / 2 + self.crop_stride[0] * i
                        crop_center_y = self.crop_size[1] / 2 + self.crop_stride[1] * j
                        crop_center_z = self.crop_size[2] / 2 + self.crop_stride[2] * h

                        ct_crops.append(scans[
                                        int(crop_center_x - self.crop_size[0] / 2):int(
                                            crop_center_x + self.crop_size[0] / 2),
                                        int(crop_center_y - self.crop_size[1] / 2):int(
                                            crop_center_y + self.crop_size[1] / 2),
                                        int(crop_center_z - self.crop_size[2] / 2):int(
                                            crop_center_z + self.crop_size[2] / 2)
                                        ])

            ct_crops = np.array(ct_crops)  # (150,128,128,64)
            ct_crops = np.float32(ct_crops / 1.0)
            seg_gt_realsize = np.expand_dims(seg_gt[..., 0:ct_dim[2]], axis=0)  # (1,512,512,200)
            seg_gt_realsize = np.concatenate(((1 - seg_gt_realsize), seg_gt_realsize), axis=0)  # (2,512,512,200)
            lung_mask_realsize = np.expand_dims(lung_mask, axis=0)  # (1,512,512,200)
            lung_mask_realsize = np.concatenate(((1 - lung_mask_realsize), lung_mask_realsize),
                                                axis=0)  # (2,512,512,200)
            lung_mask_realsize = np.float32(lung_mask_realsize / 1.0)

            ct_crops = ct_crops.transpose((0, 3, 1, 2))  # (150,64,128,128)
            return torch.from_numpy(ct_scans), torch.from_numpy(ct_crops), torch.from_numpy(seg_gt_realsize), torch.from_numpy(lung_mask_realsize),ct_dim

def get_patch_seg(num_split=6,phase='Train',crop_size=[160,160,64],dataset='ours'):
    root_path='../dataset/'
    ######## segment_label ########
    mask_paths_covid = os.listdir(root_path + 'covid/segment_label')
    mask_paths_covid.sort()
    train_end_covid_segment_new= np.floor(len(mask_paths_covid) )
    train_end_covid_segment = np.floor(len(mask_paths_covid) * 0.75)

    mask_paths_cap = os.listdir(root_path + 'cap/segment_label')
    mask_paths_cap.sort()
    train_end_cap_segment_new = np.floor(len(mask_paths_cap))
    train_end_cap_segment = np.floor(len(mask_paths_cap) * 0.75)
    ct_paths=[]
    lung_path=[]
    mask_paths=[]
    if phase=='Test':
        for i in range(int(train_end_covid_segment),len(mask_paths_covid)):
            mask_paths.append(root_path + 'covid/segment_label/' + mask_paths_covid[i])
            ct_paths.append(root_path + 'covid/data-nii/' + mask_paths_covid[i][:-4]+'.nii.gz')
            lung_path.append(root_path + 'covid/lung-npy/' + mask_paths_covid[i][:-4] + '.npy')
        for i in range(int(train_end_cap_segment),len(mask_paths_cap)):
            mask_paths.append(root_path + 'cap/segment_label/' + mask_paths_cap[i])
            ct_paths.append(root_path + 'cap/data-nii/' + mask_paths_cap[i][:-4]+'.nii.gz')
            lung_path.append(root_path + 'cap/lung-npy/' + mask_paths_cap[i][:-4] + '.npy')
    return Dataset_covid(ct_paths,lung_path,mask_paths, phase, num_split,crop_size,dataset)

def remove_all_file(path):
    if os.path.isdir(path):
        for i in os.listdir(path):
            path_file = os.path.join(path, i)
            os.remove(path_file)

if __name__ == '__main__':
    remove_all_file('temp')


import pandas as pd

from common.filter import FilterInterface

from extrato.lib.extrato_columns import ExtratoColumns
from extrato.lib.extrato_dataframes_kit import ExtratoKit

class ExtratoFilter(FilterInterface):
    def __init__(self) -> None:
        """Structure to apply filters based on Extrato objects.

        Args:
        - columns_object: any object instance inherited from 'ColumnsInterface'
        - df_interface_object: any object instance inherited from 'DataframesKitInterface'
        """
        self.__columns_object = ExtratoColumns()
        self.__df_interface_object = ExtratoKit()
        super().__init__(self.__df_interface_object, self.__columns_object)

    def applyOperationFilter(self, operation: str) -> None:
        column = self.__columns_object._operation_col.getName()
        if operation != "Exibir todas":
            self._filtered_df = self._filtered_df.loc[self._filtered_df[column] == operation]
            self._filtered_fmtdf = self._filtered_fmtdf.loc[self._filtered_fmtdf[column] == operation]

    def applyDateFilter(self, start_date: pd.Timestamp, end_date: pd.Timestamp) -> None:
        column = self.__columns_object._date_col.getName()
        if start_date and end_date:
            self._filtered_df = self._filtered_df.loc[(start_date <= self._filtered_df[column]) & (self._filtered_df[column] <= end_date)]
            self._filtered_fmtdf = self._filtered_fmtdf.loc[(start_date <= self._filtered_fmtdf[column]) & (self._filtered_fmtdf[column] <= end_date)]


import logging

from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_apscheduler import APScheduler

from config import Config
from .shudeled import shudeled_task_records, sheduled_task

db = SQLAlchemy()
migrate = Migrate()
scheduler = APScheduler()

app = Flask(__name__)
app.config.from_object(Config)
app.debug = True

# logging
logging.basicConfig(
    filename="record.log",
    level=logging.INFO,
    format=f"%(asctime)s %(levelname)s %(name)s %(threadName)s : %(message)s",
)
logging.getLogger("apscheduler.executors.default").propagate = False

db.init_app(app)
migrate.init_app(app, db)

from .api import api as api_blueprint

app.register_blueprint(api_blueprint, url_prefix="/")

scheduler.add_job(
    id="Scheduled task",
    func=sheduled_task,
    trigger="interval",
    seconds=120,
    misfire_grace_time=9999999999999,
)
scheduler.add_job(
    id="Scheduled task 2",
    func=shudeled_task_records,
    trigger="interval",
    seconds=360,
    misfire_grace_time=9999999999999,
)

scheduler.start()

if __name__ == "__main__":
    app.run(host="0.0.0.0")


#Game Tree Creation
import rules, gui, copy, ai

def printBoard(board):
    #Used for testing! :D
    array = []
    for i in range(8):
        array.append([])

    for row in range(len(board.board)):
        for tile in board.board[row]:
            if tile.hasChecker():
                if tile.getChecker().getColour() == "Red":
                    array[row].append("R")
                else:
                    array[row].append("W")
            else:
                array[row].append("B")

        print(array[row])

    print("\n")
    print("\n")
    print("\n")

class TakeMoves:
    def __init__(self):
        self.moves = []

    def generateMoves(self, board, ref):
        moveGen = rules.MoveAnalyser()
        mvs = moveGen.getMoves(board, ref[-1])

        if moveGen.take == True:
            for mv in mvs:
                board.moveChecker(ref[-1], mv, True)
                temp = list(ref)
                temp.append(mv)
                self.generateMoves(board, temp)
                board.undoLastMove()

        else:
            self.moves.append(ref)

def getAllMoves(board):
    theMoves = []

    takeBefore = False
    for row in range(len(board.board)):
        for tile in range(len(board.board[row])):
            if board.getTile((row,tile)).hasChecker() and board.getTile((row,tile)).getChecker().getColour() == board.getTurn():
                moveGen = rules.MoveAnalyser()
                moves = moveGen.getMoves(board, (row,tile))
                if takeBefore == False or (takeBefore == True and moveGen.take == True):
                    for move in moves:
                        if moveGen.take == False:
                            theMoves.append(((row,tile),move))
                        else:
                            if takeBefore == False:
                                theMoves = []
                                takeBefore = True

                            moveGet = TakeMoves()
                            moveGet.generateMoves(board, [(row,tile)])
                            extraMoves = moveGet.moves
                            theMoves += extraMoves

    return (theMoves, takeBefore)

class Node:
    def __init__(self):
        self.nodes = []

    def genTree(self, board, depth, alpha, beta, maximum):
        allMoves = getAllMoves(board)
        if depth == 0:
            return ai.evaluateMove(board)

        if maximum == True:
            for moves in allMoves[0]:
                undos = 0

                for i in range(1,len(moves)):
                    board.moveChecker(moves[i-1], moves[i], allMoves[1])
                    undos+=1

                board.changeTurn()
                alpha = max(alpha, self.genTree(board, depth-1, alpha, beta, not(maximum)))

                for i in range(undos):
                    board.undoLastMove()

                if beta <= alpha:
                    break

            return alpha
        else:
            for moves in allMoves[0]:
                undos = 0
                for i in range(1,len(moves)):
                    board.moveChecker(moves[i-1], moves[i], allMoves[1])
                    undos+=1

                board.changeTurn()
                beta = min(beta, self.genTree(board, depth-1, alpha, beta, not(maximum)))

                for i in range(undos):
                    board.undoLastMove()

                if beta <= alpha:
                    break

            return beta


import pandas as pd
import requests
import os

def get_tweets(usernames: str, start: str, end: str = None, token: str = None) -> pd.DataFrame:
    """
    The twitter function makes a GET request to the Twitter API for the tweets endpoint to get the public metrics.

    SAMPLE CODE

    google_tweets = get_tweets(usernames='google', start='2023-01-01')
    # or you can pass multiple usernames
    google_amazon_tweets = get_tweets(usernames='google,amazon', start='2023-01-01')

    :param usernames: str Twitter usernames separated by a comma with not space: 'name1,name2,name3'
    :param start: str a date string that indicates the start date for posts
    :param end: str default value is None but you can enter an end date if you want
    :param token: str Twitter API bearer token if None look for the environment variable TWITTERTOKEN
    :return: Tuple[Any, List[pd.DataFrame]] Will return two frames - the first is the followers frame and the
             second is the media frame
    """

    start = start + 'T00:00:00.000Z'

    if not token:
        token = os.environ.get('TWITTERTOKEN')

    # set the Authorization header for authentication
    headers = {"Authorization": f"Bearer {token}"}
    # We some user data because we need the account id
    user_info_url = (f"https://api.twitter.com/2/users/by?"
                     f"usernames={usernames}"
                     f"&user.fields=created_at,name,public_metrics")
    # make the GET request
    data = requests.get(user_info_url, headers=headers).json()['data']
    # dict comprehension in case there are multiple usernames provided
    account_data = {d['username']: {'id': d['id'],
                                    'followers_count': d['public_metrics']['followers_count'],
                                    'tweet_count': d['public_metrics']['tweet_count']}
                    for d in data}

    # create an empty list to append DataFrames durning iteration
    all_tweets = []
    for username, val in account_data.items():
        if not end:  # URL if no end date supplied
            posts_url = (f"https://api.twitter.com/2/users/{val['id']}/tweets?"
                         "max_results=100"
                         f"&start_time={start}"
                         "&tweet.fields=created_at,public_metrics")
        else:  # URL if end date supplied
            end = end + 'T00:00:00.000Z'
            posts_url = (f"https://api.twitter.com/2/users/{val['id']}/tweets?"
                         "max_results=100"
                         f"&start_time={start}"
                         f"&end_time={end}"
                         "&tweet.fields=created_at,public_metrics")

        try:
            posts = requests.get(posts_url, headers=headers).json()
            post_data = pd.json_normalize(posts['data'])
            post_data['account'] = username
            all_tweets.append(post_data)
            # this while loop handles the pagination as the Twitter API will return a max of 100 results
            while True:
                try:
                    posts = requests.get(posts_url + f"&pagination_token={posts['meta']['next_token']}",
                                         headers=headers).json()['data']
                    next_df = pd.json_normalize(posts)
                    next_df['account'] = username
                    all_tweets.append(next_df)
                except TypeError:  # if no next_token is found break
                    break
        # if there are no twitter posts `posts['data']` it will raise a KeyError
        except KeyError:
            pass
    # concat all the frames together
    tweets_df = pd.concat(all_tweets)
    tweets_df['created_at'] = pd.to_datetime(tweets_df['created_at']).dt.date
    tweets_df = tweets_df.sort_values('created_at')

    return tweets_df


# Library imports
import os
import json
import argparse
import discord
from discord.ext import commands

# Cog imports
from cogs.util.config import write_config_value, get_config_value

def parse_cmd_arguments():
    parser = argparse.ArgumentParser(description="Discord Selfbot")
    parser.add_argument('--reset-config', action="store_true",
                        help="Reruns the setup")
    return parser

args = parse_cmd_arguments().parse_args()
_reset_cfg = args.reset_config

def wizard():
    config = {}
    print("Welcome to the Hades selfbot setup wizard.\nAll choices you make during this wizard can be changed by editing the settings/config.json file.")

    print("Enter your user token here.")
    print("-------------------------------------------------------------")
    config["token"] = input("| ").strip().strip(" ")

    config["cmd_prefix"] = False
    while not config["cmd_prefix"]:
        print("Please enter the prefix you would like to use for triggering commands.")
        print("-------------------------------------------------------------")
        config["cmd_prefix"] = input("| ").strip()
        if not config["cmd_prefix"]:
            print("Empty command prefixes are invalid.")

    input(
        "Your settings:\nPrefix: {prefix} | e.g. {prefix}ping\n\nPress enter to start.".format(prefix=config["cmd_prefix"]))
    with open("settings/config.json", encoding="utf8", mode="w") as f:
        json.dump(config, f, sort_keys=True, indent=4)

samples = os.listdir("settings")
for f in samples:
    if f.endswith("sample") and f.rsplit('.', 1)[0] not in samples:
        with open("settings/%s" % f, 'r', encoding="utf8") as template:
            with open("settings/%s" % f.rsplit('.', 1)[0], "w", encoding="utf8") as g:
                fields = json.load(template)
                json.dump(fields, g, sort_keys=True, indent=4)

bot = commands.Bot(command_prefix=get_config_value(
    'config', 'cmd_prefix'), description='''Hades Selfbot''', self_bot=True)

@bot.event
async def on_ready():
    print(f"huEHEHUHUUE\nlogged in as {bot.user.name}")

@bot.command()
async def ping(ctx):
    await ctx.send("PONG AHAHAHHA")

if _reset_cfg:
    wizard()

if __name__ == "__main__":
    for extension in os.listdir("cogs"):
        if extension.endswith(".py"):
            try:
                bot.load_extension("cogs." + extension[:-3])
            except:
                print("Failed to load extension {}\n{}: {}".format(
                    extension, type(e).__name__, e))

    while True:
        token = get_config_value("config", "token")
        try:
            bot.run(token, bot=False)
        except discord.errors.LoginFailure:
            print("Token is incorrect!")
            print("Enter your token below.")
            print("-------------------------------------------------------------")
            token = input("| ").strip('"')
            with open("settings/config.json", "r+", encoding="utf8") as fp:
                config = json.load(fp)
                config["token"] = token
                fp.seek(0)
                fp.truncate()
                json.dump(config, fp, indent=4)
            continue
        break


import secrets

from django.contrib import auth
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.auth.models import User
from django.contrib.auth.views import LoginView, LogoutView
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render, get_object_or_404
from django.urls import reverse_lazy
from django.views import generic
from django.views.generic import FormView, ListView, DetailView
from rest_framework.generics import get_object_or_404, CreateAPIView, ListAPIView, RetrieveAPIView, \
    RetrieveUpdateAPIView, RetrieveUpdateDestroyAPIView, ListCreateAPIView
from rest_framework import viewsets
from django.shortcuts import get_object_or_404
from django.contrib.auth.decorators import login_required
from django.utils import timezone
from datetime import timedelta

from movies.forms import UserForm, DownloadForm
from movies.models import Movie, UserUniqueToken, MovieRate
from movies.api.serializer import MovieSerializer
from rest_framework.permissions import IsAuthenticated

from movies.tasks import download_task, send_email

def index(request):
    return render(request, 'index.html')

def single(request):
    return render(request, 'single.html')

def horror(request):
    return render(request, 'horror.html')

def genres(request):
    return render(request, 'genres.html')

def list_view(request):
    return render(request, 'list.html')

def comedy(request):
    return render(request, 'comedy.html')

class DownloadView(FormView):
    template_name = 'download.html'
    form_class = DownloadForm

    def form_valid(self, form):
        option = form.data.get('option')
        movie_name = form.cleaned_data['search']
        download_task.delay(option, movie_name)

        return HttpResponseRedirect(reverse_lazy('movies:index'))

# Despues de login
class BestView(ListView):
    template_name = 'index2.html'
    # extra_context = {'title': 'My Internet movie database'}
    # context_object_name = 'object_list'
    queryset = Movie.objects.all()
    # paginate_by = 6

    def get_queryset(self):
        qs = super(BestView, self).get_queryset()
        return qs.order_by('-id')

    def get_context_data(self, **kwargs):
        data = super(BestView, self).get_context_data(**kwargs)
        best_movie = MovieRate.objects.get_best_rated()
        if best_movie:
            for j in range(len(best_movie)):
                num = str(j)
                movie = Movie.objects.filter(pk=best_movie[j]['movie__id'])
                data.update({
                    'best_rated_movie_' + num: movie,
                    'best_rated_value_' + num: best_movie[j]['rate'],
                })
            return data

# @login_required
class IndexView(ListView):
    template_name = 'principal.html'
    queryset = Movie.objects.all()
    # paginate_by = 6

    def get_queryset(self):
        qs = super(IndexView, self).get_queryset()
        return qs.order_by('-release_date')

    def get_context_data(self, **kwargs):
        data = super(IndexView, self).get_context_data(**kwargs)
        best_movie = MovieRate.objects.get_best_rated()
        if best_movie:
            for j in range(len(best_movie)):
                num = str(j)
                movie = Movie.objects.filter(pk=best_movie[j]['movie__id'])
                data.update({
                    'best_rated_movie_' + num: movie,
                    'best_rated_value_' + num: best_movie[j]['rate'],
                })
            return data

# @login_required
class MovieDetailView(LoginRequiredMixin, DetailView):
    queryset = Movie.objects.all()
    template_name = 'single.html'
    slug_field = 'slug'
    query_pk_and_slug = False

    def get_context_data(self, **kwargs):
        data = super(MovieDetailView, self).get_context_data(**kwargs)
        best_movie = MovieRate.objects.get_best_rated()
        for dic in best_movie:
            # for key, value in dic.items():
            #    print(key, value, dic['rate'])
            if data['movie'].id == dic['movie__id']:
                data.update({
                    'rated_value': dic['rate'],
                })
                return data

class Login(LoginView):
    template_name = 'index.html'

    def form_valid(self, form):
        response = super(Login, self).form_valid(form)
        try:
            UserUniqueToken.objects.get(user_id=self.request.user.pk)
        except UserUniqueToken.DoesNotExist:
            UserUniqueToken.objects.create(token=secrets.token_hex(8), user_id=self.request.user)

        return response

class Logout(LogoutView):
    next_page = '/'


"""5.	Який відсоток синсетів іменників не мають гіпонімів? До всіх
 синсетів можна доступитися
 за допомогою wn.all_synsets('n').  """
from nltk.corpus import wordnet as wn
list_synsets = []
no_hyponims_list = []
for word in wn.all_synsets('n'):
    list_synsets.append(word)
    val = word.hyponyms()
    if len(val) ==0:
        no_hyponims_list.append(word)
print(len(no_hyponims_list)/len(list_synsets)*100, ' %')


import cv2
import numpy as np
import time

def color_area():
    video_capture = cv2.VideoCapture(1)
    img_size = 500
    while 1:
        grabbed, src = video_capture.read()

        img_shape = src.shape[:2]
        if img_shape[1] > img_shape[0]: img = cv2.resize(src, (img_size, int(img_shape[0]*(img_size/img_shape[1]))))
        else: img = cv2.resize(src, (int(img_shape[1]*(img_size/img_shape[0])), img_size))
        img = cv2.rotate(img, cv2.ROTATE_180)

        # dst = cv2.fastNlMeansDenoisingColored(blur,None,10,10,7,21)
        # src1 = dst
        # hsv1 = cv2.cvtColor(src1, cv2.COLOR_BGR2HSV)
        # lowerb1=(170, 100, 100)
        # upperb1=(180, 255, 255)
        # dst1 = cv2.inRange(hsv1, lowerb1, upperb1)
        # dilation = cv2.dilate(dst1, kernel, iterations=2)

        cv2.imshow("Origin", img)

        key = cv2.waitKey(1)
        if key == ord("q"):
            break
    cv2.destroyAllWindows()
color_area()


from django.contrib import admin
from django.urls import path,include
from clients import views

urlpatterns = [
    path('admin/', admin.site.urls),
    #business type url
    path('business/type/list/', views.types_list),
    path('business/type/create', views.create_types),
    path('business/type/edit/<int:id>/', views.edit_types),
    path('business/type/update/<int:id>/', views.update_type),
    path('business/type/delete/<int:id>', views.delete_type),

    # business owner url
    path('business/owner/list/', views.owners_list),
    path('business/owner/create', views.create_owners),
    path('business/owner/edit/<int:id>/', views.edit_owners),
    path('business/owner/update/<int:id>/', views.update_owner),
    path('business/owner/delete/<int:id>', views.delete_owner),

    # client url
    path('client/list/', views.clients_list),
    path('client/create', views.create_clients),
    path('client/edit/<int:id>/', views.edit_clients),
    path('client/update/<int:id>/', views.update_client),
    path('client/delete/<int:id>', views.delete_client),

]


import numpy as np
import cv2
import torch
import glob as glob
import os
import time
from model_direct import create_model
from config import (
    NUM_CLASSES, DEVICE, CLASSES,model_version
)
from custom_utils import NormalizeData
# this will help us create a different color for each class
COLORS = np.random.uniform(0, 255, size=(len(CLASSES), 3))
OUT_DIR = '/users2/local/datasets/h22elhaf/datasets/iSAID/inference_'+model_version

if os.path.exists(OUT_DIR):
    print('========================')
    print('Directory already exists')
    print('========================')
else:
    os.makedirs(OUT_DIR)

# load the best model and trained weights
model = create_model(num_classes=NUM_CLASSES)
checkpoint = torch.load('/users2/local/datasets/h22elhaf/datasets/iSAID/outputs/output_fusion_exp_'+model_version+'/last_model.pth', map_location=DEVICE)
model.load_state_dict(checkpoint['model_state_dict'])
model.to(DEVICE).eval()
# directory where all the images are present
DIR_TEST = '/users2/local/h22elhaf/Faster_RCNN_for_DOTA/test/images'
test_images = glob.glob(f"{DIR_TEST}/*.png")
print(f"Test instances: {len(test_images)}")
# define the detection threshold...
# ... any detection having score below this will be discarded
detection_threshold = 0.5 #0.8
# to count the total number of images iterated through
frame_count = 0
# to keep adding the FPS for each image
total_fps = 0

def metadata_(image_prefix,image_suffix,classe,x,y): #,metadata_(image_prefix,image_suffix,classe,x,y)
    img_link = os.path.join('/users2/local/h22elhaf/Faster_RCNN_for_DOTA/val/Prob_images_exp' ,image_prefix + '_' + classe + '_instance_color_RGB' + image_suffix)
    img = cv2.imread(img_link)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY).astype(np.float32)
    img = cv2.resize(img, (x,y),interpolation = cv2.INTER_AREA)
    img /= 255.0
    return img

for i in range(len(test_images)):
    # get the image file name for saving output later on
    image_name = test_images[i].split(os.path.sep)[-1].split('.')[0]
    image = cv2.imread(test_images[i])
    w=image.shape[0]
    h=image.shape[1]
    orig_image = image.copy()
    # BGR to RGB
    image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB).astype(np.float32)
    # make the pixel range between 0 and 1
    image /= 255.0
    # bring color channels to front
    image = np.transpose(image, (2, 0, 1)).astype(np.float32)
    image_prefix = test_images[i].split('/')[-1][:5]
    image_suffix = test_images[i].split('/')[-1][5:]

    plane = metadata_(image_prefix,image_suffix,'plane',h,w)
    ship = metadata_(image_prefix,image_suffix,'ship',h,w)
    storage_tank = metadata_(image_prefix,image_suffix,'storage_tank',h,w)
    baseball_diamond = metadata_(image_prefix,image_suffix,'baseball_diamond',h,w)
    tennis_court = metadata_(image_prefix,image_suffix,'tennis_court',h,w)
    basketball_court = metadata_(image_prefix,image_suffix,'basketball_court',h,w)
    ground_track_field = metadata_(image_prefix,image_suffix,'ground_track_field',h,w)
    harbor = metadata_(image_prefix,image_suffix,'harbor',h,w)
    bridge = metadata_(image_prefix,image_suffix,'bridge',h,w)
    large_vehicle = metadata_(image_prefix,image_suffix,'large_vehicle',h,w)
    small_vehicle = metadata_(image_prefix,image_suffix,'small_vehicle',h,w)
    helicopter = metadata_(image_prefix,image_suffix,'helicopter',h,w)
    roundabout = metadata_(image_prefix,image_suffix,'roundabout',h,w)
    soccer_ball_field = metadata_(image_prefix,image_suffix,'soccer_ball_field',h,w)
    swimming_pool = metadata_(image_prefix,image_suffix,'swimming_pool',h,w)

    metadata = np.stack([plane, ship, storage_tank, baseball_diamond, tennis_court,
                        basketball_court, ground_track_field, harbor, bridge, large_vehicle, small_vehicle,
                        helicopter, roundabout, soccer_ball_field, swimming_pool])
    fusion = np.concatenate([image, metadata], axis=0)

    # convert to tensor
    fusion = torch.tensor(fusion, dtype=torch.float).cuda()
    # add batch dimension
    fusion = torch.unsqueeze(fusion, 0)
    start_time = time.time()
    with torch.no_grad():
        outputs = model(fusion.to(DEVICE))
    end_time = time.time()
    # get the current fps
    fps = 1 / (end_time - start_time)
    # add `fps` to `total_fps`
    total_fps += fps
    # increment frame count
    frame_count += 1
    # load all detection to CPU for further operations
    outputs = [{k: v.to('cpu') for k, v in t.items()} for t in outputs]
    # carry further only if there are detected boxes
    if len(outputs[0]['boxes']) != 0:
        boxes = outputs[0]['boxes'].data.numpy()
        scores = outputs[0]['scores'].data.numpy()
        # filter out boxes according to `detection_threshold`
        boxes = boxes[scores >= detection_threshold].astype(np.int32)
        draw_boxes = boxes.copy()
        # get all the predicited class names
        pred_classes = [CLASSES[i] for i in outputs[0]['labels'].cpu().numpy()]
        pred_classes_ = np.array(pred_classes)[scores >= detection_threshold]
        scores_ = scores[scores>= detection_threshold]
        # prediction = np.concatenate([boxes, pred_classes_[:,None]],scores_, axis=1)
        prediction = np.concatenate([boxes, pred_classes_[:,None],scores_[:,None]], axis=1)
        # np.savetxt(f"/users2/local/datasets/h22elhaf/datasets/iSAID/inference_all_50epochs_test/{image_prefix}.txt", prediction, delimiter=",",fmt='%s',header="x_min,y_min,x_max,y_max,class,score")
        np.savetxt(f"{OUT_DIR}/{image_name}.txt", prediction, delimiter=",",fmt='%s',header="x_min,y_min,x_max,y_max,class,score")

        # draw the bounding boxes and write the class name on top of it
        for j, box in enumerate(draw_boxes):
            class_name = pred_classes[j]
            color = COLORS[CLASSES.index(class_name)]
            cv2.rectangle(orig_image,
                        (int(box[0]), int(box[1])),
                        (int(box[2]), int(box[3])),
                        color, 2)
            cv2.putText(orig_image, class_name,
                        (int(box[0]), int(box[1]-5)),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, color,
                        2, lineType=cv2.LINE_AA)
        # cv2.imshow('Prediction', orig_image)
        # cv2.waitKey(1)
        # cv2.imwrite(f"/users2/local/datasets/h22elhaf/datasets/iSAID/inference_all_50epochs_test/{image_name}.png", orig_image)
        cv2.imwrite(f"{OUT_DIR}/{image_name}.png", orig_image)
    else:
        prediction = np.empty((0,6))
        np.savetxt(f"{OUT_DIR}/{image_name}.txt", prediction, delimiter=",",fmt='%s',header="x_min,y_min,x_max,y_max,class,score")
        cv2.imwrite(f"{OUT_DIR}/{image_name}.png", orig_image)

    print(f"Image {i+1} done...")
    print('-'*50)
print('TEST PREDICTIONS COMPLETE')
# cv2.destroyAllWindows()
# calculate and print the average FPS
avg_fps = total_fps / frame_count
print(f"Average FPS: {avg_fps:.3f}")


#!/usr/bin/env python3

import json
import boto3

arn = "arn:aws:sns:us-east-1:460782256770:plates-PendingReviews"
message = {"foo": "bar"}
client = boto3.client('sns')

response = client.publish(
    TargetArn=arn,
#    Message=json.dumps({'default': json.dumps(message)}),
    Message=json.dumps({'default': "The quick brown fox is a snarky ass-munch."}),
    MessageStructure='json'
)


from flask_restful import Resource
from flask import request
import os
from app import app
import sqlite3

VALID_EXTENSIONS = ['jpg','jpeg','png','PNG','JPG','JPEG']
class Products(Resource):
    def __init__(self):
        self.conn = sqlite3.connect('products.db')
        self.cur = self.conn.cursor()

    def get(self):
        products = []
        for row in (self.cur.execute("SELECT * FROM products")):
            products.append({
                'name':row[0],
                'price':row[1],
                'quantity':row[2],
                'image_path':row[3],
                })
        return products,200

    def is_valid_extension(self,filename):
        extension = filename.split('.')[-1]
        if extension in VALID_EXTENSIONS:
            return True
        return False

    def post(self):
        file = request.files['file']
        filename = file.filename
        if not self.is_valid_extension(filename):
            return {"message":"Not a valid file"},400
        filepath = os.path.join(app.config['UPLOAD_FOLDER'],filename)
        file.save(filepath)
        self.cur.execute("INSERT INTO products (name,price,quantity,img_path) VALUES (?,?,?,?)",
        (
            request.form.get('name'),
            request.form.get('price'),
            request.form.get('quantity'),
            filepath,
        ))
        self.conn.commit()
        tmp = {
            "name": request.form.get('name'),
            "price": request.form.get('price'),
            "quantity": request.form.get('quantity'),
            "file": filepath
        }
        return tmp,200

    def put(self,id):
        sql = "UPDATE products SET %s = ? WHERE id = ?"
        if request.form.get('name'):
            self.cur.execute(sql%('name'),(request.form.get('name'),id))
            self.conn.commit()
        if request.form.get('price'):
            self.cur.execute(sql%('price'),(request.form.get('price'),id))
            self.conn.commit()
        if request.form.get('quantity'):
            self.cur.execute(sql%('quantity'),(request.form.get('quantity'),id))
            self.conn.commit()
        return {"message": "commited"},200

    def delete(self,id):
        sql = "DELETE FROM products WHERE id = ?"
        self.cur.execute(sql,id)
        self.conn.commit()
        return {"message": "deleted"}


import time
import progressbar

data = []
count = 0
sum_len = 0
bar = progressbar.ProgressBar(max_value = 1000000)
with open('reviews.txt','r') as f:
	for line in f:
		data.append(line)
		count += 1
		sum_len += len(line)
		bar.update(count)
print('档案读取完了，总共有', len(data), '笔资料')

new = []
for d in data:
	if len(d) < 100:
		new.append(d)

good = [d for d in data if 'good' in d]
bad = ['bad' in d for d in data]

print('留言平均长度为', sum_len / len(data))
print('一共有', len(new), '笔留言长度小于100')
print('一共有', len(good), '笔留言提到了good')

#文字计数功能
start_time = time.time()
wc = {}
for d in data:
	words = d.split()
	for word in words:
		if word in wc:
			wc[word] += 1
		else:
			wc[word] = 1

for word in wc:
	if wc[word] > 10000:
		print(word, wc[word])

end_time = time.time()
print('花了', end_time - start_time, 'seconds')

while True:
	word = input('请问你想查什么字：')
	if word == 'q':
		break
	if word in wc:
		print(word, '出现的次数为', wc[word])
	else:
		print('这个字没有出现！')
print('感谢使用查询功能')


import pandas as pd
import tensorflow as tf
from tensorflow import keras
import numpy as np
import matplotlib.pyplot as plt
import os

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from tensorflow.keras.layers import Embedding
from sklearn.utils import shuffle
import seaborn as sns

import re, string
from keras.models import Sequential
from keras.layers import LSTM, Dense

dataset = pd.read_csv('news_classification_train.csv')
dataset.drop(['authors', 'link', 'date'], axis = 1, inplace = True)

df = dataset.copy()
df.drop_duplicates(keep='last', inplace=True)
df.duplicated(subset=['short_description_new','headline_new']).sum()
df.drop_duplicates(subset=['short_description_new','headline_new'],keep='last',inplace=True)

df.loc[df['headline_new'] == "", 'headline'] = np.nan
df.dropna(subset=['headline_new'], inplace=True)
df.loc[df['short_description_new'] == "", 'short_description_new'] = np.nan
df.dropna(subset=['short_description_new'], inplace=True)

df = shuffle(df)
df.reset_index(inplace=True, drop=True)

df['desc'] = df['headline_new'].astype(str)+"-"+df['short_description_new']
df.drop(columns =['headline_new','short_description_new'],axis = 1, inplace=True)
df.astype(str)

test = pd.read_csv('news_classification_test.csv')
test.drop(['authors', 'link', 'date'], axis = 1, inplace = True)

df_test = test.copy()
df_test.drop_duplicates(keep='last', inplace=True)
df_test.duplicated(subset=['short_description_new','headline_new']).sum()
df_test.drop_duplicates(subset=['short_description_new','headline_new'],keep='last',inplace=True)

df_test.loc[df_test['headline_new'] == "", 'headline'] = np.nan
df_test.dropna(subset=['headline_new'], inplace=True)
df_test.loc[df_test['short_description_new'] == "", 'short_description_new'] = np.nan
df_test.dropna(subset=['short_description_new'], inplace=True)

df_test = shuffle(df_test)
df_test.reset_index(inplace=True, drop=True)

df_test['desc'] = df_test['headline_new'].astype(str)+"-"+df_test['short_description_new']
df_test.drop(columns =['headline_new','short_description_new'],axis = 1, inplace=True)
df_test.astype(str)

X_train = df['desc'] #train_data
y_train = df['category'] #train_label
X_val = df_test['desc'] #test_data
y_val = df_test['category'] #test_label

X_val, X_test , y_val, y_test= train_test_split(X_val,y_val, test_size=0.5, random_state=42)

vocab_size = 25000
max_length = 50
trunc_type ='post'
padding_type ='post'
oov_tok = "<OOV>"

tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words = vocab_size, oov_token=oov_tok)
tokenizer.fit_on_texts(X_train)

word_index = tokenizer.word_index

X_train = tokenizer.texts_to_sequences(X_train)
X_train = pad_sequences(X_train,maxlen= max_length,padding=padding_type, truncating=trunc_type)
y_train = np.asarray(y_train)
y_train = pd.get_dummies(y_train)

X_val = tokenizer.texts_to_sequences(X_val)
X_val = pad_sequences(X_val,maxlen= max_length,padding=padding_type, truncating=trunc_type)
y_val = np.asarray(y_val)
y_val = pd.get_dummies(y_val)

train_set = np.array(X_train)
val_set = np.array(X_val)

train_label = np.array(y_train)
val_label = np.array(y_val)

y_test = pd.get_dummies(y_test)
y_test = np.asarray(y_test)
y_test = np.argmax(y_test,axis=1)   #this would be our ground truth label while testing

print(train_set.shape)
print(train_label.shape)

print(val_set.shape)
print(val_label.shape)

path_to_glove_file =  'glove.6B.100d.txt'
num_tokens = len(tokenizer.word_index.items()) + 2
embedding_dim = 100
hits = 0
misses = 0

embeddings_index = {}
with open(path_to_glove_file) as f:
    for line in f:
        word, coefs = line.split(maxsplit=1)
        coefs = np.fromstring(coefs, "f", sep=" ")
        embeddings_index[word] = coefs

print("Found %s word vectors." % len(embeddings_index))

# Prepare embedding matrix
embedding_matrix = np.zeros((num_tokens, embedding_dim))
for word, i in tokenizer.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector
        hits += 1
    else:
        misses += 1
print("Converted %d words (%d misses)" % (hits, misses))

tf.keras.backend.clear_session()
embed_size = 100
model = keras.models.Sequential([

        Embedding(num_tokens,
        embedding_dim,
        embeddings_initializer=keras.initializers.Constant(embedding_matrix),
        mask_zero=True,input_shape=[None],trainable=False),
        keras.layers.Bidirectional(keras.layers.LSTM(256, dropout = 0.4)),
        keras.layers.Dense(28, activation="softmax")

        ])

model.summary()

opt = keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"])
history = model.fit( train_set,train_label,
                     batch_size = 32,
                     steps_per_epoch=len(X_train) // 32,
                     validation_data = (val_set , val_label),
                     validation_steps = len(val_set)//32, epochs=20)

classes = dataset['category'].value_counts().index

def prediction(inference_data):
    X = tokenizer.texts_to_sequences(inference_data)
    X = pad_sequences(X,maxlen= max_length,padding=padding_type, truncating=trunc_type)
    pred = model.predict(X)
    pred_value = tf.argmax(pred,axis =1).numpy()
    return pred_value

y_pred = prediction(X_test)
print(classification_report(np.asarray(y_test),np.asarray(y_pred)))


# python -m pip install pip install git+https://github.com/baxterisme/pytube
# installed from this repo because of problems with the upstream version

from pytube import YouTube
from pytube.cli import on_progress
from pytube.exceptions import RegexMatchError
from pytube.extract import video_id

def download(video_url, destination_path='./', on_callback=None, attempts=3):
    print(f'Downloading video from {video_url}...')
    for attempt in range(attempts):
        try:
            print(f'...attempt {attempt}...')
            YouTube(video_url,
                    on_progress_callback=on_progress,
                    on_complete_callback=on_callback) \
                .streams \
                .filter(progressive=True) \
                .get_highest_resolution() \
                .download(output_path=destination_path, filename=video_id(video_url)+'.mp4')
        except RegexMatchError:
            print(f'...attempt {attempt} failed...')
            continue
        break
    else:
        print('...attempts exceeded, failed miserably.')
    print('...all done.')

def print_info(stream, file_path):
    print(f'downloaded from stream: {stream} to {file_path}')

if __name__ == '__main__':
    URL = 'https://www.youtube.com/watch?v=UnoE8M5qbrk'
    DESTINATION_PATH = r'C:\Users\phili\private_workspace\engineer\db\raw'

    download(URL, DESTINATION_PATH, on_callback=print_info)


## MLP - Homework #3 - MoniKa Vyas CS 59000 14 NLP

########## Note:: Acurracy is still not consisent, each time the model is trained,
# accuracy is different every time ############

import os
import pandas as pd
import numpy as np
import re
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from gensim.parsing.preprocessing import strip_punctuation
from gensim.models import word2vec

from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
import tensorflow as tf
from sklearn.metrics import classification_report

#nltk.download("stopwords") #Only need to run this once
#nltk.download('punkt')

# Get the current working directory path
current_dir_path = os.getcwd()

twitter_dataset = current_dir_path+r'\olid-training-v1.0.tsv'

# Preprocess the data.
def preprocessed_data(input_string):
    new_string = strip_punctuation(input_string)
    preprocessed_data = [x.strip(" '") for x in re.split(r"[^a-z'.]",new_string.lower())if len(x) > 0]
    preprocess_data_string = " ".join(preprocessed_data)
    preprocessed_data = word_tokenize(preprocess_data_string)
    return preprocessed_data

# Tried simple ways to pre-process training data for better accuracy.
def simple_preprocessing(raw_text, remove_stopwords=False):
    # 1. Remove non-letters, but including numbers
    letters_only = re.sub("[^0-9a-zA-Z]", " ", raw_text)
    # 2. Convert to lower case, split into individual words
    words = letters_only.lower().split()
    if remove_stopwords:
        stops = set(stopwords.words("english"))
        # Remove stop words
        meaningful_words = [w for w in words if not w in stops]
        words = meaningful_words
    return words

# `evaluate_model` method to handle probability, prediction of each test texts,
# calculate score and create report.
# It returns the prediction, score and report.
def evaluate_model(model,x,y):
  p_pred = model.predict_proba(x)
  y_pred = model.predict(x)
  score_ = model.score(x,y)
  report = classification_report(y,y_pred)
  return p_pred,y_pred,score_,report

# helper function to format and output the file for method test_LR_model
def format_and_output_test_File(test_dataframe,probability,prediction):
  df = test_dataframe
  probability_array = [x[1] for x in probability]
  df['off_prob'] = probability_array

  df['prediction'] = prediction

  #output the file
  df.to_csv(current_dir_path+'\\'+'MLP_test_model_results'+'.tsv',sep="\t",index=False)

# Create feature vector for the words.
def makeFeatureVec(words, model, num_features):
    # Pre-initialize an empty numpy array
    featureVec = np.zeros((num_features,), dtype="float32")
    nwords = 0

    # This is a list that contains the names of the words in
    # the model's vocabulary.
    word_list = set(model.wv.index_to_key)

    # Loop over each word in the tweet and, if it is in the model's
    # vocabulary, add its feature vector to the total
    for word in words:
        if word in word_list:
            nwords = nwords + 1
            featureVec = np.add(featureVec, model.wv[word])
    # Divide the result by the number of words to get the average
    if nwords == 0:
        nwords = 1
    featureVec = np.divide(featureVec, nwords)
    return featureVec

# This method is used to calculate the average of vectors.
def getVectorsAverage(tweets, model, num_features):
    # Given a set of tweets (each one a list of words), calculate
    # the average feature vector for each one and return a 2D numpy array
    tweetFeatureVecs = np.zeros((len(tweets), num_features), dtype="float32")
    counter = 0
    # Loop through the reviews
    for tweet in tweets:
        # Call the function to calculate the average of feature vectors
        tweetFeatureVecs[counter] = makeFeatureVec(tweet, model, num_features)
        counter = counter + 1
    return tweetFeatureVecs

# `train_MLP_model` method to trains a multi-layer perceptron model
# on the training data and returns that trained model.
#-------------------------------------------------------------------------------------------------
# @parameters -
#   `path_to_train_file`: path of file which can be used for
#                        training the MLP model, input file could be olid(train_file),
#   `num_layers`: Number of layers use to train an MLP model on the entire dataset.
#-------------------------------------------------------------------------------------------------
def train_MLP_model(path_to_train_file, num_layers=2):
    # Read file one
    train_file = pd.read_csv(path_to_train_file, delimiter = '\t' )

    # Preprocessing file one.
    train_file['tweet'] = train_file['tweet'].apply(lambda x: preprocessed_data(x))

    # Extract pre-processed tweet values to train MLP model.
    preprocessed_tweets = train_file['tweet']

    # Train a word2vec model using training tweets.
    model = word2vec.Word2Vec(preprocessed_tweets,
        vector_size=300
    )

    # Training dataset average vectors in matrix form.
    avg_tweets = getVectorsAverage(preprocessed_tweets, model, 300)

    y = train_file['subtask_a'].values

    if num_layers == 1:
        layer_size = (50)
    elif num_layers == 3:
        layer_size = (30,50,50)
    else:
        layer_size = (30,30)

    clf = MLPClassifier(hidden_layer_sizes=layer_size,max_iter=1000)
    clf.fit(avg_tweets, y)

    return clf

# `test_MLP_model` method to test a multi-layer perceptron model
# on single, two and three layers.
# It provides the classification report and accuracy of the MLP model.
# This also outputs a test file (i.e. MLP_test_model_results.tsv)
# with probability and prediction labels.
#-------------------------------------------------------------------------------------------------
# @parameters -
#   `path_to_test_file`: path of file which can be used for
#                        testing the trained MLP model, test file could be testset-levela(test_file),
#   `MLP_model`: This is the trained MLP model on desired number of layers.
#-------------------------------------------------------------------------------------------------
def test_MLP_model(path_to_test_file, MLP_model):
    #get the data file and create a pandas dataframe
    test_file_data = pd.read_csv(path_to_test_file, delimiter = '\t' )

    #clean the data using the function preprocessed_data()
    test_file_data['tweet'] = test_file_data['tweet'].apply(lambda x: preprocessed_data(x))

    #get the full labels dataset
    labels_a = current_dir_path+r'\labels-levela.csv'
    labels_df_full = pd.read_csv(labels_a,names = ['id','subtask_a'],header = None)

    #Extract the test data
    preprocessed_tweets = test_file_data['tweet']

    # Train Word2Vec model.
    model = word2vec.Word2Vec(preprocessed_tweets,
        vector_size=300
    )

    avg_tweets = getVectorsAverage(preprocessed_tweets, model, 300)

    # extract the classifications
    y_test = labels_df_full['subtask_a'].values

    # get the data from the classifier
    probabilty, prediction,score, report = evaluate_model(MLP_model,avg_tweets,y_test)

    # Create the output test file with probability and prediction labels
    format_and_output_test_File(test_file_data,probabilty,prediction)

    print(report)
    print('score:',score)

Machine learning models - Neural Networks/MLP/mlp.py


import cv2
import numpy as np
from imutils.object_detection import non_max_suppression

if __name__ == '__main__':
    img = cv2.imread('test.JPG')
    hog = cv2.HOGDescriptor()
    hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
    (rects, weight) = hog.detectMultiScale(img, winStride=(4, 4), padding=(16, 16), scale=1.05)
    rects = np.array([[x, y, x + w, y + h] for (x, y, w, h) in rects])
    pick = non_max_suppression(rects, probs=None, overlapThresh=0.65)
    for (xA, yA, xB, yB) in pick:
        cv2.rectangle(img, (xA, yA), (xB, yB), (0, 0, 255), 2)
    cv2.imshow('image', img)
    cv2.waitKey(0)


import models
import database
import json

def lambda_handler(event, context):
    gmae_version = event.get("pathParameters", {}).get("game_version", None)

    if not gmae_version:
        return {
            "statusCode": "400",
            "headers": {
                'Content-Type': 'application/json',
                'Access-Control-Allow-Origin': '*'
            },
            "body": json.dumps(
                {
                    "message": "gmae_version is required"
                }
            )
        }

    db = next(database.get_db())
    db_game_version: models.GameVersion = db.query(models.GameVersion).filter(
        models.GameVersion.game_version == gmae_version).first()

    if not db_game_version:
        return {
            "statusCode": "404",
            "headers": {
                'Content-Type': 'application/json',
                'Access-Control-Allow-Origin': '*'
            },
            "body": json.dumps(
                {
                    "message": f"GameVersion (id : {gmae_version}) Not Found"
                }
            )
        }

    return {
        "statusCode": "200",
        "headers": {
            'Content-Type': 'application/json',
            'Access-Control-Allow-Origin': '*'
        },
        "body": json.dumps(
            {
                "game_version": db_game_version.game_version,
                "total_match_count": db_game_version.total_match_count,
                "created_at": db_game_version.created_at.strftime("%Y-%m-%d %H:%M:%S"),
                "updated_at": db_game_version.updated_at.strftime("%Y-%m-%d %H:%M:%S")
            }
        )
    }


import argparse

import torch
import os
from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.modeling.detector import build_detection_model

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Extract and save weights")
    folder = '.'
    parser.add_argument(
	"--input",
        default="expr/nyuv2_baseline_26_maskrcnn/model_final.pth",
        help="The path to the checkpoint for test saved during pre-training",
    )
    parser.add_argument(
        "--output",
        default="pretrained/rgb_baseline_26_imagenet.pth",
        help="Output directory for storing model weights",
    )

    args = parser.parse_args()

    checkpoint_path =  args.input
    weight_path = args.output

    print("%s => %s" % (checkpoint_path, weight_path))
    checkpoint = torch.load(checkpoint_path)
    torch.save(checkpoint['model'], weight_path)


# -*- coding: utf-8 -*-

import os
import shutil
from subprocess import Popen
import sys
import zipfile

try:
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

from nsis import NsisScript

def statusbar(current, total):
    step = total / 20.0
    sys.stdout.write("\r[%-20s] %3.2f%%" % (
        '=' * int(current / step),
        current * 100.0 / total
    ))
    sys.stdout.flush()

def copyfileobj(fsrc, fdst, callback=None, length=16*1024):
    copied = 0

    while True:
        chunk = fsrc.read(length)

        if not chunk:
            break

        fdst.write(chunk)
        copied += len(chunk)

        if callback:
            callback(copied)

def download_file(url, dest=None):
    if dest and not os.path.exists(dest):
        os.makedirs(dest)

    filename = os.path.abspath(os.path.join(dest or '', os.path.basename(url)))
    response = urlopen(url)
    info = response.info()
    total = int(info.get('Content-Length'))

    if os.path.exists(filename) and (os.path.getsize(filename) == total):
        return filename

    with open(filename, 'wb') as out_file:
        copyfileobj(response, out_file)
        return filename

def extract_file(filename, dest=None):
    name, ext = os.path.splitext(os.path.basename(filename))

    if not dest:
        dest = os.getcwd()

    if ext == '.zip':
        with zipfile.ZipFile(filename) as z:
            z.extractall(dest)
            return z.namelist()

    if ext == '.tar.gz':
        return

files = extract_file(
    download_file('https://nssm.cc/release/nssm-2.24.zip'), 'for_copy'
)
nssm = files[0].strip('/')

files = extract_file(
    download_file('http://nginx.org/download/nginx-1.11.6.zip'), 'for_copy'
)
filename = files[0].strip('/')

LANG_RUSSIAN = 'LANG_RUSSIAN'

script = NsisScript(template='installer.nsi')

section = script.add_section(nssm, template='install/nssm.nsi')
section.add_description(u'The Non-Sucking Service Manager')
section.add_description(u'Сервис-менеджер', lang=LANG_RUSSIAN)

section = script.add_section(filename, template='install/nginx.nsis')
section.add_description(u'HTTP and reverse proxy server')
section.add_description(u'HTTP-сервер и обратный прокси-сервер', lang=LANG_RUSSIAN)

section = script.add_uninstall_section(filename, template='uninstall/nginx.nsi')
section = script.add_uninstall_section(nssm, template='uninstall/nssm.nsi')

script.update_context({
    'APP_NAME': u'dev-workflow',
    'APP_TITLE': u'DevWorkflow',
    'APP_VERSION': u'1.0.0.0',
    'ICON': u'resources/favicon.ico',
    'MUI_HEADERIMAGE_BITMAP': u'',
    'COMPANY': u'ЦДПО НИУ ИТМО',
    'COPYRIGHT': u'profi.ifmo.ru',
    'DESCRIPTION': u'The developer workflow installer',
    'APP_WEB_SITE': 'http://profi.ifmo.ru',
    'NSSMDIR': nssm,
    'NGINXDIR': filename,
})

script.generate(u'script.nsis')
Popen(['makensis', 'script.nsis'])


#!/usr/bin/env python

# SPDX-License-Identifier: Apache-2.0

import getpass
from jinja2 import Template
import dateutil.parser
from datetime import datetime, timezone
import requests
import argparse

def query_github(gh_org, token):
  q = Template('''
    {
      organization(login: "{{org}}") {
        repositories(first: 100, orderBy: {field: NAME, direction: ASC}{{after}}) {
          totalCount
          pageInfo {
            endCursor
            hasNextPage
          }
          edges {
            node {
              ... on Repository {
                name
                isArchived
                defaultBranchRef {
                  target {
                    ... on Commit {
                      committedDate
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
    ''')

  has_next_page = True
  after_clause = ''
  repos = []
  while has_next_page:
    j = { 'query' : q.render(org=gh_org, after=after_clause) }
    url = 'https://api.github.com/graphql'
    h = {'Authorization': 'token %s' % token}
    r = requests.post(url, json=j, headers=h)
    r.raise_for_status()

    results = r.json()

    for i in results['data']['organization']['repositories']['edges']:
      repos.append({'name':i['node']['name'],
                    'isArchived':i['node']['isArchived'],
                    'committedDate':i['node']['defaultBranchRef']['target']['committedDate']})

    has_next_page = results['data']['organization']['repositories']['pageInfo']['hasNextPage']
    after_clause =', after:"{}"'.format(results['data']['organization']['repositories']['pageInfo']['endCursor'])

  return repos

def list_old_repositories(org, token, when, details):
  repos = query_github(org, token)

  print("Repos where HEAD commit occurred prior to {}".format(when))
  for r in repos:
    if not r['isArchived']:
      last_commit_date = dateutil.parser.isoparse(r['committedDate'])
      if last_commit_date <= when.replace(tzinfo=timezone.utc):
          if details:
              print("{},{}".format(last_commit_date, r['name']))
          else:
              print(r['name'])

def main():
  parser = argparse.ArgumentParser(description="List repositories that have not had a commit in a certain time period.")
  parser.add_argument("-t", "--token",
      help="Github access token to use for API calls",
      required=False)
  parser.add_argument("-o", "--org", help="GitHub organization (required)",
      required=True)
  parser.add_argument("-d", "--details", help="Include last commit date in output",
      default=False, action='store_true')
  parser.add_argument("-w", "--when", help="Include repos with last commit prior to date (required YYYY-mm-dd)",
      required=True, type=lambda s: datetime.strptime(s, '%Y-%m-%d'))
  args = parser.parse_args()

  token = args.token

  if not token:
      token = getpass.getpass("Please enter github access token:")

  list_old_repositories(args.org, token, args.when, args.details)

if __name__ == "__main__":
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-
import os

from helper_functions_chessbot import *
import chessyTensorflow_core # For neural network model

from flask import Flask, jsonify, request
from werkzeug import secure_filename

app = Flask(__name__)

# This is the path to the upload directory
app.config['UPLOAD_FOLDER'] = 'img/puzzles/'
# These are the extension that we are accepting to be uploaded
app.config['ALLOWED_EXTENSIONS'] = set(['png', 'jpg', 'jpeg', 'gif'])

# For a given file, return whether it's an allowed type or not
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in app.config['ALLOWED_EXTENSIONS']

def invert(fen):
    return ''.join(reversed(fen))

predictor = chessyTensorflow_core.ChessboardPredictor()

@app.route('/api', methods=['POST'])
def chessyPredict():
    imageUrl = request.args.get('img_url')
    fileName = request.args.get('file_name')
    sideToPlay = request.args.get('side')
    print("\n---\nImage URL: %s - side: %s" % (imageUrl, sideToPlay))

    if not imageUrl and not fileName:
        print("> %s - Couldn't generate FEN - no img url neither file name")
        return jsonify(result=[dict(fen="")])
    else:
        # Start up Tensorflow CNN with trained model
        if imageUrl:
            fen, certainty = predictor.makePrediction(imageUrl)
        else:
            path = os.path.join(app.config['UPLOAD_FOLDER'], fileName)
            fen, certainty = predictor.makePredictionFromFile(path)

        fen = shortenFEN(fen) # ex. '111pq11r' -> '3pq2r'

        if sideToPlay == 'b':
            # Flip FEN if black to play, assumes image is flipped
            fen = invert(fen)

        print("Predicted FEN: %s" % fen)
        print("Side to Play: %s" % sideToPlay)
        print("Certainty: %.4f%%" % (certainty*100))

        castle_status = getCastlingStatus(fen)
        fen = "%s %s %s -" % (fen, sideToPlay, castle_status)

        # Generate response message
        #msg = generateMessage(fen, certainty, side)
        print("final FEN: %s" % fen)
        return jsonify(result=[dict(fen=fen, certainty=certainty*100)])
        # return jsonify(result=[dict(a=1, b=2), dict(c=3, d=4)])
        #return jsonify(result=[dict(imageUrl=imageUrl)])

@app.route('/upload', methods=['POST'])
def upload():
    # Get the name of the uploaded file
    file = request.files['puzzle']
    # Check if the file is one of the allowed types/extensions
    if file and allowed_file(file.filename):
        # Make the filename safe, remove unsupported chars
        filename = secure_filename(file.filename)
        print("uploaded: %s", filename)
        # Move the file form the temporal folder to
        # the upload folder we setup
        path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
        print("file saved to %s", path)
        file.save(path)
        # Redirect the user to the uploaded_file route, which
        # will basicaly show on the browser the uploaded file
        return jsonify(result=[dict(status="ok")])

if __name__ == '__main__':
    app.run(port = 9001, debug = True)


import logging
import random
import re
import time

repetition_pattern = re.compile(r"([^A-Za-z])\1{1,}")
# debug_pattern = re.compile(r'[0-9۰۱۲۳۴۵۶۷۸۹]')
# debug_pattern = re.compile(r'^گرون$|^میدون$|^خونه$|^نون$|^ارزون$|^اون$|^قلیون$')
# debug_pattern = re.compile(r'هایمان')
debug_pattern = re.compile(r'^(.)\1{5}$')

logger = None
normalizers = {}
validators = {}
taggers = {}
token_set = set()
repetition_word_set = set()
compile_patterns = lambda patterns: [(re.compile(pattern), repl) for pattern, repl in patterns]
typographies = r'&*@‱\\/•^†‡⹋°〃=※×#÷%‰¶§‴~_\|‖¦٪'
punctuations = r'\.:!،؛?؟»\]\)\}«\[\(\{\'\…¡¿'
num_punctuations = r':!،؛?؟»\]\)\}«\[\(\{\'\…¡¿'
hashtag = r'#'
numbers = r'۰۱۲۳۴۵۶۷۸۹'
persians = 'اآب‌پتثجچحخدذرزژسشصضطظعغفقکگلمنوهی'
has_persian_character_pattern = re.compile(rf"([{persians}{numbers}])")
link = r'((https?|ftp):\/\/)?(?<!@)([wW]{3}\.)?(([a-zA-Z۰-۹0-9-]+)(\.([a-zA-Z۰-۹0-9]){2,})+([-\w@:%_\+\/~#?&=]+)?)'
emojies = r'\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F4CC\U0001F4CD'
email = r'[a-zA-Z۰-۹0-9\._\+-]+@([a-zA-Z۰-۹0-9-]+\.)+[A-Za-z]{2,}'
id = r'@[a-zA-Z_]+'
num = r'[+-]?[\d۰-۹]+'
numf = r'[+-]?[\d۰-۹,]+[\.٫,]{1}[\d۰-۹]+'
tag = r'\#([\S]+)'
nj = ''
# tag_set_token_tags = dict()
all_token_tags = dict()
import path
import os
from utils import load_file

if os.path.isfile(path.mohaverekhan_parsed_dir):
    # tag_set_token_tags = load_obj(path.mohaverekhan_parsed_fir)
    all_token_tags = load_file(path.mohaverekhan_parsed_dir)
    # print(all_token_tags)
else:
    # file = read_bijan_files(path.bijan_files_dir)
    # (file, path.mohaverekhan_parsed_dir)
    # # tag_set_token_tags = file
    # all_token_tags = file
    print("FILE DOES NOT EXIST")

# for t in tag_set_token_tags

is_number_pattern = re.compile(rf"^({num})|(numf)$")

###############################################################################
# باید بررسی کنیم نشانه‌های مورد نظر در مجموعه داده موجود وجود دارد یا نه
# ممکنه نشانه مورد نظر، یک نشانه بی‌نهایت باشد و یک نشانه بی‌نهایت برای ما معتبر هست.
def is_token_valid(token_content):
    # اگر نشانه عدد بود، آن را قبول و برای بهبود سرعت آن را در کش ذخیره می‌کنیم.
    if is_number_pattern.fullmatch(token_content):
        # logger.info(f'> Number found and added : {token_content}')
        # tag_set_token_tags['mohaverekhan-tag-set'][token_content] = {'U': 1}
        all_token_tags[token_content] = {'U': 1}
        return True

    # برچسب آر به معنای معتبر بودن نشانه نیست و اگر نشانه فقط برچسب آر داشت آن را معتبر نمی‌خوانیم.
    if token_content in all_token_tags and list(all_token_tags[token_content].keys()) != ['R']:
        return True

    return False


# -*- coding:utf-8 -*-
# Author: ssdcxy
# Date: 2020-03-11 19:39:48
# LastEditTime: 2020-12-05 21:19:23
# LastEditors: ssdcxy
# Description: 连续子数组的最大和
# FilePath: /arithmetic_oj/JianzhiOffer/42.py

import json
from typing import List

class Solution:
    def maxSubArray(self, nums: List[int]) -> int:
        for i in range(1, len(nums)):
            nums[i] += nums[i-1] if nums[i-1] > 0 else 0
        return max(nums)

def stringToIntegerList(input):
    return json.loads(input)

def main():
    import sys
    import io
    def readlines():
        for line in io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8'):
            yield line.strip('\n')

    lines = readlines()
    while True:
        try:
            line = next(lines)
            nums = stringToIntegerList(line);

            ret = Solution().maxSubArray(nums)

            out = str(ret);
            print(out)
        except StopIteration:
            break

if __name__ == '__main__':
    main()


# remain no good result

import argparse
import os
import numpy as np
import tqdm
import torch
import torchvision
from torchvision import datasets, transforms
from torch.autograd import Variable
import torch.nn.functional as F
import torch.nn as nn
from tensorboardX import SummaryWriter
from itertools import product

parser = argparse.ArgumentParser(description="plain GAN for mnist")
parser.add_argument("--root", type=str, default="/home/ubuntu/lijiguo/workspace/pytorch_examples",
                    help="root path for pytorch examples. default /home/lijiguo/workspace/pytorch_examples/")
parser.add_argument("--batch_size", type=int, default=64, help="batch size for training data. default 64")
parser.add_argument("--no_cuda", action="store_true", default=False, help="disable cuda use")
parser.add_argument("--epoches", type=int, default=100, help="training epoches, default 10")
parser.add_argument("--hidden_dim", type=int, default=1024, help="hidden dimension of gan. (default 1024, 3 layers total)")
parser.add_argument("--z_dim", type=int, default=50, help="dimension of z. default 10")
parser.add_argument("--condition_dim", type=int, default=2, help="dimension of c. default 2")
parser.add_argument("--condition_embedding_dim", type=int, default=12, help="embedding dimension of c. default 2")
parser.add_argument("--class_num", type=int, default=10, help="class number for info gan, default 10")
parser.add_argument("--x_feature_dim", type=int, default=512, help="x_feature dim for D, default 512")
args = parser.parse_args()

args.cuda = not args.no_cuda and torch.cuda.is_available()
if args.cuda:
    print("cuda in available, use cuda")
else:
    print("cuda in not available, use cpu")

model_dir = "./info_gan"
log_dir = os.path.join(args.root, model_dir, "./log")
if not os.path.exists(log_dir):
    os.mkdir(log_dir)

writer = SummaryWriter(log_dir=log_dir)

#torch.manual_seed(1)
#if args.cuda:
 #   torch.cuda.manual_seed(1)

train_loader = torch.utils.data.DataLoader(
    datasets.MNIST(root=os.path.join(args.root, "../data/mnist"), train=True, download=True,
                    transform=transforms.Compose([
                    transforms.ToTensor(),
                    transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5,0.5,0.5))
                    ]) ),
    batch_size=args.batch_size, shuffle=True
)
test_loader = torch.utils.data.DataLoader(
    datasets.MNIST(root=os.path.join(args.root, "../data/mnist"), train=False, download=True,
                    transform=transforms.Compose([
                    transforms.ToTensor(),
                    transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5,0.5,0.5))
                    ]) ),
    batch_size=args.batch_size, shuffle=True
)

class Generator(nn.Module):
    """
    generator for GAN
    """
    def __init__(self, noise_dim=10, condition_dim=10, channel=1):
        super(Generator, self).__init__()
        self.noise_dim = noise_dim
        self.condition_dim=condition_dim
        self.channel = channel
        self.net = nn.Sequential(
            # batch x 10 x 1 x 1 --> batch x 512 x 3 x 3
            nn.ConvTranspose2d(self.noise_dim+self.condition_dim, 512, 3, 1, 0, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(True),
            # batch x 512 x 3 x 3 --> batch x 256 x 6 x 6
            nn.ConvTranspose2d(512, 256, 4, 2, 1, bias=False),
            nn.BatchNorm2d(256),
            nn.ReLU(True),
            # batch x 256 x 6 x 6 --> batch x 128 x 12 x 12
            nn.ConvTranspose2d(256, 128, 4, 2, 1, bias=False),
            nn.BatchNorm2d(128),
            nn.ReLU(True),
            # batch x 128 x 12 x 12 --> batch x  64 x 24 x 24
            nn.ConvTranspose2d(128, 64, 4, 2, 1, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            # batch x 64 x 24 x 24 --> batch x channel x 28 x 28
            nn.ConvTranspose2d(64, self.channel, 5, 1, 0)
        )
        self.output = nn.Tanh()

    def forward(self, z, c):
        z_c = torch.cat([z,c.unsqueeze(2).unsqueeze(3)], 1)
        x = self.net(z_c)
        x = self.output(x)
        return x

class Discriminator(nn.Module):
    """
    discriminator for GAN
    """
    def __init__(self, image_size=28, channel=1, x_feature_dim=512):
        super(Discriminator, self).__init__()
        self.image_size = image_size
        self.channel = channel
        self.x_feature_dim = x_feature_dim
        self.net = nn.Sequential(
            # 3 x 28 x 28 --> 64 x 24 x 24
            nn.Conv2d(self.channel, self.x_feature_dim//8, 5, 1, 0, bias=False),
            nn.BatchNorm2d(self.x_feature_dim//8),
            nn.LeakyReLU(0.2, True),
            # 64 x 24 x 24 --> 128 x 12 x 12
            nn.Conv2d(self.x_feature_dim//8, self.x_feature_dim//4, 3, 2, 1, bias=False),
            nn.BatchNorm2d(self.x_feature_dim//4),
            nn.LeakyReLU(0.2, True),
            # 128 x 12 x 12 --> 256 x 6 x 6
            nn.Conv2d(self.x_feature_dim//4, self.x_feature_dim//2, 3, 2, 1, bias=False),
            nn.BatchNorm2d(self.x_feature_dim//2),
            nn.LeakyReLU(0.2, True),
            # 256 x 6 x 6 --> 512 x 3 x 3
            nn.Conv2d(self.x_feature_dim//2, self.x_feature_dim, 3, 2, 1, bias=False),
            nn.BatchNorm2d(self.x_feature_dim),
            nn.LeakyReLU(0.2, True)
        )
        self.image_size_temp = (self.image_size-4)//8
        self.project_layer = nn.Sequential(
            nn.Linear(self.x_feature_dim*self.image_size_temp**2, self.x_feature_dim*3**2, bias=False),
            nn.BatchNorm1d(self.x_feature_dim*3**2),
            nn.LeakyReLU(0.2, True)
        )
        self.output = nn.Sequential(
            nn.Conv2d(self.x_feature_dim, self.x_feature_dim, 3, 1, 1, bias=False),
            nn.BatchNorm2d(self.x_feature_dim),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(self.x_feature_dim, 1, 3, 1, 0),
            nn.Sigmoid()
        )

    def forward(self, z):
        """
        input: z (batch_size x 1 x 28 x 28)
        output: pred (batch_size x 1)
        """
        x_feature_temp = self.net(z).view(-1, self.x_feature_dim*self.image_size_temp**2)
        x_feature = self.project_layer(x_feature_temp).view(-1,self.x_feature_dim, 3, 3)
        x = self.output(x_feature)
        return x.squeeze(), x_feature

class Q_net(nn.Module):
    def __init__(self, feature_shape, class_num, condition_dim):
        """
        x_feature: batch_size x feature_shape
        """
        super(Q_net, self).__init__()
        self.feature_shape = feature_shape
        self.class_num = class_num
        self.condition_dim = condition_dim

        self.classifier = nn.Sequential(
            nn.Conv2d(self.feature_shape[0], self.feature_shape[0], 3,1,1, bias=False),
            nn.BatchNorm2d(self.feature_shape[0]),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(self.feature_shape[0], self.class_num, self.feature_shape[2],1,0, bias=True)
        )

        self.distribution = nn.Sequential(
            nn.Conv2d(self.feature_shape[0], self.feature_shape[0], 3,1,1, bias=False),
            nn.BatchNorm2d(self.feature_shape[0]),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(self.feature_shape[0], self.condition_dim*2, self.feature_shape[2],1,0, bias=True)
        )

    def forward(self, x):
        """
        Input:
            x: batch_size x feature_shape tensor

        Return:
            pred_class:
            mu:
            log_var:
        """
        pred_class = self.classifier(x).squeeze(3).squeeze(2)
        temp = self.distribution(x)
        mu, log_var = temp[:,:self.condition_dim], temp[:,self.condition_dim:]
        mu, log_var = mu.squeeze(3).squeeze(2), log_var.squeeze(3).squeeze(2)
        return pred_class, mu, log_var

def loss_func_D(input, target):
    error = F.binary_cross_entropy(input, target)
    return error

def loss_func_Q(input, target):
    error = F.cross_entropy(input, target)
    return error

def loss_func_guanssian(x, mu, log_var):
    """get the PDF of N(mu, log_var**2) as x
    reference: https://github.com/pianomania/infoGAN-pytorch/blob/master/trainer.py
    """
    # logli = -0.5*torch.log(log_var.pow(2).mul(2*np.pi)+1e-6) - \
    #         (x-mu).pow(2).div(log_var.pow(2).mul(2)+1e-6)
    # logli = -0.5*torch.log(torch.Tensor([2*np.pi])) - \
    #         0.5*(x-mu).pow(2).div(1)
    return 0.5*F.mse_loss(mu, x)
    # return 1 - logli.mean(dim=1).mean()
    # logli = F.binary_cross_entropy_with_logits(mu, x)
    # return logli

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

model_G = Generator(noise_dim=args.z_dim, condition_dim=args.condition_dim+args.class_num)
model_D = Discriminator()
model_Q = Q_net((args.x_feature_dim, 3,3), args.class_num, args.condition_dim)

model_G, model_D, model_Q = model_G.apply(weights_init), model_D.apply(weights_init), model_Q.apply(weights_init)
if args.cuda:
    model_G.cuda()
    model_D.cuda()
    model_Q.cuda()

optimizer_G = torch.optim.Adam(model_G.parameters(), lr=1e-3, betas=(0.5, 0.999))# torch.optim.SGD(model_G.parameters(), lr=1e-3, momentum=0.9)
optimizer_D = torch.optim.Adam(model_D.parameters(), lr=2e-4, betas=(0.5, 0.999))# torch.optim.SGD(model_D.parameters(), lr=1e-4, momentum=0.9)
optimizer_Q = torch.optim.Adam(model_Q.parameters(), lr=2e-4, betas=(0.5, 0.999))

save_idx = 0
batch_cnt = 0
def train(epoch):
    """
    """
    global batch_cnt
    global save_idx

    model_D.train()
    model_G.train()
    loader_bar = tqdm.tqdm(train_loader)
    for (data, label) in loader_bar:
        batch_cnt += 1
        batch_size = data.size(0)
        #real data
        data_real = data
        label_real = torch.Tensor(batch_size).requires_grad_(False).fill_(1)
        data_real = Variable(data_real)
        condition_class_idx = torch.randint(args.class_num, size=(batch_size,)).long()  # target for Q
        condition_class = torch.eye(args.class_num)[condition_class_idx.long()].reshape(batch_size, args.class_num)
        # condition_class[range(batch_size), condition_class_idx] = 1.0
        condition_other = (torch.rand(size=(batch_size, args.condition_dim))-0.5)/0.5
        #condition_other = torch.rand(size=(batch_size, args.condition_dim))
        condition = torch.cat([condition_class, condition_other], 1).requires_grad_(True)
        #condition = torch.eye(10)[label].reshape(batch_size, 10)

        if args.cuda:
            data_real, label_real, condition, condition_class_idx, condition_other = \
                data_real.cuda(), label_real.cuda(), condition.cuda(), condition_class_idx.cuda(), condition_other.cuda()

        pred_real, _ = model_D.forward(data_real)

        #fake data
        z = torch.randn(size=(batch_size, args.z_dim)).unsqueeze(2).unsqueeze(3).requires_grad_(True)
        label_fake = torch.Tensor(batch_size).requires_grad_(False).fill_(0)
        if args.cuda:
            z = z.cuda()
            label_fake = label_fake.cuda()

        data_fake = model_G.forward(z, condition)

        #update D
        model_D.zero_grad()
        model_Q.zero_grad()
        pred_D_real, _ = model_D.forward(data_real)
        pred_D_fake, x_feature_fake = model_D.forward(data_fake.detach())
        pred_class, mu, log_var = model_Q.forward(x_feature_fake)
        loss_D_real = loss_func_D(pred_D_real, label_real)
        loss_D_fake = loss_func_D(pred_D_fake, label_fake)
        loss_D_gan = (loss_D_real+loss_D_fake)/2
        loss_D_Q = loss_func_Q(pred_class, condition_class_idx)
        loss_D_guassian = loss_func_guanssian(condition_other, mu, log_var)

        loss_D = loss_D_gan + loss_D_Q + loss_D_guassian
        loss_D.backward()
        optimizer_Q.step()
        optimizer_D.step()

        #update G
        z = torch.randn(size=(batch_size, args.z_dim)).unsqueeze(2).unsqueeze(3).requires_grad_()
        if args.cuda:
            z = z.cuda()
        data_fake = model_G.forward(z, condition)
        pred_G, x_feature_G = model_D.forward(data_fake)
        pred_class, mu, log_var = model_Q.forward(x_feature_G)

        model_G.zero_grad()
        model_Q.zero_grad()
        loss_G_gan = loss_func_D(pred_G, label_real)
        loss_G_Q = loss_func_Q(pred_class, condition_class_idx.long())
        loss_G_guassian = loss_func_guanssian(condition_other, mu, log_var)
        loss_G = loss_G_gan + loss_G_Q + loss_G_guassian
        loss_G.backward()
        optimizer_G.step()
        optimizer_Q.step()
        loader_bar.set_description("==>epoch:{:2d}, loss_D:{:6.4f}[{:6.4f},{:6.4f},{:6.4f},{:6.4f}], loss_G:{:6.4f}[{:6.4f},{:6.4f},{:6.4f}]".format(
                    epoch,
                    loss_D.cpu().item(), loss_D_real.cpu().item(), loss_D_fake.cpu().item(), loss_D_Q.cpu().item(), loss_D_guassian.cpu().item(),
                    loss_G.cpu().item(), loss_G_gan.cpu().item(), loss_G_Q.cpu().item(), loss_G_guassian.cpu().item(),
                    ))
        writer.add_scalars(main_tag="loss_d", tag_scalar_dict={
            "loss_d_real":loss_D_real.cpu().item(),
            "loss_d_fake":loss_D_fake.cpu().item(),
            "loss_d_q":loss_D_Q.cpu().item(),
            "loss_d_guassian":loss_D_guassian.cpu().item()
        }, global_step=batch_cnt)
        writer.add_scalars(main_tag="loss_g", tag_scalar_dict={
            "loss_g_gan":loss_G_gan.cpu().item(),
            "loss_g_q":loss_G_Q.cpu().item(),
            "loss_g_guassian":loss_G_guassian.cpu().item()
        }, global_step=batch_cnt)
        writer.add_scalars(main_tag="loss", tag_scalar_dict={
            "loss_d":loss_D.cpu().item(),
            "loss_g":loss_G.cpu().item()
        }, global_step=batch_cnt)
        if batch_cnt%100 == 0:
            #save image
            if not os.path.exists(os.path.join(args.root, model_dir, "./sample/")):
                os.system("mkdir {}".format(os.path.join(args.root, model_dir, "./sample/")))
            # sample
            sample_data1, sample_data2 = sample(model_G)
            torchvision.utils.save_image(tensor=sample_data1.view(-1,1,28,28).data.cpu(),
                    filename=os.path.join(args.root, model_dir, "./sample/", "sample_fake1_{}.png".format(save_idx)), nrow=10)
            torchvision.utils.save_image(tensor=sample_data2.view(-1,1,28,28).data.cpu(),
                    filename=os.path.join(args.root, model_dir, "./sample/", "sample_fake1_{}.png".format(save_idx)), nrow=10)
            writer.add_image(tag="fake_image1", img_tensor=torchvision.utils.make_grid(sample_data1, normalize=True, scale_each=True, nrow=10, range=(-1,1)), global_step=batch_cnt)
            writer.add_image(tag="fake_image2", img_tensor=torchvision.utils.make_grid(sample_data2, normalize=True, scale_each=True, nrow=10, range=(-1,1)), global_step=batch_cnt)
            writer.add_image(tag="real_image", img_tensor=torchvision.utils.make_grid(data_real, normalize=True, scale_each=True, range=(-1,1)), global_step=batch_cnt)
            #save model
            if not os.path.exists(os.path.join(args.root, model_dir, "./model/")):
                os.system("mkdir {}".format(os.path.join(args.root, model_dir, "./model/")))

            #torch.save(model_G.state_dict(), f=os.path.join(args.root, model_dir, "./model/model_G_{}.pytorch".format(save_idx)))
            #torch.save(model_D.state_dict(), f=os.path.join(args.root, model_dir, "./model/model_D_{}.pytorch".format(save_idx)))
            save_idx += 1

def sample(net_G):

    condition_class_idx = np.array(list(range(args.class_num)))
    condition_class = np.eye(args.class_num)[condition_class_idx.astype(np.int32)]

    condition_other1 = np.stack([np.linspace(-1,1,10), np.zeros(10)]).transpose()
    condition_other2 = np.stack([np.zeros(10), np.linspace(-1,1,10)]).transpose()
    # condition_other1 = np.stack([np.linspace(0,1,10), np.zeros(10)]).transpose()
    # condition_other2 = np.stack([np.zeros(10), np.linspace(0,1,10)]).transpose()
    condition_np1 = [np.concatenate([_class, _other]) for _class in condition_class for _other in condition_other1]
    condition_np2 = [np.concatenate([_class, _other]) for _class in condition_class for _other in condition_other2]
    #condition_np = list(product(condition_class, condition_other))

    condition1 = torch.Tensor(condition_np1)
    condition2 = torch.Tensor(condition_np2)

    #condition = torch.Tensor([torch.cat([_class, _other], 0) for _class in condition_class for _other in condition_other]).requires_grad_(False)
    z = torch.randn(size=(condition1.shape[0], args.z_dim)).unsqueeze(2).unsqueeze(3).requires_grad_(False)
    if args.cuda:
        condition1, condition2, z = condition1.cuda(), condition2.cuda(), z.cuda()
    data_fake1, data_fake2 = net_G.forward(z, condition1), net_G.forward(z, condition2)
    return data_fake1, data_fake2

if __name__=="__main__":
    for epoch in range(0, args.epoches):
        train(epoch)