import pandas as pd
from datetime import datetime
import backtrader as bt
import Strategy
import sqlite3

#set the period
start = datetime(2020, 1, 1)
end = datetime(2020, 12, 31)
d1=start.strftime('%Y-%m-%d')
d2=end.strftime('%Y-%m-%d')
stock_tag = "MSFT"
#collect to historical data
db = sqlite3.connect('db3.db')
msft = pd.read_sql(con=db, sql=f'SELECT Date, Open, High, Low, Close, Volume  FROM "{stock_tag}" WHERE Date BETWEEN  "{d1}" and "{d2}" ')

msft.index = pd.to_datetime(msft['Date'])
msft['openinterest'] = 0
msft = msft[['Open', 'High', 'Low', 'Close','Volume', 'openinterest']]

#feed the data to dataframe
data = bt.feeds.PandasData(dataname=msft,fromdate=start,todate=end)

#initial the cilent
cerebro = bt.Cerebro()
cerebro.adddata(data)
cerebro.addstrategy(Strategy.OldBuyAndHold)
startCash = 100000
cerebro.broker.setcash(startCash)
cerebro.broker.setcommission(commission=0.0005)
print(f'Initial cash: { startCash }')

cerebro.addanalyzer(bt.analyzers.SharpeRatio, _name = 'SharpeRatio')
cerebro.addanalyzer(bt.analyzers.SQN, _name='SQN')

results = cerebro.run()
portValue = cerebro.broker.getvalue()
netProfit = portValue - startCash
strat = results[0]
print('Final Portfolio Value: %.2f' % portValue)
print('SR:', strat.analyzers.SharpeRatio.get_analysis())
print('SQN:', strat.analyzers.SQN.get_analysis())
print(f'Net Profit: {round(netProfit,2)}')
print(f'Profit Rate: {round(netProfit/startCash,3)}')

cerebro.plot()


import csv
import os
import pandas as pd
from glob import glob
from PIL import Image
import numpy as np
import imghdr

# Define folders
# folder = "E:\\bird img\\"

dB_location = "E:\\ML Training Data\\"
dB_filename = "dB of all bird images from all datasets 3.csv"

# Load data into data frames
image_db = pd.read_csv(dB_location + dB_filename, index_col=None, header=0)
ML_training_map = pd.read_csv(dB_location + 'ML training map.csv', index_col=None, header=0)

fileNum = '13'

doExtract = True

if doExtract:

    try:
        # Generate IDs
        full_bird_list_df = pd.DataFrame(columns=['Label', 'ID'])
        ID = -1
        for i in range(len(ML_training_map)):

            if not ML_training_map['Include'][i]:
                continue

            common_name = ML_training_map['Common name'][i]

            if ML_training_map['Unisex'][i]:

                label = common_name
                ID += 1

                # Add to dataframe
                full_bird_list_df = full_bird_list_df.append({'Label': label, 'ID': ID}, ignore_index=True)

            if ML_training_map['Male'][i]:

                label = common_name + ' Male'
                ID += 1

                # Add to dataframe
                full_bird_list_df = full_bird_list_df.append({'Label': label, 'ID': ID}, ignore_index=True)

            if ML_training_map['Female'][i]:

                label = common_name + ' Female'
                ID += 1

                # Add to dataframe
                full_bird_list_df = full_bird_list_df.append({'Label': label, 'ID': ID}, ignore_index=True)

        full_bird_list_df.to_csv(r'E:\ML Training Data\\' + 'Class label to ID map.csv', index=False)

        birds = full_bird_list_df['Label'].unique()
        # with open('E:\\ML Training Data\\bird list.csv', 'w') as f:
        #     csv.writer(f).writerow(birds)

        image_db = image_db[image_db['Common name'] != 'Yellow-rumped Warbler']
        image_db = image_db[image_db['Common name'] != 'Yellow-rumped Warbler (Myrtle)']
        image_db = image_db.reset_index(drop=True)

        partial_load = False
        if partial_load:
            df_train_formatted = pd.read_csv(dB_location + 'files extracted for training 12.csv', index_col=None, header=0)

            where_were_we = len(df_train_formatted)
            last_image_path_and_filename = df_train_formatted['path_and_filename'].iloc[-1]
            while last_image_path_and_filename != image_db['Full Path'].iloc[where_were_we]:
                where_were_we += 1
            else:
                where_were_we += 1

        else:
            where_were_we = 0
            df_train_formatted = pd.DataFrame(columns=['Label', 'path_and_filename', 'ID'])

        # Process each DB entry and make a new CSV
        for i in range(where_were_we, len(image_db)):

            path_filename = image_db['Full Path'][i]

            test_read_image = True
            if test_read_image:
                try:
                    # Check if PIL can read it without error
                    image = Image.open(path_filename)
                    if image.format not in ['JPEG', 'PNG', 'GIF']:
                        print('PIL test: wrong filetype, skipping ' + path_filename + ' (' + image.format + ')')
                        continue

                    # Check for filetype
                    filetype = imghdr.what(path_filename)
                    if filetype not in ['jpeg', 'png', 'gif']:
                        print('IMGHDR test: wrong filetype, skipping ' + path_filename + ' (' + filetype + ')')
                        continue

                except:
                    print("Can't read " + path_filename + ". Skipping")
                    continue

            sex = image_db['Sex'][i]

            common_name = image_db['Common name'][i]

            # Get map for current species
            map = ML_training_map[ML_training_map['Common name'] == common_name]

            if map['Unisex'].values:

                label = common_name

            elif map['Male'].values == 1 and sex == 'Male':

                label = common_name + ' Male'

            elif map['Female'].values == 1 and sex == 'Female':

                label = common_name + ' Female'

            else:
                # print(str(i) + ": skipping " + common_name + " | " + path_filename)

                continue
                # raise Exception("Not able to categorize entry. Sumthin's bad wrong!")

            tmp_ID = full_bird_list_df[full_bird_list_df['Label'] == label]['ID'].values
            if tmp_ID.size == 0:
                continue
            else:
                ID = int(tmp_ID)

            data_dict = {'Label': label, 'path_and_filename': path_filename, 'ID': ID}
            df_train_formatted = df_train_formatted.append(data_dict, ignore_index=True)

            if (i % 1000) == 0:
                print(str(i) + ": " + label)

                df_train_formatted.to_csv(r'E:\ML Training Data\\' + 'files extracted for training ' + fileNum + '.csv',
                                          index=False)

    except:
        print("Mofo exeption")
        print("Saving what can be saved ... ")
        df_train_formatted.to_csv(r'E:\ML Training Data\\' + 'files extracted for training ' + fileNum + '.csv',
                                  index=False)
        print('Bye')
        raise

else:
    df_train_formatted = pd.read_csv(dB_location + 'files extracted for training ' + fileNum + '.csv',
                                     index_col=None, header=0)

# df_train_formatted['ID'] = pd.to_numeric(df_train_formatted['ID'])

# Create train, eval, and test CSVs

num_images = len(df_train_formatted)
train_eval_test_proportions = [0.7, 0.2, 0.1]
filenames = ['train', 'eval', 'test']
high_idx = 0

birds = df_train_formatted['Label'].unique()
print("")
print("*********************")
print("List of the " + str(len(birds)) + " classes:")
for b in birds:
    print(b)
print("*********************")
print("")

# Randomly shuffle dataset
df_train_formatted = df_train_formatted.sample(frac=1).reset_index(drop=True)

for i, fn in enumerate(filenames):

    low_idx = high_idx
    high_idx = low_idx + int(train_eval_test_proportions[i] * num_images)

    df = df_train_formatted.iloc[low_idx : high_idx]
    csv_filename = filenames[i] + '.csv'
    df.to_csv(r'E:\ML Training Data\\' + csv_filename, index=False)

print("Done!")


from odoo import fields, models, api, _

import logging
_logger = logging.getLogger(__name__)

class ResPartner(models.Model):
    _inherit = "res.partner"

    subcontractor_pos_number = fields.Integer(string=_('Subcontractor POS'),
                                            help=_('Point of sale used to post subcontractor invoices'))
    subcontractor_invoice_ids = fields.One2many('account.move','subcontractor_partner_id',_('Subcontractor invoices'))
    subcontractor_invoice_count = fields.Integer('Subcontractor invoice count', compute="_compute_subcontractor_invoice_count", store=True)

    def action_subcontractor_fetch_pos(self):
        journal = self.env['account.move']._get_subcontractor_journal(['sale'])
        print("company", self.company_id)
        company = journal.company_id
        afip_ws = journal.afip_ws
        for record in self:
            context = {
                'default_afip_invoicing_partner': record,
            }
            ws = company.with_context(context).get_connection(afip_ws).connect()

            res = ws.ParamGetPtosVenta()
            _logger.info("Subcontractor %s pos %s", record.name, res)
            print(res)
            print("Resultado:", ws.Resultado)
            if not res:
                _logger.warning("Subcontractor %s has no POS", record.name)
            elif len(res)>1:
                _logger.warning("Subcontractor %s has %s POS: %s", record.name, len(res), res)
            else:
                pos = res[0].split('|')[0]
                _logger.info("Subcontractor %s has POS=%s", record.name, pos)
                record.subcontractor_pos_number= pos
        # if res:
        #     self.pos = res[0].split('|')[0]
        #     self.partner_id.towing_driver_point_of_sale = self.pos
        # elif not self.pos:
        #     self.pos = self.partner_id.towing_driver_point_of_sale

    @api.depends('subcontractor_invoice_ids')
    def _compute_subcontractor_invoice_count(self):
        for record in self:
            record.subcontractor_invoice_count = len(record.subcontractor_invoice_ids)

    def action_open_subcontractor_invoices(self):
        self.ensure_one()
        action = self.env.ref("l10n_ar_subcontractor_invoices.action_subcontractor_invoice_type").read()[0]
        action["context"] = {}
        action["domain"] = [("id", "in", self.subcontractor_invoice_ids.ids)]

        return action


import torch
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import functional as F
import json
import random
import os
import cv2

class FishDataset(Dataset):
    def __init__(self, img_dir, lab_path, transforms, keypoint_names=("Head", "Dorsal1", "Dorsal2", "Pectoral", "Gluteal", "Caudal")):
        self.img_dir = img_dir
        with open(lab_path, "r") as f:
            lab = json.load(f)
        lab = [v for k, v in lab.items() if
               len([reg for reg in v["regions"] if reg["shape_attributes"]["name"] == "rect"]) > 0]
        self.imgs = []
        self.boxess = []
        self.keypointsss = []
        for e in lab:
            img = e["filename"]
            self.imgs.append(img)
            boxes = [[region["shape_attributes"]["x"],
                      region["shape_attributes"]["y"],
                      region["shape_attributes"]["x"] + region["shape_attributes"]["width"],
                      region["shape_attributes"]["y"] + region["shape_attributes"]["height"]]
                     for region in e["regions"] if region["shape_attributes"]["name"] == "rect"]
            self.boxess.append(boxes)
            keypointss = []
            for box in boxes:
                xmin, ymin, xmax, ymax = box
                keypoints = [[0, 0, 0] for k in keypoint_names]
                for i, k in enumerate(keypoint_names):
                    for region in e["regions"]:
                        if region["shape_attributes"]["name"] == "point":
                            key = region["region_attributes"]["Keypoint"]
                            x, y = region["shape_attributes"]["cx"], region["shape_attributes"]["cy"]
                            if key == k and xmin <= x <= xmax and ymin <= y <= ymax:
                                keypoints[i] = [x, y, 1]  # 1 means visibility = True
                keypointss.append(keypoints)
            self.keypointsss.append(keypointss)
        self.transforms = transforms

    def __getitem__(self, idx):
        img_path = os.path.join(self.img_dir, self.imgs[idx])
        image = cv2.imread(img_path)
        b, g, r = cv2.split(image)
        image = cv2.merge([r, g, b])
        boxes = self.boxess[idx].copy()
        keypointss = self.keypointsss[idx].copy()
        labels = torch.ones((len(boxes),), dtype=torch.int64)  # Only have one class
        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        keypointss = torch.as_tensor(keypointss, dtype=torch.float32)
        image_id = torch.tensor([idx])
        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["keypoints"] = keypointss
        target["image_id"] = image_id
        if self.transforms is not None:
            image, target = self.transforms(image, target)
        return image, target

    def __len__(self):
        return len(self.imgs)

class ToTensor(object):
    def __call__(self, image, target):
        image = F.to_tensor(image)
        return image, target

class RandomHorizontalFlip(object):
    def __init__(self, prob):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            height, width = image.shape[-2:]
            image = image.flip(-1)

            bbox = target["boxes"]
            bbox[:, [0, 2]] = width - bbox[:, [2, 0]]
            target["boxes"] = bbox
            keypoints = target["keypoints"]
            for keypoint in keypoints:
                for kp in keypoint:
                    kp[0] = width - kp[0]
            target["keypoints"] = keypoints
        return image, target

class Compose(object):
    def __init__(self, transforms):
        self.transforms = transforms

    def __call__(self, image, target):
        for t in self.transforms:
            image, target = t(image, target)
        return image, target

def get_transforms(train):
    transforms = []
    transforms.append(ToTensor())
    if train:
        transforms.append(RandomHorizontalFlip(0.5))
    return Compose(transforms)

def collate_fn(batch):
    return tuple(zip(*batch))  # Default collate_fn needs same size of images, so should re-define it here

def get_dataloader(train, batch_size, val_split=0.0, shuffle=True, num_workers=0):
    dataset = FishDataset("data/Frames", "data/Annotation/annotation_keypoint.json", get_transforms(train=train))
    if val_split == 0.0:
        dataset_train = dataset
        dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, collate_fn=collate_fn)
        dataloader_val = None
    else:
        indices = torch.randperm(len(dataset)).tolist()
        where_train = int(len(dataset) * (1 - val_split))
        dataset_train = torch.utils.data.Subset(dataset, indices[:where_train])
        dataset_val = torch.utils.data.Subset(dataset, indices[where_train:])
        dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, collate_fn=collate_fn)
        dataloader_val = DataLoader(dataset_val, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, collate_fn=collate_fn)
    return dataloader_train, dataloader_val

if __name__ == '__main__':
    dataset = FishDataset("data/Frames", "data/Annotation/annotation_keypoint.json", get_transforms(train=True))
    a = next(iter(dataset))
    pass


from django.urls import  path
from .views import  PostDetailView, PostLike,AddPostDataset,CreatePostView, PostListView, CommentList,CommentDetail,PostCommentsView
urlpatterns = [

    path('postcreate/', CreatePostView.as_view()), #to add post data like img and content bla
    path('postcreate/<int:id>/',AddPostDataset.as_view()),#to add dataset to be predicted
    path('list/', PostListView.as_view() ), #to list all posts
    path('list/<int:id>/',  PostDetailView.as_view()), #to get post, update and delete
    path('list/like/<int:id>/',  PostLike.as_view()),

    path('comments/' , PostCommentsView.as_view()),

]


from django.contrib.auth.decorators import login_required
from django.http import Http404
from django.http import HttpResponse
from django.shortcuts import render, redirect, reverse, get_object_or_404

from .forms import (
    HomeForm,
    CandidateForm,
    UpseeForm,
    UgOrDiplomaForm,
    IntermediateForm,
    PCMForm,
    HighSchoolForm,
    BranchFrom
)
from .models import Candidate
from account.models import SiteContent
from utils.tasks import send_sms, convert_thumbnail, send_email

def home_view(request):
    homeForm = HomeForm(request.POST or None)
    if request.method == 'POST':
        if homeForm.is_valid():
            year = request.POST.get('applyYear')
            course = request.POST.get('course')
            return redirect(
                reverse('admission:form', kwargs={
                        'year': year, 'course': course})
            )
    siteContent = SiteContent.objects.filter(name='notice_admission').first()
    content = {
        'title': 'Home',
        'siteContent': siteContent.content,
        'homeForm': homeForm
    }
    return render(request, 'home.html', content)

def save_with_key(formValue, key):
    temp = formValue.save(commit=False)
    temp.candidate = key
    temp.save()
    return temp

def form_view(request, year, course):
    candidateForm = CandidateForm(request.POST or None, request.FILES or None)
    highSchoolForm = HighSchoolForm(
        request.POST or None, request.FILES or None)
    intermediateForm = IntermediateForm(
        request.POST or None, request.FILES or None)
    pcmForm = PCMForm(request.POST or None)
    ugOrDiplomaForm = UgOrDiplomaForm(
        request.POST or None, request.FILES or None)
    upseeForm = UpseeForm(request.POST or None, request.FILES or None)
    branchFrom = BranchFrom(request.POST or None, request.FILES or None)

    if request.method == 'POST':
        # checking vaildtion
        vaild = True
        if not (
                candidateForm.is_valid() and highSchoolForm.is_valid() and
                upseeForm.is_valid):
            vaild = False
        if not (year == '2' and course == 'BTech'):
            if not intermediateForm.is_valid():
                vaild = False
        if not course == 'MCA':
            if not branchFrom.is_valid():
                vaild = False
        if year == '1' and course == 'BTech':
            if not pcmForm.is_valid():
                vaild = False
        else:
            if not ugOrDiplomaForm.is_valid():
                vaild = False
        if vaild:
            instance = candidateForm.save()
            instance.registrationNo = "21187"+"{:04}".format(instance.id)
            instance.save()
            save_with_key(highSchoolForm, instance)
            save_with_key(upseeForm, instance)
            temp = None
            if not (course == 'BTech' and year == '2'):
                temp = save_with_key(intermediateForm, instance)
                if (course == 'BTech' and year == '1'):
                    pcm = pcmForm.save(commit=False)
                    pcm.intermediate = temp
                    pcm.save()
            if not (course == 'BTech' and year == '1'):
                save_with_key(ugOrDiplomaForm, instance)
            if course == 'BTech':
                save_with_key(branchFrom, instance)

            # sending sms and email to user
            mgs = "Dear! "+instance.name+", You're registered successfully and \
                Reg No is " + \
                instance.registrationNo+". For more Information \
                    visit website http://www.fgiet.ac.in"
            send_sms.delay(instance.mobileNo, mgs)
            convert_thumbnail.delay(instance.image.path, (420, 560))
            convert_thumbnail.delay(instance.signImage.path, (560, 160))
            mail_context = {
                'name': instance.name,
                'registrationNo': instance.registrationNo
            }
            send_email.delay('FGIET Admission Registration', '',
                             'mails/success_mail.html',
                             mail_context, None, [instance.email])

            # return responce
            request.session['pk'] = instance.id
            request.session['aadhar'] = instance.aadharNo
            return redirect(reverse('admission:success'))
    content = {
        'title': 'Admission Form',
        'candidateForm': candidateForm,
        'highSchoolForm': highSchoolForm,
        'intermediateForm': intermediateForm,
        'pcmForm': pcmForm,
        'ugOrDiplomaForm': ugOrDiplomaForm,
        'upseeForm': upseeForm,
        'branchFrom': branchFrom,
        'type': {
            'year': year,
            'course': course
        }
    }
    return render(request, 'form.html', content)

def success_view(request):
    if 'pk' not in request.session:
        raise Http404()
    instance = get_object_or_404(Candidate, pk=request.session['pk'])
    return render(
        request, 'success.html', {'instance': instance, 'applicant': True})

@login_required(login_url='/account/login/')
def application_view(request, id):
    instance = get_object_or_404(Candidate, id=id)
    if not request.user.is_superuser:
        if request.user.profile.branch != instance.course:
            return HttpResponse(
                "User is not vaild to access this application details")
    return render(request, 'success.html', {'instance': instance})

def fqs_view(request):
    return HttpResponse("fqs")


import torch
import mrcfile
import argparse
import numpy as np
import pandas as pd
from torch import nn
from tqdm import tqdm
from math import ceil
from torch import FloatTensor as FT
from torch.autograd import Variable as V

from DenseNetB import DenseNet

np.set_printoptions(threshold = np.inf)
np.set_printoptions(suppress = True)

use_gpu = torch.cuda.is_available()
if use_gpu:
    torch.cuda.empty_cache()

def get_args():
    parser = argparse.ArgumentParser(description="This script predict amino acid and secondary structure type for LDPs",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--mcv", "-m", type=str, required=True,
                        help="input voxel density file")
    parser.add_argument("--ldp", "-l", type=str, required=True,
                        help="input LDP file in .pdb format")
    parser.add_argument("--bsize", "-b", type=int, default=128,
                        help="number of data points to predict in one batch")
    parser.add_argument("--model", "-M", type=str, default="../models/DenseNetB.pkl",
                        help="file name of trained model")
    args = parser.parse_args()
    return args

def read_file(file_name):
	f = open(file_name,'r')
	source = []
	for line in f.readlines():
		source.append(line.strip())
	f.close()
	return source

def main():
    vsize = 10

    args = get_args()
    mcv_file = args.mcv
    ldp_file = args.ldp
    batch_size = args.bsize
    model_file = args.model

    ldp=read_file(ldp_file)

    df = pd.read_csv(mcv_file, sep=' ', header=None)
    data = np.array(df.iloc[:,1:], dtype='float32')

    X = data.reshape(-1, vsize ** 3)

    X = V(FT(X), requires_grad=False).view(-1, 1, vsize, vsize, vsize)

    data_num = len(X)

    n_batches = ceil(data_num/batch_size)

    model = DenseNet().cuda()
    model = nn.DataParallel(model)
    model.module.load_state_dict(torch.load(model_file))

    model.eval()
    y = np.zeros((data_num,  7), dtype='float32')
    for i in tqdm(range(n_batches)):
        X_batch = X[i * batch_size : (i + 1) * batch_size].cuda()
        y_batch = model(X_batch).cpu().detach().numpy()
        y[i * batch_size : (i + 1) * batch_size] = y_batch

    i = 0
    for line in ldp:
        if line[0:4] == "ATOM":
            print("{}{:6d}{:6d}".format(line, np.argmax(y[i,:4]), np.argmax(y[i,4:])) )
            i += 1
        else:
            print(line)

if __name__ == "__main__":
    main()


"""
This is the third script to parse the comments for rozetka themselves.
You might need to run script_to_keep_parsing before it

Also each time parsing fails you MUST change rerun_id variable (increase it) so that new
files where parsed data is stored have different name
"""

from tqdm import tqdm
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.relative_locator import locate_with
import time
import random
from selenium.common.exceptions import NoSuchElementException
from multiprocessing import Process

import multiprocessing, logging
#mpl = multiprocessing.log_to_stderr()
#mpl.setLevel(logging.INFO)

#NOTE: notdone_data is generated with script_to_keep_parsing in case
#parsing fails and we need to continue

prodlinks_data = pd.read_pickle("notdone_data.pkl")

random.shuffle(prodlinks_data)

comms_data = []

N_threads = 8

def chunks_gen(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

def task(chunk, chunk_id):

    rerun_id = 0  # NOTE: increase manually after each code restart when continue parsing

    #mpl.info("Process number: " + str(chunk_id))

    try:
        test_driver = webdriver.Firefox()
        test_driver.set_window_size(2048, 1200)

        with open("comments_files/parsed_data_" + str(rerun_id) + "_" + str(chunk_id) + ".txt", "w") as f:
            f.write("\n")

        for prod in tqdm(chunk):

            try:
                #print(prod)
                test_driver.get(prod["prod_link"])
                time.sleep(4)

                xpath_all_comments_page = "/html/body/app-root/div/div/rz-product/div/rz-product-tab-main/div[1]/div[2]/div[2]/rz-product-comments-tab-main/section/a"

                more_comments_was_clicked = False
                try:
                    all_comments_button = test_driver.find_element(By.XPATH, xpath_all_comments_page)
                    all_comments_button.click()
                    time.sleep(3)
                    more_comments_was_clicked = True
                except NoSuchElementException:
                    pass

                comment_ind = 1
                xpath_comment_template = "/html/body/app-root/div/div/rz-product/div/rz-product-tab-feedback/div/div/main/rz-product-comments/section/section/rz-comment-list/ul/li[<comment_ind>]/rz-product-comment/rz-comment/div/div/div[2]"

                more_comments_exists = True
                while (more_comments_exists):
                    try:
                        xpath_comment = xpath_comment_template.replace('<comment_ind>', str(comment_ind))
                        comment_element = test_driver.find_element(By.XPATH, xpath_comment)
                        #mpl.info("comment_element")
                        num_stars = -1
                        general_comment_text = ""
                        extra_comment_text = ""

                        general_comment_element_xpath = xpath_comment + '/p'
                        #mpl.info(general_comment_element_xpath)

                        if len(test_driver.find_elements(By.XPATH, general_comment_element_xpath)) != 0:
                            #mpl.info("general_comment_element_xpath")
                            general_comment_element = comment_element.find_element(By.XPATH, general_comment_element_xpath)
                            general_comment_text = general_comment_element.text
                            #mpl.info(general_comment_text)

                        #mpl.info("mid 1")

                        extra_comment_element_xpath = xpath_comment + '/dl'
                        if len(test_driver.find_elements(By.XPATH, extra_comment_element_xpath)) != 0:
                            #mpl.info("extra_comment_element_xpath")
                            extra_comment_element = comment_element.find_element(By.XPATH, extra_comment_element_xpath)
                            extra_comment_text = extra_comment_element.text
                            #mpl.info(extra_comment_text)

                        #mpl.info("mid 2")

                        total_comment_text = general_comment_text + "\n" + extra_comment_text

                        #mpl.info(total_comment_text)

                        stars_element_xpath_v1 = xpath_comment + '/rz-comment-rating/div/ul/li[1]'
                        stars_element_xpath_v2 = xpath_comment + '/rz-comment-rating/div'

                        if len(test_driver.find_elements(By.XPATH, stars_element_xpath_v1)) != 0:
                            #mpl.info("stars_element_xpath_v1")
                            stars_element = comment_element.find_element(By.XPATH, stars_element_xpath_v1)
                            stars_element_html = stars_element.get_attribute("innerHTML")
                            num_stars = stars_element_html.count('#ffa900')

                        elif len(test_driver.find_elements(By.XPATH, stars_element_xpath_v2)) != 0:
                            #mpl.info("stars_element_xpath_v2")
                            stars_element = comment_element.find_element(By.XPATH, stars_element_xpath_v2)
                            stars_element_html = stars_element.get_attribute("innerHTML")
                            num_stars = stars_element_html.count('#ffa900')

                        #full_html = comment_element.get_attribute("innerHTML")

                        #num_stars = full_html.count('#ffa900')

                        # print("NUM STARS: " + str(num_stars))
                        # print(comment_element.text)

                        #mpl.info("NUM STARS: " + str(num_stars))
                        #mpl.info(total_comment_text)

                        doc = prod.copy()
                        doc["comment"] = total_comment_text
                        doc["rating"] = str(num_stars)

                        #print(doc)

                        with open("comments_files/parsed_data_" + str(rerun_id) + "_" + str(chunk_id) + ".txt", "a") as f:
                            f.write(str(doc) + "\n")

                        comment_ind += 1

                    except NoSuchElementException:
                        try:
                            xpath_more_comments = "/html/body/app-root/div/div/rz-product/div/rz-product-tab-feedback/div/div/main/rz-product-comments/section/section/button"
                            more_comments_button = test_driver.find_element(By.XPATH, xpath_more_comments)
                            more_comments_button.click()
                        except NoSuchElementException:
                            more_comments_exists = False
                            break
            except:
                pass
    except:
        print("Chunk " + chunk_id + " is broken")

prodlinks_chunks = list(chunks_gen(prodlinks_data, len(prodlinks_data) // N_threads))

print("prodlinks_chunks")
print(len(prodlinks_chunks))
print(prodlinks_chunks[0][:10])
time.sleep(10)

for ind, c in enumerate(prodlinks_chunks):
    print("Length of chunk " + str(ind))
    print(len(c))

processes = []
for ind, chunk in enumerate(prodlinks_chunks):
    t = Process(target=task, args=(chunk, ind))
    processes.append(t)
    t.start()

print("Join called")
for t in processes:
    t.join()

print("After join")


import sqlite3
from tabulate import tabulate
from customer import Customer
from service import Service
from service_type import ServiceType
from appointment import Appointment

connection = None
cursor = None

def initialise_database():
    global connection, cursor
    connection = sqlite3.connect("OMG_DB.db")
    cursor = connection.cursor()

    CREATE_CUSTOMER_TABLE_QUERY = """
    CREATE TABLE IF NOT EXISTS customer (
        customerID INTEGER PRIMARY KEY AUTOINCREMENT,
        customerFirstName TEXT NOT NULL,
        customerLastName TEXT NOT NULL,
        customerPhoneNumber TEXT NOT NULL,
        customerEmail TEXT NOT NULL,
        customerAddress TEXT NOT NULL
    )"""

    CREATE_USER_TABLE_QUERY = """
    CREATE TABLE IF NOT EXISTS users (
        userID INTEGER PRIMARY KEY AUTOINCREMENT,
        username TEXT UNIQUE NOT NULL,
        password TEXT  NOT NULL
    )"""

    CREATE_SERVICE_TYPE_TABLE_QUERY = """
    CREATE TABLE IF NOT EXISTS serviceType(
        serviceTypeID INTEGER PRIMARY KEY AUTOINCREMENT,
        serviceTypeName TEXT NOT NULL
    )"""

    CREATE_SERVICE_TABLE_QUERY = """
    CREATE TABLE IF NOT EXISTS service(
        serviceID INTEGER PRIMARY KEY AUTOINCREMENT,
        serviceName TEXT NOT NULL,
        servicePrice TEXT NOT NULL,
        serviceDescription TEXT DEFAULT 'No description provided',
        serviceTypeID INTEGER,
        FOREIGN KEY(serviceTypeID) REFERENCES serviceType(serviceTypeID)
    )"""

    CREATE_APPOINTMENT_TABLE_QUERY = """
    CREATE TABLE IF NOT EXISTS appointment (
        appointmentID INTEGER PRIMARY KEY AUTOINCREMENT,
        appointmentDate DATE,
        appointmentTime TIME,
        appointmentStatus TEXT NOT NULL,
        customerID INTEGER,
        serviceNumber INTEGER,
        FOREIGN KEY (customerID) REFERENCES customer(customerID),
        FOREIGN KEY (serviceNumber) REFERENCES service(serviceNumber)
        )"""

    cursor.execute(CREATE_CUSTOMER_TABLE_QUERY)
    cursor.execute(CREATE_USER_TABLE_QUERY)
    cursor.execute(CREATE_SERVICE_TYPE_TABLE_QUERY)
    cursor.execute(CREATE_SERVICE_TABLE_QUERY)
    cursor.execute(CREATE_APPOINTMENT_TABLE_QUERY)
    connection.commit()

def AddCustomer(fname, lname, phoneNum, email, address):
    try:
        initialise_database()
        cursor.execute("""
            INSERT INTO customer (
                customerFirstName, customerLastName, customerPhoneNumber, customerEmail, customerAddress
            ) VALUES (?, ?, ?, ?, ?)
        """, (fname, lname, phoneNum, email, address))
        connection.commit()
        CloseDatabase()
    except Exception as e:
        print(e)
        raise e

def AddUser(username, password):
    try:
        initialise_database()
        cursor.execute("""
            INSERT INTO users (username, password) VALUES (?, ?)
        """, (username, password))
        connection.commit()
        CloseDatabase()
    except Exception as e:
        print(e)
        raise e

def Login(username, password):
    try:
        initialise_database()
        cursor.execute("""
            SELECT userID FROM users WHERE username=? AND password=?
        """, (username, password))
        user_id = cursor.fetchone()

        if user_id:
            cursor.execute("""
                SELECT * FROM customer WHERE customerID=?
            """, (user_id[0],))
            record = cursor.fetchone()

            if record:
                num, fname, lname, cell, email, addr = record
                User = Customer(fname, lname, email, cell, addr)
                CloseDatabase()
                return User
        CloseDatabase()
        return None
    except Exception as e:
        print(e)
        raise e

def getCustomerNumber(customer):
    initialise_database()
    fname, lname, phone, email = customer.first_name, customer.last_name, customer.phoneNum, customer.email
    cursor.execute("""SELECT customerID FROM customer
        WHERE customerFirstName=? AND customerLastName=?
        AND customerPhoneNumber=? AND customerEmail=?""",
        (fname,lname,phone,email))
    id = cursor.fetchone()
    CloseDatabase()
    return id

def CloseDatabase(): connection.close()

initialise_database()
CloseDatabase()

# How to do services: they will be saved manually by admin.
# 1. Fade Cut

def AddService(service: Service):
    try:
        id, name, price, descr, type_id = service.serviceID, service.serviceName, service.servicePrice, service.serviceDescription, service.serviceTypeID
        cursor.execute(""""INSERT INTO service VALUES(?, ?, ?, ?, ?)""",
        (id, name, price, descr, type_id))
        connection.commit()
    except Exception as e:
        print(e)
        raise e

def AddAppointmentRequest(appointment: Appointment, cust_id):
    try:
        initialise_database()
        sched, cust_num, serv_num = appointment.schedule, appointment.customer_num, appointment.service_num
        date, time, status = appointment.appointmentDate, appointment.appointmentTime, str(appointment.appointmentStatus)
        print(date, time, status, cust_num, serv_num)
        cursor.execute("INSERT INTO appointment (appointmentDate, appointmentTime, appointmentStatus, customerID, serviceNumber) VALUES(?, ?, ?, ?, ?)", (date, time, status, cust_id, serv_num))
        connection.commit()
        CloseDatabase()
    except Exception as e:
        print(e)
        raise e

def getServiceNumber(service_name: str):
    try:
        initialise_database()
        cursor.execute('SELECT serviceID FROM service WHERE serviceName=?', service_name)
        record = cursor.fetchone()[0]
    except Exception as e:
        raise e

initialise_database()
cursor.execute("SELECT * FROM appointment")
res = cursor.fetchall()
headers = ['App_ID', 'App_date', 'App_time', 'App_status', 'Cust_ID', 'Serv_num']
print(tabulate(res, headers, "fancy_grid"))


from asyncore import read
import cv2
import numpy as np

def stackImages(scale,imgArray,lables=[]):
    sizeW= imgArray[0][0].shape[1]
    sizeH = imgArray[0][0].shape[0]
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shape[0]
    if rowsAvailable:
        for x in range ( 0, rows):
            for y in range(0, cols):
                imgArray[x][y] = cv2.resize(imgArray[x][y], (sizeW, sizeH), None, scale, scale)
                if len(imgArray[x][y].shape) == 2: imgArray[x][y]= cv2.cvtColor( imgArray[x][y], cv2.COLOR_GRAY2BGR)
        imageBlank = np.zeros((height, width, 3), np.uint8)
        hor = [imageBlank]*rows
        hor_con = [imageBlank]*rows
        for x in range(0, rows):
            hor[x] = np.hstack(imgArray[x])
            hor_con[x] = np.concatenate(imgArray[x])
        ver = np.vstack(hor)
        ver_con = np.concatenate(hor)
    else:
        for x in range(0, rows):
            imgArray[x] = cv2.resize(imgArray[x], (sizeW, sizeH), None, scale, scale)
            if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
        hor= np.hstack(imgArray)
        hor_con= np.concatenate(imgArray)
        ver = hor
    if len(lables) != 0:
        eachImgWidth= int(ver.shape[1] / cols)
        eachImgHeight = int(ver.shape[0] / rows)
        print(eachImgHeight)
        for d in range(0, rows):
            for c in range (0,cols):
                cv2.rectangle(ver,(c*eachImgWidth,eachImgHeight*d),(c*eachImgWidth+len(lables[d][c])*13+27,30+eachImgHeight*d),(255,255,255),cv2.FILLED)
                cv2.putText(ver,lables[d][c],(eachImgWidth*c+10,eachImgHeight*d+20),cv2.FONT_HERSHEY_COMPLEX,0.7,(255,0,255),2)
    return ver

#alternative to cv2 trackbar. Basically allows you to change certain values of any array
def showValues(img,valNames, valRanges,vals,option):
    i=0
    cv2.putText(img, str(option[0]), (20,100),cv2.FONT_HERSHEY_COMPLEX, 0.7, (0,255,0), 2)
    for x in valNames:
        isOption=" "
        if(i==option[0]):
            isOption="#"
            k = cv2.waitKey(1) & 0xff
            if k == ord('w') and option[0]>0:
                option[0]-=1
            elif k == ord('s') and option[0]<len(valNames)-1:
                option[0]+=1
            elif k == ord('a') and vals[i]>valRanges[i][0]:
                vals[i]-=1
            elif k == ord('d') and vals[i]<valRanges[i][1]:
                vals[i]+=1
        a=valRanges[i][0]
        b=valRanges[i][1]
        cv2.putText(img, isOption+x+" "+str(a)+" - "+str(vals[i])+" - "+str(b), (20, (i+1)*20 + 100), cv2.FONT_HERSHEY_COMPLEX, 0.7, (0,255,0), 2)
        i+=1

# def mss_screencap():
#     with mss.mss() as sct:#starts mss
#         while 'Screen capturing':
#             monitor_1 = sct.monitors[1]#monitor_1 represents screen monitor
#             sct_img = np.array(sct.grab(monitor_1))#grabs input from screen
#             img = cv2.cvtColor(np.array(sct_img), cv2.COLOR_BGRA2BGR)#converts screen input to BGR format
#             h = img.shape[0] #get height of image
#             w = img.shape[1] #get width of image
#             img=cv2.resize(img, (int(w/2),int(h/2)))#resize image so it won't take up a huge portion of the screen
#             cv2.imshow('img',img) # show img on window named 'img'
#             k=cv2.waitKey(27) & 0xff# checks for key press
#             if k==ord('q'):# if key pressed is q, exit
#                 break
#     cv2.destroyAllWindows()

def color_detect():
    cap = cv2.VideoCapture(0)

    names = ["hue1", "sat1", "value1", "hue2", "sat2", "value2"]
    values = [40, 0, 0, 50, 255, 255]
    ranges =[[0, 100], [0, 255], [0, 255], [0, 100], [0, 255], [0, 255]]
    option = [0]

    lowerbound = np.array([50, 30, 30])
    upperbound = np.array([80, 255, 255])
    # hue range is 0 -180
    # sat range is 0 -255
    # 0 is dark max is light

    # maybe values hue 7 - 21; sat 100 - 255; val 100 - 255
    names = ["hue1", "sat1", "value1", "hue2", "sat2", "value2"]
    values = [5, 100, 100, 16, 255, 255]
    ranges = [[0, 100], [0, 255], [0, 255], [0, 100], [0, 255], [0, 255]]
    option = [0]

    while cap.isOpened():

        lowerbound = np.array([values[0], values[1], values[2]])
        upperbound = np.array([values[3], values[4], values[5]])

        ret, img = cap.read()
        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        # determines what pixels remain in image
        mask = cv2.inRange(hsv, lowerbound, upperbound)
        output = cv2.bitwise_and(img, img, mask=mask)
        showValues(img, names, ranges, values, option)
        imgStack = stackImages(0.3, ([output, mask],[img, img]))
        cv2.imshow('img', imgStack)
        k = cv2.waitKey(27)

        if k == ord('q'):
            break
    cap.release()
    cv2.destroyAllWindows()
    print("end")

if __name__ == "__main__":
    color_detect()

# show vals
# names[]
# vals[]
#ranges[]
# option = [0]


# -*- coding: utf-8 -*-
#!/usr/bin/env python
"""Unit tests for Rake.py"""

from collections import defaultdict

import nltk

from rake_nltk import Metric, Rake

###########################################################

text = """Compatibility of systems of linear constraints
over the set of natural numbers. Criteria of compatibility of a system
of linear Diophantine equations, strict inequations, and nonstrict
inequations are considered. Upper bounds for components of a minimal
set of solutions and algorithms of construction of minimal generating
sets of solutions for all types of systems are given. These criteria
and the corresponding algorithms for constructing a minimal supporting
set of solutions can be used in solving all the considered types of
systems and systems of mixed types."""

def test_build_frequency_dist():
    r = Rake()

    phrase_list = [['red', 'apples'], ['good'], ['red'], ['flavour']]
    freq = defaultdict(lambda: 0)
    freq['apples'] = 1
    freq['good'] = 1
    freq['flavour'] = 1
    freq['red'] = 2
    r._build_frequency_dist(phrase_list)
    assert r.get_word_frequency_distribution() == freq

def test_build_word_co_occurance_graph():
    r = Rake()

    phrase_list = [['red', 'apples'], ['good'], ['red'], ['flavour']]
    degree = defaultdict(lambda: 0)
    degree['apples'] = 2
    degree['good'] = 1
    degree['flavour'] = 1
    degree['red'] = 3
    r._build_word_co_occurance_graph(phrase_list)
    assert r.get_word_degrees() == degree

def test_generate_phrases():
    r = Rake()

    sentences = [
        'Red apples, are good in flavour.',
        'Keywords, which we define as a sequence of one or more words, '
        + "provide a compact representation of a document's content",
        'Criteria of compatibility of a system of linear Diophantine ' + 'equations',
        'whose gloomy beams transfigured the village into a lunar '
        + 'encampment of glowing houses and streets of volcanic deserts '
        + 'every fifteen seconds ',
    ]
    phrase_list = [
        ('red', 'apples'),
        ('good',),
        ('flavour',),
        ('keywords',),
        ('define',),
        ('sequence',),
        ('one',),
        ('words',),
        ('provide',),
        ('compact', 'representation'),
        ('document',),
        ('content',),
        ('criteria',),
        ('compatibility',),
        ('system',),
        ('linear', 'diophantine', 'equations'),
        ('whose', 'gloomy', 'beams', 'transfigured'),
        ('village',),
        ('lunar', 'encampment'),
        ('glowing', 'houses'),
        ('streets',),
        ('volcanic', 'deserts', 'every', 'fifteen', 'seconds'),
    ]
    assert r._generate_phrases(sentences) == phrase_list

def test_generate_phrases_with_length_limit():
    sentences = [
        'Red apples, are good in flavour.',
        'Keywords, which we define as a sequence of one or more words, '
        + "provide a compact representation of a document's content",
        'Criteria of compatibility of a system of linear Diophantine ' + 'equations',
        'whose gloomy beams transfigured the village into a lunar '
        + 'encampment of glowing houses and streets of volcanic deserts '
        + 'every fifteen seconds ',
    ]

    # Consider phrases which have minimum of 2 words and max of 4
    r = Rake(min_length=2, max_length=4)
    phrase_list = [
        ('red', 'apples'),
        ('compact', 'representation'),
        ('linear', 'diophantine', 'equations'),
        ('whose', 'gloomy', 'beams', 'transfigured'),
        ('lunar', 'encampment'),
        ('glowing', 'houses'),
    ]
    assert r._generate_phrases(sentences) == phrase_list

    # Consider phrases which have minimum of 5 words and max of 5
    r = Rake(min_length=5, max_length=5)
    phrase_list = [('volcanic', 'deserts', 'every', 'fifteen', 'seconds')]
    assert r._generate_phrases(sentences) == phrase_list

def test_get_phrase_list_from_words():
    r = Rake()

    word_list = ['red', 'apples', ',', 'are', 'good', 'in', 'flavour']
    phrase_list = [('red', 'apples'), ('good',), ('flavour',)]
    assert r._get_phrase_list_from_words(word_list) == phrase_list

    word_list = [
        'keywords',
        ',',
        'which',
        'we',
        'define',
        'as',
        'a',
        'sequence',
        'of',
        'one',
        'or',
        'more',
        'words',
        ',',
        'provide',
        'a',
        'compact',
        'representation',
        'of',
        'a',
        'document',
        "'",
        's',
        'content',
    ]
    phrase_list = [
        ('keywords',),
        ('define',),
        ('sequence',),
        ('one',),
        ('words',),
        ('provide',),
        ('compact', 'representation'),
        ('document',),
        ('content',),
    ]
    assert r._get_phrase_list_from_words(word_list) == phrase_list

def test_extract_keywords_from_text_degree_to_frequency_metric():
    r = Rake()
    r.extract_keywords_from_text(text)
    ranked_phrases = [
        'minimal generating sets',
        'linear diophantine equations',
        'minimal supporting set',
        'minimal set',
        'linear constraints',
        'upper bounds',
        'strict inequations',
        'nonstrict inequations',
        'natural numbers',
        'mixed types',
        'corresponding algorithms',
        'considered types',
        'set',
        'types',
        'considered',
        'algorithms',
        'used',
        'systems',
        'systems',
        'systems',
        'systems',
        'system',
        'solving',
        'solutions',
        'solutions',
        'solutions',
        'given',
        'criteria',
        'criteria',
        'construction',
        'constructing',
        'components',
        'compatibility',
        'compatibility',
    ]
    assert r.get_ranked_phrases() == ranked_phrases
    assert [phrase for _, phrase in r.get_ranked_phrases_with_scores()] == ranked_phrases

def test_extract_keywords_from_text_word_degree_metric():
    r = Rake(ranking_metric=Metric.WORD_DEGREE)
    r.extract_keywords_from_text(text)
    ranked_phrases = [
        'minimal supporting set',
        'minimal set',
        'minimal generating sets',
        'linear diophantine equations',
        'considered types',
        'mixed types',
        'linear constraints',
        'strict inequations',
        'set',
        'nonstrict inequations',
        'types',
        'corresponding algorithms',
        'upper bounds',
        'systems',
        'systems',
        'systems',
        'systems',
        'natural numbers',
        'solutions',
        'solutions',
        'solutions',
        'considered',
        'algorithms',
        'criteria',
        'criteria',
        'compatibility',
        'compatibility',
        'used',
        'system',
        'solving',
        'given',
        'construction',
        'constructing',
        'components',
    ]
    assert r.get_ranked_phrases() == ranked_phrases
    assert [phrase for _, phrase in r.get_ranked_phrases_with_scores()] == ranked_phrases

def test_extract_keywords_from_text_word_frequency_metric():
    r = Rake(ranking_metric=Metric.WORD_FREQUENCY)
    r.extract_keywords_from_text(text)
    ranked_phrases = [
        'minimal supporting set',
        'minimal set',
        'minimal generating sets',
        'considered types',
        'systems',
        'systems',
        'systems',
        'systems',
        'mixed types',
        'linear diophantine equations',
        'types',
        'strict inequations',
        'solutions',
        'solutions',
        'solutions',
        'set',
        'nonstrict inequations',
        'linear constraints',
        'corresponding algorithms',
        'upper bounds',
        'natural numbers',
        'criteria',
        'criteria',
        'considered',
        'compatibility',
        'compatibility',
        'algorithms',
        'used',
        'system',
        'solving',
        'given',
        'construction',
        'constructing',
        'components',
    ]

    assert r.get_ranked_phrases() == ranked_phrases
    assert [phrase for _, phrase in r.get_ranked_phrases_with_scores()] == ranked_phrases

def test_load_portuguese_stopwords():
    r = Rake(language='portuguese')
    assert r.stopwords is not None

def test_rake_default_metric():
    r = Rake(ranking_metric=1)
    assert r.metric == Metric.DEGREE_TO_FREQUENCY_RATIO

def test_allow_repeated_phrases():
    sentences = ['Magic systems is a company.', 'Magic systems was built in a garage']
    expected_phrase_list_with_repetitions = [
        (
            'magic',
            'systems',
        ),
        ('company',),
        ('magic', 'systems'),
        ('built',),
        ('garage',),
    ]
    expected_phrase_list_without_repetitions = [
        (
            'magic',
            'systems',
        ),
        ('company',),
        ('built',),
        ('garage',),
    ]

    r = Rake()
    assert r._generate_phrases(sentences) == expected_phrase_list_with_repetitions

    r = Rake(include_repeated_phrases=True)
    assert r._generate_phrases(sentences) == expected_phrase_list_with_repetitions

    r = Rake(include_repeated_phrases=False)
    assert r._generate_phrases(sentences) == expected_phrase_list_without_repetitions

def test_sent_tokenizer_config():
    text = 'This is a sentence with a #sent hashtag. This is another sentence with a a@rake.com email address.'

    punct_tokenized_sentences = [
        'This is a sentence with a #sent hashtag.',
        'This is another sentence with a a@rake.com email address.',
    ]

    # Default.
    r = Rake()
    assert punct_tokenized_sentences == r._tokenize_text_to_sentences(text)

    # Punct tokenize.
    r = Rake(sentence_tokenizer=nltk.tokenize.sent_tokenize)
    assert punct_tokenized_sentences == r._tokenize_text_to_sentences(text)

    # Custom tokenizer.
    def custom_tokenizer(text):
        return text.split('with')

    r = Rake(sentence_tokenizer=custom_tokenizer)
    assert [
        'This is a sentence ',
        ' a #sent hashtag. This is another sentence ',
        ' a a@rake.com email address.',
    ] == r._tokenize_text_to_sentences(text)

def test_word_tokenizer_config():
    sentence = 'This is a cooool #dummysmiley: :-) :-P <3 and some arrows < > -> <--'

    punct_tokenized_words = [
        'This',
        'is',
        'a',
        'cooool',
        '#',
        'dummysmiley',
        ':',
        ':-)',
        ':-',
        'P',
        '<',
        '3',
        'and',
        'some',
        'arrows',
        '<',
        '>',
        '->',
        '<--',
    ]

    # Default
    r = Rake()
    assert punct_tokenized_words == r._tokenize_sentence_to_words(sentence)

    # Punct tokenize.
    r = Rake(word_tokenizer=nltk.tokenize.wordpunct_tokenize)
    assert punct_tokenized_words == r._tokenize_sentence_to_words(sentence)

    # Custom tokenizer. (Tweet)
    r = Rake(word_tokenizer=nltk.tokenize.TweetTokenizer().tokenize)
    assert [
        'This',
        'is',
        'a',
        'cooool',
        '#dummysmiley',
        ':',
        ':-)',
        ':-P',
        '<3',
        'and',
        'some',
        'arrows',
        '<',
        '>',
        '->',
        '<--',
    ] == r._tokenize_sentence_to_words(sentence)


from DB_Operations.database_connection import db_connection
from sklearn.model_selection import train_test_split
import pandas as pd
from sklearn.impute import KNNImputer
from Log.Log import App_Logger

class preprocessing:
    def __init__(self):
        self.connection = db_connection()
        self.file = open("Log/File_Logs/Db_error.txt", 'a+')
        self.logger = App_Logger()

    def remove_comma_from_cost(self, session, useful_rating_data):
        """
        Cost column value is given as 1,500 type. Conversion into float and removing the comma is done in this method
        :param session: database connectivity
        :param useful_rating_data: training data

        Written by : Priya Ganesan
        Version : 1.0

        """
        try:
            self.connection.log_insert_into_db(session, 'DebugLog', 'Trying to do preprocessing step in cost column')
            #comma string is replaced by empty. After executing the following statement, 1,000 is converted to 1000
            useful_rating_data['approx_cost(for two people)'] = useful_rating_data[
                'approx_cost(for two people)'].str.replace(",", "")
            #converting the column to float since it will be easy for computation while creating model
            useful_rating_data['approx_cost(for two people)'] = useful_rating_data[
                'approx_cost(for two people)'].astype(float)
            useful_rating_data = useful_rating_data.reset_index(drop=True)
            self.connection.log_insert_into_db(session, 'InfoLog', 'Preprocessing step in cost column completed')
            return useful_rating_data

        except Exception as e:
            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))

    def train_test_split(self, session, useful_rating_data):
        """
        This method is used to split the data into train and test. This will be useful in calculating the accuracy and test the model
        :param session: database connectivity
        :param useful_rating_data: training data

        Written by : Priya Ganesan
        Version : 1.0

        """
        try:
            self.connection.log_insert_into_db(session, 'DebugLog', 'Trying to split train and test data')
            X = useful_rating_data.drop(columns=['rate'])
            y = useful_rating_data['rate']
            #80% of the data is taken for training and 20% for testing
            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=200)
            self.connection.log_insert_into_db(session, 'InfoLog', 'Train , Test split completed successfully')
            return X_train, X_test, y_train, y_test

        except Exception as e:
            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))

    def one_hot_encoding(self, session, X_train, X_test):
        """
        There are number of categorical columns in this dataset. Conversion of categorical columns into numerical values is necessary
        :param session: database connectivity
        :param X_train: Training features
        :param X_test: Testing features
        :param y_train: Training target
        :param y_test: Testing target
        """

        try:
            self.connection.log_insert_into_db(session, 'DebugLog', 'One hot encoding part started')
            #List of columns which are categorical are passed
            column_list = ['location', 'rest_type', 'cuisines', 'listed_in(type)']
            X_train,X_test = self.one_hot_encoding_conversion(session, X_train, X_test,column_list)
            self.connection.log_insert_into_db(session, 'InfoLog', 'One hot encoding part completed successfully')
            return X_train,X_test

        except Exception as e:
            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))

    def one_hot_encoding_conversion(self, session, X_train, X_test, column_list):

        try:
            self.connection.log_insert_into_db(session, 'DebugLog', 'One hot encoding conversion started')
            for column in column_list:
                #pandas get dummies method is used to create dummies for the data. Number of columns increase while doing this method
                X_train = pd.concat([X_train.drop(column, 1), X_train[column].str.get_dummies(sep=",")], 1)
                X_test = pd.concat([X_test.drop(column, 1), X_test[column].str.get_dummies(sep=",")], 1)
            self.connection.log_insert_into_db(session, 'InfoLog', 'One hot encoding conversion completed successfully')
            return X_train,X_test

        except Exception as e:

            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))

    def creating_same_number_of_columns_in_train_test(self,session,X_train,X_test):
        """
        Since we are doing one hot encoding there might be mismatch in the train and test data.
        So we are performing some operations to make sure there is same number of columns in train and test data

        Written by : Priya Ganesan
        Version : 1.0

        """
        try :
            self.connection.log_insert_into_db(session, 'DebugLog', 'Matching equal number of train and test columns started')
            #removing extra space present in the train data column name
            X_train.columns = X_train.columns.str.strip()
            #removing extra space present in the test data column name
            X_test.columns = X_test.columns.str.strip()
            #Sometimes columns can be duplicated. So duplicate columns are removed in this step
            X_train = X_train.loc[:, ~X_train.columns.duplicated()]
            X_test = X_test.loc[:, ~X_test.columns.duplicated()]
            #If the column present only in the train and not in the test, then that columns are mapped to zero in the test
            missing_cols = set(X_train.columns) - set(X_test.columns)
            for c in missing_cols:
                X_test[c] = 0
            #If the column present only in the test, then that columns are removed to match the number of columns in the train data and test data
            missing_cols_train = set(X_test.columns) - set(X_train.columns)

            if len(missing_cols_train) > 0:
                for value in missing_cols_train:
                    X_test = X_test.drop(columns=[value])
            #to arrange train data and test data in the same order
            X_test = X_test[X_train.columns]
            #train data is stored to act as a reference to the input data from prediction
            X_train.to_csv('X_train.csv',index=False)
            self.connection.log_insert_into_db(session, 'InfoLog','Matching equal number of train and test columns completed')

            return X_train, X_test

        except Exception as e:
            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))

    def mapping_encoding(self, session, useful_rating_data):
        """
        For few columns encoding can be done directly by mapping values. Those type of encoding is done in this method
        :param X_train: Training data
        :param X_test: Testing data

        """
        try :
            self.connection.log_insert_into_db(session, 'DebugLog','Trying to start mapping_encoding')
            #rate is present as 4.1/5. After executing the below statement it will be stored as 4.1
            useful_rating_data['rate'] = useful_rating_data['rate'].str[0:3]
            #online_order is mapped to numerical number
            useful_rating_data['online_order'] = useful_rating_data['online_order'].map({'Yes': 1, 'No': 0})
            #book_table column is mapped to numeric value
            useful_rating_data['book_table'] = useful_rating_data['book_table'].map({'Yes': 1, 'No': 0})
            useful_rating_data['phone'] = useful_rating_data['phone'].fillna(0)
            #if phone number is present then it is mapped to 1, or else it is mapped to 0
            useful_rating_data['phone'] = useful_rating_data['phone'].apply(lambda x: 1 if x != 0 else x)
            self.creating_dish_liked_list(session,useful_rating_data)
            useful_rating_data['dish_liked'] = useful_rating_data['dish_liked'].fillna(0)
            useful_rating_data['dish_liked'] = useful_rating_data['dish_liked'].apply(lambda x: 1 if x != 0 else x)
            self.connection.log_insert_into_db(session, 'InfoLog', 'Mapping encoding completed successfully')
            return useful_rating_data
        except Exception as e :
            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))

    def creating_dish_liked_list(self,session,useful_rating_data):
        try :
            """
            User gives input for recommended dish for their restaurant. We are taking some of the top liked dishes from the
            training data. If the recommended dish is present in the data, we are mapping it to 1 or else we will map to 0.
            This information is useful while choosing the restaurant. For example if we are ordering food from a particular restaurant,
            the first thing which we see is recommended dish. If we have our liked dish, then we will go ahead and order the food in that
            particular restaurant. So this information is important in rating.

            Written by: Priya Ganesan
            Version : 1.0
            """
            self.connection.log_insert_into_db(session, 'DebugLog', 'Trying to create dish liked file')
            #since dish_liked column is comma seperated, we are trying to take individual dishes
            useful_rating_data = useful_rating_data.assign(dish_liked=useful_rating_data['dish_liked'].str.split(',')).explode('dish_liked')
            #removing the extra space present in the dish
            useful_rating_data['dish_liked'] = useful_rating_data['dish_liked'].str.strip()
            #converting to upper case as it will be easy to compare the user input to the list of the dish liked
            useful_rating_data['dish_liked'] = useful_rating_data['dish_liked'].str.upper()
            dish_liked_counts = useful_rating_data['dish_liked'].value_counts().rename('liked_dish_count')
            df1 = pd.DataFrame(dish_liked_counts)
            #taking the top dish liked and the value counts. Dishes whose value count > 0 is chosen in the list
            df2 = df1[df1['liked_dish_count'] > 50]
            dish_liked_list = list(df2.index)
            dish_liked_file = open("Training_data_information/dish_liked.txt", 'w+')
            for item in dish_liked_list:
                    # write each item on a new line
                    dish_liked_file.write("%s\n" % item)
            dish_liked_file.close()

            self.connection.log_insert_into_db(session, 'InfoLog', 'dish liked file successfully created')

        except Exception as ex:
            self.connection.log_insert_into_db(session, 'ErrorLog', str(ex))

    def KNNImputer(self,session,X_train,X_test):
        """
        The dataset is having null values. This method imputes null values
        Written by : Priya Ganesan
        Version : 1.0
        """
        try :
            self.connection.log_insert_into_db(session, 'DebugLog','Trying to start imputation')
            #giving the value of k =2 for performing knn
            imputer = KNNImputer(n_neighbors=2)
            impute = imputer.fit(X_train)
            #the imputation done for train is also followed for the test data
            X_train = impute.transform(X_train)
            X_test = impute.transform(X_test)
            return X_train,X_test

        except Exception as e :
            self.connection.log_insert_into_db(session, 'ErrorLog', str(e))


import sys
import os
from cx_Freeze import setup, Executable

# TARGET
target = Executable(
    script="main.py",
    base="Win32GUI",
    icon="icon.ico"
)

# SETUP CX FREEZE
setup(
    name="Right Posture",
    version="1.0",
    description="Right Posture",
    author="Yuul B. Alwright",
    executables=[target]
)

options = {
    'build_exe': {
        'build_exe': './/build'
    }
}
# Type python setup.py build to build.exe


import logging

def get_logger(sheet_title):
    logger = logging.getLogger("exampleApp")
    logger.setLevel(logging.INFO)

    # create the logging file handler
    fh = logging.FileHandler(f"{sheet_title.replace('/', '.')}.log")
    fh.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    ch.setLevel(logging.ERROR)

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)

    # add handler to logger object
    logger.addHandler(fh)
    logger.addHandler(ch)
    return logger


# -*- coding: utf-8 -*-
"""Created on Wed Oct 28 05:21:08 2020.

@author: ben
"""
import pytest
import logging
import pathlib
import context
from dataclasses import asdict
from typing import List, Dict, Tuple
from palitype.txt_files \
    import build_path, read_file, read_grammar, write_file, readfile
from palitype.strings import camel_to_snake
from palitype.palilex import delimiter_locations
from palitype.palilex import group_into_sections, get_settings
from palitype.db_ref import populate_sections
from palitype.palilex import markup_substitution
from palitype.classes import Delim, ModCounter, Setting
from palitype.constants import VERSE_DELIM

#import lark
#lark.logger.setLevel(logging.DEBUG)
import re
from hypothesis import given
import hypothesis.strategies as st
d = context.set_path()
log_fn = d / 'tests' / 'error.log'
logging.basicConfig(filename=log_fn, level=logging.DEBUG, filemode='w')

# ONLY APPEND NEW TEXT TO THE END OTHERWISE EXISTING TESTS WILL FAIL!!!
def get_pali_shorthand(file_nr: int = 0):
    """Example text.

    Notes:
        1. extra '{' before 'striving' that has no '}'
    """
    _l = (('{', '=', '}'), read_file(f"input_text_{file_nr}.txt", 'tests'))
    return _l

def test_write_read_file():
    """Test read_file()"""

    filename = "test.txt"
    dir_name = pathlib.Path(__file__).parent
    delimiters, text = get_pali_shorthand()
    nr_char = write_file(text, filename, dir_name)
    _fn = build_path(filename, dir_name)
    assert nr_char == len(text) and _fn.is_file()
    file_text = read_file(filename, dir_name)
    logging.info('test')
    assert file_text[0] == text[0]
    _fn.unlink()
    assert not _fn.is_file()

    grammar = read_grammar('restructuredtext.grammar')
    assert grammar[0] == '/'

def test_camel_to_snake():
    """
    Tests:
        Changing camel style strings to snake-style strings.
    """
    test_str = "sectionElement"
    assert camel_to_snake(test_str) == "section_element"
    assert test_str == "sectionElement"
    test_str = "SectionElement"
    assert camel_to_snake(test_str) == "Section_element"
    grammar = read_grammar('restructuredtext.grammar')
    new_grammar = camel_to_snake(grammar)
    assert re.search('comment_line', new_grammar)
    assert re.search('Line_break', new_grammar)

def group_id_loop(delims, tokens):
    counter = -1
    for token in delims:
        if token.str == tokens[-1]:  # end delimiter
            counter += 1
            assert token.group_id == counter
    assert counter > -1

def test_group_sections():
    delimiters, text = get_pali_shorthand()
    delims = delimiter_locations(delimiters, text)
    delims = group_into_sections(delimiters, delims)
    assert len(delims) > 0
    group_id_loop(delims, delimiters)

pattern = re.compile(r".*v=[^=]*?e=[^=]*?p=[^=]*?\.=[^=]*")
verse_pattern = [st.from_regex(pattern, fullmatch=True)]

@given(*verse_pattern)
def test_group_sections2(text):
    delimiters = ['v=', 'e=', "p=", ".="]
    delims = delimiter_locations(delimiters, text)
    delims = group_into_sections(delimiters, delims)
    assert len(delims) > 0
    group_id_loop(delims, delimiters)

@pytest.mark.parametrize('filename, directory',
                         [('palitype_instr_0.yml', 'tests')])
def test_settings(filename, directory):
    yaml_text = read_file(filename, directory)
    delim_dict = get_settings(yaml_text).delim_dict
    assert delim_dict['==.'].tag == "Pali"

@pytest.mark.parametrize(
    'text, text_to_markup, finished_text, tok',
    [("""Appatiṭṭhaṃ khvāhaṃ anāyūhaṃ
[#āyūhaṃ] ogham atari ti.""", """anāyūhaṃ
[#āyūhaṃ]_ ogham atari""", """Appatiṭṭhaṃ khvāhaṃ anāyūhaṃ
[#āyūhaṃ] ogham atari ti.""", '*'),
     ("""e=By not reaching out, is the flood crossed.
      p=Appatiṭṭhaṃ khvāhaṃ anāyūhaṃ [#āyūhaṃ]_ ogham atari.=.i=SN1.1.1
      e=By not reaching out, is the flood crossed.""",
      """Appatiṭṭhaṃ khvāhaṃ anāyūhaṃ [#āyūhaṃ]_ ogham atari.""",
      """e=By not reaching out, is the flood crossed.
      p=*Appatiṭṭhaṃ khvāhaṃ anāyūhaṃ [#āyūhaṃ]_ ogham atari.*=.i=SN1.1.1
      e=By not reaching out, is the flood crossed.""", '*')])
def test_surround(text, text_to_markup, finished_text, tok):

    istart = text.find(text_to_markup)
    iend = istart + len(text_to_markup)
    mu = Delim.surround(tok, text[istart:iend])
    print(mu, istart, iend)
    assert text.replace(text[istart:iend], mu) == finished_text

def verse_line(start_whitesp: str,
               phrases: str,
               end_whitesp: str,
               markup: str = '',
               verse: str = '') -> str:

    return '\n'.join(
        [start_whitesp + verse + markup + p + markup + end_whitesp for p in phrases])

def make_verse(start_whitesp: str,
               phrases: List[str],
               end_whitesp: str,
               m: str = '',
               verse: str = '') -> Tuple[str, str]:
    """Make a verse and its corresponding marked-up version.

    Parameters
    ----------
    start_whitesp : str
    phrases : List[str]
    end_whitesp : str
    m : str
        The token for inline markup.

    Returns
    -------
    test_text : str
        Original text.
    answer : str
        Marked-up version
    """

    answer = verse_line(start_whitesp, phrases, end_whitesp, markup=m, verse=verse)
    test_text = verse_line(start_whitesp, phrases, end_whitesp)
    return test_text, answer

def verse_n_lang(delims: Dict[str, Delim],
                 start_whitesp: str,
                 phrases: List[str],
                 end_whitesp: str,
                 verse: str = '') -> Tuple[str, str]:
    orig_tot = []
    marked_tot = []
    for delim in list(delims.values())[:-1]:
        orig, marked = make_verse(start_whitesp,
                                  phrases,
                                  end_whitesp,
                                  m=delim.inline_markup,
                                  verse=verse)
        orig_tot.append(
            orig.replace(orig.strip(), delim.token + orig.strip(), 1))
        marked_tot.append(marked)
    orig_tot.append(delims[list(delims.keys())[-1]].token)

    return '\n'.join(orig_tot), '\n'.join(marked_tot)

INLINE_MARKUP = '*'

def make_delims(nr_lang: int) -> Dict[str, Delim]:
    """Make `nr_lang` delims plus verse delimiter and end delimiter.

    The first delimiter is the verse delimiter.
    The second last delimiter has inline markup '*'.
    The last delimiter is the end delimiter.

    Parameters
    ----------
    nr_lang : int
        Number of languages.

    Returns
    -------
    delim : Delim
        DESCRIPTION.
    """

    if nr_lang > 24:
        nr_lang = 24
    elif not nr_lang:
        nr_lang = 1
    inline_markup = [''] * (nr_lang + 2)  # extra verse and end delimiters
    inline_markup[-2] = INLINE_MARKUP  # 2nd last delimiter has inline markup
    verse_token = 'VT='
    VERSE_DELIM.token = 'VT='
    delim = {verse_token: Delim(**asdict(VERSE_DELIM))}
    for lang_ix, m in enumerate(inline_markup[1:]):
        ix_s = ord('a') + lang_ix
        token = chr(ix_s) + '='
        delim[token] = (Delim(token=token,
                              tag=''.join(
                                  [chr(i) for i in range(ix_s, ix_s + 3)]),
                              inline_markup=m))
    return delim

phrase_pattern = re.compile(r"[^\s=|]+[ \w]+[^\s=|]+")
white_space_pattern = re.compile(r" +")
verse_pattern = [
    st.from_regex(white_space_pattern, fullmatch=True),
    st.lists(st.from_regex(phrase_pattern, fullmatch=True), min_size=1),
    st.from_regex(white_space_pattern, fullmatch=True)
]

@given(*verse_pattern)  # type: ignore
def test_delim_modify_lines(start_whitesp, phrases, end_whitesp):
    delim = Delim()
    delim.inline_markup = '*'
    test_text, answer = make_verse(start_whitesp, phrases, end_whitesp,
                                   delim.inline_markup)
    mod = ModCounter()
    mod_text = delim.get_modified_lines(test_text, mod)
    assert mod_text == answer
    assert getattr(mod, "inline_markup", 0) == 1

@given(*verse_pattern)  # type: ignore
def test_verse_n(start_whitesp, phrases, end_whitesp):
    settings = Setting({})
    settings.delim_dict = make_delims(2)
    next(iter(settings.delim_dict.values())).tag = 'dont add verse'
    test_text, answer = verse_n_lang(settings.delim_dict, start_whitesp,
                                     phrases, end_whitesp)
    mod = ModCounter()

    delims = list(settings.delim_dict.keys())
    mod_text, mod = markup_substitution(
        settings,
        group_into_sections(delims, delimiter_locations(delims, test_text)),
        test_text)
    assert mod_text == answer

def delims_1(delim_dict):
    tokens = list(delim_dict.keys())
    assert delim_dict[tokens[0]].tag == VERSE_DELIM.tag
    assert delim_dict[tokens[-2]].inline_markup == INLINE_MARKUP
    return

@given(*verse_pattern)  # type: ignore
def test_verse_markup(start_whitesp, phrases, end_whitesp):
    settings = Setting({})
    delim_dict = make_delims(2)
    delims_1(delim_dict)
    test_text, answer = verse_n_lang(delim_dict,
                                     start_whitesp,
                                     phrases,
                                     end_whitesp,
                                     verse='| ')
    mod = ModCounter()
    settings.delim_dict = delim_dict
    delims = list(delim_dict.keys())
    delim_list = group_into_sections(delims,
                                     delimiter_locations(delims, test_text))
    group_id_loop(delim_list, delims)
    mod_text, mod = markup_substitution(settings, delim_list, test_text)
    assert mod_text == answer

@pytest.mark.parametrize('filename, directory, nr_pali',
                         [('palitype_instr_1.yml', 'tests', 3),
                          ('palitype_instr_0.yml', 'tests', 7)])
def test_inline_markup(filename, directory, nr_pali):
    yaml_text = read_file(filename, directory)
    settings = get_settings(yaml_text)
    delim_dict = settings.delim_dict
    _, text = get_pali_shorthand()
    delims = list(delim_dict.keys())
    d = delimiter_locations(delims, text)
    group_into_sections(delims, d)
    mod_text, mod = markup_substitution(settings, d, text)
    assert mod.inline_markup == nr_pali

@pytest.mark.parametrize('filename, directory', [
    ('input_text_0.txt', 'tests'),
])
def test_newlines(filename, directory):
    t, n = readfile(filename, directory)
    assert True


#!/usr/bin/env python
import os
import glob
import optparse

import cv2
import torch
import numpy as np

from PIL import Image
from torch import nn
from torchvision import transforms
from torch.utils.data import Dataset, DataLoader

# - Fixed seed (PyTorch, Numpy, Python)
# - What is accuracy, precicion, recall?
# - Model checkpoints
# - Use nn.Sequential in Model.forward()
# - Have a description on the final YOLO layers, how would we implement it?
#   Tranforming(Rotation, Scaling)?
# - Learn about regularization. Implement in model, save as different checkpoints.

## Things we should need in the final result
# - Save error and verification error to file so we can inspect it as a graph
# -

class CatsAndDogsDataset(Dataset):
    def __init__(self, f_name, transform):
        self.transform = transform
        self.dogcat_list = []

        cats_paths = os.path.join(f_name, "cats", "*.jpg")
        cats = glob.glob(cats_paths)
        dog_paths = os.path.join(f_name, "dogs", "*.jpg")
        dogs = glob.glob(dog_paths)
        ## 0 = dogs
        ## 1 = cats
        dog_constant = torch.zeros(1)
        cat_constant = torch.ones(1)

        if torch.cuda.is_available():
            dog_constant = dog_constant.cuda()
            cat_constant = cat_constant.cuda()

        for path in dogs:
            item = (dog_constant, path)
            self.dogcat_list.append(item)

        for path in cats:
            item = (cat_constant, path)
            self.dogcat_list.append(item)

    def __len__(self):
        return len(self.dogcat_list)

    def __getitem__(self, idx):
        filename = self.dogcat_list[idx][1]
        classCategory = self.dogcat_list[idx][0]
        im = Image.open(filename)
        if self.transform:
            im = self.transform(im)
        return im.view(-1), classCategory, filename

class CatsAndDogsModel(nn.Module):
    def __init__(self, image_size, out_dim):
        super(CatsAndDogsModel, self).__init__()
        in_dim = image_size[0] * image_size[1]

        self.layer = nn.Sequential(
            nn.Linear(in_dim, 20000),
            nn.Linear(20000, out_dim),
            nn.Sigmoid(),
        )

    def forward(self, x):
        return self.layer(x)

def serialize_model(model, epoch, loss, validate_loss):
    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'loss': loss,
        'validate_loss': validate_loss,
    }, 'model.{}.pt'.format(epoch))

def load_model_from_file(model, path):
    "Returns the saved epoch of the model"
    checkpoint = torch.load(path)
    model.load_state_dict(checkpoint['model_state_dict'])
    return checkpoint['epoch']

def train_model(model, epochs, dataset, test_dataset, save_every=10, start_epoch=0):
    dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

    print(f"Max mem: {torch.cuda.max_memory_allocated()}, allocated: {torch.cuda.memory_allocated()}")

    model.train(True)

    learning_rate = 0.001
    optimiser = torch.optim.SGD(model.parameters(), lr=learning_rate)
    criterion = nn.MSELoss()

    try:
        inter_epoch_loss = 0
        for epoch in range(epochs):
            print("epoch #{}".format(epoch+start_epoch+1))
            print("Herro people")
            running_loss = 0
            for images, labels, path in dataloader:
                optimiser.zero_grad()

                if torch.cuda.is_available():
                    images = images.cuda()
                    labels = labels.cuda()
                    #print(f"device: {torch.cuda.get_device_name(images.get_device())}")

                output = model.forward(images)
                loss = criterion(output, labels)
                loss.backward()
                optimiser.step()

                running_loss += loss.item()
                inter_epoch_loss += loss.item()
            else:
                print(f"Total epoch loss: {running_loss/len(dataloader)}")

            print(f"Max mem: {torch.cuda.max_memory_allocated()}, allocated: {torch.cuda.memory_allocated()}")

            if epoch % save_every == 0:
                validate_loss = validate_model(model, test_dataset)
                serialize_model(model, epoch+start_epoch, inter_epoch_loss, validate_loss)
                inter_epoch_loss = 0
    except KeyboardInterrupt:
        pass

    model.train(False)

def validate_model(model, dataset):
    "Returns the loss of a single validation epoch"
    dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

    model.train(False)

    criterion = nn.MSELoss()

    total = 0

    cats_total = 0
    cats_total_tp = 0
    cats_total_fp = 0

    dogs_total = 0
    dogs_total_tp = 0
    dogs_total_fp = 0

    running_loss = 0
    for images, labels, _ in dataloader:

        if torch.cuda.is_available():
            images = images.cuda()
            labels = labels.cuda()

        output = model.forward(images)
        loss = criterion(output, labels)
        loss.backward()

        results = output > 0.5
        results = list(zip(results, labels))

        for c in results:
            tp = 0
            fp = 0

            if c[0] == c[1].byte():
                tp += 1
            elif c[0] != c[1].byte():
                fp += 1

            if c[0] == 0:
                dogs_total += 1
                dogs_total_tp += tp
                dogs_total_tp += fp
            elif c[0] == 1:
                cats_total += 1
                cats_total_tp += tp
                cats_total_fp += fp

        total += len(results)

        running_loss += loss.item()

    # test_model is called from withing train_model, as such we should return to training mode

    model.train(True)

    cats_precision = cats_total_tp / (cats_total_tp + cats_total_fp)
    dogs_precision = dogs_total_tp / (dogs_total_tp + dogs_total_fp)

    cats_recall = cats_total_tp / (cats_total)
    dogs_recall = dogs_total_tp / (dogs_total)

    print("cats: ", cats_precision, ", dogs: ", dogs_precision)
    print("cats: ", cats_recall, ", dogs:", dogs_recall)

    return running_loss

if __name__ == "__main__":
    torch.manual_seed(69)
    np.random.seed(69)

    print(f"Using cuda: {torch.cuda.is_available()}")

    image_size = (100, 100)

    transform = transforms.Compose([
        transforms.Resize(image_size),
        transforms.Grayscale(),
        transforms.ToTensor(),
        transforms.Normalize((0.5,), (0.5,))
    ])

    training_dataset = CatsAndDogsDataset('dataset/training_set/', transform)
    test_dataset = CatsAndDogsDataset('dataset/test_set/', transform)

    model = CatsAndDogsModel(image_size, 1)
    if torch.cuda.is_available():
        model = model.cuda()
        model = nn.DataParallel(model)

#    start_epoch = load_model_from_file(model, 'first_model/model.2300.pt')
    start_epoch = 0
    train_model(model, 1, training_dataset, test_dataset, save_every=5, start_epoch=start_epoch)
    #validate_model(model, test_dataset, device)

    dataloader = DataLoader(test_dataset, batch_size=1, shuffle=True)
    total_tested = 0
    total_correct = 0

    for img, label, path in dataloader:
        output = model(img.cuda())
        cv_img = cv2.imread(path[0])
        #cv_img = img.detach().reshape((100, 100)).numpy()

        cv_img = cv2.resize(cv_img, (400, 400))
#        cv_img = cv2.cvtColor(cv_img, cv2.COLOR_GRAY2RGBA)
        cv2.putText(cv_img, f"truth: {'cat' if label > 0.5 else 'dog'}, {label.item()}", (0, 30), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 255, 255))

        correct = bool(output.item() > 0.5) == bool(label.item() > 0.5)

        total_tested += 1
        total_correct += 1 if correct else 0

        if correct:
            color = (0, 255, 0)
        else:
            color = (0, 0, 255)

        cv2.putText(cv_img, f"guess: {'cat' if output.item() > 0.5 else 'dog'}, {output.item():.5f}", (0, 60), cv2.FONT_HERSHEY_DUPLEX, 1, color)
        cv2.putText(cv_img, f"{total_correct}/{total_tested}, {float(total_correct)/float(total_tested)*100.0:.2f}%", (0, 90), cv2.FONT_HERSHEY_DUPLEX, 1, color)
        cv2.imshow("random", cv_img)
        cv2.waitKey(400)

    print(f"{total_correct}/{total_tested}, {float(total_correct)/float(total_tested)*100.0:.2f}%")

    # Find Learning rate - 0.001
    # Find optmiser      - SGD
    # Find loss function - MSE(Mean squared error), BCE(binary cross entropy)

    # for (images, cats) in dataloader:
    #     for i in range(len(cats)):
    #         out = model.forward(images[i])
    #         print("model out: {}, is_cat: {}".format(float(out), float(cats[i])))


import pyperclip
import re
import time
import os
from difflib import SequenceMatcher
# Get the directory of the executed script
script_dir = os.path.dirname(os.path.abspath(__file__))

# Construct the file path relative to the script's directory
file_path = os.path.join(script_dir, 'sanat.txt')

# Open the file for reading with utf-8 encoding
with open(file_path, encoding='utf-8') as input_file:
    sanat = input_file.read()
    lines = input_file.readlines()

#pharse sanat to list
spellingremove = re.sub('/.*?/', "00", sanat)

#replace \n0\n with \n
testi = re.sub(r'\n00\n', ':', spellingremove)

#replace \n: with :
testi2 = re.sub(r'\n:', ':', testi)

#make two way dict pair devided by :
testi33 = re.sub(r':\n', ':', testi2)
testi3 = testi33.replace(", ", ",")

#remove all empty lines from testi3
lines = testi3.split("\n")
non_empty_lines = [line for line in lines if line.strip() != ""]
word_translations = "\n".join(non_empty_lines).replace(", ", "")
#print(word_translations)

while True:
    pyperclip.waitForNewPaste()
    clipboard = pyperclip.paste()
    search_string = clipboard

    best_match = None
    best_similarity = 0

    for line in word_translations.splitlines():
        word, translation = line.split(':')
        similarity_word = SequenceMatcher(None, search_string, word).ratio()
        similarity_translation = SequenceMatcher(None, search_string, translation).ratio()
        if similarity_word > best_similarity:
            best_similarity = similarity_word
            best_match = translation
        if similarity_translation > best_similarity:
            best_similarity = similarity_translation
            best_match = word

    if best_match is not None:
        print(best_match)
        pyperclip.copy(best_match)

    #wait for next copy
    time.sleep(1)


"""
File: no_duplicates_test_issue#103_issue#105.py
Author: Dhruv Duggal
Description: This file tests that duplicates are not being created for tabgroups or active tabs
Tests Contained: This file contains test #7,#8 and #9
"""

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from time import sleep
import json
import os
all_tests_check = 0
CRED = '\033[91m'
CGREEN = '\33[32m'
CYELLOW = '\33[33m'
CEND = '\033[0m'

print(os.getcwd())
options = Options()
print("loading packed extension")
options.add_argument("load-extension=./project/build/")
# overcome limited resource problems
options.add_argument("--disable-dev-shm-usage")
# Bypass OS security model
options.add_argument("--no-sandbox")
# starting window maximized to prevent any scrolling issues
options.add_argument("--start-maximized")
print("set up driver")
# creating chrome driver
driver = webdriver.Chrome(options=options)

# assigning the extensions uid
uid = "flfgpjanhbdjakbkafipakpfjcmochnp"

# opening multiple of the same websites in different tabs
driver.get("https://facebook.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[1])
driver.get("https://facebook.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[2])
driver.get("https://facebook.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[3])
driver.get("https://facebook.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[4])
driver.get("https://facebook.com")
driver.switch_to.window(driver.window_handles[4])
# going to the extensions menu page
driver.get("chrome-extension://"+ uid +"/menu.html")

'''
Test Number #7

Description: This test checks that active tabs are not being populated with duplicates when the same website is opened
multiple times in different tabs

'''
print(CYELLOW + "Running Test 7" + CEND)
# finding active tabs element by class name
active_tabs_div = driver.find_element_by_class_name("activeTabs")
# finding all tabContainer class named elements
active_tabs = active_tabs_div.find_elements_by_class_name("tabContainer")
# creating a counter to check the number of tabs for a particular website
facebook_count = 0
# checking the active tabs and updating this count value
for i in active_tabs:
    if "Facebook" in i.text:
        facebook_count +=1
try:
    assert facebook_count == 1
except AssertionError:
    print(CRED + "Test 7 Failed" + CEND)
    print(CRED + "Duplicate active tabs were rendered" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 7 Passed" + CEND)
print("-----------------------")

'''
Test Number #8

Description: This test is checking that opening a new tab after *after* checking for duplicates does not cause
the code to break and render duplicate tabs by mistake

'''
print(CYELLOW + "Running Test 8" + CEND)
# getting a different website in the current tab
driver.get("https://instagram.com")
# opening a new tab and getting the menu.html page of the chrome extension
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[5])
driver.get("chrome-extension://"+uid+"/menu.html")

# checking the active tabs
active_tabs_div = driver.find_element_by_id("activeTabs")
active_tabs = active_tabs_div.find_elements_by_class_name("tabContainer")
# creating a counter for the number of tabs for a website opened multiple times
facebook_second_count = 0
# looping through all the active tabs
for i in active_tabs:
    if "Facebook" in i.text:
        facebook_second_count += 1
        facebook_tab = 1
    elif "Instagram" in i.text:
        insta_tab = 1
# checking if all tabs are rendered properly and there are no duplicates
try:
    assert facebook_tab == 1 and insta_tab == 1 and facebook_count == 1
except AssertionError:
    print(CRED + "Test 8 failed" + CEND)
    if facebook_tab != 1 or insta_tab != 1:
        print(CRED + "Active tabs were not rendered correctly" + CEND)
    if facebook_count == 0:
        print(CRED + "Duplicate active tabs were rendered" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 8 Passed" + CEND)
assert facebook_tab == 1 and insta_tab == 1 and facebook_second_count == 1
print("-----------------------")

'''
Test Number #9

Description: This test checks that duplicate tab groups are not being created when two tab groups are
given the same name

'''
print(CYELLOW + "Running Test 9" + CEND)
# creating a new tab group called Test Group
add_group_button = driver.find_element_by_class_name("addGroup")
add_group_button.click()
model_body = driver.find_element_by_class_name("modal-body")
input_name = model_body.find_element_by_id("groupName")
# sending "Test Group" string to input
input_name.send_keys("Test Group")
# sending enter key to page
ActionChains(driver).send_keys(Keys.ENTER).perform()

# creating another tab group called Test Group
add_group_button = driver.find_element_by_class_name("addGroup")
add_group_button.click()
model_body = driver.find_element_by_class_name("modal-body")
input_name = model_body.find_element_by_id("groupName")
# sending "Test Group" string to input
input_name.send_keys("Test Group")
# sending enter key to page
ActionChains(driver).send_keys(Keys.ENTER).perform()

# creating another tab group called Test Group
add_group_button = driver.find_element_by_class_name("addGroup")
add_group_button.click()
model_body = driver.find_element_by_class_name("modal-body")
input_name = model_body.find_element_by_id("groupName")
# sending string "Test Group" to input
input_name.send_keys("Test Group")
# sending enter key to page
ActionChains(driver).send_keys(Keys.ENTER).perform()

# creating a tab group called Test Group3
add_group_button = driver.find_element_by_class_name("addGroup")
add_group_button.click()
model_body = driver.find_element_by_class_name("modal-body")
input_name = model_body.find_element_by_id("groupName")
# sending string "Test Group3" to input
input_name.send_keys("Test Group3")
# sending enter keys to page
ActionChains(driver).send_keys(Keys.ENTER).perform()

# creating another tab group called Test Group3
add_group_button = driver.find_element_by_class_name("addGroup")
add_group_button.click()
model_body = driver.find_element_by_class_name("modal-body")
input_name = model_body.find_element_by_id("groupName")
# sending string "Test Group3" to input
input_name.send_keys("Test Group3")
# sending enter key to page
ActionChains(driver).send_keys(Keys.ENTER).perform()
# finding all cards by class name
cards = driver.find_elements_by_class_name("card")
# creating counter to ensure that only one group with a unique name is created
test_group_count = 0
test_group_3_count = 0
# looping through all the tab groups
for i in cards:
    header = i.find_element_by_class_name("card-header")
    if header.text == "Test Group":
        test_group_count += 1
    elif header.text == "Test Group3":
        test_group_3_count += 1
try:
    assert test_group_3_count == 1 and test_group_count == 1
except AssertionError:
    print(CRED+"Test 9 failed" + CEND)
    print(CRED+"Duplicate tab groups were rendered"+CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 9 Passed" + CEND)
print("-----------------------")
assert all_tests_check == 0, CRED + "Some Tests Failed" + CEND
print(CGREEN + "All Tests Passed" + CEND)

# writing coverage details
coverage_json_file = open("./project/.nyc_output/#103_#105.json","w+")
json.dump(driver.execute_script("return window.__coverage__;"), coverage_json_file)
coverage_json_file.close()
driver.quit()


import os
import socket

from pyngrok import ngrok

host = ""
port = int(os.environ.get("PORT"))

# Create a TCP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Bind a local socket to the port
server_address = ("", port)
sock.bind(server_address)
sock.listen(1)

# Open a ngrok tunnel to the socket
public_url = ngrok.connect(port, "tcp").public_url
tunnel_url = socket.gethostbyname(public_url.strip("tcp://").split(":")[0])
tunnel_port = public_url.strip("tcp://").split(":")[1]
print(f" * ngrok tunnel available, access with `telnet {tunnel_url} {tunnel_port}`")
print("ngrok tunnel \"{}\" -> \"tcp://127.0.0.1:{}\"".format(public_url, port))

while True:
    connection = None
    try:
        # Wait for a connection
        print("\nWaiting for a connection ...")
        connection, client_address = sock.accept()

        print("... connection established from {}".format(client_address))

        # Receive the message, send a response
        while True:
            data = connection.recv(1024)
            if data:
                print("Received: {}".format(data.decode("utf-8")))

                message = "pong"
                print("Sending: {}".format(message))
                connection.sendall(message.encode("utf-8"))
            else:
                break
    except KeyboardInterrupt:
        print(" Shutting down server.")

        if connection:
            connection.close()
        break

sock.close()


import webapp2
import jinja2
import os
import models
import datetime
from google.appengine.api import users

template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.getcwd()))

class Confirm(webapp2.RequestHandler):
	def get(self):
		user = users.get_current_user()
		login_url = users.create_login_url(self.request.path)
		logout_url = users.create_logout_url(self.request.path)
		userprefs = models.get_userprefs()

		template = template_env.get_template('confirm.html')
		invites = []
		for element in userprefs.created:
			new_obj = models.get_meeting(element)
			invites.append(new_obj)
		context = {
		'user' : user,
		'login_url' : login_url,
		'logout_url' :logout_url,
		'userprefs' : userprefs,
		'invites' : invites,
		}
		self.response.out.write(template.render(context))

	def post(self):
		userprefs = models.get_userprefs()
		try:
			meet_id = int(self.request.get('identify'))
			userprefs.fix(meet_id)
			userprefs.put()
		except ValueError:
			# User entered a value that wasn't a int. Ignore for now.
			pass
		self.redirect('/')

application = webapp2.WSGIApplication([('/confirm', Confirm)],debug=True)