import os
import yaml
import lasagne
import pandas as pd
import numpy as np

from network import Network
import architectures as arches

L = lasagne.layers

# loads data with names according to autoload_data.py
from autoload_data import *

# load specs for all networks
with open('arch_specs.yaml') as archfile:
    arch_dict = yaml.load(archfile)

### Compiling results from saved parameters ###

def compute_pretrained_results(net, archname, idx, test_data, fake=False):
    """
    Compute pre-tuning results for a given arch/network on appropriate test data
    """
    Xt, yt = test_data

    if fake:
        fname = '{} {} split fake data.npz'
        fname = fname.format('fake_' + archname, idx)
        paramsdir = os.path.join(paramsdir_, 'fake_' + archname)

    else:
        fname = '{} {} split agg fit exp 1-4.npz'
        fname = fname.format(archname.replace('_', ' '), idx)
        paramsdir = os.path.join(paramsdir_, archname[:-1])

    results_df = pd.DataFrame(index=np.arange(Xt.shape[0]), columns=[idx])

    net.load_params(os.path.join(paramsdir, fname))

    nlls = net.itemized_test_fn(Xt, yt)
    predictions = net.output_fn(Xt)
    results_df[idx] = nlls

    return results_df, predictions

def compute_tuned_results(net, archname, idx, test_idx, test_data, df):
    """
    Compute post-tuning results for a given architecture/network on appropriate
        test data
    """
    Xt, yt = test_data
    group_idx = (test_idx - 1) % 5 # fix eventually to take df/groupidx/selection passed independently?
    selection = df.loc[df['group']==(group_idx+1)].index.values
    results_df = pd.DataFrame(index=np.arange(Xt.shape[0]), columns=[idx])
    predictions_df = pd.DataFrame(index=selection, columns=np.arange(36))

    fname = '{} {} agg fit exp 1-4 {} tune fit exp 0.npz'
    fname = fname.format(archname.replace('_', ' '), idx, test_idx)

    net.load_params(os.path.join(paramsdir_, archname[:-1], fname))

    nlls = net.itemized_test_fn(Xt[selection, :, :, :], yt[selection])
    predictions = net.output_fn(Xt[selection, :, :, :])
    predictions_df.loc[selection, :] = predictions
    results_df.loc[selection, idx] = nlls
    return results_df, predictions_df

def compute_net_results(net, archname, test_data, df):
    """
    For a given network, test on appropriate test data and return dataframes
    with results and predictions (named obviously)
    """
    pretrain_results = []
    pretrain_predictions = []
    tune_results = []
    tune_predictions = []

    for idx in range(5):
        results_df, predictions_df = compute_pretrained_results(net, archname, idx, test_data)
        pretrain_results.append(results_df)
        pretrain_predictions.append(predictions_df)

    pretrain_results = pd.concat(pretrain_results, axis=1)

    for idx in range(5):
        for test_idx in range(5):
            results_df, predictions_df  = compute_tuned_results(net, archname, idx, test_idx, test_data, df)
            tune_results.append(results_df)
            tune_predictions.append(predictions_df)

    tune_results = pd.concat(tune_results, axis=1, join='inner').stack().unstack()

    return pretrain_results, pretrain_predictions, tune_results, tune_predictions

def rehydrate(verbose=False):
    """
    Recompile networks, load params, and run on appropriate test data

    outputs dictionaries with keys = names of architectures as in arch_specs.yaml

    outputs:
    PTx is pretrained on bulk but untuned
    Tx is tuned
    xR is results (nlls)
    xP is predictions (distributions)

    param_counts is what it sounds like.
    """
    Xt, yt, _, _, _ = loading.unpack_data(df)       # get Xs and ys

    PTR = {}                # results and predictions holders
    TR = {}
    PTP = {}
    TP = {}
    param_counts = {}       # counter of parameters per net

    for archname in arch_dict.keys():
        # for each network
        arch_dir = archname[:-1]        # get directory

        if arch_dir not in os.listdir(paramsdir_):
            # if it doesn't exist
            print("{} not started".format(archname[:-1]))       # alert us
            continue

        files = os.listdir(os.path.join(paramsdir_, arch_dir))
        if not any(archname.replace('_', ' ') in f for f in files):
            # if a network doesn't have a full set of parameter fits
            print("{} not completed".format(archname))      # let us know
            continue

        if verbose:
            print(archname)
        arch = arch_dict[archname]
        af = getattr(arches, arch['type'])
        arch_func = lambda input_var: af(input_var, **arch['kwargs'])
        net = Network(arch_func)        # compile network from specs in arch_specs.yaml

        param_counts[archname] = L.count_params(net.net)        # count the params
        pretrain_R, pretrain_P, tune_R, tune_P = compute_net_results(net, archname, (Xt, yt), df)

        PTR[archname] = pretrain_R      # insert results into respective holders
        TR[archname] = tune_R
        PTP[archname] = pretrain_P
        TP[archname] = tune_P

        pretrain_R.to_csv(os.path.join(resultsdir, 'pretrain {}.csv'.format(archname)))     # save results into respective directories
        tune_R.to_csv(os.path.join(resultsdir, 'train {}.csv'.format(archname)))

    return PTR, TR, PTP, TP, param_counts

### Statistics and summaries ###

def entropy_zets(zets):
    """Shannon entropy"""
    z = np.histogram(zets, bins=np.arange(37), normed=True)[0]
    z = z[z > 0]
    return -(z * np.log2(z)).sum()

def count_pieces(row):
    """Counts pieces in a given binstring state representation"""
    bp, wp = row[['bp', 'wp']]
    n_bp = np.array(list(bp)).astype(int).sum()
    n_wp = np.array(list(wp)).astype(int).sum()

    return n_bp + n_wp

def aggregate_results(PTR, TR, param_counts):
    """
    Args:
    PTR, TR, are dictionaries of results dataframes with archnames as keys,
        archnames as in arch_specs.yaml and PTR/TR as produced by rehydrate()
    param_counts also keyed by archnames and as produced by rehydrate; contains
        parameter counts

    outputs:
    F is a dataframe containing individual networks in rows with mean
        performances in columns

    """
    pc_series = pd.Series(param_counts)
    pc_series.to_csv(os.path.join(resultsdir, 'params per net.csv'))
    F = pd.DataFrame(index=np.arange(len(pc_series.index)), columns=['net name'])
    F['net name'] = pc_series.index
    F['num params'] = pc_series.values

    for k, v in PTR.items():
        idx = F['net name'] == k
        v['subject'] = Ss
        v['mean'] = v.mean()
        F.loc[idx, 'pretrained'] = v['mean'].mean()
        pvt = v.pivot_table(index='subject', values='mean', aggfunc=np.mean).mean().values
        F.loc[idx, 'pretrained subject'] = pvt

    for k, v in TR.items():
        idx = F['net name'] == k
        v['subject'] = Ss
        v['mean'] = v.mean()
        F.loc[idx, 'tuned'] = v['mean'].mean()
        pvt = v.pivot_table(index='subject', values='mean', aggfunc=np.mean).mean().values
        F.loc[idx, 'tuned subject'] = pvt

    F['tuning improvement'] = -(F['tuned'] - F['pretrained'])

    return F

def error_per_piece(archname, resultdict, datadf):
    """
    Args:
    archname is name of architecture as in arch_specs.yaml
    resultdict should be a dictionary of predictions where keys = archnames
    datadf is the original data containing positions, subjects, moves, etc

    Outputs:
    preds is a list of dataframes containing predictions for reach observation
        in datadf and training split for archname
    per_pieces is a df with the mean chance-relative nll for each subject and
        number of pieces in position

    """

    chancenll = lambda x: -np.log(1/(36-x))
    cross_entropy = lambda row: -np.log(row[int(row['zet'])])
    preds = [pd.concat(resultdict[archname][i:i+5]).sort_index() for i in range(5)]

    for pred in preds:
        pred['subject'] = df['subject']
        pred['zet'] = df['zet']
        pred['num pieces'] = df['np']
        pred['rt'] = df['rt']
        pred['chance nll'] = chancenll(df['np'].values)
        pred['error'] = pred.apply(cross_entropy, axis=1)
        pred['relative error'] = pred['chance nll'] - pred['error']

    per_pieces = pd.concat([
        p.pivot_table(index='subject', values='relative error', columns='num pieces').mean(axis=0)
        for p in preds
    ], axis=1)

    return preds, per_pieces


from flask import Flask, request
import statistics

app = Flask(__name__)

@app.route('/')
def index():
    return "Usage;\n<Operation>?X=<Value1, Value2, ..., ValueN>\n"

def Inputs_method():
    try:
        inputs = request.args.get('X', type=str)
        inputs = inputs.split(',')
        values = []
        for value in inputs:
            value = float(value)
            values.append(value)
        return values
    except ValueError:
        Error_msg = "There are error values in inputs, Please provide a list of values,for example 1,2,3,4\n"
        return Error_msg

@app.route('/max')
def maximum():
    values = Inputs_method()
    if not type(values) is str:
        values.sort(reverse=True)
        if values[0].is_integer():
            return str(int(values[0])) + ' \n'
        else:
            return str(float(round(values[0], 3))) + ' \n'
    else:
        return values

@app.route('/min')
def minimum():
    values = Inputs_method()
    if not type(values) is str:
        values.sort()
        if values[0].is_integer():
            return str(int(values[0])) + ' \n'
        else:
            return str(float(round(values[0], 3))) + ' \n'

    else:
        return values

@app.route('/average')
def average():
    values = Inputs_method()
    if not type(values) is str:
        avrg = statistics.mean(values)
        if avrg.is_integer():
            return str(int(avrg)) + ' \n'
        else:
            return str(float(round(avrg, 3))) + ' \n'
    else:
        return values

@app.route('/avg')
def avg():
    values = Inputs_method()
    if not type(values) is str:
        avrg = statistics.mean(values)
        if avrg.is_integer():
            return str(int(avrg)) + ' \n'
        else:
            return str(float(round(avrg, 3))) + ' \n'
    else:
        return values

@app.route('/mean')
def mean():
    values = Inputs_method()
    if not type(values) is str:
        avrg = statistics.mean(values)
        if avrg.is_integer():
            return str(int(avrg)) + ' \n'
        else:
            return str(float(round(avrg, 3))) + ' \n'

@app.route('/median')
def median():
    values = Inputs_method()
    if not type(values) is str:
        med = statistics.median(values)
        if med.is_integer():
            return str(int(med)) + ' \n'
        else:
            return str(float(round(med, 3))) + ' \n'

    else:
        return values

if __name__ == "__main__":
    app.run()


# evaluate the mask rcnn model on the kangaroo dataset
from os import listdir
from xml.etree import ElementTree
from numpy import zeros
from numpy import asarray
from numpy import expand_dims
from numpy import mean
from mrcnn.config import Config
from mrcnn.model import MaskRCNN
from mrcnn.utils import Dataset
from mrcnn.utils import compute_ap
from mrcnn.model import load_image_gt
from mrcnn.model import mold_image
import matplotlib.pyplot as plt
import matplotlib.patches as pt

# class that defines and loads the kangaroo dataset
class KangarooDataset(Dataset):
	# load the dataset definitions
	def load_dataset(self, dataset_dir, is_train=True):
		# define one class
		self.add_class("dataset", 1, "algae")
		self.add_class("dataset",2,"Trash_on_water")
		# define data locations
		images_dir = dataset_dir + '/algae/'
		annotations_dir = dataset_dir + '/annotation/'
		# find all images
		count=0
		dict1={}
		for filename in listdir(images_dir):
			# extract image id
			image_id = filename[19:32]
			# if(image_id in discard_id):
			# 	continue
			# dict1[image_id]=count/
			count+=1
			# skip bad images
			#if image_id in ['00090']:
			#	continue
			# skip all images after 150 if we are building the train set
			if is_train and count >= 1001:
				continue
			# skip all images before 150 if we are building the test/val set
			if not is_train and count < 1001:
				continue
			img_path = images_dir + filename
			ann_path = annotations_dir + filename[:-4]+ '.xml'
			# add to dataset
			self.add_image('dataset', image_id=image_id, path=img_path, annotation=ann_path)

	# extract bounding boxes from an annotation file
	def extract_boxes(self, filename):
		# load and parse the file
		tree = ElementTree.parse(filename)
		# get the root of the document
		root = tree.getroot()
		# extract each bounding box
		boxes = list()
		for box in root.findall('.//bndbox'):
			xmin = int(box.find('xmin').text)
			ymin = int(box.find('ymin').text)
			xmax = int(box.find('xmax').text)
			ymax = int(box.find('ymax').text)
			coors = [xmin, ymin, xmax, ymax]
			boxes.append(coors)
		# extract image dimensions
		width = int(root.find('.//size/width').text)
		height = int(root.find('.//size/height').text)
		return boxes, width, height

	# load the masks for an image
	def load_mask(self, image_id):
		# get details of image
		info = self.image_info[image_id]
		# define box file location
		path = info['annotation']
		# load XML
		boxes, w, h = self.extract_boxes(path)
		# create one array for all masks, each on a different channel
		masks = zeros([h, w, len(boxes)], dtype='uint8')
		# create masks
		class_ids = list()
		for i in range(len(boxes)):
			box = boxes[i]
			row_s, row_e = box[1], box[3]
			col_s, col_e = box[0], box[2]
			masks[row_s:row_e, col_s:col_e, i] = 1
			class_ids.append(self.class_names.index('algae'))
			class_ids.append(self.class_names.index('Trash_on_water'))
			# print(self.class_names.index('Trash_on_water'))
			# print(self.class_names.index('algae'))
		return masks, asarray(class_ids, dtype='int32')

	# load an image reference
	def image_reference(self, image_id):
		info = self.image_info[image_id]
		return info['path']

# define the prediction configuration
class PredictionConfig(Config):
	# define the name of the configuration
	print("hey reached prediction config")
	NAME = "algae_cfg"
	# number of classes (background + kangaroo)
	NUM_CLASSES = 1 + 2
	# simplify GPU config
	GPU_COUNT = 1
	IMAGES_PER_GPU = 1

# calculate the mAP for a model on a given dataset
def plot_actual_vs_predicted(dataset, model, cfg, n_images=5):
	# load image and mask
	for i in range(n_images):
		# load the image and mask
		image = dataset.load_image(i)
		mask, _ = dataset.load_mask(i)
		# convert pixel values (e.g. center)
		scaled_image = mold_image(image, cfg)
		# convert image into one sample
		sample = expand_dims(scaled_image, 0)
		# make prediction
		yhat = model.detect(sample, verbose=0)[0]
		# define subplot
		plt.subplot(n_images, 2, i*2+1)
		# plot raw pixel data
		plt.imshow(image)
		plt.title('Actual')
		# plot masks
		for j in range(mask.shape[2]):
			plt.imshow(mask[:, :, j], cmap='gray', alpha=0.3)
		# get the context for drawing boxes
		plt.subplot(n_images, 2, i*2+2)
		# plot raw pixel data
		plt.imshow(image)
		plt.title('Predicted')
		ax = plt.gca()
		# plot each box
		for box in yhat['rois']:
			# get coordinates
			y1, x1, y2, x2 = box
			# calculate width and height of the box
			width, height = x2 - x1, y2 - y1
			# create the shape
			rect = pt.Rectangle((x1, y1), width, height, fill=False, color='red')
			# draw the box
			ax.add_patch(rect)
	# show the figure
	plt.show()

# load the train dataset
train_set = KangarooDataset()
train_set.load_dataset('./Dataset', is_train=True)
train_set.prepare()
print('Train: %d' % len(train_set.image_ids))
# load the test dataset
test_set = KangarooDataset()
test_set.load_dataset('./Dataset', is_train=False)
test_set.prepare()
print('Test: %d' % len(test_set.image_ids))
cfg = PredictionConfig()

model = MaskRCNN(mode='inference', model_dir='./mode/algae_cfg/', config=cfg)
# load model weights
model_path = './mode/algae_cfg/mask_rcnn_algae_cfg_0050.h5'
#model.load_weights(model_path, by_name=True)
# plot predictions for train dataset
image = plt.imread('img1.jpg')

plt.imshow(image)
# convert pixel values (e.g. center)
scaled_image = mold_image(image, cfg)
# convert image into one sample
sample = expand_dims(scaled_image, 0)
# make prediction
yhat = model.detect(sample, verbose=0)[0]
ax = plt.gca()
# plot each box
for box in yhat['rois']:
	# get coordinates
	y1, x1, y2, x2 = box
	# calculate width and height of the box
	width, height = x2 - x1, y2 - y1
	# create the shape
	rect = pt.Rectangle((x1, y1), width, height, fill=False, color='red')
	# draw the box
	ax.add_patch(rect)
#plt.imshow(yhat)

# create config

# define the model
model = MaskRCNN(mode='inference', model_dir='./mode/algae_cfg/', config=cfg)
# load model weights
model_path = './mode/algae_cfg/mask_rcnn_algae_cfg_0050.h5'
model.load_weights(model_path, by_name=True)
# plot predictions for train dataset
plot_actual_vs_predicted(train_set, model, cfg)
# plot predictions for test dataset
plot_actual_vs_predicted(test_set, model, cfg)


import os
from os.path import join, dirname
from dotenv import load_dotenv
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when, row_number, last, count
from pyspark.sql.window import Window
from fuzzywuzzy import fuzz
import pymongo
import boto3
import re

# load variables from .env file
dotenv_path=join(dirname(__file__),"dags/.env")
load_dotenv(dotenv_path=dotenv_path)

# Accessing environment variables
AWS_ACCESS_KEY=os.getenv('AWS_ACCESS_KEY')
AWS_SECRET_KEY=os.getenv('AWS_SECRET_KEY')
CRM_PATH=os.getenv('CRM_PATH')
DEVICE_PATH=os.getenv('DEVICE_PATH')

# Creating singleton class ,so that it creates only one spark instance
class SingletonSparkSession:
    """
        Creates a SparkSession with necessary configurations.

        Returns:
            SparkSession: The created SparkSession object.
    """

    _instance = None

    def __new__(cls):
        if not cls._instance:
            cls._instance = super().__new__(cls)
            cls._instance.spark = SparkSession.builder \
                .appName("read_S3_write_mongodb") \
                .config("spark.jars.packages", "org.mongodb.spark:mongo-spark-connector_2.12:3.0.1,org.apache.hadoop:hadoop-aws:3.2.2,com.amazonaws:aws-java-sdk-bundle:1.12.180") \
                .config("spark.hadoop.fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") \
                .config("spark.driver.memory", "15g").config("spark.driver.maxResultSize", "4g") \
                .config("spark.hadoop.fs.s3a.access.key", AWS_ACCESS_KEY)\
                .config("spark.hadoop.fs.s3a.secret.key", AWS_SECRET_KEY)\
                .getOrCreate()
        return cls._instance.spark

# Writing the data to moongodb
class SingletonMongoWriter:
    """
    Singleton class for writing data to MongoDB.
    """
    _instance = None

    def __new__(cls):
        """
        Create and return a new instance of the class if it doesn't exist.

        Returns:
            SingletonMongoWriter: The instance of the class.
        """

        if not cls._instance:
            cls._instance = super().__new__(cls)
        return cls._instance

    def write_to_Mongodb(self, dataframe, output_uri):
        """
        Write a DataFrame to MongoDB.

        Args:
            dataframe (pyspark.sql.DataFrame): The DataFrame containing the data to be written.
            output_uri (str): The MongoDB connection URI specifying the target database and collection.

        Returns:
            None
        """
        dataframe.write.format("com.mongodb.spark.sql.DefaultSource") \
            .mode("append") \
            .option("spark.mongodb.output.uri", output_uri) \
            .save()

# For validating data
class DataProcessor:
    """
    A class that validates and processes data.

    This class provides methods for checking file existence in an S3 bucket,
    performing fuzzy matching on words to determine gender, and processing
    CRM and device data.

    Attributes:
        spark (SparkSession): The singleton SparkSession object.
        mongo_writer (SingletonMongoWriter): The singleton MongoDB writer object.

    Methods:
        check_file_exists_in_S3(bucket_name, key):
            Checks if a file exists in the specified S3 bucket.

        fuzzy_ratio(word):
            Determines the gender based on a fuzzy matching ratio of the input word.

        process_crm_data(crm_s3_path):
            Processes CRM data from an S3 bucket.

        process_device_data(device_s3_path):
            Processes device data from an S3 bucket.
    """
    def __init__(self):
        """
        Initializes the DataProcessor object.

        It creates instances of the SingletonSparkSession and SingletonMongoWriter
        classes to access the SparkSession and MongoDB writer objects.
        """
        self.spark = SingletonSparkSession()
        self.mongo_writer = SingletonMongoWriter()

    def check_file_exists_in_S3(self, bucket_name, key):
        """
        Checks if a file exists in the specified S3 bucket.

        Args:
            bucket_name (str): The name of the S3 bucket.
            key (str): The key or path of the file in the S3 bucket.

        Returns:
            bool: True if the file exists in the S3 bucket, False otherwise.
        """

        # Create a session using AWS access key and secret key
        session = boto3.Session(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY)

        # Create an S3 client using the session
        s3_client = session.client('s3')

        try:
            # Use the head_object method to check if the file exists
            s3_client.head_object(Bucket=bucket_name, Key=key)
            print(f"The file '{key}' exists in the bucket '{bucket_name}'.")
            return True

        except s3_client.exceptions.ClientError as e:
            if e.response['Error']['Code'] == '404':
                # If the file does not exist, handle the 404 error
                print(f"The file '{key}' does not exist in the bucket '{bucket_name}'.")
            else:
                # Handle other exceptions
                print(f"We got an exception: {e}")

            return False

    @staticmethod
    def fuzzy_ratio(word):
        """
        Determines the gender based on a fuzzy matching ratio of the input word with 'male' and 'female'.

        Args:
            word (str): The input word.

        Returns:
            str or None: The gender ('male' or 'female') if a match is found with a sufficient ratio, or None otherwise.
        """

        if word is None:
            return None

        male_pattern = "ma(l)*(e)*"
        female_pattern = "fe(m)*(a)*(l)*(e)*"

        # Convert the word to lowercase for case-insensitive matching
        word = word.lower()

        # Check if the word directly matches 'f' or matches the female pattern
        if word == "f" or re.search(female_pattern, word):
            return "female"

        # Check if the word directly matches 'm' or matches the male pattern
        if word == "m" or re.search(male_pattern, word):
            return "male"

        # Calculate fuzzy matching ratios between the word and 'male' and 'female'
        ratio1 = fuzz.ratio(word, "male")
        ratio2 = fuzz.ratio(word, "female")

        # Determine the gender based on the fuzzy matching ratios
        if ratio1 >= 70:
            res = "male"
        elif ratio2 >= 70:
            res = "female"
        else:
            res = None

        return res

    def process_crm_data(self, crm_s3_path):
        """
        Processes CRM data from an S3 bucket.

        Args:
            crm_s3_path (str): The S3 path of the CRM data file.

        Returns:
            DataFrame or None: The processed CRM data as a DataFrame if successful, None otherwise.
        """

        # Checking file exits or not in s3 bucket
        bucket_name = "telecom-data-source"
        key = "crm1.csv"

        file_flag = self.check_file_exists_in_S3(bucket_name=bucket_name, key=key)

        if not file_flag:
            # If the file doesn't exist, print an error message and return None
            print("Some error occured while getting data from s3 bucket")
            return None

        # Creating user define function to transform gender column.
        fuzzy_ratio_udf = self.spark.udf.register("fuzzy_ratio", self.fuzzy_ratio)

        #  Read CRM data from the provided s3 file
        crm_df = self.spark.read.option('header','true').option('inferSchema','true').format("csv").load(crm_s3_path)

        # # Clean and normalize 'gender' column values and replacing null values in year_of_birth with not available
        crm_df = crm_df.withColumn("gender", fuzzy_ratio_udf(col("gender")))\
                       .withColumn("year_of_birth", when(col("year_of_birth").isNull(), 'Not Available').otherwise(col("year_of_birth")))

        # Fill the last non-null value in the 'gender' column for each 'msisdn'
        window_spec = Window.partitionBy("msisdn").orderBy(col("year_of_birth"))
        crm_df = crm_df.withColumn("row_number1", row_number().over(window_spec))

        window_spec = Window.partitionBy("msisdn").orderBy(col("row_number1")).rowsBetween(Window.unboundedPreceding, Window.currentRow)
        crm_df = crm_df.withColumn('gender', last('gender', ignorenulls=True).over(window_spec))

        window_spec = Window.partitionBy("msisdn").orderBy(col("row_number1").desc()).rowsBetween(Window.unboundedPreceding, Window.currentRow)
        crm_df = crm_df.withColumn('gender', last('gender', ignorenulls=True).over(window_spec))
        # Filling all null with Not Available in gender column
        crm_df = crm_df.drop("row_number1")
        crm_df = crm_df.withColumn("gender", when(col("gender").isNull(), "Not Available").otherwise(col("gender")))

        # Create an outer window partitioned by "msisdn"
        outer_window = Window.partitionBy("msisdn")

        # Create a window partitioned by multiple columns including "msisdn", "year_of_birth", "mobile_type", and "gender"
        window = outer_window.partitionBy("msisdn", "year_of_birth", "mobile_type", "gender")

        # Drop duplicates
        crm_df = crm_df.dropDuplicates()

        # Adding required columns to the crm_df
        crm_df = crm_df.withColumn("outer_window", count("*").over(outer_window))\
                        .withColumn("no_of_deactive", count(when(col("system_status").isin("DEACTIVE"), True)).over(outer_window)) \
                        .withColumn("count_except_deactive", count(when(col("system_status").isin("ACTIVE", "SUSPEND", "IDLE"), True)).over(window))\
                        .withColumn("total_active", count(when((col("system_status") == "ACTIVE"), True)).over(outer_window))\
                        .withColumn("count_active", count(when(col("system_status") == 'ACTIVE', True)).over(window)) \
                        .withColumn("count_idle", count(when(col("system_status").isin("IDLE"), True)).over(window))\
                        .withColumn("total_idle", count(when((col("total_active") == 0) & (col("system_status") == "IDLE"), True)).over(outer_window))

        # Replacing the count_except_deactive columm value if it satisfy the given condition
        crm_df = crm_df.withColumn("count_except_deactive", when(col("outer_window") == col("count_except_deactive"), 1).otherwise(col("count_except_deactive")))

        # Replacing the count_except_deactive columm value if it satisfy the given conditions
        crm_df = crm_df.withColumn("count_except_deactive", when(((col("count_active") == 0) & (col("count_except_deactive") == 1) & (col("total_active") >= 1) & (col("no_of_deactive") == 0)), -1).
                                                when(((col("count_active") == 0) & (col("count_except_deactive") == 1) & (col("total_active") == 0) & (col("total_idle") >= 1) & (col("count_idle") == 0)), -1).
                                                otherwise(col("count_except_deactive")))\
                        .withColumn("taken", count(when((col("count_except_deactive") == 1) & (col("system_status") == "ACTIVE"), 1)).over(outer_window))

        # Filtering the based on count_except_deactive column value
        crm_df = crm_df.filter(~(col("count_except_deactive") == -1))

        #  # Replacing the count_except_deactive columm value if it satisfy the given conditions
        crm_df = crm_df.withColumn("count_except_deactive", when((col("total_active") >= 1) & (col("count_active") >= 1) & (col("taken") == 0), 1)
                                                            .when(((col("total_active") == 0) & (col("total_idle") >= 1) & (col("system_status") == "IDLE") & (col("count_idle") >= 1)), 1)
                                                            .when(((col("total_active") == 0) & (col("total_idle") == 0) & (col("system_status") == "SUSPEND") & (col("count_idle") == 0)), 1)
                                                            .otherwise(col("count_except_deactive")))
        # Filtering the records based on conditions
        crm_df = crm_df.filter(((col("system_status") == "DEACTIVE") & (col("no_of_deactive") != 0)) |
                                            ((col("count_except_deactive") == 1) & (col("system_status") == "ACTIVE") & (col("no_of_deactive") == 0)) |
                                            ((~(col("system_status") == "ACTIVE") & (col("count_active") == 0)  & (col("count_except_deactive") == 1) & (col("system_status") == "IDLE") & (col("no_of_deactive") == 0 ) &(col("total_active") == 0))) |
                                            (((col("count_except_deactive") == 1) & (col("count_idle") == 0) & (col("count_active") == 0 ) & (col("system_status") == "SUSPEND") & (col("no_of_deactive") == 0 ))) |
                                            (((col("total_active") >= col("count_active")) & (col("system_status") == "ACTIVE") & (col("count_except_deactive") == 1) & (col("no_of_deactive") == 0) & (col("taken") == 0))) |
                                            ((col("count_active") >= 1) & (col("system_status") == 'ACTIVE') & (col("no_of_deactive") == 0) & (col("taken") == 0)))

        # Drop the unnnessary columns
        crm_df= crm_df.drop("outer_window", "no_of_deactive", "count_except_deactive", "count_active", "count_idle", "total_idle", "total_active", "taken")

        # Filter duplicate 'msisdn' entries based on 'system_status'
        window_spec = Window.partitionBy("msisdn").orderBy(col("system_status").asc())
        crm_df = crm_df.withColumn("row_number", row_number().over(window_spec))
        crm_df = crm_df.filter(col("row_number") == 1).drop("row_number")

        # Write the processed CRM data to MongoDB
        self.mongo_writer.write_to_Mongodb(crm_df, 'mongodb://127.0.0.1/ProjectDB.crm1')

        # Writing to local
        # crm_df.coalesce(1).to_csv('results/crm_validate.csv')
        return crm_df

    def process_device_data(self, device_s3_path):
        """
        Processes device data from an S3 bucket.

        Args:
            device_s3_path (str): The S3 path of the device data file.

        Returns:
            DataFrame or None: The processed device data as a DataFrame if successful, None otherwise.
        """

        # Checking if the file exists in the S3 bucket
        bucket_name = "telecom-data-source"
        key = "device1.csv"
        file_flag = self.check_file_exists_in_S3(bucket_name=bucket_name, key=key)
        if not file_flag:
            print("Some error occurred while getting data from the S3 bucket")
            return None

        # Read device data from CSV
        device_df = self.spark.read.option("header", "true").option("inferSchema", "true").format("csv").load(device_s3_path)

        # Finding the number of nulls in each row
        device_df = device_df.withColumn("no_of_nulls", sum([col(c).isNull().cast('int') for c in device_df.columns]))

        # Remove duplicate 'msisdn' entries based on 'imei_tac' and select the record with fewer null values
        window_spec = Window.partitionBy("msisdn").orderBy('no_of_nulls')
        device_df = device_df.withColumn("row_number", row_number().over(window_spec))
        device_df = device_df.filter(col('row_number') == 1)
        device_df = device_df.drop("row_number").drop("no_of_nulls")

        # Fill missing values in the dataframe with "Not Available"
        device_df = device_df.fillna('Not Available')

        # Write the processed device data to MongoDB
        self.mongo_writer.write_to_Mongodb(device_df, 'mongodb://127.0.0.1/ProjectDB.device1')

        # Writing to local
        # device_df.coalesce(1).to_csv('results/device_validate.csv')

        return device_df

def databaseValidation(db_name, collection_name):
    """
    Validates the existence of a MongoDB database and collection, and checks if a document exists in the collection.

    Args:
        db_name (str): The name of the MongoDB database.
        collection_name (str): The name of the collection within the database.

    Returns:
        int: 1 if the database is validated, 0 otherwise.
    """

    # Create a MongoClient instance
    mongo_client = pymongo.MongoClient('mongodb://127.0.0.1:27017')

    try:
        # Get the list of database names from the MongoClient instance
        db_list = mongo_client.list_database_names()

        # Check if the specified database exists
        if db_name in db_list:
            # Get the specified database
            db = mongo_client.get_database(db_name)

            # Get the list of collection names within the database
            collections_list = db.list_collection_names()

            # Check if the specified collection exists
            if collection_name in collections_list:
                # Get the specified collection
                collection = db.get_collection(collection_name)

                # Find one document in the collection
                document = collection.find_one()

                # Check if a document exists in the collection
                if document is not None:
                    print("Database is validated")
                    return 1

    except pymongo.errors.PyMongoError as e:
        # Handle specific exceptions raised by PyMongo
        print(f"An error occurred during database validation: {str(e)}")

    return 0

if __name__ == '__main__':
    # Creating an object for initilizing spark
    processor = DataProcessor()

    # Validating the CRM database and processing CRM data if validation fails
    if databaseValidation('ProjectDB', "crm1") == 0:
        crm_df = processor.process_crm_data(CRM_PATH)

    # Validating the CRM database and processing device data if validation fails
    if databaseValidation('ProjectDB', "device1") == 0:
        device_df = processor.process_device_data(DEVICE_PATH)


import os
import random
from typing import *

import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from argparse import ArgumentParser
from tqdm import tqdm

import scallopy

class MNISTSort2Dataset(torch.utils.data.Dataset):
  mnist_img_transform = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize(
      (0.1307,), (0.3081,)
    )
  ])

  def __init__(
    self,
    root: str,
    train: bool = True,
    target_transform: Optional[Callable] = None,
    download: bool = False,
    max_digit: Optional[int] = None,
  ):
    # Contains a MNIST dataset
    self.mnist_dataset = torchvision.datasets.MNIST(
      root,
      train=train,
      transform=self.mnist_img_transform,
      target_transform=target_transform,
      download=download,
    )
    self.index_map = list(range(len(self.mnist_dataset)))
    random.shuffle(self.index_map)

    # Check if we want limited labels
    if max_digit is not None:
      self.index_map = [i for i in self.index_map if self.mnist_dataset[i][1] <= max_digit]

  def __len__(self):
    return int(len(self.index_map) / 2)

  def __getitem__(self, idx):
    # Get two data points
    (a_img, a_digit) = self.mnist_dataset[self.index_map[idx * 2]]
    (b_img, b_digit) = self.mnist_dataset[self.index_map[idx * 2 + 1]]

    # Each data has two images and the GT is the comparison result of two digits
    return (a_img, b_img, 0 if a_digit < b_digit else 1)

  @staticmethod
  def collate_fn(batch):
    a_imgs = torch.stack([item[0] for item in batch])
    b_imgs = torch.stack([item[1] for item in batch])
    cmp = torch.stack([torch.tensor(item[2]).long() for item in batch])
    return ((a_imgs, b_imgs), cmp)

def mnist_sort_2_loader(data_dir, batch_size, max_digit):
  train_loader = torch.utils.data.DataLoader(
    MNISTSort2Dataset(
      data_dir,
      train=True,
      download=True,
      max_digit=max_digit,
    ),
    collate_fn=MNISTSort2Dataset.collate_fn,
    batch_size=batch_size,
    shuffle=True,
  )

  test_loader = torch.utils.data.DataLoader(
    MNISTSort2Dataset(
      data_dir,
      train=False,
      download=True,
      max_digit=max_digit,
    ),
    collate_fn=MNISTSort2Dataset.collate_fn,
    batch_size=batch_size,
    shuffle=True,
  )

  return train_loader, test_loader

class MNISTNet(nn.Module):
  def __init__(self, num_classes=10):
    super(MNISTNet, self).__init__()
    self.conv1 = nn.Conv2d(1, 32, kernel_size=5)
    self.conv2 = nn.Conv2d(32, 64, kernel_size=5)
    self.fc1 = nn.Linear(1024, 1024)
    self.fc2 = nn.Linear(1024, num_classes)

  def forward(self, x):
    x = F.max_pool2d(self.conv1(x), 2)
    x = F.max_pool2d(self.conv2(x), 2)
    x = x.view(-1, 1024)
    x = F.relu(self.fc1(x))
    x = F.dropout(x, p=0.5, training=self.training)
    x = self.fc2(x)
    return F.softmax(x, dim=1)

class MNISTSort2Net(nn.Module):
  def __init__(self, provenance, train_k, test_k, max_digit=9):
    super(MNISTSort2Net, self).__init__()
    self.max_digit = max_digit
    self.num_classes = max_digit + 1

    # MNIST Digit Recognition Network
    self.mnist_net = MNISTNet(num_classes=self.num_classes)

    # Scallop Context
    self.scl_ctx = scallopy.ScallopContext(provenance=provenance, train_k=train_k, test_k=test_k)
    self.scl_ctx.add_relation("digit_1", int, input_mapping=list(range(self.num_classes)))
    self.scl_ctx.add_relation("digit_2", int, input_mapping=list(range(self.num_classes)))
    self.scl_ctx.add_rule("less_than(a < b) = digit_1(a), digit_2(b)")

    # The `less_than` logical reasoning module
    self.less_than = self.scl_ctx.forward_function("less_than", [True, False])

  def forward(self, x: Tuple[torch.Tensor, torch.Tensor]):
    (a_imgs, b_imgs) = x

    # First recognize the two digits
    a_distrs = self.mnist_net(a_imgs)
    b_distrs = self.mnist_net(b_imgs)

    # Then execute the reasoning module; the result is a size 2 tensor
    return self.less_than(digit_1=a_distrs, digit_2=b_distrs)

def bce_loss(output, ground_truth):
  (_, dim) = output.shape
  gt = torch.stack([torch.tensor([1.0 if i == t else 0.0 for i in range(dim)]) for t in ground_truth])
  return F.binary_cross_entropy(output, gt)

def nll_loss(output, ground_truth):
  return F.nll_loss(output, ground_truth)

class Trainer():
  def __init__(self, train_loader, test_loader, model_dir, learning_rate, loss, train_k, test_k, provenance, max_digit=9):
    self.network = MNISTSort2Net(provenance, train_k=train_k, test_k=test_k, max_digit=max_digit)
    self.optimizer = optim.Adam(self.network.parameters(), lr=learning_rate)
    self.train_loader = train_loader
    self.test_loader = test_loader
    self.model_dir = model_dir
    self.min_test_loss = 100000000.0
    if loss == "nll": self.loss = nll_loss
    elif loss == "bce": self.loss = bce_loss
    else: raise Exception(f"Unknown loss function `{loss}`")

  def train(self, epoch):
    self.network.train()
    iter = tqdm(self.train_loader, total=len(self.train_loader))
    for (data, target) in iter:
      self.optimizer.zero_grad()
      output = self.network(data)
      loss = self.loss(output, target)
      loss.backward()
      self.optimizer.step()
      iter.set_description(f"[Train Epoch {epoch}] Loss: {loss.item():.4f}")

  def test(self, epoch):
    self.network.eval()
    num_items = len(self.test_loader.dataset)
    test_loss = 0
    correct = 0
    with torch.no_grad():
      iter = tqdm(self.test_loader, total=len(self.test_loader))
      for (data, target) in iter:
        output = self.network(data)
        test_loss += self.loss(output, target).item()
        pred = output.data.max(1, keepdim=True)[1]
        correct += pred.eq(target.data.view_as(pred)).sum()
        perc = 100. * correct / num_items
        iter.set_description(f"[Test Epoch {epoch}] Total loss: {test_loss:.4f}, Accuracy: {correct}/{num_items} ({perc:.2f}%)")
    if test_loss < self.min_test_loss:
      self.min_test_loss = test_loss
      torch.save(self.network, os.path.join(self.model_dir, "mnist_sort_2_net.pkl"))

  def run(self, n_epochs):
    self.test(0)
    for epoch in range(1, n_epochs + 1):
      self.train(epoch)
      self.test(epoch)

if __name__ == "__main__":
  # Argument parser
  parser = ArgumentParser("mnist_sort_2")
  parser.add_argument("--n-epochs", type=int, default=10)
  parser.add_argument("--batch-size", type=int, default=64)
  parser.add_argument("--learning-rate", type=float, default=0.0001)
  parser.add_argument("--loss-fn", type=str, default="bce")
  parser.add_argument("--seed", type=int, default=1234)
  parser.add_argument("--provenance", type=str, default="difftopkproofs")
  parser.add_argument("--train-k", type=int, default=3)
  parser.add_argument("--test-k", type=int, default=3)
  parser.add_argument("--max-digit", type=int, default=9)
  args = parser.parse_args()

  # Parameters
  torch.manual_seed(args.seed)
  random.seed(args.seed)

  # Directories
  data_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "../../../data"))
  model_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "../../../model/mnist_sort_2"))
  if not os.path.exists(model_dir): os.mkdir(model_dir)

  # Dataloaders
  train_loader, test_loader = mnist_sort_2_loader(data_dir, args.batch_size, max_digit=args.max_digit)

  # Create trainer and train
  trainer = Trainer(train_loader, test_loader, model_dir, args.learning_rate, args.loss_fn, args.train_k, args.test_k, args.provenance, max_digit=args.max_digit)
  trainer.run(args.n_epochs)


"""A setuptools based setup module.
See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

# Always prefer setuptools over distutils
import os
from setuptools import setup, find_packages

here = os.path.abspath(os.path.dirname(__file__))

# Get the long description from the README file
with open(os.path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='opinions',
    version=os.getenv("OPINIONS_VERSION", "Unknown"),
    description='Flask opinions',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/erewok/flask-opinions',
    author='Erik Aker',
    author_email='eraker@gmail.com',
    packages=find_packages(exclude=['test']),  # Required
    install_requires=[
        'requests',
        'Flask',
        'Jinja2',
        'gunicorn',
        'python-json-logger',
        'structlog',
        'ecological',
        'python-rapidjson'
    ],
    extras_require={
        'test': ['pytest',
                 'ipdb',
                 'coverage',
                 'pytest-cov',
                 'flake8',
                 'pytest-flake8',
                 'mypy',
                 'pytest-mypy'],
    }
)


# Right off
# https://docs.opencv.org/3.4/da/d97/tutorial_threshold_inRange.html
from __future__ import print_function
import cv2 as cv
import argparse
from pathlib import Path

max_value = 255
max_value_H = 360 // 2
low_H = 0
low_S = 0
low_V = 0
high_H = max_value_H
high_S = max_value
high_V = max_value
window_capture_name = "Video Capture"
window_detection_name = "Object Detection"
low_H_name = "Low H"
low_S_name = "Low S"
low_V_name = "Low V"
high_H_name = "High H"
high_S_name = "High S"
high_V_name = "High V"

def on_low_H_thresh_trackbar(val):
    global low_H
    global high_H
    low_H = val
    low_H = min(high_H - 1, low_H)
    cv.setTrackbarPos(low_H_name, window_detection_name, low_H)

def on_high_H_thresh_trackbar(val):
    global low_H
    global high_H
    high_H = val
    high_H = max(high_H, low_H + 1)
    cv.setTrackbarPos(high_H_name, window_detection_name, high_H)

def on_low_S_thresh_trackbar(val):
    global low_S
    global high_S
    low_S = val
    low_S = min(high_S - 1, low_S)
    cv.setTrackbarPos(low_S_name, window_detection_name, low_S)

def on_high_S_thresh_trackbar(val):
    global low_S
    global high_S
    high_S = val
    high_S = max(high_S, low_S + 1)
    cv.setTrackbarPos(high_S_name, window_detection_name, high_S)

def on_low_V_thresh_trackbar(val):
    global low_V
    global high_V
    low_V = val
    low_V = min(high_V - 1, low_V)
    cv.setTrackbarPos(low_V_name, window_detection_name, low_V)

def on_high_V_thresh_trackbar(val):
    global low_V
    global high_V
    high_V = val
    high_V = max(high_V, low_V + 1)
    cv.setTrackbarPos(high_V_name, window_detection_name, high_V)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Code for Thresholding Operations using inRange tutorial."
    )
    parser.add_argument(
        "image_path",
        help="Path[s] to images",
        nargs="+",
        type=Path,
    )
    args = parser.parse_args()

    cv.namedWindow(window_capture_name)
    cv.namedWindow(window_detection_name)
    cv.createTrackbar(
        low_H_name,
        window_detection_name,
        low_H,
        max_value_H,
        on_low_H_thresh_trackbar,
    )
    cv.createTrackbar(
        high_H_name,
        window_detection_name,
        high_H,
        max_value_H,
        on_high_H_thresh_trackbar,
    )
    cv.createTrackbar(
        low_S_name,
        window_detection_name,
        low_S,
        max_value,
        on_low_S_thresh_trackbar,
    )
    cv.createTrackbar(
        high_S_name,
        window_detection_name,
        high_S,
        max_value,
        on_high_S_thresh_trackbar,
    )
    cv.createTrackbar(
        low_V_name,
        window_detection_name,
        low_V,
        max_value,
        on_low_V_thresh_trackbar,
    )
    cv.createTrackbar(
        high_V_name,
        window_detection_name,
        high_V,
        max_value,
        on_high_V_thresh_trackbar,
    )

    for image_path in args.image_path:
        while True:
            image = cv.imread(str(image_path))

            image_HSV = cv.cvtColor(image, cv.COLOR_BGR2HSV)
            image_threshold = cv.inRange(
                image_HSV, (low_H, low_S, low_V), (high_H, high_S, high_V)
            )

            cv.imshow(window_capture_name, image)
            cv.imshow(window_detection_name, image_threshold)

            (22, 79, 147), (41, 161, 215)  # Image 16
            (21, 15, 134), (66, 255, 255)  # Image 35

            key = cv.waitKey(30)
            is_quit = key == ord("q") or key == 27  # 27 is escape
            is_next = key == ord(" ")

            if is_quit or is_next:
                print(f"==== Final HSVs for {image_path.name} ====")
                print(f"lowH = {low_H}")
                print(f"lowS = {low_S}")
                print(f"lowV = {low_V}")
                print(f"highH = {high_H}")
                print(f"highS = {high_S}")
                print(f"highV = {high_V}")
                print()
                print(
                    f"({low_H}, {low_S}, {low_V}), ({high_H}, {high_S}, {high_V})"
                )
                print("=======================================================")

                if is_quit:
                    print("Quitting")
                    exit(0)
                elif is_next:
                    print("Continuing to next image")
                    break


# -*- coding: utf-8 -*-
"""
Created on Mon Mar  6 19:37:49 2017

@author: Simon
"""
from  scipy import *
from  pylab import *
import scipy.misc as sm
import scipy.sparse as sparse

v = sm.imread('kvinna.jpg', True)

#cut away pixels to have even number of y and x pixels
if len(v[:, 0]) % 2 != 0:
    v = v[1:, :]

if len(v[0, :]) % 2 != 0:
    v = v[:, 1:]

y = len(v[:, 0])
x = len(v[0, :])

"""
Algoritmen tar W*v = y.
Vi önskar konstruera W. Första gör vi v endimensionell och sedan konstruerar
vi en allmän W från denna längd
"""

print(x)
print(y)

v = v.reshape(1, -1)[0]

def createW(n):
	W = zeros([n,n])
	i = 0
	for j in range(int(n/2)):
		W[j, i] = 1/sqrt(2)
		W[j, i+1] = 1/sqrt(2)
		i += 2
	i = 0
	for j in range(int(n/2), n):
		W[j, i] = -1/sqrt(2)
		W[j, i+1] = 1/sqrt(2)
		i += 2
	return W

#override for test
v = array([100, 200, 44, 50, 20, 20, 4, 2])

W = createW(len(v))

#print(W)
print(abs(W.T - inv(W)) <  0.0001)
#print(inv(W))

y = dot(W, v)

print(y)


# Discord imports
import discord
from discord.ext import tasks

#Misc imports
from datetime import datetime
import json, os

# File imports
from src.indicators import TA
from src.chart import Charter
from src.stream import Streamer
from src.trade_finder import TradeFinder
from src.alert_logger import Logger
import src.helper as Helper

#-------------------------------------- Bot initiation and loop --------------------------------------#

bot = discord.Bot()

# Read config
TOKEN, PARAMETERS, ids, PAIRS = Helper.read_config()
# Make pairs upper case
PAIRS = [pair.upper() for pair in PAIRS]
GUILD_ID = ids["guild_id"]

# Initate classes
TechnicalAnalyzer = TA()
DataStreamer = Streamer(PARAMETERS["rank_by_marketcap"])
TradeScraper = TradeFinder(DataStreamer, TechnicalAnalyzer, PARAMETERS["vol_breakout_threashold"], PARAMETERS["vol_breakout_timeframe"])
AlertLogger = Logger()

@bot.event
async def on_ready():
    print("Bot is ready to scrape coins!!")
    if not scrape_market.is_running():
        scrape_market.start()

@tasks.loop(seconds = PARAMETERS["scrape_interval"])
async def scrape_market():
    print(f"{datetime.utcnow()} - Busy scraping market!")

    # Do market scrape if channel is supplied in config.json
    vol_breakout_alerts = None if ids["volume_breakout_channel"] == "" else ids["volume_breakout_channel"]
    if vol_breakout_alerts is not None:
        # TODO Perform the stuff in a subproces to still be able to access the bot commands
        # Check for high volume moves to the upside
        # Try/except is due to weird extra curly bracket error
        try:
            tfs_triggers = await TradeScraper.checkVol(PAIRS)
        except Exception as e:
            if e.__class__.__name__ == "JSONDecodeError":
                Helper.clean_tfs_log()
                tfs_triggers = await TradeScraper.checkVol(PAIRS)
            else:
                print(f"Error occured: {e}")
                tfs_triggers = None

        if tfs_triggers is None:
            print("No high volume breakouts!")
        else:
            channel = bot.get_channel(vol_breakout_alerts)
            await channel.send(f":alarm_clock: Caught high volume breakouts! :alarm_clock: \n{tfs_triggers.to_string()}")

    # Check if any alerts have been triggered
    if os.path.isfile("alerts.json"):
        symbols = AlertLogger.get_symbols()
        price_triggers = TradeScraper.checkPriceAlerts(symbols)
        vol_triggers = TradeScraper.checkVolumeAlerts(symbols)

        # Alert users that their alert(s) has/have been triggered
        alerts = price_triggers + vol_triggers
        channel = bot.get_channel(ids["alerts_channel"])
        await Helper.alert_user(channel, alerts)

        # Remove alerts after being triggered
        [AlertLogger.rm_alert(alert) for alert in alerts if alert["delete"] == "true"]

    print(f"{datetime.utcnow()} - Done scraping!")

@scrape_market.before_loop
async def set_status():
    await bot.change_presence(activity = discord.Activity(type = discord.ActivityType.watching, name = "candlesticks"))
    await bot.wait_until_ready()

#--------------------------------------------- Commands ----------------------------------------------#

# Add an alert for a symbol pair when certain price is crossed
@bot.slash_command(guild_ids = [GUILD_ID], name = "pricealert", description = "Add an alert for a symbol pair price movement")
async def price_alert(ctx, symbol, type, price, delete_when_triggered = "true"):
    symbol = symbol.upper()
    delete_when_triggered = delete_when_triggered.lower()
    if not DataStreamer.checkSymbol(symbol):
        await ctx.respond("That sucks.... I looked everywhere on Binance, but I can't find this symbol pair:cry:")
    elif type not in ("up", "down", "volume"):
        await ctx.respond("Please add a valid type for this alert: up, down, volume")
    elif delete_when_triggered not in ("true", "false"):
        await ctx.respond("Incorrect value for \"delete_when_triggered\". Should be either true or false...")
    else:
        # Get user ID
        uid = ctx.author.id
        AlertLogger.add_alert(uid, symbol, type, price, timeframe = '', volume_multiple = 0, delete = delete_when_triggered)
        await ctx.respond(f"Set an alert for {symbol}!")

# Add a volume breakout alert for certain symbol on chosen timeframe
@bot.slash_command(guild_ids = [GUILD_ID], name = "volumealert", description = "Add a volume breakout alert for a symbol pair on certain timeframe")
async def volume_alert(ctx, symbol, timeframe, volume_multiple, delete_when_triggered = "true"):
    symbol = symbol.upper()
    delete_when_triggered = delete_when_triggered.lower()
    if not DataStreamer.checkSymbol(symbol):
        await ctx.respond("That sucks.... I looked everywhere on Binance, but I can't find this symbol pair:cry:")
    elif not DataStreamer.checkTimeframe(timeframe):
        await ctx.respond("Please add a valid timeframe...")
    elif delete_when_triggered not in ("true", "false"):
        await ctx.respond("Incorrect value for \"delete_when_triggered\". Should be either true or false...")
    else:
        # Get user ID
        uid = ctx.author.id
        AlertLogger.add_alert(uid, symbol, type = "volume", price = 0, timeframe = timeframe, volume_multiple = volume_multiple, delete = delete_when_triggered)
        await ctx.respond(f"Set an alert for {symbol}!")

# Remove alert
@bot.slash_command(guild_ids = [GUILD_ID], name = "removealert", description = "Remove an alert. Get available alerts with \"/getalerts\"")
async def removemalert(ctx, alert):
    uid = ctx.author.id
    alert = json.loads(alert)
    if uid == alert["userid"]:
        try:
            AlertLogger.rm_alert(alert)
            await ctx.respond(f"Alert deleted!")
        except:
            await ctx.respond("An error occured... Did you include the \"{ *alert* }\" (curley brackets) around the alert?")
    else:
        await ctx.respond("Unfortunately you do not have permission to remove this alert (not created by you) :lock:")

# Sends all active alerts for some symbol
@bot.slash_command(guild_ids = [GUILD_ID], name = "getalerts", description = "Get all current alerts for chosen symbol")
async def get_alerts(ctx, symbol):
    symbol = symbol.upper()
    if not symbol in AlertLogger.get_symbols():
        await ctx.respond("There are currently no alerts for this symbol pair. Will you be the first to add one?")
    else:
        alerts = AlertLogger.get_alerts(symbol)
        parsed_alerts = json.dumps(alerts, indent = 4)
        await ctx.respond(f"Active alerts for {symbol}:\n ```json\n{parsed_alerts}\n```\n Copy one of the following if you wish to remove an alert:", delete_after = 30)
        for alert in alerts:
            await ctx.send(f"`{json.dumps(alert)}`")

# Chart symbol pair last 24H
@bot.slash_command(guild_ids = [GUILD_ID], name = "chart", description = "Plot a currency pair 24H back")
async def chart(ctx, symbol):
    symbol = symbol.upper()
    if not DataStreamer.checkSymbol(symbol):
        await ctx.respond("That sucks.... I looked everywhere on Binance, but I can't find this symbol pair:cry:")
    else:
        await ctx.respond(f"Drawing chart for {symbol}...")
        # 5m candles over a span of 24H: 24*60/5 = 288
        candles = DataStreamer.getKlines(symbol, 288)
        chart = Charter(symbol, candles)
        chart_path = chart.createChart()

        # Send chart in given path and delete after done
        with open(chart_path, 'rb') as image:
            img = discord.File(image)
            # TODO reconsider this: await ctx.send(file = img, delete_after = 60)
            image.close()

        os.remove(chart_path)

# Get the value of a certain indicator from at certain symbol pair
@bot.slash_command(guild_ids = [GUILD_ID], name = "indicator", description = "Get the latest value of a certain indicator on a certain timeframe")
async def indicator(ctx, symbol, indicator, window='0', timeframe = '1h'):
    symbol = symbol.upper()
    if not DataStreamer.checkSymbol(symbol):
        await ctx.respond("That sucks.... I looked everywhere on Binance, but I can't find this symbol pair:cry:")
    elif not DataStreamer.checkTimeframe(timeframe):
        await ctx.respond("Whooops, that looks like an incorrect timeframe:alarm_clock: Please check your command!")
    elif window == '0' and indicator != 'bms':
        await ctx.respond(f"Did you add an incorrect window for the indicator? For example rsi **14** or ema **21**.")
    else:
        candles = DataStreamer.getKlines(symbol, 100, timeframe)
        val = TechnicalAnalyzer.do_indicator(indicator, candles, window)
        await ctx.respond(f"Indicator, \"{indicator}\", for symbol, {symbol}, is currently at:\n{val}\n- on {timeframe} timeframe.")

# Get latest TFS from certain coin
@bot.slash_command(guild_ids = [GUILD_ID], name = "tfs", describtion = "Get time of last TFS from symbol")
async def tfs(ctx, symbol):
    symbol = symbol.upper()
    path = "tfs_log.json"

    with open(path, "r") as log_file:
        file = json.load(log_file)

        # Check if a trigger is logged
        if symbol in list(file.keys()):
            latest_trigger = file[symbol]["Time"]
            await ctx.respond(f"Latest trigger for {symbol} was at: {latest_trigger}")
        else:
            await ctx.respond(f"No triggers has been logged for {symbol} yet!!")

        log_file.close()

# For clearing chat
@bot.slash_command(guild_ids = [GUILD_ID], name = "clear", describtion = "Clear the previous, \"n\", bot messages")
async def clear(ctx, amount = 10):
    # First check if user is moderator/admin and has permission to delete messages
    if ctx.author.guild_permissions.administrator:
        await ctx.respond(f"Deleting {amount} messages!")
        channel = bot.get_channel(ids["alerts_channel"])
        await channel.purge(limit = int(amount))
    else:
        await ctx.respond("Unfortunately you do not have permission to use this command :cry:")

# Run bot
bot.run(TOKEN)


import requests
import json
import os
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta
from tqdm import tqdm
import re
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError

# eprint RSS
base_url = 'https://eprint.iacr.org/rss/rss.xml'
xml_file = 'eprint.xml'
response = requests.get(base_url)

# last eprint
last_file = 'last.txt'

# Slack APIトークン
SLACK_API_TOKEN = 'xoxb-Slack API Token'
# Slackに投稿するチャンネル名を指定する
SLACK_CHANNEL = "#eprintまとめ"

# 翻訳APIのURL
# [参考：Google翻訳APIを無料で作る方法](https://qiita.com/satto_sann/items/be4177360a0bc3691fdf)
trancerate_url='Google scriptのURL'

## 英文を翻訳APIに流し込む関数
def get_summary(result):
    _params = {'text': f'{result}', 'source': 'en', 'target': 'ja'}
    r = requests.get(trancerate_url, params=_params)
    text_json = r.json()
    out_text = text_json['text']
    return out_text

with open(xml_file, "wb") as file:
    file.write(response.content)
    print("Complete DL rss file!")

with open(last_file, 'r') as file:
    lastnum = file.read()
    lastnum = lastnum.replace("/", "")
maxnum = lastnum

client = WebClient(token=SLACK_API_TOKEN)

tree = ET.parse(xml_file)
root = tree.getroot()

# itemタグの情報を抽出する
for item in tqdm(root.findall(".//item")):

    # ex) 'https://eprint.iacr.org/2023/548' => '2023548'
    eprintnum=re.findall('\d+', item.find("link").text)
    eprintnum = ''.join(eprintnum)

    # last.txtの値よりも大きい場合のみ投稿する
    if eprintnum > lastnum:

        # last.txtの値を更新する
        if maxnum < eprintnum:
            maxnum = eprintnum

        pub_date = item.find("pubDate").text
        pub_date = datetime.strptime(pub_date, "%a, %d %b %Y %H:%M:%S %z").strftime("%Y-%m-%d")

        title = item.find("title").text
        link = item.find("link").text
        description = item.find("description").text
        message = title+"について要約しました！ \n発行日："+ pub_date + "\n" + link + "\n" + get_summary(description)

        try:
            response = client.chat_postMessage(
            channel=SLACK_CHANNEL,
            text=message
            )
            # last.txtの値を更新する
            with open(last_file, 'w') as file:
                file.write(maxnum)
            print("Complete post message!")

        except SlackApiError as e:
            print(f"Error posting message: {e}")


import click
import google.generativeai as palm
import llm
from pprint import pprint

@llm.hookimpl
def register_models(register):
    register(Palm("chat-bison-001"), aliases=("palm", "palm2"))

@llm.hookimpl
def register_commands(cli):
    @cli.group(name="palm")
    def palm_():
        "Commands for working directly with PaLM"

    @palm_.command()
    @click.option("--key", help="PaLM API key")
    def models(key):
        "List models available in the PaLM API"
        api_key = llm.get_key(key, "palm", "PALM_API_KEY")
        palm.configure(api_key=api_key)
        models = palm.list_models()
        pprint(list(models))

class Palm(llm.Model):
    needs_key = "palm"
    key_env_var = "PALM_API_KEY"

    def __init__(self, model_id):
        self.model_id = model_id

    def build_prompt_messages(self, prompt, conversation):
        if not conversation:
            return prompt
        messages = []
        for response in conversation.responses:
            messages.append(response.prompt.prompt)
            messages.append(response.text())
        messages.append(prompt)
        return messages

    def execute(self, prompt, stream, response, conversation):
        palm.configure(api_key=self.get_key())

        kwargs = {"messages": self.build_prompt_messages(prompt.prompt, conversation)}
        if prompt.system:
            kwargs["context"] = prompt.system

        palm_response = palm.chat(**kwargs)
        last = palm_response.last
        yield last or ""
        response._prompt_json = kwargs

    def __str__(self):
        return "PaLM 2: {}".format(self.model_id)


import pandas as pd
import argparse
import re

'''transfer Root_Content.txt to a csv file'''
parser = argparse.ArgumentParser()
parser.add_argument('--input_file', type=str, default='./data/weibo_dataset/weibocontents/Retweet_Content.txt')
args = parser.parse_args()
opt = vars(args)

MID_PATTERN = '^[0-9]{14,20}\ '
MID_REGEX = re.compile(MID_PATTERN)

UID_PATTERN = '^[0-9]{6,12}$'
UID_REGEX = re.compile(UID_PATTERN)

def isOrigStartLine(line):
    return len(MID_REGEX.findall(line)) > 0

def isRepostStartLine(line):
    return len(UID_REGEX.findall(line)) > 0

def main(opt):
    headers = ['weiboID', 'content', 'at', 'link']
    df = pd.DataFrame(columns=headers)

    with open(opt['input_file'], 'r', encoding='gb18030') as f:
        buff = []
        idx = 0
        currentLine = None
        while True:
            try:
                lastLine = currentLine
                currentLine = f.readline().strip()
                if isOrigStartLine(currentLine) and lastLine:
                    next_first_line = getRepost(df, f)
                    df.loc[idx] = buff
                    del buff
                    buff = [line]
                    idx += 1
                    if idx % 100 == 0:
                        df.to_csv('./Root_Content.csv', index=False)
                else:
                    buff.append(line)
            except:
                print('End of File')
                break

        print(df.head())
    df.to_csv('./userProfile.csv', index=False)

if __name__ == '__main__':
    main(opt)


import time
import numpy as np
import torch
from utils import *
from params import *
from reconstruction import *
import scipy
import cv2
import skimage.measure
from itertools import compress

def lpf_detection(holo,mask,erode_size=20, dilate_size=60, threshold=10, A_min = 1, show_plot=False):
    # mask for low pass filter generated by create_lpf_mask (calculated outside of this function to
    #     optimize processing time)
    # erode size and dilate size are in number of pixels
    # threshold is in raw intensity
    # A_min is in mm^2

    # calculate image FFT
    #img = np.float32(holo.squeeze())
    dft = np.fft.fft2(holo)
    dft_shift = np.fft.fftshift(dft)
    f_ishift = np.fft.ifftshift(dft_shift)

    # apply mask and calculate inverse FFT
    fshift = dft_shift * mask

    proc_im = np.fft.ifft2(fshift)
    proc_im = np.abs(proc_im)

    # apply threshold to create binary image
    proc_im[proc_im < threshold] = 0
    proc_im[proc_im > 0] = 1

    # perform erosion and dilation
    if erode_size > 0:
        eroder = cv2.getStructuringElement(cv2.MORPH_RECT,(erode_size,erode_size));
        proc_im = cv2.morphologyEx(proc_im, cv2.MORPH_ERODE, eroder)

    if dilate_size > 0:
        dilater = cv2.getStructuringElement(cv2.MORPH_RECT,(dilate_size,dilate_size));
        proc_im = cv2.morphologyEx(proc_im, cv2.MORPH_DILATE, dilater)

    # find regions in binary image
    labeled_im = skimage.measure.label(proc_im,connectivity=2)
    regions = skimage.measure.regionprops(labeled_im)
    A = np.array([region.area for region in regions])
    regions = list(compress(regions,A > A_min/((pix_size*1e3)**2)))

    if show_plot:
        plt.figure(4).clf()
        plt.subplot(212)
        plt.imshow(proc_im)

        plt.subplot(211)
        plt.imshow(-holo)

        for region in regions:
            minr, minc, maxr, maxc = region.bbox
            bx = (minc, maxc, maxc, minc, minc)
            by = (minr, minr, maxr, maxr, minr)
            plt.plot(bx,by,'-r',linewidth=2)

    return regions

def create_lpf_mask(holo,keep_frac=0.05):
    # create low pass filter mask, keeping only "keep_frac"
    # fraction of lowest frequencies
    # calculating this once for all holos saves about 0.04 seconds per hologrma

    # e.g., keep_frac = 0.05 keeps 5% of frequencies

    rows, cols = holo.shape
    crow, ccol = int(rows / 2), int(cols / 2)

    mask = np.zeros((rows, cols), np.uint8)
    rr = keep_frac*(rows/2)
    rc = keep_frac*(cols/2)
    center = [crow, ccol]
    x, y = np.ogrid[:rows, :cols]
    mask_area = (x - center[0]) ** 2/rr**2 + (y - center[1]) ** 2/rc**2 <= 1
    mask[mask_area] = 1

    return mask


import uuid
import json
from unittest.mock import MagicMock

COMPONENT_CODE = -725052640
STATE_MACHINE_CODE = -2027871621
EVENT_CODE = 8
MESSAGE_TYPE = "XComponent.Devinette.UserObject.CheckWord"
ROUTING_KEY = "input.1_0.microservice1.Devinette.DevinetteChecker"
UUID = str(uuid.uuid4())
SESSION_DATA = "sessionData"
JSON_MESSAGE = {"Name": "My Name"}

uid = MagicMock(["create"], key="uid")
uid.create = MagicMock(return_value=UUID)
uid.create("create", key="uid")
uid.create.assert_called_with("create", key="uid")

def get_header(visibility):
    header = {
        "StateMachineCode": {"Case": "Some", "Fields": [STATE_MACHINE_CODE]},
        "ComponentCode": {"Case": "Some", "Fields": [COMPONENT_CODE]},
        "EventCode": EVENT_CODE,
        "IncomingType": 0,
        "MessageType": {"Case": "Some", "Fields": [MESSAGE_TYPE]}
    }
    return header

def get_correct_data(visibility):
    return {
        "event": {
            "Header": get_header(True),
            "JsonMessage": json.dumps(JSON_MESSAGE)
        },
        "routingKey": ROUTING_KEY
    }

def get_correct_websocket_input_format(visibility):
    correct_data = get_correct_data(True)
    correct_websocket_input_format = correct_data["routingKey"] + " " + str(
        correct_data["event"]["Header"]["ComponentCode"]["Fields"][0]) + " " + json.dumps(correct_data["event"])
    return correct_websocket_input_format

# Mocking configuration

configuration = MagicMock(["get_component_code", "get_state_machine_code",
                           "get_publisher_details"], key="configuration")

def component_code(component_name):
    if component_name is None:
        raise Exception("Error")
    return COMPONENT_CODE

configuration.get_component_code = MagicMock(side_effect=component_code)

def state_machine_code(component_name, state_machine_name):
    if (component_name is None) or (state_machine_name is None):
        raise Exception("Error")
    return STATE_MACHINE_CODE

configuration.get_state_machine_code = MagicMock(side_effect=state_machine_code)

def publisher_details(component_code, state_machine_code, message_type):
    return {
        "eventCode": EVENT_CODE,
        "routingKey": ROUTING_KEY
    }

configuration.get_publisher_details = MagicMock(side_effect=publisher_details)

# Mocking websocket

def create_mock_websocket():
    websocket = MagicMock(["send"], key="websocket")
    return websocket


from typing import List
import functools

floats: List[float] = [12.3554, 4.02, 5.777, 2.12, 3.13, 4.44, 11.0001]
names: List[str] = ["Vanes", "Alen", "Jana", "William", "Richards", "Joy"]
numbers: List[int] = [22, 33, 10, 6894, 11, 2, 1]

cubed_floats = list(map(lambda x: round(x ** 3, 3), floats))

filtered_names = list(filter(lambda x: len(x) >= 5, names))

product_numbers = round(functools.reduce(lambda x, y: x * y, numbers), 3)

print(cubed_floats)
print(filtered_names)
print(product_numbers)


import threading
import uuid
from functools import partial
from pathlib import Path
from typing import Generic, Optional, TypeVar

from PyQt5.QtCore import QObject, pyqtSignal

from bubblesub.api.base_stream import BaseStream, StreamUnavailable
from bubblesub.api.threading import synchronized

TStream = TypeVar("TStream", bound=BaseStream)

class BaseStreamsApi(Generic[TStream], QObject):
    """Common functions for audio and video stream manager APIs."""

    stream_lock = threading.RLock()

    current_stream_switched = pyqtSignal(object)

    stream_created = pyqtSignal(object)
    stream_changed = pyqtSignal(object)
    stream_errored = pyqtSignal(object)
    stream_loaded = pyqtSignal(object)
    stream_unloaded = pyqtSignal(object)

    def __init__(self) -> None:
        """Initialize self."""
        super().__init__()
        self._streams: list[TStream] = []
        self._current_stream: Optional[TStream] = None

    @synchronized(lock=stream_lock)
    def unload_all_streams(self) -> None:
        """Unload all loaded streams."""
        for stream in self._streams:
            self.stream_unloaded.emit(stream)
        self._streams = []
        self.switch_stream(None)

    @synchronized(lock=stream_lock)
    def load_stream(self, path: Path, switch: bool = True) -> bool:
        """Load stream from specified file.

        :param path: path to load the stream from
        :param switch: whether to switch to that stream immediately
        :return: False if stream was already loaded, True otherwise
        """
        for stream in self._streams:
            if stream.path.samefile(path):
                if switch:
                    self.switch_stream(stream.uid)
                return False

        stream = self._create_stream(path)
        stream.loaded.connect(partial(self._on_stream_load, stream))
        stream.errored.connect(partial(self._on_stream_error, stream))
        stream.changed.connect(partial(self._on_stream_change, stream))
        self._streams.append(stream)
        self.stream_created.emit(stream)

        if switch:
            self.switch_stream(stream.uid)

        return True

    def get_stream_index(self, uid: uuid.UUID) -> Optional[int]:
        """Returns index of the given stream uid.

        :param uid: stream to get the index of
        :return: stream index
        """
        uids = [stream.uid for stream in self._streams]
        try:
            return uids.index(uid)
        except ValueError:
            return None

    @synchronized(lock=stream_lock)
    def unload_stream(self, uid: uuid.UUID) -> None:
        """Unload stream with the given uid.

        :param uid: stream to unload
        """
        index = self.get_stream_index(uid)
        if index is None:
            return
        old_stream = self._streams[index]
        self._streams = [
            stream for stream in self._streams if stream.uid != uid
        ]
        if index < len(self._streams):
            self.switch_stream(self._streams[index].uid)
        else:
            self.switch_stream(None)
        self.stream_unloaded.emit(old_stream)

    @synchronized(lock=stream_lock)
    def switch_stream(self, uid: Optional[uuid.UUID]) -> None:
        """Switches streams.

        :param uid: stream to switch to
        """
        if uid is None:
            self._set_current_stream(None)
        else:
            stream: Optional[TStream]
            for stream in self._streams:
                if stream.uid == uid:
                    break
            else:
                stream = None
            if not stream:
                raise ValueError(f"stream {uid} is not loaded")
            self._set_current_stream(stream)

    @synchronized(lock=stream_lock)
    def unload_current_stream(self) -> None:
        """Unload currently loaded stream."""
        if self._current_stream is not None:
            self.unload_stream(self._current_stream.uid)

    @property
    @synchronized(lock=stream_lock)
    def current_stream(self) -> TStream:
        """Return currently loaded stream.

        Raises an exception if there is no stream loaded yet.

        :return: stream
        """
        if not self._current_stream:
            raise StreamUnavailable("stream is not available right now")
        return self._current_stream

    @property
    @synchronized(lock=stream_lock)
    def has_current_stream(self) -> bool:
        """Return whether there is a current stream.

        Should be avoided in favor of:

        ```
        try:
            current_stream = stream_api.current_stream
        except ResourceUnavailable:
            current_stream = None

        as checking has_current_stream, then accessing current_stream is prone
        to races.

        :return: whether current_stream property is available
        """
        return self._current_stream is not None

    @property
    @synchronized(lock=stream_lock)
    def streams(self) -> list[TStream]:
        """Return all loaded streams.

        :return: list of streams
        """
        return self._streams[:]

    def _set_current_stream(self, stream: Optional[TStream]) -> None:
        if stream != self._current_stream:
            self._current_stream = stream
            self.current_stream_switched.emit(self._current_stream)

    @synchronized(lock=stream_lock)
    def _on_stream_load(self, stream: TStream) -> None:
        self.stream_loaded.emit(stream)

    @synchronized(lock=stream_lock)
    def _on_stream_error(self, stream: TStream) -> None:
        self.stream_errored.emit(stream)

    @synchronized(lock=stream_lock)
    def _on_stream_change(self, stream: TStream) -> None:
        self.stream_changed.emit(stream)

    def _create_stream(self, path: Path) -> TStream:
        raise NotImplementedError

    @synchronized(lock=stream_lock)
    def cycle_streams(self) -> None:
        """Cycle to next available stream.

        Does nothing if there are no loaded streams.
        """
        if not self._current_stream:
            return
        uid = self._current_stream.uid
        idx = self.get_stream_index(uid)
        assert idx is not None
        idx += 1
        idx %= len(self.streams)
        uid = self.streams[idx].uid
        self.switch_stream(uid)


import os
import spacy
import string

def get_size_of_corpus(filepaths):
    """ Given a list of filepaths, it will return the total number of lines

        Parameters
        ----------
        filepaths : [ str ]
            A list of filepaths

        Returns
        -------
        num_lines : int
            The total number of lines in filepaths
    """

    def blocks(files, size=65536):
        while True:
            b = files.read(size)
            if not b:
                break
            yield b

    num_lines = 0
    for filepath in filepaths:
        with open(filepath, encoding="utf-8") as f:
            num_lines += sum(bl.count("\n") for bl in blocks(f))

    return num_lines

def get_tokens_from_line(line, spacy_instance):
    global PUNCTUATION_TABLE, LEGAL_CHARS_TABLE, LEGAL_CHARS

    # Make the text lowercase and remove the \n
    line = line.lower().strip()

    # Standardize single quotations
    line = (
        line.replace("\u2019", "'")
        .replace("\u0060", "'")
        .replace("\u00B4", "'")
        .replace("\u2018", "'")
        .replace("\u201A", "'")
        .replace("\u201B", "'")
        .replace("\u2039", "'")
        .replace("\u203A", "'")
        .replace("\u275B", "'")
        .replace("\u275C", "'")
        .replace("\u276E", "'")
        .replace("\u276F", "'")
    )

    # Standardize dashes
    line = (
        line.replace("\u00AD", "-")
        .replace("\u2013", "-")
        .replace("\u2014", "-")
        .replace("\u2015", "-")
        .replace("\u2212", "-")
        .replace("\u02D7", "-")
    )

    # Standardize double quotations
    line = (
        line.replace("\u201C", '"')
        .replace("\u201D", '"')
        .replace("\u2033", '"')
        .replace("\u00AB", '"')
        .replace("\u00BB", '"')
        .replace("\u201E", '"')
        .replace("\u201F", '"')
        .replace("\u275D", '"')
        .replace("\u275E", '"')
        .replace("\u301D", '"')
        .replace("\u301E", '"')
        .replace("\u301F", '"')
        .replace("\uFF02", '"')
    )

    # Standardize the ellipses
    line = line.replace("\u2026", "...")

    # Tokenize the text
    tokens = spacy_instance.tokenizer(line)

    # Remove tokens that are spaces, tabs, newlines, etc
    tokens = list(filter(lambda token: not token.is_space, tokens))

    # Extract only the text
    tokens = [token.text for token in tokens]

    # Replace tokens that contain numbers with 'NUM'
    num_vals = []
    new_tokens = []
    for token in tokens:
        new_token = token

        if any(char.isdigit() for char in token):
            new_token = "NUM"
            num_vals.append(token)

        new_tokens.append(new_token)

    tokens = new_tokens

    return tokens, num_vals

def read_transcription_files(filepaths, spacy_instance):
    """ Generate line info from data in a file for a given language

        Parameters
        ----------
        lang : {'en', 'fr'}
            Whether to tokenize the English sentences ('e') or French ('f').
        filenames : sequence
            Only tokenize sentences with matching names. If :obj:`None`, searches
            the whole directory in C-sorted order.

        Yields
        ------
        tokenized, filename, offs : list
            `tokenized` is a list of tokens for a line. `filename` is the source
            file. `offs` is the start of the sentence in the file, to seek to.
            Lines are yielded by iterating over lines in each file in the order
            presented in `filenames`.
    """
    for filepath in filepaths:
        with open(filepath, encoding="utf-8") as f:
            offs = f.tell()
            line = f.readline()
            while line:
                tokens, _ = get_tokens_from_line(line, spacy_instance)

                yield tokens, filepath, offs
                offs = f.tell()
                line = f.readline()

def get_parallel_text(dir_, langs):
    """ Get a list of all files in 'dir_' with a file extension that is in 'langs'

        Parameters
        ----------
        dir_ : str
            A path to the transcription dictionary
        langs : [str]
            A list of file extensions for the parallel texts

        Returns
        -------
        filenames : list
            A list of all parallel texts' file names without the file extension
    """
    # Get a set of files that has the lang file extension
    files = os.listdir(dir_)
    filenames = []
    for lang in langs:
        lang = "." + lang
        lang_filenames = set(
            filename[:-3] for filename in files if filename.endswith(lang)
        )
        filenames.append(lang_filenames)
    del files

    if len(filenames) == 0:
        raise ValueError(
            f"Directory {dir_} contains no transcriptions ending in {lang} "
        )

    # Get the set of files that has the same name
    transcriptions_filenames = filenames[0]
    for lang_filenames in filenames:
        transcriptions_filenames = transcriptions_filenames & lang_filenames

    transcriptions_filenames = sorted(transcriptions_filenames)

    if len(transcriptions_filenames) == 0:
        raise ValueError(
            f"Directory {dir_} contains no common files ending in {lang}."
            f"Are you sure this is the right directory?"
        )

    # Create the output
    parallel_text = []
    for filename in transcriptions_filenames:
        parallel_text.append(tuple([filename + "." + lang for lang in langs]))

    return parallel_text

def get_spacy_instance(lang, spacy_namespace=None):
    """ Returns the correct Spacy instance given the lang ISO.
        If the namespace is provided, it will use the namespace instead

        Parameters
        ----------
        lang : { 'en', 'fr' }
            The lang ISO
        spacy_namespace : str, optional
            The Spacy namespace

        Returns
        -------
        spacy_instance : Spacy
            The Spacy instance with the correct language
    """
    if spacy_namespace is None:
        if lang == "en":
            spacy_namespace = "en_core_web_sm"
        elif lang == "fr":
            spacy_namespace = "fr_core_news_sm"
        else:
            raise Exception("Unknown language: " + lang)

    # For sentence segmentation, we are using dependency parse
    # To use rule-based sentence segmentation, follow https://spacy.io/usage/linguistic-features#sbd-component
    spacy_instance = spacy.load(spacy_namespace)

    return spacy_instance


# Krishna Chaitanya Naragam
# 1001836274

from tkinter import *
from datetime import datetime
import threading
from flask import Flask
import os, signal
from flask import request
import time

# declarations
root = None
information_panel = None

# serve Queue A
def serve_A(i):
    i = float(i)
    output = "\n=========================\n"
    output += "Meter: {}\n".format(i)
    output += "Millimeter: {}\n".format(i*1000)
    output += "Centimeter: {}\n".format(i*100)
    output += "Kilometer: {}\n".format(i*0.001)
    output += "Astronomical Unit: {}\n".format(i*(6.68459e-12))
    output += "=========================\n"
    return output

# serve Queue B
def serve_B(i):
    i = float(i)
    output = "\n=========================\n"
    output += "Meter: {}\n".format(i)
    output += "Parsec: {}\n".format(i*(3.24078e-17))
    output += "Light Year: {}\n".format(i*(1.057e-16))
    output += "Inch: {}\n".format(i*39.3701)
    output += "Foot: {}\n".format(i*3.28084)
    output += "Yard: {}\n".format(i*(1.0936133333333))
    output += "=========================\n"
    return output

# serve Queue A
def serve_C(i):
    i = float(i)
    output = "=========================\n"
    output += "Meter: {}\n".format(i)
    output += "Mile: {}\n".format(i*0.000621371)
    output += "Nautical Mile: {}\n".format(i*0.000539957)
    output += "American football field: {}\n".format(i/109.7)
    output += "Hand: {}\n".format(i*9.84252)
    output += "Horse: {}\n".format(i/2.4)
    output += "=========================\n"
    return output

# Quit functionality for GUI
def quit():
    global app
    global root
    app.do_teardown_appcontext()
    root.destroy()
    os.kill(os.getpid(), signal.SIGINT)

# append information to panel on the UI
def append_info_panel(info):
    global information_panel
    information_panel['state'] = NORMAL
    information_panel.insert(END, str(datetime.now())+ ": " + str(info)+"\n")
    information_panel.see(END)
    information_panel['state'] = DISABLED

# run GUI
def run_GUI():
    # GUI loop
    global information_panel
    global root
    root = Tk()
    root.title('Server')
    server_status = Label(root, text="Server: Online")
    server_status.pack()
    information_panel = Text()
    information_panel.pack()
    information_panel['state'] = DISABLED
    quit_button = Button(root, command=quit, text="Quit")
    quit_button.pack()
    root.mainloop()

app = Flask(__name__)

# home page
@app.route('/')
def home():
    return 'Server up!'

# get entire queue and delete the file
@app.route('/getQueue/<user>/<q>')
def getQueue(user,q):
    append_info_panel("User connected {}".format(user))
    if q in ['A', 'B', 'C']:

        out = ''
        try:
            f= open('{}.queue'.format(q))
            out = f.read()
            f.close()
            os.remove('{}.queue'.format(q))
        except Exception as e:
            out = "Queue {} is empty!".format(q)
        append_info_panel("Getting Queue {}\n{}".format(q, out))
        return out
    append_info_panel("User {} Disconnected!".format(user))
    return "Queue Not Found"

# put item into the queue
@app.route('/putInQueue/<user>/<q>/<metric>')
def putInQueue(user,q,metric):
    append_info_panel("User connected {}".format(user))
    append_info_panel("user {} inserterd to queue: {}, metric: {}".format(user, q, metric))
    f = open('{}.queue'.format(q),'a+')
    if q == 'A':
        metric = serve_A(metric)
    if q == 'B':
        metric = serve_B(metric)
    if q == 'C':
        metric = serve_C(metric)
    f.write(metric)
    f.close()
    append_info_panel("Converted values are,\n{}".format(metric))
    append_info_panel("User {} Disconnected!".format(user))
    return "inserted to queue: {}".format(q)

if __name__ == '__main__':
    port = 5000
    t = threading.Thread(target=run_GUI)
    t.daemon = True
    t.start()
    time.sleep(2)
    append_info_panel("Server running on port {}".format(port))
    app.run(port=port)


import random
import PySimpleGUI as sg

class ChuteONumero:
    def __init__(self):
        self.valor_aleatorio = 0
        self.valor_min = 1
        self.valor_max = 100
        self.tentar_novamente = True

    def Iniciar(self):
        #elementos do layout
        layout = [
            [sg.Text("Digite seu chute: ", size=(20,0))],
            [sg.Input(size=(18,0), key='ValorChute')],
            [sg.Button('Chutar!')],
            [sg.Output(size=(40,10))]
        ]
        #criação da janela
        self.janela = sg.Window("Chute um Número!", layout=layout)
        self.GerarNumeroAleatorio()
        try:
            while True:
                #leitura dos valores da tela
                self.evento, self.valores = self.janela.Read()
                #Tratamento dos Valores Lidos
                if self.evento == 'Chutar!':
                    self.valor_chute = self.valores['ValorChute']
                    while self.tentar_novamente == True:
                        if int(self.valor_chute) < self.valor_aleatorio:
                            print("Chute um valor mais alto!")
                            break
                        elif int(self.valor_chute) > self.valor_aleatorio:
                            print("Chute um valor mais baixo!")
                            break
                        elif int(self.valor_chute) == self.valor_aleatorio:
                            self.tentar_novamente = False
                            print("Acertou!! O numero correto era "+str(self.valor_aleatorio))
                            break
        except:
            print("Aconteceu um erro.")
            self.Iniciar()

    def GerarNumeroAleatorio(self):
        self.valor_aleatorio = random.randint(self.valor_min, self.valor_max)

chte = ChuteONumero()
chte.Iniciar()


import zipfile
import os

class unzipp:
    def __init__(self,log_writer,file_object):
        self.log_writer=log_writer
        self.file_object=file_object
    def unzipping(self):
        """""
                                                   Method Name: zipping
                                                   Description: This method unzips the data downloaded from the S3 bucket
                                                   and store it in RawTrainData and RawTestData folder
                                                   Output: None
                                                   On Failure: Raise Exception

                                                    Written By: Harsh Kumar
                                                   Version: 1.0
                                                   Revisions: None

                            """
        self.log_writer.log(self.file_object,'Entered into Unzipping method')
        path='C:/Users/Harsh/Data science/Internship/CancerPrediction/RawTrainData/training_text'
        if os.path.isfile(path) == True:
            self.log_writer.log(self.file_object, 'Files have already been unzipped')
        else:
            try:

                with zipfile.ZipFile('C:/Users/Harsh/Data science/Internship/CancerPrediction/DataFromS3/training_text.zip', 'r') as zip_ref:
                    zip_ref.extractall('C:/Users/Harsh/Data science/Internship/CancerPrediction/RawTrainData')

                with zipfile.ZipFile('C:/Users/Harsh/Data science/Internship/CancerPrediction/DataFromS3/test_text.zip', 'r') as zip_ref:
                    zip_ref.extractall('C:/Users/Harsh/Data science/Internship/CancerPrediction/RawPredictionData')

                with zipfile.ZipFile('C:/Users/Harsh/Data science/Internship/CancerPrediction/DataFromS3/training_variants.zip', 'r') as zip_ref:
                    zip_ref.extractall('C:/Users/Harsh/Data science/Internship/CancerPrediction/RawTrainData')

                with zipfile.ZipFile('C:/Users/Harsh/Data science/Internship/CancerPrediction/DataFromS3/test_variants.zip', 'r') as zip_ref:
                    zip_ref.extractall('C:/Users/Harsh/Data science/Internship/CancerPrediction/RawPredictionData')
                self.log_writer.log(self.file_object,'Exited fromm unzipping method')
            except Exception as e:
                self.log_writer.log(self.file_object,'Error : %s Occured while unzipping the files' %e)