#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Created on Thu Sep 21 16:21:35 2023

Download DSWx data

@author: km
"""
from pystac_client import Client
from shapely import wkt
import os, json, requests, re, glob
from datetime import datetime
from swampy import config
from urllib.request import urlopen
import concurrent.futures
import geopandas as gpd
import pandas as pd
from shapely.geometry import shape
from matplotlib import pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import numpy as np
from urllib.error import URLError

nproc = int(os.cpu_count()-1)

def filter_by_cloud_cover(item, threshold=10):
    '''
    Removes images with more than <threshold>% cloud cover.
    '''
    xml_url = item.assets['metadata'].href
    try:
        response = urlopen(xml_url)
        data_json = json.loads(response.read())
    except URLError:
        print("Error fetching URL:", xml_url)
        return False

    c_cover = -1
    for attribute in data_json['AdditionalAttributes']:
        if attribute['Name'] == 'PercentCloudCover':
            c_cover = int(attribute['Values'][0])
            break

    if c_cover == -1:
        print("PercentCloudCover not found in metadata for URL:", xml_url)
        return False

    return c_cover <= threshold

def return_granule(item):
    return item.assets['0_B01_WTR'].href

def searchDSWx(ps):
    '''
    Searches DSWx data given polygon AOI and date range.
    Returns:
        filtered_urls: urls to the data from sparse-cloud days
        dswx_data_df: pandas dataframe for all search results
    '''

    # convert to the datetime format
    start_date = datetime(int(ps.date_start[0:4]), int(ps.date_start[4:6]), int(ps.date_start[6:8]))
    stop_date = datetime(int(ps.date_stop[0:4]), int(ps.date_stop[4:6]), int(ps.date_stop[6:8]))

    # Convert the wkt polygon to a Shapely Polygon
    aoi = wkt.loads(ps.polygon)
    intersects_geometry = aoi.__geo_interface__

    # Search data through CMR-STAC API
    stac = 'https://cmr.earthdata.nasa.gov/cloudstac/'    # CMR-STAC API Endpoint
    print("Connecting to API...")
    api = Client.open(f'{stac}POCLOUD/')
    collections = ['OPERA_L3_DSWX-HLS_PROVISIONAL_V1']

    search_params = {"collections": collections,
                     "intersects": intersects_geometry,
                     "datetime": [start_date, stop_date],
                     "max_items": 10000}
    search_dswx = api.search(**search_params)
    items = search_dswx.item_collection()
    dswx_data =list(search_dswx.items())
    plot_frames(dswx_data,aoi)

    # Filter cloudy days
    print("Filtering cloudy days > " + str(ps.cloudy_threshold) + "%...")
    filtered_items = list(filter(lambda item: filter_by_cloud_cover(item, threshold=ps.cloudy_threshold), items))
    print(str(len(list(items))) + ' items were found.')
    print(str(len(filtered_items)) + ' items meet the cloud threshold')

    filtered_urls = list(map(return_granule, filtered_items))
    print(len(filtered_urls))

    # Create table of search results
    dswx_data_df = []
    for item in dswx_data:
        item.to_dict()
        fn = item.id.split('_')
        ID = fn[3]
        sensor = fn[6]
        dat = item.datetime.strftime('%Y-%m-%d')
        # spatial_coverage = intersection_percent(item, intersects_geometry)
        geom = item.geometry

        # Take all the band href information
        band_links = [item.assets[links].href for links in item.assets.keys()]
        dswx_data_df.append([ID,sensor,dat,geom,band_links])

    dswx_data_df = pd.DataFrame(dswx_data_df, columns = ['TileID', 'Sensor', 'Date', 'Footprint','BandLinks'])

    # Save the results:
    dswx_data_df.to_csv('searchResults.csv',index=False)
    with open('filteredURLs.txt', 'w') as file:
        for item in filtered_urls:
            file.write(f"{item}\n")

    return filtered_urls, dswx_data_df

def dl(url,outname):
    response = requests.get(url,stream=True,allow_redirects=True)

    # Open the local file for writing
    with open(outname, 'wb') as file:
        # Iterate through the content and write to the file
        for data in response.iter_content(chunk_size=int(2**14)):
            file.write(data)

def dlDSWx(urls,ps,dataDir):
    '''
    Download the DSWx files given urls
    '''
    # Create a list of file path/names
    outNames = []
    dl_list = []
    # dataDir2 = './westCoastData'

    # for url in urls:
    #     fname2 = os.path.join(dataDir2, url.split('/')[-1])
    #     if os.path.isfile(fname2):
    #         os.system('ln -s ' + fname2 + ' dataDir/')

    for url in urls:
        file_name = url.split('/')[-1]
        fname = os.path.join(dataDir, file_name)
        pattern = f'./data/*/{file_name}'
        matching_files = glob.glob(pattern)

        if len(matching_files) == 0:
            outNames.append(os.path.join(dataDir, url.split('/')[-1]))
            dl_list.append(url)

    if not os.path.isdir(dataDir):
        os.mkdir(dataDir)

    print('Downloading the following files:')
    print(dl_list)
    print('downloading with '  + str(nproc) + ' cpus')

    with concurrent.futures.ThreadPoolExecutor(max_workers=nproc) as executor:  # Adjust max_workers as needed
        futures = [executor.submit(dl, url, outName) for url, outName in zip(dl_list, outNames)]
        concurrent.futures.wait(futures)

    # check the files
    for url in urls:
        fname = os.path.join(dataDir, url.split('/')[-1])
        if not os.path.isfile(fname):
            print('Warning: File does not exist ' + fname)
        else:
            if os.path.getsize(fname) < 2**10: # If it's smaller than 1 Kb
                print('Warning: ' + fname + ' is too small. Try again.')

def plot_frames(dswx_data,aoi):
    '''
    Plot the footprints from the DSWx search results
    '''
    # Visualize the DSWx tile boundary and the user-defined bbox
    geom_df = []
    for d,_ in enumerate(dswx_data):
        geom_df.append(shape(dswx_data[d].geometry))
    geom_granules = gpd.GeoDataFrame({'geometry':geom_df})

    minlon = np.floor(min(coord[0] for coord in aoi.__geo_interface__['coordinates'][0]))
    maxlon = np.ceil(max(coord[0] for coord in aoi.__geo_interface__['coordinates'][0]))
    minlat = np.floor(min(coord[1] for coord in aoi.__geo_interface__['coordinates'][0]))
    maxlat = np.ceil(max(coord[1] for coord in aoi.__geo_interface__['coordinates'][0]))

    # Set up the figure and axis with a chosen map projection (e.g., PlateCarree)
    fig, ax = plt.subplots(figsize=(10, 10), subplot_kw={'projection': ccrs.PlateCarree()})
    ax.set_extent([minlon, maxlon, minlat, maxlat])  # Set extent to fit around your data's bounding box

    # Add a basemap using cartopy features
    ax.add_feature(cfeature.LAND, edgecolor='black', alpha=0.2)
    ax.add_feature(cfeature.COASTLINE, edgecolor='black')
    ax.add_feature(cfeature.BORDERS, linestyle=':')
    ax.add_feature(cfeature.STATES, linestyle=':',linewidth=.3)

    import cartopy.io.img_tiles as cimgt

    bg='World_Shaded_Relief'
    zoomLevel = 12
    url = 'https://server.arcgisonline.com/ArcGIS/rest/services/' + bg + '/MapServer/tile/{z}/{y}/{x}.jpg'
    image = cimgt.GoogleTiles(url=url)
    ax.add_image(image, zoomLevel,zorder=1) #zoom level
    # Plot the DSWx tile boundary polygons
    geom_granules.boundary.plot(ax=ax, color='green', linewidth=1)

    aoi_series = gpd.GeoSeries([aoi])

    # Plot the user-specified AOI polygon
    aoi_series.boundary.plot(ax=ax, color='#8B0000', linewidth=1,linestyle='--')

    plt.title("DSWx Tile Boundary and User-specified AOI")
    plt.show()

def extract_date_from_filename(filename):
    """Extract the date in YYYYMMDD format from the filename."""
    match = re.search(r'(\d{8})', filename)
    if match:
        return match.group(1)
    else:
        raise ValueError(f"Cannot extract date from filename: {filename}")

def organize_directories(data_dir):
    tif_files = [f for f in os.listdir(data_dir) if f.endswith('.tif')]

    # Extract unique dates from filenames
    unique_dates = set([extract_date_from_filename(f) for f in tif_files])

    # Create directories for each unique date
    for date in unique_dates:
        os.makedirs(os.path.join(data_dir, date), exist_ok=True)

    # Move the .tif files to the respective directories
    for f in tif_files:
        date = extract_date_from_filename(f)
        destination = os.path.join(data_dir, date, f)
        source = os.path.join(data_dir, f)
        os.rename(source, destination)

def main():

    ps = config.getPS()

    # Get the filtered list of urls
    filtered_urls,dswx_data_df = searchDSWx(ps)

    # Download the files
    dlDSWx(filtered_urls,ps,ps.dataDir)

    # Organize files into their respective dates directories
    organize_directories(ps.dataDir)

    return filtered_urls,dswx_data_df

if __name__ == '__main__':
    '''
    Main driver.
    '''
    main()


import os
import csv
import sys
import sqlite3

WORK_DIR = '/home/ming/PSMD'
DB_DIR = os.path.join(WORK_DIR, 'dbs')

for infile in sys.argv[1:]:
	with open(infile) as fh:
		rows = csv.reader(fh, delimiter='\t')
		for row in rows:
			if row[3] == 'Viroids':
				continue

			if ',' in row[4]:
				row[4] = row[4].split(',')[0]

			sub_dir = os.path.join(*row[3:6]).replace(' ', '_')
			db_file = os.path.join(DB_DIR, sub_dir, '{}.db'.format(row[15]))

			conn = sqlite3.connect(db_file)
			cursor = conn.cursor()

			for record in cursor.execute("SELECT COUNT(*) FROM ssr"):
				total_ssr = record[0] or 0

			if total_ssr == 0:
				continue

			for record in cursor.execute("SELECT COUNT(*) FROM ssrannot"):
				genic_ssr = record[0] or 0

			for record in cursor.execute("SELECT COUNT(*) FROM ssrannot WHERE location=1"):
				cds_ssr = record[0] or 0

			for record in cursor.execute("SELECT COUNT(*) FROM cssr"):
				total_cssr = record[0] or 0

			for record in cursor.execute("SELECT COUNT(*) FROM cssrannot"):
				genic_cssr = record[0] or 0

			for record in cursor.execute("SELECT COUNT(*) FROM cssrannot WHERE location=1"):
				cds_cssr = record[0] or 0

			for record in cursor.execute("SELECT SUM(complexity) FROM cssr"):
				complexity = record[0] or 0

			cursor.close()
			conn.close()

			#if genic_ssr == 0:
			#	continue

			res = [row[0], row[1], row[3], row[4], row[5]]
			res.extend([total_ssr, genic_ssr, cds_ssr, total_ssr-cds_ssr, total_cssr, genic_cssr, cds_cssr, total_cssr-cds_cssr, complexity])

			print("\t".join(list(map(str,res))))


#train test split
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y)

#grid search
from sklearn import svm, grid_search, datasets
iris = datasets.load_iris()
parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
svr = svm.SVC()
clf = grid_search.GridSearchCV(svr, parameters)
clf.fit(iris.data, iris.target)
print(clf.best_params_)


import json
import logging

from downloader import Downloader
from xmlconverter import XmlToCsvConverter
from uploader import S3Uploader

def lambda_handler(event: str, context: dict | None) -> dict:
    event = json.loads(event)
    url = event.get('url', None)

    if not url:
        raise ValueError("URL not present")

    # Download and extract the XML from the ZIP file
    downloader = Downloader()
    download_url = downloader.download_url(url)
    xml_file_path = downloader.download_and_extract(download_url)

    # Convert the XML to CSV
    xtc = XmlToCsvConverter()
    csv_rows = xtc.convert(xml_file_path)
    xtc.write_to_file("data.csv", csv_rows)

    # Upload the CSV to S3
    s3u = S3Uploader("data.csv")
    url = s3u.upload()

    return {"url": url}


#!/usr/bin/env python

"""Utility class for searching Twitter API for specific emojis
and saving to csv files for training/validation data"""

import pandas
import os, sys, re
import json
import tweepy
import searchtweets
from definitions import ROOT_DIR

class TweetCollector:
    def __init__(self):
        self.twitter_api_key = os.environ['TWITTER_API_KEY']
        self.twitter_api_secret = os.environ['TWITTER_API_SECRET']
        self.twitter_access_token = os.environ['TWITTER_API_TOKEN']
        self.twitter_access_token_secret = os.environ['TWITTER_API_TOKEN_SECRET']
        self.positive_emojis_csv = "{0}/data/emojis/positive/face-smiley.csv".format(ROOT_DIR)
        self.negative_emojis_csv = "{0}/data/emojis/negative/face-negative.csv".format(ROOT_DIR)

    def authenticate(self):
        self.premium_search_args = searchtweets.load_credentials("{0}/.twitter_keys.yaml".format(ROOT_DIR),
                                       yaml_key="search_tweets_30_day",
                                       env_overwrite=False)
        auth = tweepy.OAuthHandler(self.twitter_api_key, self.twitter_api_secret)
        auth.set_access_token(self.twitter_access_token, self.twitter_access_token_secret)
        self.api = tweepy.API(auth)

    def get_positive_tweets(self):
        positive_emoji_codes = get_emoji_unicodes_list(self.positive_emojis_csv)
        positive_tweets = self.get_emoji_tweets(positive_emoji_codes)
        return clean_tweets(positive_tweets, 1)
        # for positive_emoji_code in positive_emoji_codes:
        #     positive_tweets =
        #     return clean_tweets(positive_tweets, 1)
        #     # write to file
        #     # path = "{0}/data/tweets/positive/".format(ROOT_DIR)
        #     # filename = "raw_positive_tweets.csv"
        #     # write_to_csv(tweets_dict, filename)
        #     # # writeToJSONFile(path, filename, positive_tweets)

    def get_negative_tweets(self):
        negative_emoji_codes = get_emoji_unicodes_list(self.negative_emojis_csv)
        negative_tweets = self.get_emoji_tweets(negative_emoji_codes)
        return clean_tweets(negative_tweets, 0)
        # for negative_emoji_code in negative_emoji_codes:
        #     # negative_tweets =
        #     return clean_tweets(negative_tweets, 0)
        #     # write_to_csv(negative_tweet_dict, filename)
        #     # write to file
        #     # path = "{0}/data/tweets/negative/".format(ROOT_DIR)
        #     # filename = "raw_negative_tweets.json"
        #     # writeToJSONFile(path, filename, negative_tweet_group)

    def get_emoji_tweets(self, emoji_list):
        emoji_list = ' OR '.join(emoji_list);
        print(emoji_list)
        max_tweets = 100
        rule = searchtweets.gen_rule_payload(emoji_list,
                        # from_date="2017-01-01", #UTC 2017-09-01 00:00
                        # to_date="2019-02-12",#UTC 2017-10-30 00:00
                        results_per_call=max_tweets)
        print(rule)
        tweets = searchtweets.collect_results(rule, max_results=500, result_stream_args=self.premium_search_args)
        return tweets

# def writeToJSONFile(path, filename, data):
#     with open(filename, 'w') as fp:
#         json.dump(data, fp, indent=2)
#     try:
#         filePathNameWExt = path + filename
#         with open(filePathNameWExt, 'w') as fp:
#             json.dump(data, fp)
#     except Exception as e:
#         print ("writeToJSONFile exception")
#         print (e)

 # - english only
 # - no question marks
 # - no negative emoji
 # - must have a subject and a verb

# def limit_handled(cursor):
#     while True:
#         try:
#             yield cursor.next()
#         except tweepy.RateLimitError:
#             time.sleep(15 * 60)

# def clean_emoji_code(raw_code):
#     return raw_code.replace('U+','')

def remove_emojis(text):
    emoji_pattern = re.compile("["
            u"\U0001F600-\U0001F64F"  # emoticons
            u"\U0001F300-\U0001F5FF"  # symbols & pictographs
            u"\U0001F680-\U0001F6FF"  # transport & map symbols
            u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               "]+", flags=re.UNICODE)
    without_emoji = emoji_pattern.sub(r'', text) # no emoji
    return text

def clean_tweets(tweets, sentiment):
    cleaned_tweets = {'tweet_text':[], 'sentiment': []}
    for tweet in tweets:
        if tweet.lang == 'en' and "?" not in tweet.text:
            tweet_text = " ".join(filter(lambda x:x[0]!='@', tweet.text.split()))
            cleaned_tweets['sentiment'].append(sentiment)
            cleaned_tweets['tweet_text'].append(tweet_text)

    # print(cleaned_tweets['tweet_text'])
    return cleaned_tweets

def write_to_csv(tweets_dict, filename):
    tweets_df = pandas.DataFrame(tweets_dict)
    tweets_df.to_csv(filename, sep=',')

def get_emoji_unicodes_list(file):
    # reads csv of emojis
    # returns list of unicode representations of emojis
    emoji_data = pandas.read_csv(file)
    return emoji_data['Browser'].tolist()


import argparse
import secrets

import yaml
from symbolchain.CryptoTypes import Hash256, PrivateKey
from symbolchain.facade.NemFacade import NemFacade
from symbolchain.nem.Network import Network
from zenlog import log

MICROXEM_PER_XEM = 1000000

class NemesisConfigurationGenerator:
	def __init__(self, network_name, custom_network_identifier):
		network_identifier = (
			int(custom_network_identifier) if custom_network_identifier else Network.MAINNET.identifier if 'mainnet' == network_name else
			Network.TESTNET.identifier
		)
		epoch = Network.MAINNET.datetime_converter.epoch if 'mainnet' == network_name else Network.TESTNET.datetime_converter.epoch
		self.facade = NemFacade(Network(network_name, network_identifier, epoch))
		self.signer_private_key_pair = self.facade.KeyPair(PrivateKey.random())
		self.generation_hash = Hash256(self._random_bytes(Hash256.SIZE))
		self.account_key_pairs = []

	def print_header(self, amount):
		signer_address = self.facade.network.public_key_to_address(self.signer_private_key_pair.public_key)
		total_xem = (len(self.account_key_pairs) * amount) / MICROXEM_PER_XEM

		log.info('Generated Nemesis Configuration')
		log.info(f' *  SIGNER ADDRESS: {signer_address}')
		log.info(f' * GENERATION HASH: {self.generation_hash}')
		log.info(f' *  TOTAL ACCOUNTS: {len(self.account_key_pairs)}')
		log.info(f' *       TOTAL XEM: {total_xem:.6f}')

	def generate_keys(self, count):
		self.account_key_pairs = [self.facade.KeyPair(PrivateKey.random()) for _ in range(0, count)]

	def save_nemesis_configuration(self, amount, network_name, output_filepath):
		accounts = [{
			'address': str(self.facade.network.public_key_to_address(key_pair.public_key)),
			'amount': amount
		} for key_pair in self.account_key_pairs]

		configuration = {
			'signer_private_key': str(self.signer_private_key_pair.private_key),
			'generation_hash': str(self.generation_hash),
			'network': network_name,
			'epoch_time': int(self.facade.network.datetime_converter.epoch.timestamp()),
			'accounts': accounts,
			'identifier': self.facade.network.identifier
		}

		self._save_configuration_file(output_filepath, configuration)

	def save_account_configuration(self, output_filepath):
		account_descriptors = [{
			'privatekey': str(key_pair.private_key),
			'publickey': str(key_pair.public_key),
			'address': str(self.facade.network.public_key_to_address(key_pair.public_key))
		} for key_pair in self.account_key_pairs]

		configuration = {
			'generation_hash': str(self.generation_hash),
			'accounts': account_descriptors,
			'nemesis': {
				'privatekey': str(self.signer_private_key_pair.private_key),
				'publickey': str(self.signer_private_key_pair.public_key),
				'address': str(self.facade.network.public_key_to_address(self.signer_private_key_pair.public_key))
			}
		}

		self._save_configuration_file(output_filepath, configuration)

	@staticmethod
	def _save_configuration_file(output_filepath, configuration):
		with open(output_filepath, 'wt', encoding='utf8') as configuration_file:
			yaml.dump(configuration, configuration_file)

	@staticmethod
	def _random_bytes(size):
		return secrets.token_bytes(size)

def main():
	run_command_name = None
	if globals().get('__spec__'):
		program_name = __spec__.name.partition('.')[0]
		run_command_name = f'python -m {program_name}'

	parser = argparse.ArgumentParser(
		prog=run_command_name,
		description='NEM nemesis configuration generator'
	)
	parser.add_argument('-c', '--count', type=int, help='Number of accounts in the nemesis', required=True)
	parser.add_argument('-s', '--seed', type=int, help='Seed amount', required=True)
	parser.add_argument('-o', '--output', help='Nemesis configuration file', required=True)
	parser.add_argument('-a', '--accounts-output', help='Account configuration file', required=True)
	parser.add_argument('-n', '--network-name', help='Network name', choices=('mainnet', 'testnet'), required=True)
	parser.add_argument('-i', '--network-identifier', help='Network identifier', required=False)
	args = parser.parse_args()

	generator = NemesisConfigurationGenerator(args.network_name, args.network_identifier)
	generator.generate_keys(args.count)
	generator.save_nemesis_configuration(args.seed, args.network_name, args.output)
	generator.save_account_configuration(args.accounts_output)
	generator.print_header(args.seed)

if '__main__' == __name__:
	main()


import torch
from torch.utils.cpp_extension import load
from torch.nn import Parameter, init
import torch.nn.functional as F

spmm = load(name='spmm', sources=['spmm.cpp', 'spmm_kernel.cu'], verbose=True)

class SPMMFunction(torch.autograd.Function):
    @staticmethod
    def forward(ctx, rowptr, colind, colptr, rowind, feat, edge_weight_csr=None, edge_weight_csc=None):
        if edge_weight_csr==None:
            out = spmm.csr_spmm_no_edge_value(rowptr, colind, feat)
        else:
            out = spmm.csr_spmm (rowptr, colind, edge_weight_csr, feat)

        ctx.backward_csc = (colptr, rowind, feat, edge_weight_csr, edge_weight_csc)
        return out

    @staticmethod
    def backward(ctx, grad_out):
        colptr, rowind, feat, edge_weight_csr, edge_weight_csc = ctx.backward_csc
        if edge_weight_csr!=None:
            if edge_weight_csc==None:
                raise RuntimeError("Backward of SPMM require edge values in both src-first and dst-first order, \
                    and do not support gradients for edge values. \
                        Call with SPMMFunction.apply(rowptr, colind, colptr, rowind, in_feat, edge_value_row_first, edge_value_col_first"
                        )
            else:
                grad_feat = spmm.csr_spmm(colptr, rowind, edge_weight_csc, grad_out)
                grad_edge_weight = None
                print("[I] Treat edge weight as no_grad.")
        else:
            grad_feat = spmm.csr_spmm_no_edge_value(colptr, rowind, grad_out)
            grad_edge_weight = None

        return None, None, None, None, grad_feat, grad_edge_weight, None

# import numpy as np
# import scipy.sparse as scpsp

# def proc():
#     adj = scpsp.random(1000,1000,density=0.01, dtype=np.float32)
#     adj = adj.tocsr()
#     rowptr = torch.tensor(adj.indptr)
#     colind = torch.tensor(adj.indices)
#     adj = adj.tocsc()
#     colptr = torch.tensor(adj.indptr)
#     rowind = torch.tensor(adj.indices)
#     g = {}
#     g['rowptr'] = rowptr
#     g['colind'] = colind
#     g['colptr'] = colptr
#     g['rowind'] = rowind
#     return g
# g = proc()

# class TestGCN(torch.nn.Module):
#     def __init__(self, nh):
#         super(TestGCN, self).__init__()
#         self.n_hidden = nh
#         self.weight = Parameter(torch.Tensor(nh, nh))
#         self.bias = Parameter(torch.Tensor(nh))
#         self.reset_parameters()
#     def reset_parameters(self):
#         init.xavier_uniform_(self.weight)
#         init.zeros_(self.bias)

#     def forward(self, x):
#         x = torch.matmul(x, self.weight)
#         x = SPMMFunction.apply(g['rowptr'], g['colind'], g['colptr'], g['rowind'], x)
#         x = x + self.bias
#         return F.log_softmax(x, dim=1)

from torch_geometric.nn.inits import glorot, zeros

class GCNConv(torch.nn.Module):
    def __init__(self, in_channels, out_channels, improved=False, cached=False,
                 bias=True, normalize=True, **kwargs):
        super(GCNConv, self).__init__()

        self.in_channels = in_channels
        self.out_channels = out_channels
        self.improved = improved
        self.cached = cached
        self.normalize = normalize

        self.weight = Parameter(torch.Tensor(in_channels, out_channels))

        if bias:
            self.bias = Parameter(torch.Tensor(out_channels))
        else:
            self.register_parameter('bias', None)

        self.reset_parameters()

    def reset_parameters(self):
        glorot(self.weight)
        zeros(self.bias)
        self.cached_result = None
        self.cached_num_edges = None

    @staticmethod
    def in_deg_sqrt(indptr):
        return (1 / torch.sqrt((indptr[1:]-indptr[:-1]).float())).unsqueeze(dim=1)

    @staticmethod
    def out_deg_sqrt(indptr):
        return (1 / torch.sqrt((indptr[1:]-indptr[:-1]).float())).unsqueeze(dim=1)

    def forward(self, x, rowptr, colind, colptr, rowind, edge_weight_csr=None, edge_weight_csc=None):
        """"""
        x = torch.matmul(x, self.weight)
        # if self.cached and self.cached_result is not None:
        #     if edge_index.size(1) != self.cached_num_edges:
        #         raise RuntimeError(
        #             'Cached {} number of edges, but found {}. Please '
        #             'disable the caching behavior of this layer by removing '
        #             'the `cached=True` argument in its constructor.'.format(
        #                 self.cached_num_edges, edge_index.size(1)))

        if not self.cached or self.cached_result is None:
            # self.cached_num_edges = edge_index.size(1)
            # self.cached_num_edges = colind.size(0)
            if self.normalize:
                # edge_index, norm = self.norm(edge_index, x.size(
                    # self.node_dim), edge_weight, self.improved, x.dtype)
                in_deg_norm = self.in_deg_sqrt(rowptr)
                out_deg_norm = self.out_deg_sqrt(colptr)

            else:
                # norm = edge_weight
                in_deg_norm = torch.ones(rowptr.shape(0)-1, dtype=x.dtype, device=x.device)
                out_deg_norm = torch.ones(colptr.shape(0)-1, dtype=x.dtype, device=x.device)

            # self.cached_result = edge_index, norm
            self.cached_result = in_deg_norm, out_deg_norm

        # edge_index, norm = self.cached_result
        in_deg_norm, out_deg_norm = self.cached_result
        if self.normalize:
            x = x*out_deg_norm
        aggr_out = SPMMFunction.apply(rowptr, colind, colptr, rowind, x, edge_weight_csr, edge_weight_csc)
        if self.normalize:
            aggr_out = aggr_out*in_deg_norm
        if self.bias is not None:
            aggr_out = aggr_out + self.bias
        return aggr_out

    def __repr__(self):
        return '{}({}, {})'.format(self.__class__.__name__, self.in_channels,
                                   self.out_channels)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('Aprendeci', '0005_recurso'),
    ]

    operations = [
        migrations.AddField(
            model_name='recurso',
            name='concepto',
            field=models.ForeignKey(default=1, to='Aprendeci.Concepto'),
            preserve_default=False,
        ),
    ]


from elasticsearch import Elasticsearch
import pandas as pd
import numpy as np
import json, csv, time
from metaphone import doublemetaphone
from elasticsearch import exceptions

with open('../doc/config_1850.json') as json_data_file:
    config = json.load(json_data_file)

es = Elasticsearch(host=config['host'], port=config['port'])

def match_addr():
    df = pd.read_csv(config['cd_filename'])
    count, match,unmatch = 0,0,0
    with open(config['match_output_filename'],'w') as fw, open(config['unmatch_output_filename'],'w') as fw2:
        writer = csv.writer(fw, delimiter="\t",quotechar='"')
        columns = config["output_census_cols"] + config["output_city_directory_cols"]
        rows=""
        for cols in columns:
            rows = rows + cols + "\t"

        writer.writerow(rows.rstrip("\t").split("\t"))
        for index, row in df.iterrows():
            row = row.replace(np.nan,'',regex=True) #covnert nan to empty string
            data = row.to_dict()

            first_name_metaphone = [i for i in doublemetaphone(data["CD_FIRST_NAME"]) if i]
            last_name_metaphone = [i for i in doublemetaphone(data["CD_LAST_NAME"]) if i]

            if config['edit_distance'] !=0:
                if config['metaphone'] is 1:
                    query = { "bool" : { "must" : [{ "fuzzy": { "CENSUS_FIRST_NAME": { "value": data["CD_FIRST_NAME"], "fuzziness": config["edit_distance"], "max_expansions": 50, "prefix_length": 0, "transpositions": True, "rewrite": "constant_score" } } },
                            {"fuzzy": { "CENSUS_LAST_NAME": { "value": data["CD_LAST_NAME"], "fuzziness": config["edit_distance"], "max_expansions": 50, "prefix_length": 0, "transpositions": True, "rewrite": "constant_score" } }},
                            { "match" : { "CENSUS_WARD_NUM": data["CD_WARD_NUM"]} },
                            {"terms": {"METAPHONE_NAMELAST.keyword": last_name_metaphone}},
                            {"terms": {"METAPHONE_NAMEFIRST.keyword": first_name_metaphone}}]}}

                else:
                    query = { "bool" : { "must" : [{ "fuzzy": { "CENSUS_FIRST_NAME": { "value": data["CD_FIRST_NAME"], "fuzziness": config["edit_distance"], "max_expansions": 50, "prefix_length": 0, "transpositions": True, "rewrite": "constant_score" } } },
                            {"fuzzy": { "CENSUS_LAST_NAME": { "value": data["CD_LAST_NAME"], "fuzziness": config["edit_distance"], "max_expansions": 50, "prefix_length": 0, "transpositions": True, "rewrite": "constant_score" } }},
                            { "match" : { "CENSUS_WARD_NUM": data["CD_WARD_NUM"]} }]}}

            if config['edit_distance'] is 0:
                if config['metaphone'] is 1:
                   query =  { "bool" : { "must" : [{ "match": { "CENSUS_FIRST_NAME": data["CD_FIRST_NAME"] } },
                            { "match": { "CENSUS_LAST_NAME": data["CD_LAST_NAME"] } },
                            { "match" : { "CENSUS_WARD_NUM": data["CD_WARD_NUM"]} },
                            {"terms": {"METAPHONE_NAMELAST.keyword": last_name_metaphone}},
                            {"terms": {"METAPHONE_NAMEFIRST.keyword": first_name_metaphone}}]}}

                else:
                    query = { "bool" : { "must" : [{ "match": { "CENSUS_FIRST_NAME": data["CD_FIRST_NAME"] } },
                            { "match": { "CENSUS_LAST_NAME": data["CD_LAST_NAME"] } },
                            { "match" : { "CENSUS_WARD_NUM": data["CD_WARD_NUM"]}}]}}

            try:
                res = es.search(index=config["es-index"], body={ "from": 0, "size": 10000, "query":query})
            except exceptions.RequestError:
                print("Exception at row id: ", index)
                continue

            if res['hits']['total']['value']!= 0:
                for i in res['hits']['hits']:
                    i = i['_source']
                    content = ""
                    for j in config["output_census_cols"]:
                        content = content + str(i[j]) + "\t"

                    for j in config["output_city_directory_cols"]:
                        content = content + str(data[j]) + "\t"

                    writer.writerow(content.rstrip("\t").split("\t"))
                    match+=1
            else:
                fw2.write(str(data['CD_RECORD_ID'])+"\n")
                unmatch+=1

    print(count,match,unmatch)

if __name__=='__main__':
    st = time.time()
    match_addr()
    end = time.time()
    print(config["es-index"] +" "+ str(end-st))


import os
import unittest

from pages.dashboard import Dashboard
from pages.login_page import LoginPage
from utils.settings import DRIVER_PATH, IMPLICITLY_WAIT
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from PIL import Image

class TestTranslations(unittest.TestCase):

    @classmethod
    def setUp(self):
        os.chmod(DRIVER_PATH, 755)
        self.driver_service = Service(executable_path=DRIVER_PATH)
        self.driver = webdriver.Chrome(service=self.driver_service)
        self.driver.get('https://dareit.futbolkolektyw.pl/')
        self.driver.maximize_window()
        self.driver.implicitly_wait(IMPLICITLY_WAIT)

    def test_page_translation(self):
        login = LoginPage(self.driver)
        login.type_in_email('user02@getnada.com')
        login.type_in_password('Test-1234')
        login.click_on_the_Sign_In_button()
        login.wait_for_logo()
        language = Dashboard(self.driver)
        language.change_language_EN()
        language.EN_url()
        self.driver.save_screenshot(
            "C:/Users/Magda/Documents/GitHub/Challenge_Portfolio_Magda02/test_cases/Screenshots/page_translation/english-language.png")
        Image.open(
            "C:/Users/Magda/Documents/GitHub/Challenge_Portfolio_Magda02/test_cases/Screenshots/page_translation/english-language.png").show()
        language.change_language_PL()
        language.PL_url()
        self.driver.save_screenshot(
            "C:/Users/Magda/Documents/GitHub/Challenge_Portfolio_Magda02/test_cases/Screenshots/page_translation/polish-language.png")
        Image.open(
            "C:/Users/Magda/Documents/GitHub/Challenge_Portfolio_Magda02/test_cases/Screenshots/page_translation/polish-language.png").show()

    @classmethod
    def tearDown(self):
        self.driver.quit()


from abc import ABC
from evaluation import evaluation
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
from tensorflow.keras import layers, losses, activations, Model, optimizers, metrics
from tensorflow.keras.callbacks import LambdaCallback
from sklearn.metrics import confusion_matrix
import matplotlib.mlab as mlab
import sys
import os
import seaborn as sns
os.environ['KMP_DUPLICATE_LIB_OK']='TRUE'
best_val_mae_output = sys.float_info.max
EPOCHS=100

class MyModel():
    def __init__(self, input_shape=(1200, 56, 2)):
        inputs = layers.Input(shape=input_shape)
        self.cnn_layers = self.create_cnn(32, (5, 5), (1, 1), (40, 16), (20, 8)) + \
                          self.create_cnn(64, (3, 3), (1, 1), (20, 4), (10, 4)) + \
                          self.create_cnn(128, (2, 2), (1, 1), (12, 2), (6, 2))
        flatten_layer = layers.Flatten()
        self.branch_1 = [*self.cnn_layers, flatten_layer, *self.create_fc(), layers.Dense(4)]
        outputs_1 = outputs_2 = inputs
        for layer in self.branch_1:
            outputs_1 = layer(outputs_1)

        self.net = Model(inputs, outputs_1)

    @staticmethod
    def create_cnn(filters=64, kernel_size=(2, 2), kernel_strides=(1, 1), pool_size=(2, 2), pool_strides=(1, 1)):
        module = [
            layers.Conv2D(filters=filters, kernel_size=kernel_size, strides=kernel_strides, padding='same',
                          activation=activations.relu),
            layers.BatchNormalization(),
            layers.MaxPooling2D(pool_size=pool_size, strides=pool_strides, padding='same')
        ]
        return module

    @staticmethod
    def create_fc(units=(64, 32)):
        module = [layers.Dense(x, activation=activations.relu) for x in units]
        return module

def pre_processing(x, y):
    x = tf.cast((x - tf.reduce_mean(x)) / tf.math.reduce_std(x), tf.float32)
    y = tf.one_hot(tf.squeeze(y), 4)
    return x, y

def main():
    z=np.load("dataset.npz")
    data = z['arr_0']
    data1 = z['arr_1']
    print(data1.shape)

    db_size = data.shape[0]
    batch_size = 64
    db = tf.data.Dataset.from_tensor_slices((data, data1)).map(pre_processing).shuffle(10000)
    db_train = db.take(int(db_size * 0.8)).batch(batch_size)
    db_val = db.skip(int(db_size * 0.8)).batch(batch_size)

    model = MyModel()
    model.net.summary()
    checkpoint_filepath = './checkpoint_lr/sleep'
    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
        filepath=checkpoint_filepath,
        save_weights_only=True,
        monitor='val_categorical_accuracy',
        mode='max',
        save_best_only=True)
    model.net.compile(optimizer=optimizers.Adam(learning_rate=1e-4),
                      loss=losses.CategoricalCrossentropy(from_logits=True),
                      metrics=metrics.CategoricalAccuracy())
    H = model.net.fit(db_train, epochs=EPOCHS, validation_data=db_val, callbacks=[model_checkpoint_callback])

    N=np.arange(0,EPOCHS)
    plt.style.use("ggplot")
    plt.figure()
    #plt.title('val_output_mae')
    plt.title('Accuracy')
    plt.xlabel("Epoch #")
    plt.ylabel("Accuracy")
    plt.plot(N, H.history["categorical_accuracy"], label="Train acc")
    plt.plot(N, H.history["val_categorical_accuracy"], label="Val acc")
    plt.legend()
    plt.savefig('acc.png')
    plt.show()
    np.save("multioutput_H_1.npy",H.history)

def prediction():
    z=np.load("data.npz")
    data = z['arr_0']
    data1 = z['arr_1']

    sample = tf.data.Dataset.from_tensor_slices(data).map(
        lambda x: tf.cast((x - tf.reduce_mean(x)) / tf.math.reduce_std(x), tf.float32)).batch(4)
    data1 = np.squeeze(data1)
    model = MyModel()
    model.net.load_weights('./checkpoint/sleep').expect_partial()
    result1 = model.net.predict(sample)
    result1 = np.argmax(result1, axis=1)
    print(confusion_matrix(data1, result1))

    np.save("y_pred.npy",result1)
    np.save("y_true.npy",data1)

def plot_matrix():
    class_name=['Awake','Light','Deep','REM']
    num_local = np.array(range(len(class_name)))
    y_pred = np.load("y_pred.npy")
    y_true=np.load("y_true.npy")
    a=y_pred-y_true

    print(y_true.shape) #(2422,)

    fig=plt.figure()
    ax=fig.add_subplot(1,1,1)
    confusionmatrixs = confusion_matrix(y_true,y_pred)
    print(confusionmatrixs)
    confusionmatrixs = confusionmatrixs.astype(np.float)
    tota = np.sum(confusionmatrixs,axis=1)
    print(tota)
    print(confusionmatrixs[0,:])
    a1=np.around(np.divide(confusionmatrixs[0,:], tota[0])*100, decimals=1)
    a2=np.around(np.divide(confusionmatrixs[1,:], tota[1])*100, decimals=1)
    a3=np.around(np.divide(confusionmatrixs[2,:], tota[2])*100, decimals=1)
    a4=np.around(np.divide(confusionmatrixs[3,:], tota[3])*100, decimals=1)
    confusionmatrixs[0,:]=a1
    confusionmatrixs[1,:]=a2
    confusionmatrixs[2,:]=a3
    confusionmatrixs[3,:]=a4
    print(confusionmatrixs)

    a=sns.heatmap(confusionmatrixs,annot=True,fmt='.1f', linewidth=.5)
    ax.set_xticklabels(['Awake','Light','Deep','REM'])
    ax.set_yticklabels(['Awake','Light','Deep','REM'])
    plt.title('Confusion Matrix(%)')
    plt.ylabel('True')
    plt.xlabel('Predict')
    plt.show()

def data():

    data = np.load("sleep963_csi.npy")
    sleep_label = np.load("sleep963_label.npy")
    data =data[400:800,:,:,:]
    sleep_label=sleep_label[400:800]
    print(data.shape)
    np.savez("dataset.npz",data,sleep_label)

    z=np.load("dataset.npz")
    data = z['arr_0']
    data1 = z['arr_1']
    print(data1.shape)

if __name__ == '__main__':
    #main()
    evaluation()
    plot_matrix()


from MaKaC.webinterface.mail import GenericNotification
from MaKaC.common.info import HelperMaKaCInfo
from MaKaC.webinterface import urlHandlers
from MaKaC.common.mail import GenericMailer
from MaKaC.common.timezoneUtils import getAdjustedDate, nowutc
from persistent import Persistent
from MaKaC.errors import MaKaCError
from MaKaC.reviewing import ConferenceReview
from MaKaC.common.PickleJar import Retrieves
import datetime

###############################################
# Contribution reviewing classes
###############################################

class ReviewManager(Persistent):
    """ This class is the manager for the reviewing. It keeps historical of reviews on a contribution.
        A ReviewManager object is always linked to only 1 contribution and vice-versa.
    """

    def __init__( self, contribution ):
        """ Constructor
            contribution has to be a Contribution object (not an id)
        """
        self._contribution = contribution #the parent contribution for this ReviewManager object
        self._reviewCounter = 0 #the version number of the next new review (starts with 0)
        self._review = Review(self._reviewCounter, self) #the current review (a.k.a. last review), a Review object.
        self._versioning = [self._review] # a list of reviews, including the present one
        self._referee = None # the referee for this contribution. If None, it has not been assigned yet.
        self._editor = None # the editor for this contribution. If None, it has not been assigned yet.
        self._reviewersList = [] #the list of reviewers (there can be more than one) for this contribution.

    def getContribution(self):
        """ Returns the parent contribution for this ReviewManager object
        """
        return self._contribution

    def getConference(self):
        """ Convenience method that returns the Conference object
            to which the contribution belongs to.
        """
        return self._contribution.getConference()

    def getConfReview(self):
        """ Convenience method that returns the ConferenceReview object of the Conference
            to which the contribution belongs to.
        """
        return self.getConference().getConfReview()

    #Review methods
    def newReview(self):
        """ Creates a new Review object and saves it in the versioning list.
        """
        self._reviewCounter = self._reviewCounter + 1
        self._review = Review(self._reviewCounter, self)
        for reviewer in self._reviewersList:
            self._review.addReviewerJudgement(reviewer)
        self._versioning.append(self._review)
        self.notifyModification()

    @Retrieves (['MaKaC.contributionReviewing.ReviewManager'],'lastReview', isPicklableObject=True)
    def getLastReview(self):
        """ Returns the current review as a Review object
        """
        return self._review

    def getVersioning(self):
        """ Returns the list of reviews, current and past, as a list of Review objects.
        """
        return self._versioning

    def getReviewById(self, reviewId):
        return self._versioning[int(reviewId)]

    #referee methods
    @Retrieves (['MaKaC.contributionReviewing.ReviewManager'],'referee', isPicklableObject=True)
    def getReferee(self):
        """ Returns the referee for this contribution
        """
        return self._referee

    def setReferee(self, referee):
        """ Sets the referee for this contribution.
            referee has to be an Avatar object.
            The referee is notified by an email.
        """
        if self.hasReferee():
            raise MaKaCError("This contribution already has a referee")
        else:
            self._referee = referee
            referee.linkTo(self._contribution, "referee")
            self.getConfReview().addRefereeContribution(referee, self._contribution)
            self.getLastReview().setRefereeDueDate(self.getConfReview().getDefaultRefereeDueDate())
            notification = ContributionReviewingNotification(referee, 'Referee', self._contribution)
            GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", referee)

    def removeReferee(self):
        """ Removes the referee for this contribution.
            There is no 'referee' argument because there is only 1 referee by contribution.
            The ex-referee is notified by an email.
        """

        self._referee.unlinkTo(self._contribution, "referee")
        self.getConfReview().removeRefereeContribution(self._referee, self._contribution)
        notification = ContributionReviewingRemoveNotification(self._referee, 'Referee', self._contribution)
        GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", self._referee)
        self._referee = None

    def isReferee(self, user):
        """ Returns if a given user is the referee for this contribution.
            The user has to be an Avatar object.
        """
        return user is not None and user == self._referee

    def hasReferee(self):
        """ Returns if this conference has a referee already.
        """
        return self._referee is not None

    #editor methods
    @Retrieves (['MaKaC.contributionReviewing.ReviewManager'],'editor', isPicklableObject=True)
    def getEditor(self):
        """ Returns the editor for this contribution
        """
        return self._editor

    def setEditor(self, editor):
        """ Sets the editor for this contribution.
            editor has to be an Avatar object.
            The editor is notified by an email.
        """
        if self.hasEditor():
            raise MaKaCError("This contribution is already has an editor")
        elif self.hasReferee() or self.getConfReview().getChoice() == 3:
            self._editor = editor
            editor.linkTo(self._contribution, "editor")
            self.getConfReview().addEditorContribution(editor, self._contribution)
            self.getLastReview().setEditorDueDate(self.getConfReview().getDefaultEditorDueDate())
            notification = ContributionReviewingNotification(editor, 'Editor', self._contribution)
            GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", editor)
        else:
            raise MaKaCError("Please choose a editor before assigning an editor")

    def removeEditor(self):
        """ Removes the editor for this contribution.
            There is no 'editor' argument because there is only 1 editor by contribution.
            The ex-editor is notified by an email.
        """
        self._editor.unlinkTo(self._contribution, "editor")
        self.getConfReview().removeEditorContribution(self._editor, self._contribution)
        notification = ContributionReviewingRemoveNotification(self._editor, 'Editor', self._contribution)
        GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", self._editor)
        self._editor = None

    def isEditor(self, user):
        """ Returns if a given user is the editor for this contribution.
            The user has to be an Avatar object.
        """
        return user is not None and user == self._editor

    def hasEditor(self):
        """ Returns if this conference has a editor already.
        """
        return self._editor is not None

    #reviewer methods
    def addReviewer(self, reviewer):
        """ Adds a reviewer to this contribution.
            reviewer has to be an Avatar object.
        """
        if reviewer in self.getReviewersList():
            raise MaKaCError("This contribution is already assigned to the chosen reviewer")
        elif self.hasReferee():
            self._reviewersList.append(reviewer)
            self.notifyModification()
            reviewer.linkTo(self._contribution, "reviewer")
            self.getConfReview().addReviewerContribution(reviewer, self._contribution)
            self.getLastReview().setReviewerDueDate(self.getConfReview().getDefaultReviewerDueDate())
            if self.getLastReview().getAdviceFrom(reviewer) is None:
                self.getLastReview().addReviewerJudgement(reviewer)
            notification = ContributionReviewingNotification(reviewer, 'Reviewer', self._contribution)
            GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", reviewer)
        else:
            raise MaKaCError("Please choose a referee before assigning a reviewer")

    def removeReviewer(self, reviewer):
        """ Removes the reviewer for this contribution.
            The ex-reviewer is notified by an email.
        """
        if reviewer in self._reviewersList:
            reviewer.unlinkTo(self._contribution, "reviewer")
            self.getConfReview().removeReviewerContribution(reviewer, self._contribution)
            self._reviewersList.remove(reviewer)
            self.notifyModification()
            notification = ContributionReviewingRemoveNotification(reviewer, 'Reviewer', self._contribution)
            GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", reviewer)
        else:
            raise MaKaCError("Cannot remove a reviewer who is not yet assigned to this contribution")

    def removeAllReviewers(self):
        """ Removes all the reviewers for this contribution
        """
        for reviewer in self._reviewersList:
            reviewer.unlinkTo(self._contribution, "reviewer")
            self.getConfReview().removeReviewerContribution(reviewer, self._contribution)
            self.notifyModification()
            notification = ContributionReviewingRemoveNotification(reviewer, 'Reviewer', self._contribution)
            GenericMailer.sendAndLog(notification, self._contribution.getConference(), "MaKaC/reviewing.py", reviewer)
        del(self._reviewersList[:])

    @Retrieves (['MaKaC.contributionReviewing.ReviewManager'],'reviewersList', isPicklableObject=True)
    def getReviewersList(self):
        """ Returns the list of reviewers of this contribution,
            has a list of Avatar objects.
        """
        return self._reviewersList

    def isReviewer(self, user):
        """ Returns if a given user is the reviewer for this contribution.
            The user has to be an Avatar object.
        """
        return user in self._reviewersList

    def hasReviewers(self):
        """ Returns if this conference has at least one reviewer already.
        """
        return len(self._reviewersList) > 0

    def notifyModification(self):
        """ Notifies the DB that a list or dictionary attribute of this object has changed
        """
        self._p_changed = 1

class Judgement(Persistent):
    """ Parent class for RefereeJudgement, EditorJudgement and ReviewerJudgement
    """

    def __init__(self, review, author = None, judgement = None, comments = "", submitted = False, submissionDate = None):
        self._review = review #the parent Review object for this Judgement
        self._author = author #the user (Referee, Editor or Reviewer) author of the judgement
        self._judgement = judgement #the judgement, a string
        self._comments = comments #the comments, a string
        #a dictionary with the questions / criteria proposed and the author evaluation
        self._answers = {}
        self._submitted = submitted #boolean that indicates if the judgement has been submitted or not
        self._submissionDate = submissionDate #the date where the judgement was passed

    def getReview(self):
        return self._review

    def getReviewManager(self):
        return self._review.getReviewManager()

    def getConfReview(self):
        """ Convenience method that returns the ConferenceReview object of the Conference
            to which the judgment belongs to.
        """
        return self.getReviewManager().getConfReview()

    def getAuthor(self):
        return self._author

    @Retrieves (['MaKaC.contributionReviewing.RefereeJudgement',
                 'MaKaC.contributionReviewing.EditorJudgement',
                 'MaKaC.contributionReviewing.ReviewerJudgement'],'judgement')
    def getJudgement(self):
        return self._judgement

    def getComments(self):
        return self._comments

    def getAnswers(self):
        """ To be implemented by sub-classes
            Returns a list of tuples (question, answer)
        """
        pass

    def getAllAnswers(self):
        return self._answers

    @Retrieves (['MaKaC.contributionReviewing.RefereeJudgement',
                 'MaKaC.contributionReviewing.EditorJudgement',
                 'MaKaC.contributionReviewing.ReviewerJudgement'],'reviewingStatus')
    def isSubmitted(self):
        return self._submitted

    def getSubmissionDate(self):
        """ Returns the submission date for the review
        """
        return self._submissionDate

    def getAdjustedSubmissionDate(self):
        """ Returns a timezone-aware submission date given the conference's timezone.
        """
        return getAdjustedDate(self._submissionDate, self.getReviewManager().getConference())

    def setAuthor(self, user):
        self._author = user

    def setJudgement(self, judgement):
        self._judgement = judgement

    def setComments(self, comments):
        self._comments = comments

    def getAnswer(self, question):
        if not self._answers.has_key(question):
            self._answers[question] = ConferenceReview.initialSelectedAnswer
        return self._answers[question]

    def setAnswer(self, question, answer):
        self._answers[question] = answer
        self.notifyModification()

    def setSubmitted(self, submitted):
        if self._judgement is None:
            raise MaKaCError("Cannot submit an opinion without choosing the judgemenent before")
        self._submitted = submitted
        self._submissionDate = nowutc()

    def sendNotificationEmail(self, widthdrawn = False):
        """ Sends an email to the contribution's authors when the referee, editor or reviewer
            pass a judgement on the contribution.
        """
        authorList = self.getReviewManager().getContribution().getAuthorList()
        for author in authorList:
            if widthdrawn:
                notification = ContributionReviewingJudgementWithdrawalNotification(author, self, self.getReviewManager().getContribution())
            else:
                notification = ContributionReviewingJudgementNotification(author, self, self.getReviewManager().getContribution())
            GenericMailer.sendAndLog(notification, self._review.getConference(), "MaKaC/reviewing.py", author)

    def notifyModification(self):
        """ Notifies the DB that a list or dictionary attribute of this object has changed
        """
        self._p_changed = 1

class RefereeJudgement(Judgement):

    def setSubmitted(self, submitted):
        """ Sets the final judgement for a review, since this is the Referee judgement.
            The judgement is a string among the pre-defined states (Accept, Reject, To be corrected) and
            any user-defined states.
            If it's the first time that the final judgement is set for this review, the contribution materials
            are copied into the review.
            If the judgement is 'To be corrected' or one of the user-defined states, versioning takes place.
            A new Review object is then created as 'last review'.
        """
        Judgement.setSubmitted(self, submitted)

        self.getReview().copyMaterials([m for m in self.getReviewManager().getContribution().getAllMaterialList()
                                        if m.getTitle() in self.getConfReview().getReviewableMaterials()])

        if self._judgement == "To be corrected" or self._judgement in self.getReviewManager().getConference().getConfReview().getStates():
            self.getReviewManager().newReview()

    def getAnswers(self):
        return zip(self.getConfReview().getReviewingQuestions(), (self.getAnswer(k) for k in self.getConfReview().getReviewingQuestions()))

class EditorJudgement(Judgement):
    def getAnswers(self):
        return zip(self.getConfReview().getLayoutCriteria(), (self.getAnswer(k) for k in self.getConfReview().getLayoutCriteria()))

class ReviewerJudgement(Judgement):
    def getAnswers(self):
        return zip(self.getConfReview().getReviewingQuestions(), (self.getAnswer(k) for k in self.getConfReview().getReviewingQuestions()))

class Review(Persistent):
    """This class represents the judgement of a contribution made by the referee. It contains judgement and comments
    """

    def __init__( self, version, reviewManager):
        """ Constructor for the class.
            version: an integer, first version number is 0.
            reviewManager: the parent ReviewManager object
        """
        self._reviewManager = reviewManager #the parent ReviewManager object for this Review
        self._refereeJudgement = RefereeJudgement(self)
        self._editorJudgement = EditorJudgement(self)
        self._reviewerJudgements = {}

        self._isAuthorSubmitted = False #boolean that says if the author has submitted his / her materials or not
        self._version = version #the version number for this Review. Different Reviews for the same contribution have increasing version numbers.
        self._materials = [] #'snapshot' of the materials that were analyzed by the reviewing team. Copied from the Contribution materials when judgement is passed.
        self._authorComments = ""
        self._refereeDueDate = None #the due date where the referee has to pass his/her judgement
        self._editorDueDate = None #the due date where the editor has to pass his/her judgement
        self._reviewerDueDate = None #the due date where all the reviewers have to pass his/her judgement

    def notifyModification(self):
        """ Notifies the DB that a list or dictionary attribute of this object has changed
        """
        self._p_changed = 1

    def getId(self):
        """ Returns the id of this Review, which is the same as its version number
        """
        return self._version

    def getReviewManager(self):
        """ Returns the parent Review Manager object for this Review object.
        """
        return self._reviewManager

    def getContribution(self):
        """ Convenience method that returns the Contribution to which this Review belongs.
        """
        return self._reviewManager.getContribution()

    def getConference(self):
        """ Convenience method that returns the Conference to which this Review belongs.
        """
        return self._reviewManager.getContribution().getConference()

    def getOwner(self):
        return self.getContribution()

    @Retrieves (['MaKaC.contributionReviewing.Review'],'refereeJudgement', isPicklableObject=True)
    def getRefereeJudgement(self):
        return self._refereeJudgement

    def getEditorJudgement(self):
        return self._editorJudgement

    def getReviewerJudgement(self, reviewer):
        return self._reviewerJudgements[reviewer]

    @Retrieves (['MaKaC.contributionReviewing.Review'],'reviewingStatus')
    def getReviewingStatus(self, forAuthor = False):
        """ Returns a list of strings with a description of the current status of the review.
        """
        status = []
        if self.isAuthorSubmitted():
            if self._refereeJudgement.isSubmitted():
                status.append("Judged: " + str(self._refereeJudgement.getJudgement()))
            else:
                status.append("Pending referee decision")
                if not forAuthor:
                    if self._editorJudgement.isSubmitted():
                        status.append("Layout judged: " + str(self._editorJudgement.getJudgement()))
                    else:
                        status.append("Pending layout editor decision")

                    if self.anyReviewerHasGivenAdvice():
                        for a in self.getReviewerJudgements():
                            status.append("Content judged: " + str(a.getJudgement()))
                        if not self.allReviewersHaveGivenAdvice():
                            status.append("Some content reviewers have not decided yet")
                    else:
                        status.append("No content reviewers have decided yet")
        else:
            status.append("Materials not submitted yet")
        return status

    def isAuthorSubmitted(self):
        """ Returns if the author(s) of the contribution has marked the materials
            as submitted.
            When materials are marked as submitted, the review process can start.
        """
        return self._isAuthorSubmitted

    def setAuthorSubmitted(self, submitted):
        """ If submitted is True, it means that the author has marked the materials as submitted.
            If submitted is False, it means that the author has 'unmarked' the materials as submitted because
            he/she did some mistakes.
            In both cases, all the already chosen reviewing staff are notified with an email.
        """

        self._isAuthorSubmitted = submitted

        if submitted:

            if self._reviewManager.hasReferee():
                notification = MaterialsSubmittedNotification(self._reviewManager.getReferee(), 'Referee', self._reviewManager.getContribution())
                GenericMailer.sendAndLog(notification, self._reviewManager.getContribution().getConference(), "MaKaC/reviewing.py", self._reviewManager.getReferee())

            if self._reviewManager.hasEditor():
                notification = MaterialsSubmittedNotification(self._reviewManager.getEditor(), 'Editor', self._reviewManager.getContribution())
                GenericMailer.sendAndLog(notification, self._reviewManager.getContribution().getConference(), "MaKaC/reviewing.py", self._reviewManager.getEditor())

            for reviewer in self._reviewManager.getReviewersList():
                notification = MaterialsSubmittedNotification(reviewer, 'Reviewer', self._reviewManager.getContribution())
                GenericMailer.sendAndLog(notification, self._reviewManager.getContribution().getConference(), "MaKaC/reviewing.py", reviewer)

        else:
            if self._reviewManager.hasReferee():
                notification = MaterialsChangedNotification(self._reviewManager.getReferee(), self._reviewManager.getContribution())
                GenericMailer.sendAndLog(notification, self._reviewManager.getContribution().getConference(), "MaKaC/reviewing.py", self._reviewManager.getReferee())

            if self._reviewManager.hasEditor():
                notification = MaterialsChangedNotification(self._reviewManager.getEditor(), self._reviewManager.getContribution())
                GenericMailer.sendAndLog(notification, self._reviewManager.getContribution().getConference(), "MaKaC/reviewing.py", self._reviewManager.getEditor())

            for reviewer in self._reviewManager.getReviewersList():
                notification = MaterialsChangedNotification(reviewer, self._reviewManager.getContribution())
                GenericMailer.sendAndLog(notification, self._reviewManager.getContribution().getConference(), "MaKaC/reviewing.py", reviewer)

    def getVersion(self):
        """ Returns the version number for this review. The version number is an integer, starting by 0.
        """
        return self._version

    def setAuthorComments(self, authorComments):
        self._authorComments = authorComments

    def getAuthorComments(self):
        return self._authorComments

    #review materials methods, and methods necessary for material retrieval to work
    def getMaterials(self):
        """ Returns the materials stored in this Review.
        """
        return self._materials

    def copyMaterials(self, materials):
        """ Copies the materials from the contribution to this review object.
            This is done by cloning the materials, and putting the contribution as owner.
            This way, even if the author deletes materials during another review, they endure in this review.
        """
        self._materials = []
        for m in materials:
            self._materials.append(m.clone(self))

    def getMaterialById(self, materialId):
        """ Returns one of the materials of the review given its id
        """
        return self._materials[int(materialId)]

    def getLocator(self):
        """Gives back a globaly unique identification encapsulated in a Locator
           object for the Review instance
        """
        l = self.getOwner().getLocator()
        l["reviewId"] = self.getId()
        return l

    def isFullyPublic( self ):
        if hasattr(self, "_fullyPublic"):
            return self._fullyPublic
        else:
            self.setFullyPublic()
            return self._fullyPublic

    def setFullyPublic( self ):
        for mat in self.getMaterials():
            if not mat.isFullyPublic():
                self._fullyPublic = False
                return
        self._fullyPublic = True

    def updateFullyPublic( self ):
        self.setFullyPublic()
        self.getOwner().updateFullyPublic()

    def isProtected(self):
        return self.getOwner().isProtected()

    def resetAccessCache(self, UP=True, DOWN=True):
        if UP:
            self.getOwner().resetAccessCache(True, False)
        if DOWN:
            for mat in self.getMaterials():
                mat.resetAccessCache(False, True)

    def canIPAccess( self, ip ):
        return self.getOwner().canIPAccess(ip)

    #Advices methods
    def addReviewerJudgement(self, reviewer):#, questions, adviceJudgement, comments):
        """ Adds an ReviewerJudgement object to the list of ReviewerJudgements objects of the review.
            Each ReviewerJudgement object represents the opinion of a content reviewer.
        """
        self._reviewerJudgements[reviewer] = ReviewerJudgement(self, author = reviewer)
        self.notifyModification()

    def hasGivenAdvice(self, reviewer):
        """ Returns if a given user has given advice on the content of the contribution.
        """
        answer = (reviewer in self._reviewerJudgements) and (self._reviewerJudgements[reviewer].isSubmitted())
        return answer

    def getAdviceFrom(self, reviewer):
        """ Returns an the advice information from a given reviewer,
            as an Advice object.
            If the given reviewer doesn't have an Advice object yet, None is returned.
        """
        return self._reviewerJudgements.get(reviewer, None)

    def anyReviewerHasGivenAdvice(self):
        """ Returns if at least 1 reviewer has already given advice on the content of the contribution.
        """
        for reviewer in self._reviewManager.getReviewersList():
            if self.hasGivenAdvice(reviewer):
                return True

        return False

    def allReviewersHaveGivenAdvice(self):
        """ Returns if all reviewers have already given advice on the content of the contribution.
        """
        for reviewer in self._reviewManager.getReviewersList():
            if not self.hasGivenAdvice(reviewer):
                return False

        return len(self._reviewManager.getReviewersList()) > 0

    def getReviewerJudgements(self):
        """ Returns the advices from the reviewers.
        """
        return self._reviewerJudgements.values()

    def getSubmittedReviewerJudgement(self):
        """ Returns the advices from the reviewers, but only those that have been marked as submitted
        """
        return filter(lambda advice:advice.isSubmitted(), self.getReviewerJudgements())

#    #notification email methods
#    def sendNotificationEmail(self, judgement):
#        """ Sends an email to the contribution's authors when the referee, editor or reviewer
#            pass a judgement on the contribution.
#        """
#        authorList = self._reviewManager.getContribution().getAuthorList()
#        for author in authorList:
#            notification = ContributionReviewingJudgementNotification(author, judgement, self._reviewManager.getContribution())
#            GenericMailer.sendAndLog(notification, self.getConference(), "MaKaC/reviewing.py", author)

    #dates methods
    def setRefereeDueDate(self, date):
        """ Sets the due date for the referee.
        """
        self._refereeDueDate = date #datetime(year, month, day, 0, 0, 0, tzinfo=timezone(self.getConference().getTimezone()))

    def getRefereeDueDate(self):
        """ Returns the due date for the referee
        """
        return self._refereeDueDate

    def getAdjustedRefereeDueDate(self):
        """ Returns a timezeone-aware due date for the referee given the conference's timezone.
        """
        if self.getRefereeDueDate() is None:
            return None
        else:
            return getAdjustedDate(self._refereeDueDate, self.getConference())

    @Retrieves (['MaKaC.contributionReviewing.Review'], 'refereeDueDate')
    def getAdjustedRefereeDueDateFormatted(self):
        """ Returns a timezeone-aware due date for the referee given the conference's timezone,
            formatted to a string (this method is necessary due to syntax limitations of @Retrieve )
        """
        date = self.getAdjustedRefereeDueDate()
        if date:
            return datetime.datetime.strftime(date,'%d/%m/%Y %H:%M')
        else:
            return None

    def setEditorDueDate(self, date):
        """ Sets the due date for the editor.
        """
        self._editorDueDate = date #datetime(year, month, day, 0, 0, 0, tzinfo=timezone(self.getConference().getTimezone()))

    def getEditorDueDate(self):
        """ Returns the due date for the editor
        """
        return self._editorDueDate

    def getAdjustedEditorDueDate(self):
        """ Returns a timezeone-aware due date for the editor given the conference's timezone.
        """
        if self.getEditorDueDate() is None:
            return None
        else:
            return getAdjustedDate(self._editorDueDate, self.getConference())

    def setReviewerDueDate(self, date):
        """ Sets the due date for all the reviewers.
        """
        self._reviewerDueDate = date #datetime(year, month, day, 0, 0, 0, tzinfo=timezone(self.getConference().getTimezone()))

    def getReviewerDueDate(self):
        """ Returns the due date for all the reviewers.
        """
        return self._reviewerDueDate

    def getAdjustedReviewerDueDate(self):
        """ Returns a timezeone-aware due date for all the reviewers given the conference's timezone.
        """
        if self.getReviewerDueDate() is None:
            return None
        else:
            return getAdjustedDate(self._reviewerDueDate, self.getConference())

######################################
# Email notification classes
######################################
class ContributionReviewingNotification(GenericNotification):
    """ Template to build an email notification to a newly appointed PRM / Referee / Editor / Reviewer
        for a given contribution.
    """

    def __init__(self, user, role, contribution):
        GenericNotification.__init__(self)
        conference = contribution.getConference()
        self.setFromAddr("Indico Mailer<%s>"%HelperMaKaCInfo.getMaKaCInfoInstance().getSupportEmail())
        self.setToList([user.getEmail()])
        self.setSubject("""[Indico] You have been chosen as %s for the contribution "%s" (id: %s)"""
                        % (role, contribution.getTitle(), str(contribution.getId())))
        self.setBody("""Dear Indico user,

        You have been chosen as %s of the contribution "%s" (id: %s) of the conference %s (id: %s).
        You can go to the contribution reviewing page:
        %s
        If you have not already, you will have to log in to see this page.

        Thank you for using our system.
        """ % ( role, contribution.getTitle(), str(contribution.getId()), conference.getTitle(),
                str(conference.getId()), urlHandlers.UHContributionModifReviewing.getURL(contribution)
        ))

class ContributionReviewingRemoveNotification(GenericNotification):
    """ Template to build an email notification to a removed PRM / Referee / Editor / Reviewer
        for a given contribution.
    """

    def __init__(self, user, role, contribution):
        conference = contribution.getConference()
        GenericNotification.__init__(self)
        self.setFromAddr("Indico Mailer<%s>"%HelperMaKaCInfo.getMaKaCInfoInstance().getSupportEmail())
        self.setToList([user.getEmail()])
        self.setSubject("""[Indico] You have been removed as %s of the contribution "%s" (id: %s) of the conference %s (id: %s)"""
                        % (role, contribution.getTitle(), str(contribution.getId()), conference.getTitle(),str(conference.getId())))
        self.setBody("""Dear Indico user,

        We are sorry to inform you that you have been removed as %s of the contribution "%s" (id: %s).

        Thank you for using our system.
        """ % ( role, contribution.getTitle(), str(contribution.getId())
        ))

class ContributionReviewingJudgementNotification(GenericNotification):
    """ Template to build an email notification for a contribution submitter
        once the contribution has been judged
    """

    def __init__(self, user, judgement, contribution):
        GenericNotification.__init__(self)
        self.setFromAddr("Indico Mailer<%s>"%HelperMaKaCInfo.getMaKaCInfoInstance().getSupportEmail())
        self.setToList([user.getEmail()])

        if isinstance(judgement, EditorJudgement):
            self.setSubject("""[Indico] The layout of your contribution "%s" (id: %s) has been reviewed"""
                            % (contribution.getTitle(), str(contribution.getId())))
            self.setBody("""Dear Indico user,

        The layout of your contribution "%s" (id: %s) has been reviewed.
        The judgement was: %s

        The comments of the editor were:
        "%s"

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), judgement.getJudgement(),
                judgement.getComments())
        )

        elif isinstance(judgement, ReviewerJudgement):
            self.setSubject("""[Indico] The content of your contribution "%s" (id: %s) has been reviewed"""
                            % (contribution.getTitle(), str(contribution.getId())))
            self.setBody("""Dear Indico user,

        The content of your contribution "%s" (id: %s) has been reviewed.
        The judgement was: %s

        The comments of the reviewer were:
        "%s"

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), judgement.getJudgement(),
                judgement.getComments())
        )

        elif isinstance(judgement, RefereeJudgement):
            self.setSubject("""[Indico] Your contribution "%s" (id: %s) has been completely reviewed by the referee"""
                            % (contribution.getTitle(), str(contribution.getId())))
            self.setBody("""Dear Indico user,

        Your contribution "%s" (id: %s) has been completely reviewed by the assigned referee.
        The judgement was: %s

        The comments of the referee were:
        "%s"

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), judgement.getJudgement(),
                judgement.getComments())
        )

class ContributionReviewingJudgementWithdrawalNotification(GenericNotification):
    """ Template to build an email notification for a contribution submitter
        once the judgement of the contribution has been withdrawn.
    """

    def __init__(self, user, judgement, contribution):
        GenericNotification.__init__(self)
        self.setFromAddr("Indico Mailer<%s>"%HelperMaKaCInfo.getMaKaCInfoInstance().getSupportEmail())
        self.setToList([user.getEmail()])

        if isinstance(judgement, EditorJudgement):
            self.setSubject("""[Indico] The judgement for the layout of your contribution "%s" (id: %s) has been widthdrawn"""
                            % (contribution.getTitle(), str(contribution.getId())))
            self.setBody("""Dear Indico user,

        The judgement for the layout of your contribution "%s" (id: %s) has been widthdrawn.
        The judgement was: %s

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), judgement.getJudgement())
        )

        elif isinstance(judgement, ReviewerJudgement):
            self.setSubject("""[Indico] The judgement for the content of your contribution "%s" (id: %s) has been widthdrawn"""
                            % (contribution.getTitle(), str(contribution.getId())))
            self.setBody("""Dear Indico user,

        The judgement for the content of your contribution "%s" (id: %s) has been widthdrawn.
        The judgement was: %s

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), judgement.getJudgement())
        )

        elif isinstance(judgement, RefereeJudgement):
            self.setSubject("""[Indico] The judgement for your contribution "%s" (id: %s) has been widthdrawn by the referee"""
                            % (contribution.getTitle(), str(contribution.getId())))
            self.setBody("""Dear Indico user,

        The judgement for your contribution "%s" (id: %s) has been widthdrawn by the assigned referee.
        The judgement was: %s

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), judgement.getJudgement())
        )

class MaterialsSubmittedNotification(GenericNotification):

    def __init__(self, user, role, contribution):
        conference = contribution.getConference()
        GenericNotification.__init__(self)
        self.setFromAddr("Indico Mailer<%s>"%HelperMaKaCInfo.getMaKaCInfoInstance().getSupportEmail())
        self.setToList([user.getEmail()])
        self.setSubject("""[Indico] The author of the contribution %s (id: %s) of the conference %s has submitted his/her materials (id: %s)"""
                        % (contribution.getTitle(), str(contribution.getId()), conference.getTitle(), str(conference.getId())))
        self.setBody("""Dear Indico user,

        The author of the contribution %s (id: %s) of the conference %s (id: %s) has marked his / her materials as submitted.
        You can now start the reviewing process as a %s.

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), conference.getTitle(), str(conference.getId()), role))

class MaterialsChangedNotification(GenericNotification):

    def __init__(self, user, contribution):
        conference = contribution.getConference()
        GenericNotification.__init__(self)
        self.setFromAddr("Indico Mailer<%s>"%HelperMaKaCInfo.getMaKaCInfoInstance().getSupportEmail())
        self.setToList([user.getEmail()])
        self.setSubject("""[Indico] Warning: the author of the contribution %s (id: %s) of the conference %s (id: %s) has changed his/her materials """
                        % (contribution.getTitle(), str(contribution.getId()), conference.getTitle(), str(conference.getId())))
        self.setBody("""Dear Indico user,

        The author of the contribution %s (id: %s) of the conference %s (id: %s) has removed the 'submitted' mark from his / her materials.
        This means that he may have changed the content of the materials.
        Thus, you should wait until he has marked the materials again to start / continue the reviewing process.

        Thank you for using our system.
        """ % ( contribution.getTitle(), str(contribution.getId()), conference.getTitle(), str(conference.getId())))


from flask_wtf import FlaskForm
from wtforms import BooleanField, SelectField, StringField, SubmitField, TextAreaField
from flask_pagedown.fields import PageDownField
from wtforms.validators import DataRequired, Email, Length, ValidationError
from ..models import Recipe, Role

class CreateRecipeForm(FlaskForm):
    name = StringField("Recipe Name", validators=[DataRequired(), Length(1, 128)])
    ingredients = PageDownField("Ingredients", validators=[DataRequired()])
    how_to = PageDownField("Instruction", validators=[DataRequired()])
    image_url = StringField("Image URL")
    submit = SubmitField("Create")

    def validate_name(self, field):
        recipe = Recipe.query.filter_by(name=field.data).first()
        if recipe:
            raise ValidationError("Name is already exist")

class AddCommentForm(FlaskForm):
    body = TextAreaField("Write your comment")
    submit = SubmitField("Post")

class EditProfileForm(FlaskForm):
    name = StringField("Name", validators=[DataRequired(), Length(1, 64)])
    location = StringField("Location")
    about_me = TextAreaField("About me")
    submit = SubmitField("Save")

class EditProfileAdminForm(FlaskForm):
    mail = StringField("Mail", validators=[DataRequired(), Email(), Length(1, 64)])
    username = StringField("Username", validators=[DataRequired(), Length(1, 32)])
    confirmed = BooleanField("Confirmed")
    role_id = SelectField("Role", coerce=int)
    name = StringField("Name")
    location = StringField("Location")
    about_me = TextAreaField("About me")
    is_locked = BooleanField("Locked")
    submit = SubmitField("Save")

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        roles = Role.query.all()
        self.role_id.choices = [(role.id, role.name) for role in roles]

class EditRecipeForm(FlaskForm):
    name = StringField("Name", validators=[DataRequired(), Length(1, 128)])
    how_to = PageDownField("Instruction", validators=[DataRequired()])
    ingredients = PageDownField("Ingredients", validators=[DataRequired()])
    image_url = StringField("Image URL")
    submit = SubmitField("Save")

class EditRecipeAdminForm(FlaskForm):
    name = StringField("Name", validators=[DataRequired(), Length(1, 128)])
    how_to = PageDownField("Instruction", validators=[DataRequired()])
    ingredients = PageDownField("Ingredients", validators=[DataRequired()])
    image_url = StringField("Image URL")
    disable = BooleanField("Disable")
    submit = SubmitField("Save")


import logging
from http import HTTPStatus
from typing import Any, List, Literal, Mapping, Optional, TypedDict

from libcommon.dataset import DatasetNotFoundError
from libcommon.exceptions import CustomError
from libcommon.simple_cache import DoesNotExist, SplitFullName, get_response

from datasets_based.config import AppConfig
from datasets_based.worker import JobInfo, Worker
from datasets_based.workers.parquet_and_dataset_info import ParquetFileItem

ParquetWorkerErrorCode = Literal[
    "PreviousStepStatusError",
    "PreviousStepFormatError",
]

class ParquetResponse(TypedDict):
    parquet_files: List[ParquetFileItem]

class ParquetWorkerError(CustomError):
    """Base class for exceptions in this module."""

    def __init__(
        self,
        message: str,
        status_code: HTTPStatus,
        code: ParquetWorkerErrorCode,
        cause: Optional[BaseException] = None,
        disclose_cause: bool = False,
    ):
        super().__init__(message, status_code, str(code), cause, disclose_cause)

class PreviousStepStatusError(ParquetWorkerError):
    """Raised when the previous step gave an error. The job should not have been created."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "PreviousStepStatusError", cause, False)

class PreviousStepFormatError(ParquetWorkerError):
    """Raised when the content of the previous step has not the expected format."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "PreviousStepFormatError", cause, False)

def compute_parquet_response(dataset: str) -> ParquetResponse:
    """
    Get the response of /parquet for one specific dataset on huggingface.co.
    Args:
        dataset (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
    Returns:
        `ParquetResponse`: An object with the parquet_response (list of parquet files).
    <Tip>
    Raises the following errors:
        - [`~parquet.worker.PreviousStepStatusError`]
          If the the previous step gave an error.
        - [`~parquet.worker.PreviousStepFormatError`]
            If the content of the previous step has not the expected format
    </Tip>
    """
    logging.info(f"get parquet files for dataset={dataset}")

    # TODO: we should move this dependency to the Worker class: defining which are the inputs, and just getting their
    # value here
    try:
        response = get_response(kind="/parquet-and-dataset-info", dataset=dataset)
    except DoesNotExist as e:
        raise DatasetNotFoundError("No response found in previous step for this dataset.", e) from e
    if response["http_status"] != HTTPStatus.OK:
        raise PreviousStepStatusError(
            f"Previous step gave an error: {response['http_status']}. This job should not have been created."
        )
    content = response["content"]
    if "parquet_files" not in content:
        raise PreviousStepFormatError("Previous step did not return the expected content.")
    return {
        "parquet_files": content["parquet_files"],
    }

class ParquetWorker(Worker):
    @staticmethod
    def get_job_type() -> str:
        return "/parquet"

    @staticmethod
    def get_version() -> str:
        return "3.0.0"

    def __init__(self, job_info: JobInfo, app_config: AppConfig) -> None:
        job_type = job_info["type"]
        try:
            processing_step = app_config.processing_graph.graph.get_step_by_job_type(job_type)
        except ValueError as e:
            raise ValueError(
                f"Unsupported job type: '{job_type}'. The job types declared in the processing graph are:"
                f" {[step.job_type for step in app_config.processing_graph.graph.steps.values()]}"
            ) from e
        super().__init__(job_info=job_info, common_config=app_config.common, processing_step=processing_step)

    def compute(self) -> Mapping[str, Any]:
        return compute_parquet_response(dataset=self.dataset)

    def get_new_splits(self, content: Mapping[str, Any]) -> set[SplitFullName]:
        """Get the set of new splits, from the content created by the compute."""
        return {
            SplitFullName(dataset=parquet_file["dataset"], config=parquet_file["config"], split=parquet_file["split"])
            for parquet_file in content["parquet_files"]
        }


from PIL import Image
import json
import random

f = open('input.json')
data = json.load(f)
f.close()

images = []
count = 10000

def expand_props():
    data['new_props'] = {}
    for i in data['props']:
        props = data['props'][i]

        new_props = []

        for p in props:
            rarity = p['rarity']
            for y in range(0, rarity):
                new_props.append(p)

        data['new_props'][i] = new_props

expand_props()

def get_random_prop(props):
    prop_count = len(props)
    id = random.randint(0, prop_count-1)
    prop = props[id]
    return prop

# Prepare prop stats
prop_stats = {}
for i in data['props']:
    prop_stats[i] = {}

    sum = 0

    for x in data['props'][i]:
        sum += x['rarity']
        prop_stats[i][x['name']] = {
            "total": 0,
            "total_p": 0,
            "rarity": x['rarity'],
            "rarity_p": 0,
        }

    for x in data['props'][i]:
        prop_stats[i][x['name']]['rarity_p'] = format(prop_stats[i][x['name']]['rarity']/sum, '.2%')

for x in range(count):
    image = {}
    image['tokenId'] = x+1
    ip = {}

    for i in data['new_props']:
        props = data['new_props'][i]
        prop = get_random_prop(props)

        while (i == 'right' and ip['left']['class'] == prop['class']):
            prop = get_random_prop(props)

        prop_stats[i][prop['name']]['total'] = prop_stats[i][prop['name']]['total']+1
        ip[i] = prop

    for i in ip:
        ip[i] = ip[i]['name']

    image['props'] = ip

    images.append(image)
    # print(i)

for i in data['props']:
    for x in data['props'][i]:
        prop_stats[i][x['name']]['total_p'] = format(prop_stats[i][x['name']]['total']/count, '.2%')

raw = json.dumps(images)

f = open('output.json', 'w')
f.write(raw)
f.close()

f = open('prop_stats.json', 'w')
f.write(json.dumps(prop_stats))
f.close()


import json
from Common.parameter import login_js_path

class GetJson:
    def __init__(self, file_path):
        self.file_path = file_path
        self.data = self.read_json()

    # 读取json文件，转换成python对象
    def read_json(self):
        with open(self.file_path, encoding='utf-8') as fp:
            data = json.load(fp)
            return data

a = GetJson(login_js_path)

print(a.data['case_name'])


from django.db import models
from openhumans.models import OpenHumansMember

class PublicExperience(models.Model):
    experience_text = models.TextField()
    difference_text = models.TextField(default="")
    created_at = models.DateTimeField(auto_now=True)
    experience_id = models.TextField()
    open_humans_member = models.ForeignKey(OpenHumansMember,
                                           blank=True, null=True,
                                           on_delete=models.CASCADE)
    approved = models.CharField(
        blank=False,
        default='not reviewed',
        max_length=50)


from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when
from tools import download_antaq, concatena_dataframe, salva_arquivo

spark = SparkSession.builder.master('local[*]').getOrCreate()

dados_anos = ['2018', '2019', '2020']
dados_download = {
    'atracacao': {
        'nome_pasta': 'atracacao',
        'nome_arquivo': 'Atracacao',
        'url_download': 'http://web.antaq.gov.br/Sistemas/ArquivosAnuario/Arquivos/{}Atracacao.zip'
    },
    'tempos_atracacao': {
        'nome_pasta': 'tempos_atracacao',
        'nome_arquivo': 'TemposAtracacao',
        'url_download': 'http://web.antaq.gov.br/Sistemas/ArquivosAnuario/Arquivos/{}TemposAtracacao.zip'
    },
    'carga': {
        'nome_pasta': 'carga',
        'nome_arquivo': 'Carga',
        'url_download': 'http://web.antaq.gov.br/Sistemas/ArquivosAnuario/Arquivos/{}Carga.zip'
    },
    'carga_conteinerizada': {
        'nome_pasta': 'carga_conteinerizada',
        'nome_arquivo': 'Carga_Conteinerizada',
        'url_download': 'http://web.antaq.gov.br/Sistemas/ArquivosAnuario/Arquivos/{}CargaConteinerizada.zip'
    }
}

#download arquivos brutos
for dado in dados_download.values():
    for ano in dados_anos:

        download_antaq(dado, ano)

#Atracacao dataframe completo
df_atracacao_full = concatena_dataframe(spark, dados_anos, dados_download['atracacao'])

#Atracacao dataframe filtro ceara
df_atracacao_ce = df_atracacao_full.filter(df_atracacao_full.UF == 'Ceará')

df_atracacao_full = None

#Atracacao dataframe pré join carga
df_atracacao_id_ce = df_atracacao_ce.select(col('IDAtracacao'), col('Ano'), col('Mes'), col('Porto Atracação'), col('SGUF'))
df_atracacao_id_ce = df_atracacao_id_ce.withColumnRenamed('Ano', 'Ano da data de início da operação da atracação')
df_atracacao_id_ce = df_atracacao_id_ce.withColumnRenamed('Mes', 'Mês da data de início da operação da atracação')

#Tempo Atracacao dataframe completo
df_tempos_atracacao_full = concatena_dataframe(spark, dados_anos, dados_download['tempos_atracacao'])

#join Atracacao e Tempo Atracacao
df_atracacao_ce = df_atracacao_ce.join(df_tempos_atracacao_full,
    df_tempos_atracacao_full.IDAtracacao == df_atracacao_ce.IDAtracacao,
    'left').drop(df_tempos_atracacao_full.IDAtracacao)

df_atracacao_ce = df_atracacao_ce.withColumnRenamed('Ano', 'Ano da data de início da operação')
df_atracacao_ce = df_atracacao_ce.withColumnRenamed('Mes', 'Mês da data de início da operação')

#salva arquivo csv de atracacao
salva_arquivo(df_atracacao_ce, 'atracacao_fato')

#Carga dataframe completo
df_carga_full = concatena_dataframe(spark, dados_anos, dados_download['carga'])

df_carga_ce = df_atracacao_id_ce.join(df_carga_full,
    df_carga_full.IDAtracacao ==  df_atracacao_id_ce.IDAtracacao,
    'left').drop(df_carga_full.IDAtracacao)

df_carga_full = None

#Carga conteinerizada dataframe completo
df_carga_conteinerizada = concatena_dataframe(spark, dados_anos, dados_download['carga_conteinerizada'])

#join carga e carga conteinerizada
df_carga_ce = df_carga_ce.join(df_carga_conteinerizada, [df_carga_conteinerizada.IDCarga ==  df_carga_ce.IDCarga,
    df_carga_ce['Carga Geral Acondicionamento']=='Conteinerizada'], 'left').drop(df_carga_conteinerizada.IDCarga)

#inclusao codigo produto conteiner
df_carga_ce = df_carga_ce.withColumn('CDMercadoria',
    when(col('CDMercadoriaConteinerizada').isNotNull(),
    df_carga_ce['CDMercadoriaConteinerizada']).otherwise(col('CDMercadoria'))).drop(df_carga_ce.CDMercadoriaConteinerizada)

#inclusao peso liquido
df_carga_ce = df_carga_ce.withColumn('VLPesoCargaConteinerizada',
    when(col('Carga Geral Acondicionamento')=='Conteinerizada',
    df_carga_ce['VLPesoCargaConteinerizada']).otherwise(col('VLPesoCargaBruta')))

df_carga_ce = df_carga_ce.withColumnRenamed('VLPesoCargaConteinerizada', 'Peso líquido da carga')

#salva arquivo csv de atracacao
salva_arquivo(df_carga_ce, 'carga_fato')


#!/usr/bin/env python3
import sys
from pathlib import Path
from collections import namedtuple
from functools import partial
import numpy as np

Range = namedtuple("Range", "min max".split())

def rangeize(in_str, conversion):
    x, y = in_str.split('-')
    return Range(conversion(x), conversion(y))

import argparse
parser = argparse.ArgumentParser(description='Display a sum plot of select frames of a TDF file.')
parser.add_argument("path", help="TDF dataset path", type=Path)
parser.add_argument("frames", help="Comma-separated list of frames, including slices. Example: 35,12::17 will sum frames: 35 and every 17th frame starting from 12 till the end.", default=":")
parser.add_argument("-s", "--save", help="Save the image to file instead of displaying them", action="store_true")
parser.add_argument("--silent", help="Do not display progressbar", action='store_true')
parser.add_argument("-p", "--processes", help="Number of subprocesses to use. Will use as many as there are detected cores in your system if omitted.", type=int, default=None)
parser.add_argument("--mz-range", help="Custom mz range, example: 400.0-600.0", type=partial(rangeize, conversion=float), default=Range(0, 2000))
parser.add_argument("--rt-range", help="Additionally clamp to RT range (constraining further the frames argument), example: 400.0-600.0", type=partial(rangeize, conversion=float), default=None)
parser.add_argument("--scan-range", help="Custom scan range, inclusive, example: 200-300", type=partial(rangeize, conversion=int), default=None)
parser.add_argument("--frame-range", help="Custom frame range, inclusive, example: 200-300. It will further constrain the <frames> argument.", type=partial(rangeize, conversion=int), default=None)
parser.add_argument("--mz-resolution", help="Custom mz binning resolution for plot. Default: 1.0", type=float, default=1.0)
parser.add_argument("--intensity", help="Clamp all intensities below this threshold to 0 (for simple noise removal)", type=int, default=0)
parser.add_argument("--title", help="Add this title to plot", type=str, default="")
parser.add_argument("-o", "--output", help="Output file if using -s", type=Path, default=Path("plot.png"))
parser.add_argument("-t", "--transform", help="Transform the intensities before plotting", type=str, default="", choices="log10 sqrt id none None".split())

args=parser.parse_args()

from matplotlib import pyplot as plt
from opentimspy import OpenTIMS, set_num_threads, plotting
from opentimspy.misc import parse_slice
set_num_threads(1)

with OpenTIMS(args.path) as OT:
    max_intens = OT.max_intensity
    if args.scan_range is None:
        args.scan_range = Range(1, OT.max_scan+1)

    if args.frames is None:
        frames = set(OT.frames['Id'])

    else:
        frames = set()
        slicelist = list(range(OT.max_frame+1)) # Yes, inefficient and inelegant

        for frame_desc in args.frames.split(','):
            if not ":" in frame_desc:
                frames.add(int(frame_desc))
            else:
                frames.update(slicelist[parse_slice(frame_desc)])

    frames.discard(0)
    if not (args.rt_range is None):
        frames = set(frameid for frameid in frames if args.rt_range.min <= OT.frame_properties[frameid].Time <= args.rt_range.max)
    if not (args.frame_range is None):
        frames = set(frameid for frameid in frames if args.frame_range.min <= frameid <= args.frame_range.max)

    frame = OT.query(list(sorted(frames)), columns='scan mz intensity'.split())
    plotting.do_plot(
            plt,
            frame,
            axes='mz scan'.split(),
            xax_min=args.mz_range.min,
            xax_max=min(OT.max_mz+1, args.mz_range.max),
            xax_res=args.mz_resolution,
            yax_min=args.scan_range.min,
            yax_max=args.scan_range.max,
            yax_res=None,
            intens_cutoff=args.intensity,
            transform=args.transform,
            max_intens=max_intens,
            aspect='auto')
    plt.title(args.title)
    if args.save:
        plt.savefig(args.output, dpi=600, bbox_inches="tight", pad_inches=0)
    else:
        plt.show()
    plt.close()


# coding: utf-8

# In[36]:

import numpy as np
import cv2
import imageio as io
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

imgcv2=cv2.imread('galaxy.jpg',0)   # 1 as is    # 0  blackwhite  -1 alpha channel transparency .color image is three dimensional matrix
img2=cv2.imread('cat.jpg')

cv2.imshow('Cats',img2)
cv2.waitKey(5000)
cv2.destroyAllWindows()

np.shape(img2)
#img=io.imread('galaxy.jpg')
#imageio.imshow(img)
#img3=plt.imread('cat.jpg')
#plt.imshow(img3)

# In[47]:

count=0

label_map=['Anger','Disgust','Fear','Happy','Sad','Surprise','Neutral']

for line in open('fer2013_cleaned.txt'):
    count=count+1
    if(count<20):

        list_=line.replace(',',' ').split(' ')
        expr=int(list_[0])
        txt=list_[len(list_)-1]
        image_vec=np.array((list_[1:len(list_)-1]))
        image_vec_i=[int(i) for i in image_vec]
        image_vec_i=np.array(image_vec_i)
        len_N=int(np.sqrt(np.shape(image_vec_i)[0]))
        img=image_vec_i.reshape(len_N,len_N)
        img_norm=img/(np.max(img))
        plt.figure(figsize=(4,4))
        plt.imshow(img,cmap='gray')
        plt.title(label_map[expr])
        plt.show()

        plt.figure(figsize=(4,4))
        plt.imshow(img_norm,cmap='gray')
        plt.title(label_map[expr])
        plt.show()

    else:
        break

#print(expr)
#print(len(list_))
#print(txt)
#print(image_vec)
#print(len(image_vec))
#print(image_vec_i)
#print(np.shape(image_vec_i))
#print(len_N)

# In[34]:

#for line in open('first_line.txt'):
#
#    list_=line.replace(',',' ').split(' ')
#    expr=int(list_[0])
#    txt=list_[len(list_)-1]
#    image_vec=np.array((list_[1:len(list_)-1]))
#    image_vec_i=[int(i) for i in image_vec]
#    image_vec_i=np.array(image_vec_i)
#    len_N=int(np.sqrt(np.shape(image_vec_i)[0]))
#    img=image_vec_i.reshape(len_N,len_N)