__author__ = 'Li Bai'
"""data description: whether data are fetched directly from Norway Meteo
website; the forecast data are updated every 6 hours a day for the next 48
hours; the parameters are variables

1)	'air_temperature_2m',
2)	'integral_of_surface_downwelling_shortwave_flux_in_air_wrt_time',
3)	'precipitation_amount_acc',
4)	'relative_humidity_2m',
5)	'x_wind_10m',
6)	'y_wind_10m'

All the data will be processed in a way that they finally will be put into a
file of 12x31x24=8928 samples

reorganize the data in the fetched files; remove null data and fill with yesterday's data
"""

import pandas as pd
import datetime
import numpy as np
import os
import matplotlib.pyplot as plt

year=['2019','2020','2021']
month=["01","02","03","04","05","06","07","08","09","10","11","12"]
path_root="Bornholm_weather"
path_root_new='Bweather'

def path_change(path_root, year, month):
    addrs = []
    newpath = path_root + "/" + year + month
    if not os.path.exists(newpath):
        os.makedirs(newpath)
    for file in os.listdir(path_root):
            if file.endswith(year+month+".csv"):
                os.rename(path_root+"/"+file,
                          newpath+"/"+file)

                addrs.append(newpath+"/"+file)
    return addrs

def file_move(path_root, year, month):
    for ky in range(len(year)):
        for km in range(len(month)):
            path_change(path_root, year=year[ky], month=month[km])

# addrs=path_read(path_root)
# addrs_month=addrs[0]
# 50 columns: first column is the updated time and the others are for the 49
# hours including this one; 49 hours are selected because of accumulated data
# we have to convert them for single hours
#  every one should select updated hour[0,1,2,3] refers to 00 06 12 18 and
#  variables [0,1,2,3,4,5]
def file_month(addrs_month,  index_var, hour_update, names):

    var1 = pd.read_csv(addrs_month[index_var], sep=",")
    col0 = var1.columns[0]
    var1_new = var1.rename(columns={col0: 'time'})

    var1_new['time'] = [datetime.datetime.utcfromtimestamp(t) for t in var1_new[
        'time']]
    var1_new= var1_new.set_index('time')

    if names[index_var] in ['solarflux','precipitation']:

        cols_0_48=var1_new.iloc[:,0:-1].to_numpy()
        cols_1_49=var1_new.iloc[:,1:].to_numpy()

        var1_new_1=pd.DataFrame(columns=var1_new.columns[:-1],
                                index=var1_new.index)

        var1_new_1.iloc[:,0:]=cols_1_49-cols_0_48

        var1_new_x=var1_new_1.iloc[hour_update::4,:]
    else:
        var1_new_x=var1_new.iloc[hour_update::4,:-1]
    # with [:, idx:] here idx refers to the values of tomorrow.
    # update_hour=0, idx=24; update=6; idx=18:18+24 '
    # update_hour=12, idx=12:12+24; update_hour=24; idx=0:24
    # idx=24-update_hour: 24-update_hour+24
    return var1_new_x.iloc[:,(24-hour_update): (24-hour_update+24)]

def file_all(addrs, index_var, hour_update, names):
    dfs=[]
    for addrs_month in addrs:
        df=file_month(addrs_month,  index_var, hour_update, names)
        dfs.append(df)
    dfs=pd.concat(dfs)
    return dfs
# =======================================================================

# generate addrs
def addrs_gene(path_root):

    file_list=os.listdir(path_root)
    print(file_list)
    # names=['temperature','solarflux','precipitation','humidity','windx','windy']
    names=['integral_of_surface_downwelling_shortwave_flux_in_air_wrt_time',
     'air_temperature_2m', 'relative_humidity_2m',
     'precipitation_amount_acc', 'x_wind_10m',
     'y_wind_10m']
    years=['2019','2020','2021']
    months=['01', '02', '03','04', '05', '06','07', '08', '09','10', '11', '12']
    addrs=[]
    for year in years:
        for month in months:
            addrs_month=[]
            for name in names:
                file_name='Bornholm_'+name+'_'+year+month+'.csv'
                if file_name in file_list:
                    addr='Bornholm_weather/Bornholm_'+name+'_'+year+month+'.csv'
                    addrs_month.append(addr)
            if len(addrs_month)!=0:
                addrs.append(addrs_month)
    return addrs

def subtract_update_data(path_root, path_root_new, names):

    addrs = addrs_gene(path_root)
    print(addrs)
    # names=['solarflux',
    #  'temperature', 'humidity',
    #  'precipitation', 'windx',
    #  'windy']

    # names=['temperature','solarflux','precipitation','humidity','windx','windy']
    hours=['00','06','12','18']
    import os
    if not os.path.exists(path_root_new):
        os.makedirs(path_root_new)
    # every one should select updated hour[0,1,2,3] refers to 00 06 12 18 and
    #  variables [0,1,2,3,4,5] refers to names
    for hour_update in range(len(hours)):
        for index_var in range(6):
            dfs=file_all(addrs, index_var=index_var, hour_update=hour_update, names=names)
            dfs.to_csv(path_root_new+'/'+names[index_var] +'_'+hours[
                hour_update]+'.csv',
                       sep=',')
            # dfs.plot()

#
#
def path_gene(path_root_new, names):
    addrs = []
    for name in names:
        # if file.endswith("00.csv"):
        addrs.append(path_root_new + "/" + name+'_00.csv')
    return addrs
# ['./Bornholm_weather/humidity_00.csv',
#  './Bornholm_weather/precipitation_00.csv',
#  './Bornholm_weather/solarflux_00.csv',
#  './Bornholm_weather/temperature_00.csv',
#  './Bornholm_weather/windx_00.csv',
#  './Bornholm_weather/windy_00.csv']
# =================================================================================

def data_reshape_column(addrs_new, names):
    dfs=[]
    for namek in range(len(addrs_new)):
        var = pd.read_csv(addrs_new[namek], sep=",", index_col='time')
        var.index=pd.to_datetime(var.index)
        start_date=var.index[0]+datetime.timedelta(hours=24)
        end_date=var.index[-1]+datetime.timedelta(hours=47)
        dates=pd.date_range(start_date,end_date,freq='H')
        col_name=names[namek]
        df_var=pd.DataFrame(columns=['time',col_name])
        df_var['time']=dates
        df_var=df_var.set_index('time')
        print((var.iloc[:,0:].to_numpy()).shape)
        if col_name=='temperature':
            df_var[col_name]=((var.iloc[:,0:].to_numpy()).reshape(-1,1))-273.15
        else:
            df_var[col_name] = (var.iloc[:, 0:].to_numpy()).reshape(-1, 1)
        dfs.append(df_var)
    dfs=pd.concat(dfs,axis=1)
    return dfs

# ==============read data from each files in the directory of Nordhavn_weather and then put it into the Bweather as 00

# this is to read files from Nordhavn_weather and to save it to Bweather/Bornholm_weather00.csv

# followng alphabetic order
names = ['solarflux',
         'temperature', 'humidity',
         'precipitation', 'windx',
         'windy']
subtract_update_data(path_root, path_root_new, names)

addrs_new=path_gene(path_root_new, names)
dfs=data_reshape_column(addrs_new, names)
# dfs.to_csv('./Nweather/Nordhavn_weather_total.csv',
#                        sep=',')

def fill_nulls(dfs):
    dfs[dfs==-99.99]=np.nan
    dfs.loc[dfs['temperature']==-273.15-99.99,'temperature']=np.nan

    # all the data are missing for the same date; so we do it for all columns
    # together

    cols=dfs.columns
    for col in cols:
        dfs_null=dfs.loc[dfs[col].isna()==True]
        for idx in dfs_null.index:
            idx_yes=idx-datetime.timedelta(hours=24)
            dfs.loc[idx, col]=dfs.loc[idx_yes, col]
    return dfs

dfs=fill_nulls(dfs)
dfs.to_csv("Bornholm_weather00_full_2021"
           ".csv")

# # print(dfs.iloc[0,:]
# Out[8]:
# humidity           0.568547
# precipitation     -0.001953
# solarflux          0.000000
# temperature      277.342285
# windx              5.727576
# windy            -14.121002
# Name: 2019-01-01 00:00:00, dtype: float64)


import asyncio
import contextlib

from dostuff.commands import *

commands = [
    Check(exists='foo'),
    File('/foo/bar'),
    Package('p1', 'p2', 'p3'),
    Package('p2', 'p3', 'p4'),
    Package('p5', action='remove'),
    Service('s1', config=File('/etc/s1.conf')),
    Service('s2', action='disable', config=File('/etc/s2/s2.conf')),
    User('bob', homedir=True, system=True),
]

errors = []
for command in commands:
    try:
        command.validate()
    except ValidationError as e:
        errors.append((command, e))
if errors:
    for command, e in errors:
        print("error: {} in {}".format(e, command))
    raise SystemExit

stuff = asyncio.gather(*(command.do() for command in commands))

with contextlib.closing(asyncio.get_event_loop()) as loop:
    loop.run_until_complete(stuff)


from django import forms
from django.core.exceptions import ValidationError
from django.contrib.auth.models import User
from django.contrib.auth.forms import UserCreationForm
from .models import Order, Customer

class CustomerForm(forms.ModelForm):
    class Meta:
        model = Customer
        exclude = ['user']

    def __init__(self, *args, **kwargs):
        super(CustomerForm, self).__init__(*args, **kwargs)
        for fn, f in self.fields.items():
            f.widget.attrs['class'] = 'form-control form-control-sm mb-1'

class CreateUserForm(UserCreationForm):
    class Meta:
        model = User
        fields = ['username', 'email', 'first_name',
                  'last_name', 'password1', 'password2']

    def __init__(self, *args, **kwargs):
        super(CreateUserForm, self).__init__(*args, **kwargs)
        for fn, f in self.fields.items():
            f.widget.attrs['class'] = 'form-control'

    def clean_email(self):
        email = self.cleaned_data['email']
        found = User.objects.filter(email=email)
        if found:
            raise ValidationError(
                'User is already registered with this email.')
        return email

class OrderForm(forms.ModelForm):
    class Meta:
        model = Order
        exclude = ['customer']

    def __init__(self, *args, **kwargs):
        super(OrderForm, self).__init__(*args, **kwargs)
        for fn, f in self.fields.items():
            f.widget.attrs['class'] = 'form-select form-select-sm'


import tensorflow as tf

from datasets import pascalvoc_to_tfrecords

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string(
    'dataset_name', 'pascalvoc',
    'The name of the dataset to convert.')
tf.app.flags.DEFINE_string(
    'dataset_dir', None,
    'Directory where the original dataset is stored.')
tf.app.flags.DEFINE_string(
    'output_name', 'pascalvoc',
    'Basename used for TFRecords output files.')
tf.app.flags.DEFINE_string(
    'output_dir', './',
    'Output directory where to store TFRecords files.')

def main(_):
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')
    print('Dataset directory:', FLAGS.dataset_dir)
    print('Output directory:', FLAGS.output_dir)

    if FLAGS.dataset_name == 'pascalvoc':
        pascalvoc_to_tfrecords.run(FLAGS.dataset_dir, FLAGS.output_dir, FLAGS.output_name)
    else:
        raise ValueError('Dataset [%s] was not recognized.' % FLAGS.dataset_name)

if __name__ == '__main__':
    tf.app.run()


import math

from compas.geometry import Frame

from compas_fab.backends import RosClient

with RosClient() as client:
    robot = client.load_robot()
    assert robot.name == 'ur5_robot'

    frame = Frame([0.4, 0.3, 0.4], [0, 1, 0], [0, 0, 1])
    tolerance_position = 0.001
    tolerance_axes = [math.radians(1)] * 3

    start_configuration = robot.zero_configuration()
    start_configuration.joint_values = (-3.530, 3.830, -0.580, -3.330, 4.760, 0.000)
    group = robot.main_group_name

    # create goal constraints from frame
    goal_constraints = robot.constraints_from_frame(frame,
                                                    tolerance_position,
                                                    tolerance_axes,
                                                    group)

    trajectory = robot.plan_motion(goal_constraints,
                                   start_configuration,
                                   group,
                                   options=dict(
                                       planner_id='RRTConnect'
                                   ))

    print("Computed kinematic path with %d configurations." % len(trajectory.points))
    print("Executing this path at full speed would take approx. %.3f seconds." % trajectory.time_from_start)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

with open('../data/h11_data01.html') as f:
    dfs=pd.read_html(f)

#print(len(dfs)) #5となるはず。
#print(type(dfs[2]))
#print(dfs[2]) #使うのは表3なのでそちらを確認する。
#print(dfs[2].head())
#print(dfs[2][3])
#print(dfs[2][0])

#表をnumpyに変換する。
x=np.array([dfs[2][0].to_numpy(),dfs[2][3].to_numpy()])
#print(x)

#不要な行を削除する。
rx=np.delete(x,list(range(5)),1)
#print(rx)

#文字になっているので数値に変換する。
rx=rx.astype(np.float32)
#print(rx)

#回帰分析
a,b=np.polyfit(rx[0],rx[1],1)

#a,b,c=np.polyfit(rx[0],rx[1],2) #二次関数でもできる。
#y=a*rx[0]**2+b*rx[0]+c

#グラフとして描画する。
plt.plot(rx[0],rx[1],'.', markersize=10,label="data1")
y=a*rx[0]+b
plt.plot(rx[0],y,label="data1(fitting)")
plt.xlim(1955,2025)
plt.ylim(-20,20)

#別のデータも比較する。
with open('../data/h11_data04.html') as f:
    dfs2=pd.read_html(f)

#２つ目の表の6列目が必要。
x2=np.array([dfs2[1][0].to_numpy(),dfs2[1][5].to_numpy()])

#不要な行を取り除く。
#先頭から3行目までは不要
x2=np.delete(x2,list(range(3)),1)
#末尾の15行は不要。
x2=np.delete(x2,list(range(-15,0)),1)
#文字列を数値に変換する。
x2=x2.astype(np.float32)
print(x2)

#合わせてプロットする。
plt.plot(x2[0],x2[1],'+',markersize=10,label="data2")

#同じく回帰分析
a,b=np.polyfit(x2[0],x2[1],1)
y=a*x2[0]+b
plt.plot(x2[0],y,label="data2(fitting)")

#凡例をつける。
plt.legend()

plt.show()


import json
import os
import sys
import traceback

import discord
import discord.utils
from discord.ext import commands
from discord import app_commands
from bot.kagami import Kagami

def is_developer():
    def predicate(interaction: discord.Interaction) -> bool:
        return interaction.user.id == json.load(open("bot/data/config.json"))["developer"]
    return app_commands.check(predicate)

class Admin(commands.Cog):
    def __init__(self, bot):
        self.bot: Kagami = bot
        self.config = bot.config

    @commands.command(name="sync", description="syncs the command tree")
    @commands.is_owner()
    async def sync_command_tree(self, ctx):
        await self.bot.tree.sync()
        await ctx.send("Command Tree Synced", ephemeral=True)

    @commands.command(name="ping", description="checks the latency")
    @commands.is_owner()
    async def ping(self, ctx):
        latency = int(self.bot.latency * 1000)
        await ctx.send(f"Pong! {latency}ms")

    @commands.command(name="reload_all", description="reloads all cogs")
    @commands.is_owner()
    async def reload_all_cogs(self, ctx):
        for file in os.listdir("bot/cogs"):
            if file.endswith(".py"):
                name = file[:-3]
                await self.bot.reload_extension(f"cogs.{name}")
        await ctx.send("Reloaded all cogs")

    @commands.command(name="reload", description="reloads a  cog")
    @commands.is_owner()
    async def reload_cog(self, ctx, cog_name):
        for file in os.listdir("bot/cogs"):
            if file.endswith(".py"):
                name = file[:-3]
                if name.lower() == cog_name.lower():
                    await self.bot.reload_extension(f"cogs.{name}")
                    break
        else:
            await ctx.send(f"No cog with that name could be found")
            return
        await ctx.send(f"Reloaded cog: '{cog_name}'")

    @commands.command(name="close", description="closes the bot")
    @commands.is_owner()
    async def close_bot(self, ctx):
        await ctx.send("Shutting Down")
        await self.bot.close()

    @commands.command(name="save", description="saves data")
    @commands.is_owner()
    async def save_data(self, ctx):
        self.bot.save_data()
        await ctx.send("Saved the data to file")

    @commands.command(name="load", description="loads data")
    @commands.is_owner()
    async def load_data(self, ctx):
        self.bot.load_data()
        await ctx.send("Loaded data from file")

    @commands.Cog.listener()
    async def on_command_error(self, ctx, error):
        if isinstance(error, commands.NotOwner):
            await ctx.send("Only developers can use that command")
        else:
            await ctx.send(error)
            traceback.print_exception(error, error, error.__traceback__, file=sys.stderr)

async def setup(bot):
    await bot.add_cog(Admin(bot))


from flask import Flask, render_template
from flask_bootstrap import Bootstrap
from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField, SelectField
from wtforms.validators import DataRequired
from Notify import LineNotify

app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret_key'
bootstrap=Bootstrap(app)
notify = LineNotify()

notification_ls = list()
warring_ls = list()

class MsgForm(FlaskForm):
   name = StringField('輸入訊息內容', validators=[DataRequired()])
   select = SelectField('選擇訊息種類', choices=[('1', '通知'), ('2', '警告')])
   submit = SubmitField('送出')

class KeyForm(FlaskForm):
   name = StringField('輸入權杖碼', validators=[DataRequired()])
   select = SelectField('選擇註冊訊息種類', choices=[('1', '通知'), ('2', '警告')])
   submit = SubmitField('註冊')

@app.route('/', methods=['GET', 'POST'])
def index():

   msg_form = MsgForm()
   key_form = KeyForm()

   if msg_form.validate_on_submit():
      msg = msg_form.name.data
      mode = msg_form.select.data

      if mode == '1':
         for ele in notification_ls:
            notify.send_msg(msg, ele)

      else:
         for ele in warring_ls:
            notify.send_msg(msg, ele)

      msg_form.name.data = ''

   if key_form.validate_on_submit():
      key = key_form.name.data
      mode = key_form.select.data

      if mode == '1':
         notification_ls.append(key)
      else:
         warring_ls.append(key)

      key_form.name.data = ''

   return render_template('index.html', msg_form=msg_form, key_form=key_form)

if __name__ == '__main__':
   app.run(host='127.0.0.1', port=8080, debug=True)


import logging

def batcher(generator, batch_func, batch_size):
    buf = []
    for item in generator:
        buf.append(item)
        if len(buf) >= batch_size:
            output = batch_func(buf)
            for item in buf:
                yield (item, output)
            buf = []
    if buf:
        output = None
        try:
            output = batch_func(buf)
        except:
            logging.exception("batch function failed!")

        if output:
            for item in buf:
                yield (item, output)


import base64
import json
import traceback
from flask import request
from flask import current_app as app
from flask_restful import Resource
from pydub import AudioSegment
from modelos.modelos import User, db, Task
import requests
from datetime import datetime
import os
from google.cloud import storage

storage_client = storage.Client()

def download_from_bucket(blob_name, file_path_destiny):
    try:
        bucket = storage_client.get_bucket(app.config['GCP_BUCKET_NAME'])
        blob = bucket.blob(blob_name)
        with open(file_path_destiny, 'wb') as f:
            storage_client.download_blob_to_file(blob, f)
        return True
    except Exception as e:
        print(e)
        return False

def convert(request):
    print('storage preparing')
    storage_client = storage.Client()
    audio_bucket = storage_client.get_bucket(app.config['GCP_BUCKET_NAME'])
    print('bucket name')
    print(audio_bucket.name)

    print("coverter started")

    now = datetime.now()
    url = "https://api.mailgun.net/v3/sandboxddf98cda1dd84031bc13cda246e42344.mailgun.org/messages"
    auth = ("api", app.config['EMAIL_API_KEY'])

    taskTmp = Task.query.filter(Task.id == int(request.get("id"))).first()
    taskTmp.conversionTimeStamp = now.strftime("%Y-%m-%d %H:%M:%S")
    db.session.commit()
    print("commit 1 done")
    taskTmp.secondsTakedToStartConversion = (taskTmp.conversionTimeStamp  - taskTmp.timeStamp).total_seconds()
    db.session.commit()
    print("commit 2 done")

    userTmp = User.query.filter(User.id == taskTmp.id_usuario).first()

    local_filepath = "/tmp/"+request.get("filepath")
    bucket_filepath = request.get("filepath")
    download_from_bucket(bucket_filepath, local_filepath)
    nFormat = request.get("newFormat")

    locationNoFormat = bucket_filepath.split(".")[0]
    format = bucket_filepath.split(".")[1]

    jsons = {"from": "mbkane04@gmail.com",
                "to": [userTmp.email],
                "subject": "Your file Conversion is done!",
                "text": "The File: "+str(bucket_filepath)+" has been converted"}

    print("before convert")
    postR = False
    try:
        destinyPath = '/tmp/'+ locationNoFormat+"."+nFormat
        if format == "mp3":
            song = AudioSegment.from_mp3(local_filepath)
            song.export(destinyPath, format=nFormat)
            bucket = storage_client.get_bucket(app.config['GCP_BUCKET_NAME'])
            blob = bucket.blob(destinyPath[5:])
            blob.upload_from_filename(destinyPath)
            if postR:
                x = requests.post(url = url,auth = auth ,data = jsons)
                print(x)
            taskTmp.status = "processed"
            db.session.commit()
            print("converted to " + nFormat)
            os.remove(local_filepath)
            os.remove(destinyPath)
            return {"code": 200, "mensaje": "Se Realizo la conversion exitosamente"}
        elif format == "ogg":
            song = AudioSegment.from_ogg(local_filepath)
            song.export(destinyPath, format=nFormat)
            bucket = storage_client.get_bucket(app.config['GCP_BUCKET_NAME'])
            blob = bucket.blob(destinyPath[5:])
            blob.upload_from_filename(destinyPath)
            if postR:
                x = requests.post(url = url,auth = auth ,data = jsons)
                print(x)
            taskTmp.status = "processed"
            db.session.commit()
            print("converted to " + nFormat)
            os.remove(local_filepath)
            os.remove(destinyPath)
            return {"code": 200, "mensaje": "Se Realizo la conversion exitosamente"}
        elif format == "wav":
            song = AudioSegment.from_wav(local_filepath)
            song.export(destinyPath, format=nFormat)
            bucket = storage_client.get_bucket(app.config['GCP_BUCKET_NAME'])
            blob = bucket.blob(destinyPath[5:])
            blob.upload_from_filename(destinyPath)
            if postR:
                x = requests.post(url = url,auth = auth ,data = jsons)
                print(x)
            taskTmp.status = "processed"
            db.session.commit()
            print("converted to " + nFormat)
            os.remove(local_filepath)
            os.remove(destinyPath)
            return {"code": 200, "mensaje": "Se Realizo la conversion exitosamente"}
        else:
            print("incorrect format return")
            os.remove(local_filepath)
            return {"code":400, "resultado": "ERROR", "mensaje": "El formato no se reconoce"}
    except Exception:
        print("error in conversion !!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
        print(traceback.print_exc())
        taskTmp.status = "error"
        db.session.commit()
        isExist= os.path.exists(local_filepath)
        if(isExist):
            os.remove(local_filepath)
        return {"code":400, "resultado": "ERROR", "mensaje": traceback.print_exc()}

class MessageListener(Resource):
    def post(self):
        if (request.args.get('token', '') != app.config['PUBSUB_VERIFICATION_TOKEN']):
            return 'Invalid request', 400

        envelope = json.loads(request.data.decode('utf-8'))
        payload = base64.b64decode(envelope['message']['data']).decode('utf-8')
        message = payload.replace("'","\"")
        print("message: "+message)
        objectToConvert = json.loads(message)
        print("objectToConvert: "+str(objectToConvert))

        try:
            respuesta = convert(objectToConvert)
            return {"mensaje": str(respuesta)}, 200
        except Exception:
            print("Exception: "+str(traceback.print_exc()))
            return {"error": str(traceback.print_exc())}, 528


import asyncio
import json
import re
from datetime import datetime

import aiohttp
import motor.motor_asyncio
import yarl
from bs4 import BeautifulSoup
from pymongo.errors import DuplicateKeyError

match_article = re.compile(r"/\w+/\w+/[0-9]{8}/.+").match

rules = [
    lambda url: re.compile(r"^/.*").match(url),
    lambda url: not re.compile(r"\.(jpg|gif|png)$").match(url),
    lambda url: not re.compile(r".*/(files|video|taxonomy|img)/.*").match(url),
]

color = "\x1B[{}".format
blue = color("94m")
red = color("31m")
green = color("32m")

yellow = color("93m")
reset = color("0m")

async def put(fg=blue, **kwargs):
    print(fg, json.dumps(kwargs), reset, flush=True)

async def url_match(url):
    return bool(all((rule(url) for rule in rules)))

def get_collection():
    client = motor.motor_asyncio.AsyncIOMotorClient(
        "mongodb://mongodb:27017", ssl=False
    )
    return client.lostiempos.responses

async def find_response(session, url, follow=False):
    responses = get_collection()
    record = await responses.find_one({"url": url})

    if record:
        return record["content"]

    content = await fetch(session, url)

    if content is None:
        return

    try:
        await responses.insert_one(
            {
                "url": url,
                "content": content,
                "created_at": datetime.now(),
                "follow": follow,
                "is_article": match_article(url),
            }
        )
    except DuplicateKeyError:
        return

    return content

async def response_exists(url) -> bool:
    responses = get_collection()
    result = await responses.find_one({"url": url})
    return not (result is None)

async def fetch(session, url):
    async with session.get(url) as response:
        if response.status == 200:
            return await response.read()

founded_links = set()

async def find_links(html, fg):
    soup = BeautifulSoup(html, "html5lib")

    for link in soup.select("a"):
        if await url_match(link.attrs["href"]):
            if link.attrs["href"] in founded_links:
                continue

            url = yarl.URL(f'https://www.lostiempos.com{link.attrs["href"]}')

            if url.fragment or url.query:
                continue

            if await response_exists(str(url)):
                continue

            await put(fg, url=url.path, new=True)

            founded_links.add(url.path)

            yield url

crawled = set()

async def crawl(url, follow=False, depth=1, fg=blue):
    if url in crawled:
        return

    if not follow:
        return

    await put(fg, crawl=url, follow=follow, depth=depth)

    async with aiohttp.ClientSession() as session:
        html = await find_response(session, url, follow=follow)

        if html is None:
            return

        crawled.add(url)

        async for link in find_links(html, fg):
            await crawl(str(link), follow=depth <= 3, depth=depth + 1, fg=fg)

async def main():
    await asyncio.wait([
            crawl("https://www.lostiempos.com", follow=True, fg=blue),
            crawl("https://www.lostiempos.com/actualidad", follow=True, fg=red),
            crawl("https://www.lostiempos.com/ultimas-noticias", follow=True, fg=green),

    ])

if __name__ == "__main__":
    asyncio.run(main())


import setuptools
import io

with open("README.md", "r") as fh:
    long_description = fh.read()

dependencies = []
with io.open('requirements.txt') as f:
    dependencies = [l.strip() for l in f if not l.startswith('#')]

setuptools.setup(
    name="vscext",
    version="0.2.2",
    author="Irakli Nadareishvili",
    #author_email="author@example.com",
    description="Gives installed VS Code extensions as a linked markdown list",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/inadarei/vscode-extensions-list",
    packages=setuptools.find_packages(),
    python_requires='>=3.6.*',
    install_requires=dependencies,
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Development Status :: 6 - Mature',
        'Intended Audience :: Developers',
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.signIn, name='signIn'),
    path('signUp', views.signUp, name='signUp'),
    path('logOut', views.signOut, name='signOut'),
    path('home', views.home, name='home'),
    path('viewImage/<str:pk>/', views.viewImage, name='viewImage'),
    path('profile', views.profile, name='profile'),
    path('upload', views.upload, name='upload'),
]


import configparser
from datetime import datetime
import os
from pyspark.sql.types import StructType as R, StructField as Fld, DoubleType as Dbl, StringType as Str, IntegerType as Int, DateType as Date, LongType as Lng, TimestampType as TST
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf, col,  to_timestamp, monotonically_increasing_id
from pyspark.sql.functions import year, month, dayofmonth, hour, weekofyear, date_format

config = configparser.ConfigParser()
config.read('dl.cfg')

os.environ['AWS_ACCESS_KEY_ID']=config.get('AWS_ID', 'AWS_ACCESS_KEY_ID')
os.environ['AWS_SECRET_ACCESS_KEY']=config.get('AWS_SEC', 'AWS_SECRET_ACCESS_KEY')

def create_spark_session():
    """Creates a spark session
       with spark.jars and apache.hadoop packages

    Parameters:No arguments

    Returns: spark session named "spark"

    """
    spark = SparkSession \
        .builder \
        .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:2.7.0") \
        .getOrCreate()
    return spark

def process_song_data(spark, input_data, output_data):
    """import Song dataset extract columns and create songs and artist tables
    write those tables to parquet files

    Parameters:
    spark: name of spark session
    input_data: location of the source data s3 bucket
    output_data: location of the destination data s3 bucket

    Returns:
    writes songs table in parquet to output_data location + songs
    writes artist_table in parquet to output_dat location + artists

    """

    # Setting up the JSON table structure for the Song dataset
    song_dataset_schema= R([
        Fld("artist_id", Str()),
        Fld("artist_latitude", Dbl()),
        Fld("artist_longitude", Dbl()),
        Fld("artist_location", Str()),
        Fld("artist_name", Str()),
        Fld("song_id", Str()),
        Fld("title", Str()),
        Fld("duration", Dbl()),
        Fld("year", Str()),
        ])

    """get filepath to song data file
    use "song_data/*/*/*/*.json" for full dataset
    use "song_data/A/B/C/TRABCEI128F424C983.json" to pull a single record

    """
    song_data = input_data + "song_data/*/*/*/*.json"

    # read song data file with dataset_schema
    df = spark.read.json(song_data, schema=song_dataset_schema)

    # extract columns to create songs table
    songs_table = df.select('song_id', 'artist_id', 'year', 'duration')

    # drop duplicate rows in songs table
    songs_table = songs_table.dropDuplicates()

    # write songs table to parquet files partitioned by year and artist
    songs_table.write.mode('append').partitionBy('year', 'artist_id').parquet(output_data + "songs")

    # extract columns to create artists table
    artists_table = df.select('artist_id', 'artist_name', 'artist_location', 'artist_latitude', 'artist_longitude')

    # drop duplicate rows in artists table
    artists_table = artists_table.dropDuplicates()

    # write artists table to parquet files
    artists_table.write.mode('append').parquet(output_data + "artists")

def process_log_data(spark, input_data, output_data):
    """import log dataset extract columns create users, time, and songplays table

    Parameters:
    spark: name of spark session
    input_data: location of the source data s3 bucket
    output_data: location of the destination data s3 bucket

    Returns:
    writes users table in parquet to output_data location + users
    writes time_table in parquet to output_data location + time
    writes songplays table in parquest to output_data location + songplays

    """

    """Setting up the JSON table structure for the log dataset"""
    log_dataset_schema= R([
       Fld("artist", Str()),
       Fld("auth", Str()),
       Fld("firstName", Str()),
       Fld("gender", Str()),
       Fld("iteminSession", Lng()),
       Fld("lastName", Str()),
       Fld("length", Dbl()),
       Fld("level", Str()),
       Fld("location", Str()),
       Fld("method", Str()),
       Fld("page", Str()),
       Fld("registration", Dbl()),
       Fld("sessionId", Lng()),
       Fld("song", Str()),
       Fld("status", Lng()),
       Fld("ts", Lng()),
       Fld("userAgent", Str()),
       Fld("userId", Str()),
   ])

    # get filepath to log data file
    log_data = input_data + "log_data/*/*/*.json"

    # read log data file
    df = spark.read.json(log_data, schema = log_dataset_schema)

    # filter by actions for song plays
    df = df.filter(col("page") == 'NextSong')

    # extract columns for users table
    users_table = df.select(col("userId").alias("user_id"), col("firstName").alias("first_name"),
                            col("lastName").alias("last_name"), "gender", "level")

    # drop duplicate rows in users table
    users_table = users_table.dropDuplicates()

    # write users table to parquet files
    users_table.write.mode('append').parquet(output_data+"users")

    # create timestamp column from original timestamp column
    dfTimestamp = df.withColumn("start_time", to_timestamp(col("ts") /1000 ))
    dfTimestamp = dfTimestamp \
    .withColumn("hour", hour("start_time")) \
    .withColumn("day", dayofmonth("start_time")) \
    .withColumn("week", weekofyear("start_time")) \
    .withColumn("month", month("start_time")) \
    .withColumn("year", year("start_time")) \
    .withColumn("weekday", date_format("start_time", 'E'))

    # extract columns to create time table
    time_table = dfTimestamp.select(col("start_time"), \
                                    col("hour"), \
                                    col("day"), \
                                    col("week"), \
                                    col("month"), \
                                    col("year"), \
                                    col("weekday"))

    # drop duplicate rows in time table
    time_table = time_table.dropDuplicates()

    # write time table to parquet files partitioned by year and month
    time_table.write.mode('append').partitionBy("year","month").parquet(output_data+"time")

    # read in song data to use for songplays table
    song_data = input_data + "song_data/*/*/*/*.json"
    song_df = spark.read.json(song_data)

    # extract columns from joined song and log datasets to create songplays table
    songplays_table = song_df.join(df, song_df.artist_name==df.artist). \
        withColumn("songplay_id", monotonically_increasing_id()). \
        withColumn('start_time', to_timestamp(col("ts") /1000 )). \
        select("songplay_id",
        "start_time",
        col("userId").alias("user_id"),
        "level",
        "song_id",
        "artist_id",
        col("sessionId").alias("session_id"),
        col("artist_location").alias("location"),
        "userAgent",
        month(col("start_time")).alias("month"),
        year(col("start_time")).alias("year"))

    # drop duplicate rows in songplays table
    songplays_table = songplays_table.dropDuplicates()

    # write songplays table to parquet files partitioned by year and month
    songplays_table.write.mode('append').partitionBy('year', 'month').parquet(output_data + "songplays")

def main():
    """main function executes create_spark_session, process_song_data function, and
    process_log_data function with input_data and output_data defined.

    Parameters: No arguments

    Returns: No Returns

    """
    spark = create_spark_session()
    # input_data defines the location of the source s3 bucket
    input_data = "s3a://udacity-dend/"
    # output_data defines the location of the destination s3 bucket

    output_data = "s3a://udacity-dend"

    process_song_data(spark, input_data, output_data)
    process_log_data(spark, input_data, output_data)

if __name__ == "__main__":
    main()


import streamlit as st
import pandas as pd
import folium
from matplotlib import image
from geopy.geocoders import Nominatim
from sklearn.neighbors import DistanceMetric
from streamlit_folium import folium_static
from PIL import Image

# Create a multiselect widget for the user to choose a page
page_options = ['Home Page', 'Pub Locations', 'Find the nearest Pub']
selected_page = st.sidebar.selectbox('Select a page', page_options)

pubs_df = pd.read_csv('pub_df.csv')

# Use if else statements to display the selected page

###################PAGE-1#####################

if selected_page == 'Home Page':
    st.title(':beers: :red[Welcome to the Pub Finder Web Application]:beers:')
    st.write('This application displays information and statistics about pubs in the UK.')
    # Add an image
    image = Image.open('pub2.jpg')
    st.image(image)
    st.write(f'Total number of pubs: {len(pubs_df)}')
    st.write(f'Number of cities: {pubs_df["city"].nunique()}')
    st.write(f'Number of postal codes: {pubs_df["postcode"].nunique()}')
    city_count = pubs_df['city'].value_counts()
    highest_city = city_count.nlargest(1)
    st.write(f'City with highest number of pubs: {highest_city.index[0]} ({highest_city.values[0]} pubs)')

    st.subheader("About me :point_down:")
    st.write("Nandini Shukla :wave:")
    st.write("Data Science intern @Innomatics research lab :computer:")

    st.subheader("Contect me :coffee:")
    st.markdown("Gmail  - nandini0212shukla@gmail.com")
    st.markdown("Github  - https://github.com/OkayMini")
    st.markdown("Linkedin  - https://www.linkedin.com/in/nandinishuklads/")

###################PAGE-2#####################

elif selected_page == 'Pub Locations':
    # Set up the app layout
    st.title(":beer: :blue[Pub Locations] :beer:")

    # Get user inputs for postal code and local authority
    location_pc = st.text_input('Enter your Postal Code:', 'SW1A 2AA')
    location_la = st.text_input('Enter your Local Authority:', 'Westminster')

    # Geocode the user's location
    geolocator = Nominatim(user_agent='my_app')
    location_data = geolocator.geocode(f'{location_pc} {location_la}', exactly_one=False)[0]
    lat, lon = location_data.latitude, location_data.longitude

    # Create a folium map object centered on the user's location
    m = folium.Map(location=[lat, lon], zoom_start=13)

    # Filter the pubs based on the chosen area
    filtered_pubs = pubs_df[(pubs_df['postcode'] == location_data.raw.get('postcode', '')) |
                            (pubs_df['city'] == location_data.raw.get('city', '')) |
                            (pubs_df['local_authority'] == location_la)]

    # Add a marker for each pub in the filtered dataset
    for i, pub in filtered_pubs.iterrows():
        folium.Marker(location=[pub['latitude'], pub['longitude']],
                    popup=pub['name']).add_to(m)

    # Display the map using Streamlit
    st.markdown('### Pub Locations')
    st.write(f'Number of pubs in {location_pc} {location_la}: {len(filtered_pubs)}')
    folium_static(m)

    # Display the list of pubs in the filtered dataset
    st.markdown('### List of Pubs')
    for i, pub in filtered_pubs.iterrows():
        st.write(pub['name'])

###################PAGE-3#####################

elif selected_page == 'Find the nearest Pub':
    # Ask the user to enter their latitude and longitude
    st.title('Find the Nearest Pubs')
    lat = st.number_input('Enter your latitude:')
    lon = st.number_input('Enter your longitude:')

    # Find the nearest 5 pubs using Haversine distance
    dist = DistanceMetric.get_metric('haversine')
    distances = dist.pairwise(pubs_df[['latitude', 'longitude']], [[lat, lon]])
    pubs_df['Distance'] = distances[:, 0]
    nearest_pubs = pubs_df.nsmallest(5, 'Distance')

        # Display a map with the nearest pubs
    st.title('Nearest Pubs')
    m = folium.Map(location=[lat, lon], zoom_start=13)
    for i, pub in nearest_pubs.iterrows():
        icon = folium.Icon(icon='beer', prefix='fa')
        folium.Marker(location=[pub['latitude'], pub['longitude']],
                    popup=pub['name'], icon=icon).add_to(m)
    folium_static(m)

    # Display a table with the nearest pubs
    st.title('Nearest Pubs List')
    st.write(nearest_pubs[['name', 'address', 'Distance']])


""" Script for removing NotificationSubscriptions from registrations.
    Registrations shouldn't have them!
"""
import logging
import sys

import django
django.setup()

from website.app import init_app
from django.apps import apps

logger = logging.getLogger(__name__)

def remove_notification_subscriptions_from_registrations(dry_run=True):
    Registration = apps.get_model('osf.Registration')
    NotificationSubscription = apps.get_model('osf.NotificationSubscription')

    notifications_to_delete = NotificationSubscription.objects.filter(node__type='osf.registration')
    registrations_affected = Registration.objects.filter(
        id__in=notifications_to_delete.values_list(
            'node_id', flat=True
        )
    )
    logger.info('{} NotificationSubscriptions will be deleted.'.format(notifications_to_delete.count()))
    logger.info('{} Registrations will be affected: {}'.format(
        registrations_affected.count(),
        list(registrations_affected.values_list('guids___id', flat=True)))
    )

    if not dry_run:
        notifications_to_delete.delete()
        logger.info('Registration Notification Subscriptions removed.')

if __name__ == '__main__':
    dry_run = '--dry' in sys.argv
    init_app(routes=False)
    remove_notification_subscriptions_from_registrations(dry_run=dry_run)


"""
File: focus_mode_timer_test_issue#36_issue#30.py
Author: Dhruv Duggal
Description: This file tests the functionality for the timer on popup.html
Tests Contained: This file contains test #17, #18, #19, #20, #21, #22, #23, #24, and #25
"""

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from time import sleep
import json
import os

all_tests_check = 0
CRED = '\033[91m'
CGREEN = '\33[32m'
CYELLOW = '\33[33m'
CEND = '\033[0m'

print(os.getcwd())
options = Options()
print("loading packed extension")
options.add_argument("load-extension=./project/build/")
# overcome limited resource problems
options.add_argument("--disable-dev-shm-usage")
# Bypass OS security model
options.add_argument("--no-sandbox")
# starting window maximized to prevent any scrolling issues
options.add_argument("--start-maximized")
print("set up driver")
# creating chrome driver
driver = webdriver.Chrome(options=options)

# assigning the extensions driver
uid = "flfgpjanhbdjakbkafipakpfjcmochnp"

# opening multiple websites on different tabs
driver.get("https://cse.ucsd.edu")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[1])
driver.get("https://facebook.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[2])
driver.get("https://twitter.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[3])
driver.get("https://stackoverflow.com")
driver.execute_script("window.open('');")
driver.switch_to.window(driver.window_handles[4])
driver.get("https://piazza.com")
driver.switch_to.window(driver.window_handles[4])
# opening menu page of chrome extension
driver.get("chrome-extension://"+ uid +"/menu.html")

# creating a new Tab Group called CSE12
add_group_button = driver.find_element_by_class_name("addGroup")
add_group_button.click()
master_elem = driver.find_element_by_id("selectedTabs")
elements = master_elem.find_elements_by_tag_name("option")
old_count = len(driver.find_elements_by_class_name("card"))
for i in elements:
    ActionChains(driver) \
        .key_down(Keys.SHIFT) \
        .click(i) \
        .key_up(Keys.SHIFT) \
        .perform()
model_body = driver.find_element_by_class_name("modal-body")
input_name = model_body.find_element_by_id("groupName")
input_name.send_keys("CSE112")
ActionChains(driver).send_keys(Keys.ENTER).perform()
new_count = len(driver.find_elements_by_class_name("card"))

'''
Test Number #17

Description: This test checks that the tab group is properly displaying on the popup page
of the extension
'''
print(CYELLOW + "Running Test 17" + CEND)
# going to popup page
driver.get("chrome-extension://" + uid + "/popup.html")

# searching for tabgroup named CSE 112
menuContainer = driver.find_element_by_class_name("popupContainer")
cards = driver.find_elements_by_class_name("card")
target_group = None
found_on_popup = 0
for i in cards:
    if i.find_element_by_class_name("card-header").text == "CSE112":
        target_group = i
        found_on_popup = 1
try:
    assert found_on_popup == 1
except AssertionError:
    print(CRED + "Test 17 Failed" + CEND)
    print(CRED + "Tab group is not rendering on popup page" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 17 PASSED" + CEND)
print("-----------------------")

'''
Test Number #18

Description: This test checks that clicking the play button renders the correct elements
the popup page of the extension

'''
print(CYELLOW + "Running Test 18" + CEND)
# finding and clicking on the play_button for the correct tab group
play_button_group = target_group.find_element_by_class_name("buttonGroup")
play_button = play_button_group.find_element_by_tag_name("button")
play_button.click()
# Checking if all the text elements are rendered
first_check_el = driver.find_element_by_class_name("popupFocusMode")
title_timer = driver.find_element_by_class_name("popupFocusModeTitle")
name_timer = driver.find_element_by_class_name("popupFocusModeTabGroupName")
first_check = 0
second_check = 0
third_check = 0
final_check = 0
if title_timer.text == "Focus Mode":
    first_check = 1
if name_timer.text == "CSE112":
    second_check = 1
try:
    assert second_check == 1 and first_check == 1
except AssertionError:
    print(CRED + "Test 18 Failed" + CEND)
    if first_check == 1:
        print(CRED + "Focus mode title did not render properly" + CEND)
    if second_check == 1:
        print(CRED + "Tab groups title did not render properly" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 18 Passed" + CEND)
print("-----------------------")

'''
Test Number #19

Description: This button checks that the start button is being rendered properly

'''
print(CYELLOW + "Running Test 19" + CEND)
# finding button and checking if it is a start button
pop_up_div = driver.find_element_by_class_name("popupFocusMode")
button_timer_start = pop_up_div.find_element_by_class_name("popupFocusModeButton")
original_display = pop_up_div.text
try:
    assert button_timer_start.text == "Start Focus"
except AssertionError:
    print(CRED + "Test 19 Failed" + CEND)
    print(CRED + "Start button not rendered properly" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 19 Passed" + CEND)
print("-----------------------")

'''
Test Number #20

Description: Checking if button changes to End when it is initially clicked

'''
print(CYELLOW + "Running Test 20" + CEND)
# clicking the start button found in previous test
button_timer_start.click()
# opening popup.html again after focus mode has begun
driver.switch_to.window(driver.window_handles[4])
driver.get("chrome-extension://"+ uid +"/popup.html")
# searching for the end button
pop_up_div = driver.find_element_by_class_name("popupFocusMode")
pop_up_div = driver.find_element_by_class_name("popupFocusMode")
end_focus = pop_up_div.find_element_by_class_name("popupFocusModeButton")
try:
    assert end_focus.text == "End Focus"
except AssertionError:
    print(CRED + "Test 20 Failed" + CEND)
    print(CRED + "Button did not change to 'End' upon starting focus mode" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 20 Passed" + CEND)
print("-----------------------")

'''
Test Number #21

Description: Checking if end button takes us back to the original view of popup.html

'''
print(CYELLOW + "Running Test 21" + CEND)
# click end focus button found in previous test
end_focus.click()
# Checking if tabs are still present after clicking end focus
menuContainer = driver.find_element_by_class_name("popupContainer")
cards = driver.find_elements_by_class_name("card")
final_check = 0
for i in cards:
    if i.find_element_by_class_name("card-header").text == "CSE112":
        final_check = 1
try:
    assert final_check == 1
except AssertionError:
    print(CRED + "Test 21 Failed" + CEND)
    print(CRED + "Enc focus does not make popup.html go to its original state"+CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 21 Passed" + CEND)
print("-----------------------")

'''
Test Number #22

Description: This test checks whether the slider to change the time of focus mode is working

'''
print(CYELLOW + "Running Test 22" + CEND)
# go to popup.html
driver.get("chrome-extension://" + uid + "/popup.html")
# search for tab group cse112
cards = driver.find_elements_by_class_name("card")
for i in cards:
    if i.find_element_by_class_name("card-header").text == "CSE112":
        target_group = i
# click play button for this group
play_button_group = target_group.find_element_by_class_name("buttonGroup")
play_button = play_button_group.find_element_by_tag_name("button")
play_button.click()
# move slider and check if time has changed
slider_thumb = driver.find_element_by_class_name("sliderThumb")
ActionChains(driver).click_and_hold(slider_thumb).move_by_offset(10,0).release().perform()
try:
    assert "1:10:00" in driver.find_element_by_class_name("popupFocusModeTimer").text
except AssertionError:
    print(CRED + "Test 22 Failed" + CEND)
    print(CRED + "Timer not changing properly with slider" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 22 Passed" + CEND)
print("-----------------------")

'''
Test Number #23

Description: This test checks the max time possible for the timer

'''
print(CYELLOW + "Running Test 23")
# move the slider to the max
slider_thumb = driver.find_element_by_class_name("sliderThumb")
ActionChains(driver).click_and_hold(slider_thumb).move_by_offset(200,0).release().perform()
try:
    assert "3:00:00" in driver.find_element_by_class_name("popupFocusModeTimer").text
except AssertionError:
    print(CRED + "Test 23 Failed" + CEND)
    print(CRED + "Max time not correct" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 23 Passed" + CEND)
print("-----------------------")

'''
Test Number #24

Description: This test checks the min time possible for the timer

'''
print(CYELLOW + "Running Test 24" + CEND)
# move the slider all the way to the bottom
slider_thumb = driver.find_element_by_class_name("sliderThumb")
ActionChains(driver).click_and_hold(slider_thumb).move_by_offset(-200,0).release().perform()
try:
    assert "0:05:00" in driver.find_element_by_class_name("popupFocusModeTimer").text
except AssertionError:
    print(CRED + "Test 24 Failed" + CEND)
    print(CRED +"Min time not correct" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 24 Passed" + CEND)
print("-----------------------")

'''
Test Number #25

Description: Checking persistence of timer and whether it is counting down properly or not

'''
print(CYELLOW + "Running Test 25" + CEND)
# clicking start button
start_button = driver.find_element_by_class_name("popupFocusModeButton")
start_button.click()
# re-opening the popup extension (this takes 1-10 seconds)
driver.switch_to.window(driver.window_handles[4])
driver.get("chrome-extension://" + uid + "/popup.html")
sleep(5)
# checking if time elapsed on the timer is between 6-16 seconds
final_time = driver.find_element_by_class_name("popupFocusModeTimer").text
try:
    assert "0:04:50" in final_time or "0:04:51" in final_time or "0:04:49" in final_time or "0:04:48" in final_time or \
           "0:04:46" in final_time or "0:04:47" in final_time or "0:04:52" in final_time or "0:04:53" in final_time
except AssertionError:
    print(CRED + "Test 25 Failed" + CEND)
    print(CRED + "Timer counting down properly" + CEND)
    all_tests_check = 1
else:
    print(CGREEN + "Test 25 Passed" + CEND)
print("-----------------------")
assert all_tests_check == 0, CRED + "Some Tests Failed" + CEND
print(CGREEN + "All Tests Passed" + CEND)
# writing coverage report
coverage_json_file = open("./project/.nyc_output/#30_#36.json","w+")
json.dump(driver.execute_script("return window.__coverage__;"), coverage_json_file)
coverage_json_file.close()
driver.quit()


from PyQt5.QtGui import QFont, QRegExpValidator
from PyQt5.QtSql import QSqlTableModel, QSqlDatabase, QSqlQuery, QSqlQueryModel
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *

class ChangeDialog(QDialog):
    customerAddedSignal = pyqtSignal()
    def __init__(self):
        super().__init__()
        self.query = QSqlQuery()
        caption = []
        benefit = []
        position = []

        self.setWindowTitle("Просмотр сведений об абоненте")
        self.centralwidget = QWidget()
        self.centralwidget.setObjectName(u"centralwidget")
        self.widget = QWidget(self.centralwidget)
        self.widget.setObjectName(u"widget")
        self.widget.setGeometry(QRect(441, 212, 173, 154))
        self.layout = QVBoxLayout()
        self.gridlayout = QGridLayout()

        self.model = QSqlQueryModel(self)
        self.box = QComboBox()
        district = []
        self.query = QSqlQuery()
        self.query.exec("""SELECT name_district FROM district""")
        while self.query.next():
            district.append(str(self.query.value(0)))
        self.district = QComboBox()
        self.box.addItems(district)

        self.box2 = QComboBox()
        state = ['False', 'True']
        self.box2.addItems(state)

        self.atc = QLineEdit()
        self.license = QLineEdit()
        self.address = QLineEdit()
        self.data = QDateEdit()

        self.label1 = QLabel("Фамилия:")
        self.label2 = QLabel("Имя:")
        self.label3 = QLabel("Отчество:")
        self.label4 = QLabel("АТС:")
        self.label5 = QLabel("Номер телефона:")
        self.label6 = QLabel("Адрес:")
        self.label7 = QLabel("Социальное положение:")
        self.label8 = QLabel("Льгота:")
        self.label9 = QLabel("Документ на льготу:")

        self.surname = QLineEdit()
        self.name = QLineEdit()
        self.middlename = QLineEdit()
        self.address = QLineEdit()
        self.number = QLineEdit()
        self.atc = QComboBox()
        self.position = QComboBox()
        self.benefit = QComboBox()
        self.file = QLabel()

        self.number.setMaxLength(13)
        rx = QRegExp("[а-яА-Яa-zA-Z]+")
        validator = QRegExpValidator(rx, self)
        self.surname.setValidator(validator)
        self.name.setValidator(validator)
        self.middlename.setValidator(validator)
        self.surname.setMaxLength(20)
        self.name.setMaxLength(20)
        self.middlename.setMaxLength(20)
        self.address.setMaxLength(40)

        self.query.exec("""SELECT caption FROM atc""")
        while self.query.next():
            caption.append(self.query.value(0))

        self.query.exec("""SELECT type_positon FROM position """)
        while self.query.next():
            position.append(self.query.value(0))

        benefit.append(None)
        self.query.exec("""SELECT type FROM benefit""")
        while self.query.next():
            benefit.append(self.query.value(0))

        self.atc.addItems(caption)
        self.position.addItems(position)
        self.benefit.addItems(benefit)

        self.save_btn = QPushButton("Сохранить изменения")
        self.cancel_btn = QPushButton("Cancel")

        self.gridlayout.addWidget(self.label4, 1, 0)
        self.gridlayout.addWidget(self.label1, 2, 0)
        self.gridlayout.addWidget(self.label2, 3, 0)
        self.gridlayout.addWidget(self.label3, 4, 0)
        self.gridlayout.addWidget(self.label5, 5, 0)
        self.gridlayout.addWidget(self.label6, 6, 0)
        self.gridlayout.addWidget(self.label7, 7, 0)
        self.gridlayout.addWidget(self.label8, 8, 0)
        self.gridlayout.addWidget(self.label9, 9, 0)

        self.gridlayout.addWidget(self.atc, 1, 1, 1, 3)
        self.gridlayout.addWidget(self.surname, 2, 1, 1, 3)
        self.gridlayout.addWidget(self.name, 3, 1, 1, 3)
        self.gridlayout.addWidget(self.middlename, 4, 1, 1, 3)
        self.gridlayout.addWidget(self.number, 5, 1, 1, 3)
        self.gridlayout.addWidget(self.address, 6, 1, 1, 3)
        self.gridlayout.addWidget(self.position, 7, 1, 1, 3)
        self.gridlayout.addWidget(self.benefit, 8, 1, 1, 3)
        self.gridlayout.addWidget(self.file, 9, 2)

        self.gridlayout.addWidget(self.save_btn, 12, 0)
        self.gridlayout.addWidget(self.cancel_btn, 12, 1)

        self.layout.setGeometry(QRect(441, 212, 173, 154))
        self.layout.addLayout(self.gridlayout)
        self.setLayout(self.layout)
        self.layout.setAlignment(Qt.AlignVCenter | Qt.AlignHCenter)

        self.save_btn.clicked.connect(self.save)
        self.cancel_btn.clicked.connect(self.cancel)

    def save(self):
        atc = self.atc.currentText()
        surname = self.surname.text()
        name = self.name.text()
        middlename = self.middlename.text()
        address = self.address.text()
        number = self.number.text()
        position = self.position.currentText()
        benefit = self.benefit.currentText()
        document = self.file.text()

        self.query.prepare("""SELECT id_atc FROM atc
                                   WHERE caption = ?;""")
        self.query.addBindValue(atc)
        self.query.exec_()
        while self.query.next():
            atc_id = self.query.value(0)

        self.query.prepare("""SELECT id FROM position
                                           WHERE type_positon = ?;""")
        self.query.addBindValue(position)
        self.query.exec_()
        while self.query.next():
            position_id = self.query.value(0)

        if len(benefit) == 0:
            benefit_id = None
        else:
            self.query.prepare("""SELECT id FROM benefit
                                               WHERE type = ?;""")
            self.query.addBindValue(benefit)
            self.query.exec_()
            while self.query.next():
                benefit_id = self.query.value(0)
        if self.data_validate():
            if not surname or not name or not middlename or not address:
                messageBox = QMessageBox.critical(self, self.tr("Ошибка!"), self.tr("Необходимо заполнить все поля формы!"),
                                                  QMessageBox.Ok | QMessageBox.Cancel)
            elif benefit_id is None and len(document) > 2:
                messageBox = QMessageBox.critical(self, self.tr("Ошибка!"),
                                                  self.tr("Выберите тип льготы или удалите фото льготы!"),
                                                  QMessageBox.Ok)
            elif benefit_id is not None and len(document) < 2:
                messageBox = QMessageBox.critical(self, self.tr("Ошибка!"),
                                                  self.tr("Добавьте фото льготы или уберите льготу!"),
                                                  QMessageBox.Ok)
            elif len(number) != 13:
                messageBox = QMessageBox.critical(self, self.tr("Ошибка!"),
                                                  self.tr("Номер телефона не может быть больше или меньше 13 символов!"),
                                                  QMessageBox.Ok | QMessageBox.Cancel)
            else:
                self.query.prepare(
                    "UPDATE abonents SET id_atc = ?, surname = ?, name = ?, middlename = ?, phone_number = ?, address = ?, "
                    "id_position = ?, id_benefit = ?, document = ? "
                    " WHERE id = ? RETURNING *;")
                self.query.addBindValue(atc_id)
                self.query.addBindValue(surname)
                self.query.addBindValue(name)
                self.query.addBindValue(middlename)
                self.query.addBindValue(number)
                self.query.addBindValue(address)
                self.query.addBindValue(position_id)
                self.query.addBindValue(benefit_id)
                self.query.addBindValue(document)
                self.query.addBindValue(self.index)
                if not self.query.exec_():
                    messageBox = QMessageBox.critical(self, self.tr("Ошибка!"),
                                                      self.tr("Абонент уже существует!"),
                                                      QMessageBox.Ok | QMessageBox.Cancel)
                else:
                    messageBox = QMessageBox.information(self, self.tr("Успех"),
                                                         self.tr("Запись изменена!"),
                                                         QMessageBox.Ok)
                    self.close()

    def cancel(self):
        self.close()

    def setIndex(self, index):
        self.index = index
        self.setData()

    def setData(self):
        query = QSqlQuery()
        data = []
        query.prepare("""select atc.caption, abonents.surname, abonents.name, abonents.middlename,
                        abonents.phone_number, abonents.address,
                        position.type_positon, benefit.type, abonents.document
                        from abonents
                        join atc on abonents.id_atc = atc.id_atc
                        join position on abonents.id_position = position.id
                        left join benefit on abonents.id_benefit = benefit.id
                        where abonents.id = ?;""")
        query.addBindValue(self.index)
        query.exec_()
        while query.next():
            for i in range(9):
                data.append(query.value(i))
        self.atc.setCurrentText(data[0])
        self.surname.setText(data[1])
        self.name.setText(data[2])
        self.middlename.setText(data[3])
        self.number.setText(data[4])
        self.address.setText(data[5])
        self.position.setCurrentText(data[6])
        self.benefit.setCurrentText(data[7])
        self.file.setText(data[8])

    def data_validate(self):
        surname = self.surname.text()
        name = self.name.text()
        middlename = self.middlename.text()
        address = self.address.text()
        if len(surname) < 3 or len(address) < 3 or len(name) < 3 or len(middlename) < 3:
            messageBox = QMessageBox.warning(self, self.tr("Ошибка"),
                                             self.tr("ФИО или адрес не могут быть меньше 3 символов!"),
                                             QMessageBox.Ok)
            return False
        elif address.isdigit():
            messageBox = QMessageBox.warning(self, self.tr("Ошибка"),
                                             self.tr("Адрес не может состоять только из цифр!"),
                                             QMessageBox.Ok)
        else:
            return True


"""
Author: AruniRC
Date: Feb 2019
"""
import time
from src.utils import visualize_point_cloud_from_labels, visualize_point_cloud
import ipdb
import os
import os.path as osp
import data_utils
from data_utils.ShapeNetDataLoader import PartNormalDataset, SelfSupPartNormalDataset, ACDSelfSupDataset
from tensorboard_logger import configure, log_value
import itertools
import torch
from torch import nn
import datetime
import logging
from pathlib import Path
import sys
import importlib
import shutil
from tqdm import tqdm
import provider
import numpy as np
import sys, ipdb
from args_parser import parse_args
from testing import evaluation
from itertools import product

torch.backends.cudnn.enabled = False

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = BASE_DIR
sys.path.append(os.path.join(ROOT_DIR, 'models'))

seg_classes = {'Airplane': [0, 1, 2, 3], 'Bag': [4, 5], 'Cap': [6, 7], 'Car': [8, 9, 10, 11],
               'Chair': [12, 13, 14, 15], 'Earphone': [16, 17, 18], 'Guitar': [19, 20, 21], 'Knife': [22, 23],
               'Lamp': [24, 25, 26, 27], 'Laptop': [28, 29], 'Motorbike': [30, 31, 32, 33, 34, 35], 'Mug': [36, 37],
               'Pistol': [38, 39, 40], 'Rocket': [41, 42, 43], 'Skateboard': [44, 45, 46], 'Table': [47, 48, 49]}
classes = ['Airplane', 'Bag', 'Cap', 'Car', 'Chair', 'Earphone', 'Guitar', 'Knife', 'Lamp', 'Laptop', 'Motorbike', 'Mug', 'Pistol', 'Rocket', 'Skateboard', 'Table']
classes_parts = {0: [0, 1, 2, 3], 1: [4, 5], 2: [6, 7], 3: [8, 9, 10, 11],
                 4: [12, 13, 14, 15], 5: [16, 17, 18], 6: [19, 20, 21], 7: [22, 23],
                 8: [24, 25, 26, 27], 9: [28, 29], 10: [30, 31, 32, 33, 34, 35], 11: [36, 37],
                 12: [38, 39, 40], 13: [41, 42, 43], 14: [44, 45, 46], 15: [47, 48, 49]}

# ipdb.set_trace()

def to_categorical(y, num_classes):
    """ 1-hot encodes a tensor """
    new_y = torch.eye(num_classes)[y.cpu().data.numpy(),]
    if (y.is_cuda):
        return new_y.cuda()
    return new_y

def train_init_class(classifier, criterion, trainDataLoader, num_classes, num_part):
    """ Pre-train the classifier layer using logistic regression """
    optim = torch.optim.SGD(classifier.conv2.parameters(), lr=0.1, momentum=0.5)
    num_epoch = 500

    for epoch in range(num_epoch):
        print('Init Classifier: Epoch (%d/%d):' % (epoch + 1, num_epoch))
        mean_correct = []
        mean_loss = []
        for batch_id, (points, label, target) in enumerate(trainDataLoader):
            cur_batch_size, NUM_POINT, _ = points.size()
            points = points.data.numpy()
            points[:, :, 0:3] = provider.random_scale_point_cloud(points[:, :, 0:3])
            points[:, :, 0:3] = provider.shift_point_cloud(points[:, :, 0:3])
            points = torch.Tensor(points)
            points, label, target = points.float().cuda(), label.long().cuda(), target.long().cuda()
            points = points.transpose(2, 1)
            optim.zero_grad()

            classifier = classifier.eval()  # batch stats aren't updated

            '''applying supervised cross-entropy loss'''
            seg_pred, trans_feat, feat = classifier(points, to_categorical(label, num_classes))
            seg_pred = seg_pred.contiguous().view(-1, num_part)
            target = target.view(-1, 1)[:, 0]
            pred_choice = seg_pred.data.max(1)[1]
            correct = pred_choice.eq(target.data).cpu().sum()

            mean_correct.append(correct.item() / (cur_batch_size * NUM_POINT))

            loss = criterion(seg_pred, target, trans_feat)
            loss.backward()
            optim.step()
            mean_loss.append(loss.item())
            print('classifier: batch (%d/%s) Loss: %f Acc:%f' % (batch_id,
                                                                 len(trainDataLoader),
                                                                 loss.item(),
                                                                 mean_correct[-1]))

        log_value('init_cls_loss', np.mean(mean_loss), epoch)
        log_value('init_cls_acc', np.mean(mean_correct), epoch)
    classifier = classifier.train()

    return classifier

def main(args):
    metrics = {'best_acc': 0, 'best_epoch': 0,'best_class_avg_miou': 0, 'best_instance_avg_miou': 0, 'best_chamfer_loss': np.inf}

    def log_string(str):
        logger.info(str)
        print(str)

    '''CUDA ENV SETTINGS'''
    if args.gpu is not None:
        os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu

    if args.cudnn_off:
        torch.backends.cudnn.enabled = False  # needed on gypsum!

    # --------------------------------------------------------------------------
    '''CREATE DIR'''
    # --------------------------------------------------------------------------
    timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
    experiment_dir = Path('log/')
    experiment_dir.mkdir(exist_ok=True)
    experiment_dir = experiment_dir.joinpath('part_seg_shapenet')
    experiment_dir.mkdir(exist_ok=True)
    if args.log_dir is None:
        experiment_dir = experiment_dir.joinpath(timestr)
    else:
        # if args.k_shot > 0:
        dir_name = args.model + '_ShapeNet_' + \
                   '_k-%d_seed-%d_lr-%.6f_lr-step-%d_lr-decay-%.2f_wt-decay-%.6f_l2norm-%d' \
                   % (args.k_shot, args.seed, args.learning_rate,
                      args.step_size, args.lr_decay, args.decay_rate,
                      int(args.l2_norm))
        if args.normal:
            dir_name = dir_name + '_normals'
        if args.category:
            dir_name = dir_name + '_category-label'
        if args.selfsup:
            dir_name = dir_name + '_selfsup-%s_margin-%.2f_lambda-%.2f' \
                       % (args.ss_dataset, args.margin, args.lmbda)
        if args.anneal_lambda:
            dir_name = dir_name + '_anneal-lambda_step-%d_rate-%.2f' \
                       % (args.anneal_step, args.anneal_rate)

        if args.quantile or args.msc_iterations:
            dir_name = dir_name + '_quantile-{}_msc-its-{}_max-num-clusters-{}_alpha-{}_beta-{}'.format(args.quantile,
                                                                                args.msc_iterations,
                                                                                args.max_num_clusters, args.alpha, args.beta)

        experiment_dir = experiment_dir.joinpath(dir_name)
        # else:
        #     experiment_dir = experiment_dir.joinpath(args.log_dir)

    experiment_dir.mkdir(exist_ok=True)
    checkpoints_dir = experiment_dir.joinpath('checkpoints/')
    checkpoints_dir.mkdir(exist_ok=True)
    log_dir = experiment_dir.joinpath('logs/')
    log_dir.mkdir(exist_ok=True)
    # --------------------------------------------------------------------------
    '''LOG'''
    # --------------------------------------------------------------------------
    logger = logging.getLogger("Model")
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    log_string('PARAMETERS ...')
    log_string(args)
    configure(log_dir)                                             # tensorboard logdir

    # --------------------------------------------------------------------------
    '''DATA LOADERS'''
    # --------------------------------------------------------------------------
    root = 'ShapeSelfSup/dataset/shapenetcore_partanno_segmentation_benchmark_v0_normal'

    TRAIN_DATASET = PartNormalDataset(root=root, npoints=args.npoint, split=args.train_split, normal_channel=args.normal, k_shot=args.k_shot)
    trainDataLoader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=args.batch_size, shuffle=True, num_workers=4)

    trainDataIterator = iter(trainDataLoader)

    TEST_DATASET = PartNormalDataset(root=root, npoints=args.npoint, split=args.eval_split, normal_channel=args.normal)

    log_string("The number of training data is: %d" % len(TRAIN_DATASET))

    num_classes = args.num_classes
    num_part = args.num_parts

    if args.selfsup:
        log_string('Use self-supervision - alternate batches')
        if not args.retain_overlaps:
            log_string('\tRemove overlaps between labeled and self-sup datasets')
            labeled_fns = list(itertools.chain(*TEST_DATASET.meta.values())) \
                          + list(itertools.chain(*TRAIN_DATASET.meta.values()))
        else:
            log_string('\tUse all files in self-sup dataset')
            labeled_fns = []
        if args.ss_dataset == 'dummy':
            log_string('Using "dummy" self-supervision dataset (rest of labeled ShapeNetSeg)')
            SELFSUP_DATASET = SelfSupPartNormalDataset(root=root, npoints=args.npoint,
                                                       split='train', normal_channel=args.normal,
                                                       k_shot=args.n_cls_selfsup, labeled_fns=labeled_fns)
        elif args.ss_dataset == 'acd':
            log_string('Using "ACD" self-supervision dataset (ShapeNet Seg)')
            ACD_ROOT = args.ss_path
            SELFSUP_DATASET = ACDSelfSupDataset(root=ACD_ROOT, npoints=args.npoint,
                                                normal_channel=args.normal,
                                                k_shot=args.n_cls_selfsup,
                                                exclude_fns=labeled_fns, prefetch=False)

        selfsupDataLoader = torch.utils.data.DataLoader(SELFSUP_DATASET,
                                                        batch_size=args.batch_size,
                                                        shuffle=True, num_workers=1)
        selfsupIterator = iter(selfsupDataLoader)

    '''MODEL LOADING'''

    MODEL = importlib.import_module(args.model)
    shutil.copy('ShapeSelfSup/models/%s.py' % args.model, str(experiment_dir))
    shutil.copy('ShapeSelfSup/models/pointnet_util.py', str(experiment_dir))
    if args.reconstruct:
        log_string('Reconstruction................................')
    if args.extra_layers:
        log_string('Extra Layers..................................')
    if 'dgcnn' in args.model:
        print('DGCNN params')
        classifier = MODEL.get_model(num_part, normal_channel=args.normal, k=args.dgcnn_k).cuda()
    else:
        classifier = MODEL.get_model(num_part, normal_channel=args.normal, reconstruct=args.reconstruct, extra_layers=args.extra_layers, num_charts=args.num_charts, num_points=args.num_points).cuda()

    criterion = MODEL.get_loss().cuda()

    if args.selfsup:
        selfsupCriterion = MODEL.get_selfsup_loss(margin=args.margin).cuda()
        log_string("The number of self-sup data is: %d" % len(SELFSUP_DATASET))
    else:
        log_string("No Self Supervision..........................................")

    def weights_init(m):
        classname = m.__class__.__name__
        if classname.find('Conv2d') != -1:
            torch.nn.init.xavier_normal_(m.weight.data)
            torch.nn.init.constant_(m.bias.data, 0.0)
        elif classname.find('Linear') != -1:
            torch.nn.init.xavier_normal_(m.weight.data)
            torch.nn.init.constant_(m.bias.data, 0.0)
    if torch.cuda.device_count() > 1:
        log_string("Let's use {} GPUs!".format(torch.cuda.device_count()))
        classifier = nn.DataParallel(classifier)

    if args.optimizer == 'Adam':
        optimizer = torch.optim.Adam(
            classifier.parameters(),
            lr=args.learning_rate,
            betas=(0.9, 0.999),
            eps=1e-08,
            weight_decay=args.decay_rate
            )
    else:
        optimizer = torch.optim.SGD(classifier.parameters(), lr=args.learning_rate, momentum=0.9)

    if args.pretrained_model is None:
        # Default: load saved checkpoint from experiment_dir or start from scratch
        try:
            checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
            start_epoch = checkpoint['epoch']
            classifier.load_state_dict(checkpoint['model_state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
            log_string('Use pretrained model from checkpoints')
        except:
            log_string('No existing model, starting training from scratch...')
            start_epoch = 0
            classifier = classifier.apply(weights_init)
    else:
        # Path to a pre-trained model is provided (self-sup)
        log_string('Loading pretrained model from %s' % args.pretrained_model)
        start_epoch = 0
        ckpt = torch.load(args.pretrained_model)
        classifier.load_state_dict(ckpt['model_state_dict'])
    def bn_momentum_adjust(m, momentum):
        if isinstance(m, torch.nn.BatchNorm2d) or isinstance(m, torch.nn.BatchNorm1d):
            m.momentum = momentum

    LEARNING_RATE_CLIP = 1e-5
    MOMENTUM_ORIGINAL = 0.1
    MOMENTUM_DECAY = 0.5
    MOMENTUM_DECAY_STEP = args.step_size

    #if torch.cuda.device_count() > 1:
    #    log_string("Let's use {} GPUs!".format(torch.cuda.device_count()))
    #    classifier = nn.DataParallel(classifier)
    if args.category:
        log_string("Using one hot")

    # --------------------------------------------------------------------------
    ''' MODEL TRAINING '''
    # --------------------------------------------------------------------------

    global_epoch = 0

    if args.pretrained_model is not None:
        if args.init_cls:
            # Initialize the last layer of loaded model using logistic regression
            classifier = train_init_class(classifier, criterion, trainDataLoader, num_classes, num_part)
    if args.if_cuboid:
        print('Using Cuboid as Primitive..................')
    else:
        print('Using Ellipsoid as Primitive...............')

    if args.include_convex_loss:
        print('Using Convex Fitting/Convex Loss with lambda - {}.........................'.format(args.lmbda))
    if args.include_intersect_loss:
        print('Using Intersection Loss with alpha - {}..................................'.format(args.alpha))
        if args.include_pruning:
            print('Pruning Ellipsoids...................................................')
    if args.include_entropy_loss:
        print('Using Entropy Loss with beta - {}........................................'.format(args.beta))
    print('Using Categorical Cross Entropy Loss for Semantic Segmentation')

    for epoch in range(start_epoch, args.epoch):
        log_string('Epoch %d (%d/%s):' % (global_epoch + 1, epoch + 1, args.epoch))

        '''Adjust learning rate and BN momentum'''
        lr = max(args.learning_rate * (args.lr_decay ** (epoch // args.step_size)), LEARNING_RATE_CLIP)
        log_string('Learning rate:%f' % lr)
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr
        mean_correct = []
        momentum = MOMENTUM_ORIGINAL * (MOMENTUM_DECAY ** (epoch // MOMENTUM_DECAY_STEP))
        if momentum < 0.01:
            momentum = 0.01
        print('BN momentum updated to: %f' % momentum)
        classifier = classifier.apply(lambda x: bn_momentum_adjust(x, momentum))

        ''' Adjust (anneal) self-sup lambda '''
        if args.anneal_lambda:
            lmbda = args.lmbda * (args.anneal_rate ** (epoch // args.anneal_step))
        else:
            lmbda = args.lmbda

        '''learning one epoch'''
        num_iters = len(trainDataLoader)  # num iters in an epoch
        if args.selfsup:
            num_iters = len(selfsupDataLoader)  # calc an epoch based on self-sup dataset
        for i in tqdm(list(range(num_iters)), total=num_iters, smoothing=0.9, desc='Training'):
            # ------------------------------------------------------------------
            #   SUPERVISED LOSS
            # ------------------------------------------------------------------
            try:
                data = next(trainDataIterator)
            except StopIteration:
                # reached end of this dataloader
                trainDataIterator = iter(trainDataLoader)
                data = next(trainDataIterator)

            points, label, target = data

            class_list = [classes[label[idx, :].data] for idx in range(label.size()[0])]
            '''label_list = [label[label_id].item() for label_id in range(label.size()[0])]
            target_list = [torch.unique(target[idx, :]).data.tolist() for idx in range(target.size()[0])]
            for idx in range(len(label_list)):
                for item in target_list[idx]:
                    if item not in classes_parts[label_list[idx]]:
                        print(target_list[idx], '-', classes_parts[label_list[idx]])
                        print('incorrect-exiting................................ at its - ', i)
                        break
            '''

            cur_batch_size, NUM_POINT, _ = points.size()
            points = points.data.numpy()
            points[:, :, 0:3] = provider.random_scale_point_cloud(points[:, :, 0:3])
            points[:, :, 0:3] = provider.shift_point_cloud(points[:, :, 0:3])
            points = torch.Tensor(points)
            points, label, target = points.float().cuda(), label.long().cuda(), target.long().cuda()
            points = points.transpose(2, 1)
            if args.category:
                category_label = to_categorical(label, num_classes).contiguous()
            else:
                category_label = torch.zeros([label.shape[0], 1, num_classes]).cuda()

            optimizer.zero_grad()
            classifier.train()

            '''applying supervised cross-entropy loss'''
            try:                                                                  #iteration issue at 583 its. uneven distribution of datapoints among GPUs.
                seg_pred, trans_feat, feat, _, _ = classifier(points.contiguous(), category_label, include_convex_loss=False)
            except TypeError as t:
                continue
            seg_pred = seg_pred.contiguous().view(-1, num_part)
            target = target.view(-1, 1)[:, 0]
            pred_choice = seg_pred.data.max(1)[1]
            correct = pred_choice.eq(target.data).cpu().sum()

            mean_correct.append(correct.item() / (cur_batch_size * NUM_POINT))
            loss_sup = criterion(seg_pred, target, trans_feat)

            loss_sup.backward()
            optimizer.step()

            # ------------------------------------------------------------------
            #   SELF-SUPERVISED LOSS
            # ------------------------------------------------------------------
            if args.selfsup:
                try:
                    data_ss = next(selfsupIterator)
                except StopIteration:
                    # reached end of this dataloader
                    selfsupIterator = iter(selfsupDataLoader)
                    data_ss = next(selfsupIterator)

                points, chamfer_points, label, target = data_ss
                #points, label, target = data_ss
                points = points.data.numpy()
                points[:, :, 0:3] = provider.random_scale_point_cloud(points[:, :, 0:3])
                points[:, :, 0:3] = provider.shift_point_cloud(points[:, :, 0:3])
                points = torch.Tensor(points)
                ############################CHAMFER POINTS###############################################
                chamfer_points = chamfer_points.data.numpy()
                chamfer_points[:, :, 0:3] = provider.random_scale_point_cloud(chamfer_points[:, :, 0:3])
                chamfer_points[:, :, 0:3] = provider.shift_point_cloud(chamfer_points[:, :, 0:3])
                chamfer_points = torch.Tensor(chamfer_points)
                ############################CHAMFER POINTS###############################################
                points, chamfer_points, label, target = points.float().cuda(), chamfer_points.float().cuda(), label.long().cuda(), target.long().cuda()
                #points, label, target = points.float().cuda(), label.long().cuda(), target.long().cuda()
                points = points.transpose(2, 1)
                chamfer_points = chamfer_points.transpose(2, 1)
                # for self-sup category label is always unknown, so always zeros:
                category_label = torch.zeros([label.shape[0], 1, num_classes]).cuda()
                if args.normal:
                    # put dummy cols of zeros for normals in self-sup data
                    cur_batch_size, _, NUM_POINT = points.size()
                    points = points[:, 0:3, :]
                    points = torch.cat([points, torch.zeros([cur_batch_size, 3, NUM_POINT]).cuda()], 1)

                optimizer.zero_grad()
                classifier.train()

                '''applying self-supervised Ellipsoid Fitting (Convex) loss'''
                #quantile_list = [0.02, 0.03, 0.05]
                points = chamfer_points[:, :, np.random.choice(5000, 2048, replace=False)]
                #print(chamfer_points.shape)
                #ipdb.set_trace()
                _, _, feat, loss_self_sup, chamfer_loss = classifier(points, category_label, if_cuboid=args.if_cuboid, chamfer_points=chamfer_points, include_convex_loss=args.include_convex_loss, include_intersect_loss=args.include_intersect_loss, include_entropy_loss=args.include_entropy_loss, include_pruning=args.include_pruning, quantile=args.quantile, msc_iterations=args.msc_iterations, max_num_clusters=args.max_num_clusters, alpha=args.alpha, beta=args.beta, batch_id=i, epoch=epoch, class_list=class_list, evaluation=False)
                ss_loss = torch.mean(loss_self_sup) * lmbda
                #print('loss: ', ss_loss.requires_grad)
                if i % 100 == 0:
                    print('Final Loss: ', ss_loss.item())
                    sys.stdout.flush()
                ss_loss.backward()
                optimizer.step()

        # ----------------------------------------------------------------------
        #   Logging metrics after one epoch
        # ----------------------------------------------------------------------
        train_instance_acc = np.mean(mean_correct)
        log_string('Train accuracy is: %.5f' % train_instance_acc)
        log_string('Supervised loss is: %.5f' % loss_sup.data)
        # log_value('train_loss', loss_sup.data, epoch)

        if args.selfsup:
            log_string('Self-sup loss is: %.5f' % ss_loss.data)
            # log_value('selfsup_loss', ss_loss.data, epoch)

        # save every epoch
        savepath = str(checkpoints_dir) + ('/model_%03d.pth' % (epoch+1))
        log_string('Saving model at %s' % savepath)
        state = {
            'epoch': epoch,
            'train_acc': train_instance_acc,
            'model_state_dict': classifier.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
        }
        torch.save(state, savepath)
        log_string('Saved model.')
        log_value('train_acc', train_instance_acc, epoch)
        log_value('train_lr', lr, epoch)
        log_value('train_bn_momentum', momentum, epoch)
        log_value('selfsup_lambda', lmbda, epoch)

        global_epoch += 1

    # ----------------------------------------------------------------------
    #   Evaluation on test-set after completing training epochs
    # ----------------------------------------------------------------------
        return_metrics = evaluation(args, epoch, classifier, metrics)

if __name__ == '__main__':

    args = parse_args()
    main(args)


from flask import Flask, request

app = Flask(__name__)

@app.route("/")
def index():
    return '''
<form action="/info" method="GET">
<input type="text" name="user" placeholder="Full Name">

<input type="email" name="email" placeholder="Email">

<input type="password" name="pass" placeholder="password">

<input type="submit" value="Send">
</form>
'''

@app.route("/info",methods=['GET','POST'])
def info():
    if request.method == "GET":
        return "
GET Method
Hello "+ request.args.get('user') + \
    " Your Email:"+request.args.get('email')
    else:
        return "
POST Method
Hello "+ request.form['user'] + \
    " Your Email:"+request.form['email']

app.run(debug=True)


import curses

from exceptions import Game2048NoFreeLocationsLeft
from game_2048.game import Game2048

class Game2048Curses(Game2048):

    KEY_QUIT = ord('q')
    KEY_RESET = ord('r')
    KEY_DOWN = 66
    KEY_UP = 65
    KEY_LEFT = 68
    KEY_RIGHT = 67

    KEY_DIRECTION_MAPPING = {
        KEY_DOWN: Game2048.DIRECTION_DOWN,
        KEY_UP: Game2048.DIRECTION_UP,
        KEY_RIGHT: Game2048.DIRECTION_RIGHT,
        KEY_LEFT: Game2048.DIRECTION_LEFT
    }

    templates = {
        'score': 'Score: {score}',
        'level_complete': 'Congratulations your highest item is {highest}!. The new threshold is now {threshold}.',
        'logo': """
 ___   ___    __   ___
(__ \ / _ \  /. | ( _ )
 / _/( (_) )(_  _)/ _ \\
(____)\___/   (_) \___/

""",
        'help_text': 'Use the arrow keys to play. Press q for quit or r to restart.',
        'game_over': 'Game over! Your score is {score}. Press r to restart the game.'
    }

    _std_scr = None
    _highest_value_threshold = 2048

    _message = None

    _line_position = 0

    def __init__(self, auto_start=True, *args, **kwargs):

        super(Game2048Curses, self).__init__(*args, **kwargs)

        if auto_start:
            self._start()

    def _start(self):

        self._std_scr = curses.initscr()
        curses.cbreak()

        try:
            self._run()
        finally:
            curses.endwin()

    def _run(self):

        self._refresh_screen()

        key_ = None

        while key_ != self.KEY_QUIT:

            key_ = self._std_scr.getch()

            if key_ in self.KEY_DIRECTION_MAPPING:

                try:
                    self.move(self.KEY_DIRECTION_MAPPING[key_])
                except Game2048NoFreeLocationsLeft:
                    self._display_game_over_screen()
                    continue

                if self._highest_value >= self._highest_value_threshold:
                    self._highest_value_threshold *= 2

                    self._message = self.templates['level_complete'].format(highest=self._highest_value,
                                                                            threshold=self._highest_value_threshold)

            if key_ == self.KEY_RESET:
                self.reset()

            self._refresh_screen()

    def _update_game(self):

        offset = 1

        self._write_line((8 * '-') * self._grid_size)

        for row_i, row in enumerate(self._grid):
            str_ = ''
            for column in row:
                str_ += '{: ^6} |'.format(column if column else '')

            self._write_line(str_)
            self._write_line((8 * '-') * self._grid_size)

    def _update_logo(self):

        for line in self.templates['logo'].split("\n"):
            self._write_line(content=line)

    def _display_game_over_screen(self):

        self._line_position = 0
        self._std_scr.clear()

        self._update_logo()
        self._write_line(self.templates['game_over'].format(score=self._score))

    def _refresh_screen(self):

        self._line_position = 0
        self._std_scr.clear()

        self._update_logo()
        self._write_line(self.templates['score'].format(score=self._score))
        self._write_line(self._message or '')
        self._message = None
        self._update_game()

        self._line_position += 1

        self._write_line(content=self.templates['help_text'])

        self._write_line('')

    def _write_line(self, content, x_offset=0):

        self._std_scr.addstr(self._line_position, x_offset, content)
        self._line_position += 1

if __name__ == '__main__':

    Game2048Curses()


import numpy as np
import random
import copy
from collections import namedtuple, deque

from model import Actor, Critic

import torch
import torch.nn.functional as F
import torch.optim as optim

BUFFER_SIZE = int(1e5)  #reply buffer size
BATCH_SIZE = 512        #minibatch_size
GAMMA = 0.99            #discount factor
TAU = 1e-3              #soft update mix rate
LR_ACTOR = 1e-4         #actor learning rate
LR_CRITIC = 1e-3        #critic learning rate
WEIGHT_DECAY = 0        #L2 weight decay

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class Agent():
    def __init__(self, state_size, action_size, random_seed):

        self.state_size = state_size
        self.action_size = action_size
        self.seed = random.seed(random_seed)

        ##Actor Networks
        self.actor_local = Actor(state_size, action_size, random_seed).to(device)
        self.actor_target = Actor(state_size, action_size, random_seed).to(device)
        self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr = LR_ACTOR)

        ##Critic Networks
        self.critic_local = Critic(state_size, action_size, random_seed).to(device)
        self.critic_target = Critic(state_size, action_size, random_seed).to(device)
        self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr = LR_CRITIC, weight_decay=WEIGHT_DECAY)

        ##Noise Process
        self.noise = OUNoise(action_size, random_seed)

        ##Replay Memory
        self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)

    def step(self, state, action, reward, next_state, done):
        ##add memory to replay buffer
        self.memory.add(state, action, reward, next_state, done)

        ##Learn if we have sufficient memories
        if len(self.memory) > BATCH_SIZE:
            experiences = self.memory.sample()
            self.learn(experiences, GAMMA)

    def act(self, state, eps, add_noise=True):
        ##Move state to device
        state = torch.from_numpy(state).float().to(device)

        ##Run actor forward and get action
        self.actor_local.eval()
        with torch.no_grad():
            action = self.actor_local(state).cpu().data.numpy()
        self.actor_local.train()

        ###Add noise if desired with dampening
        if add_noise:
            action += self.noise.sample()*eps

        ##Epsilon greedy exploration
        #if random.random() < eps:
        #    action = (np.random.random(self.action_size) * 2) - 1

        ##Clip action values
        return np.clip(action, -1, 1)

    def reset(self):
        self.noise.reset()

    def learn(self, experiences, gamma):
        states, actions, rewards, next_states, dones = experiences

        ##Update Critic
        ##Get next_state targets
        actions_next = self.actor_target(next_states)
        Q_targets_next = self.critic_target(next_states, actions_next)

        ##Calculate state targets
        Q_targets = rewards + (GAMMA * Q_targets_next * (1 - dones))

        ##Compute loss
        Q_expected = self.critic_local(states, actions)
        critic_loss = F.mse_loss(Q_expected, Q_targets)

        ## Minimise loss
        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        torch.nn.utils.clip_grad_norm(self.critic_local.parameters(), 1)
        self.critic_optimizer.step()

        ##Update Actor
        ##Compute actions and loss
        actions_pred = self.actor_local(states)
        actor_loss = -self.critic_local(states, actions_pred).mean()

        ##Minimise loss
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

        ##Update target Networks
        self.soft_update(self.critic_local, self.critic_target, TAU)
        self.soft_update(self.actor_local, self.actor_target, TAU)

    def soft_update(self, local_model, target_model, tau):
        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
            target_param.data.copy_(tau*local_param.data + (1-tau)*target_param.data)

class OUNoise:
    def __init__(self, size, seed, mu=0, theta=0.15, sigma=0.2):
        self.mu = mu*np.ones(size)
        self.theta = theta
        self.sigma = sigma
        self.seed = random.seed(seed)
        self.reset()

    def reset(self):
        self.state = copy.copy(self.mu)

    def sample(self):
        x = self.state
        dx = self.theta * (self.mu - x) + self.sigma * np.array([random.random() for i in range(len(x))])
        self.state = x + dx
        return self.state

class ReplayBuffer:
    def __init__(self, action_size, buffer_size, batch_size, seed):
        self.action_size = action_size
        self.memory = deque(maxlen=buffer_size)
        self.batch_size = batch_size
        self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])
        self.seed = random.seed(seed)

    def add(self, state, action, reward, next_state, done):
        e = self.experience(state, action, reward, next_state, done)
        self.memory.append(e)

    def sample(self):
        experiences = random.sample(self.memory, k=self.batch_size)

        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)
        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).float().to(device)
        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)
        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)
        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)

        return (states, actions, rewards, next_states, dones)

    def __len__(self):
        return len(self.memory)

3. Policy Base Methods/p2_continous_control/ddpg_agent.py