from flask import render_template, url_for, redirect, request, Blueprint, flash
from application.hams.forms import SelectMethodForm, AcquisitionForm, DeconvLibrarySearch, LibrarySearch, ImportDeconv, AlgorithmnForm
from application.models import User, Acquisition_Hrms, Ms_System, Sample_Information, Sample_Location, Analytical_Column, Chromatographic_Condition, Chrom_Time, Lc_System, Threshold_Setting
from application import db
from application.hams.utils import save_datafile_L, save_datafile_D, save_datafile_DL, DownloadFromS3ToLocal
from flask_login import login_required, current_user
from urllib.request import urlopen

import secrets
import subprocess

hams = Blueprint('hams', __name__)

details = [
    {
        'title' : 'Library Search',
        'description' : "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur."
    },
    {
        'title' : 'Import Deconv',
        'description' : "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur."
    },
    {
        'title' : 'Deconv & Library Search',
        'description' : "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur."
    }
]

def getMethodList():
    methodList = []
    for method in SelectMethodForm().methods:
        methodList.append(method)
    return methodList

def getUserIcon():
    if current_user.is_authenticated:
        user_icon = url_for('static', filename='imgs/' + current_user.user_icon)
        return user_icon

def getJobID():
    random_hex = secrets.token_hex(8)
    while Sample_Information.query.filter_by(sample_job_number=random_hex).first():
        random_hex = secrets.token_hex(8)

    return random_hex

@hams.route("/manageJobs")
@login_required
def manageJobs():
    user_icon = getUserIcon()
    return render_template('manageJobs.html',icon = user_icon, title = "HAMS")

@hams.route("/chooseMethod", methods=['GET', 'POST'])
@login_required
def chooseMethod():
    form = SelectMethodForm()
    methodList = getMethodList()
    user_icon = getUserIcon()
    if form.validate_on_submit():
        chosenMethod = form.methods.data
        return redirect(url_for('hams.acquisition', chosenMethod = chosenMethod))

    return render_template('chooseMethod.html', title = "HAMS", form = form,
                             details = details, methods = methodList, icon = user_icon)

@hams.route("/acquisition/<chosenMethod>", methods=['GET', 'POST'])
@login_required
def acquisition(chosenMethod):
    user_icon = getUserIcon()

    if chosenMethod == 'LibrarySearch':
        form = LibrarySearch()
    elif chosenMethod == 'ImportDeconv':
        form = ImportDeconv()
    else:
        form = DeconvLibrarySearch()

    if form.validate_on_submit():
        j_id = getJobID()

        if chosenMethod == 'ImportDeconv':
            fileList = []
            fileList.append(form.xlsxFile)
            fileList.append(form.VANFileLow)
            fileList.append(form.VANFileHigh)

            d_set = []
            d_set.append(form.Max_W.data)
            d_set.append(form.Min_Int.data)
            d_set.append(form.Retention_Window.data)
            d_set.append(form.R_Min.data)
            d_set.append(form.P_Max.data)
            d_set.append(form.Retention_Time_Tollerance.data)
            d_set.append(form.Ms_P_W.data)
            d_set.append(form.Mass_Tol.data)
            d_set.append(form.Mass_Window.data)
            d_set.append(form.Signal_to_Noise.data)

            save_datafile_D(fileList, current_user.user_email, j_id, d_set)

        elif chosenMethod == 'LibrarySearch':
            #source =
            #mode =

            save_datafile_L(form.txtFile, current_user.user_email, j_id, 'ESI', 'POSITIVE')

        else:
            fileList = []
            fileList.append(form.xlsxFile)
            fileList.append(form.VANFileLow)
            fileList.append(form.VANFileHigh)
            fileList.append(form.txtFile)

            d_set = []
            d_set.append(float(form.Max_W.data))
            d_set.append(int(form.Min_Int.data))
            d_set.append(float(form.Retention_Window.data))
            d_set.append(float(form.R_Min.data))
            d_set.append(float(form.P_Max.data))
            d_set.append(int(form.Retention_Time_Tollerance.data))
            d_set.append(float(form.Ms_P_W.data))
            d_set.append(float(form.Mass_Tol.data))
            d_set.append(float(form.Mass_Window.data))
            d_set.append(float(form.Signal_to_Noise.data))

            #source =
            #mode =

            save_datafile_DL(fileList, current_user.user_email, j_id, d_set, 'ESI', 'POSITIVE')

        #Main tables
        if chosenMethod == 'ImportDeconv' or chosenMethod == 'DeconvLibrarySearch':
            Threshold_Setting = Threshold_Setting(
                mas_w = form.Max_W.data,
                min_int = form.Min_Int.data,
                retention_window = form.Retention_Window.data,
                r_min = form.R_Min.data,
                p_max = form.P_Max.data,
                retention_time_tollerance = form.Retention_Time_Tollerance.data,
                ms_p_w = form.Ms_P_W.data,
                mass_tol = form.Mass_Tol.data,
                mass_window = form.Mass_Window.data,
                signal_to_noise = form.Signal_to_Noise.data
            )

            db.session.add(Threshold_Setting)

        acquisition_hrms = Acquisition_Hrms(hrms_mode = form.hrms_mode.data,
                    hrms_source_gas1 = form.hrms_source_gas1.data,
                    hrms_source_gas2 = form.hrms_source_gas2.data,
                    hrms_curtain_gas = form.hrms_curtain_gas.data,
                    hrms_ionisation = form.hrms_ionisation.data,
                    hrms_voltage = form.hrms_voltage.data,
                    hrms_polarity = form.hrms_polarity.data,
                    hrms_source = form.hrms_source.data)

        sample_information = Sample_Information(user_id = getattr(current_user,'id'),
                    sample_private = form.sample_private.data,
                    sample_type = form.sample_type.data,
                    sample_city = form.sample_city.data,
                    sample_job_number = j_id,
                    sample_country  = form.sample_country.data)

        analytical_column = Analytical_Column(user_id = getattr(current_user,'id'),
                    column_phase = form.column_phase.data,
                    column_praticle_size = form.column_particle_size.data,
                    column_length = form.column_length.data,
                    column_pore_size  = form.column_pore_size.data,
                    column_inner_diameter  = form.column_inner_diameter.data,
                    column_brand  = form.column_brand.data,
                    column_model  = form.column_model.data)

        db.session.add(acquisition_hrms)
        db.session.add(sample_information)
        db.session.add(analytical_column)
        db.session.commit()

        #Sub tables
        ms_system = Ms_System(acqui_hrms_id = getattr(acquisition_hrms,'id'),
                    system_class = form.hrms_system_class.data,
                    system_model = form.hrms_system_model.data,
                    system_brand = form.hrms_system_brand.data)

        sample_location = Sample_Location(sample_id = getattr(sample_information,'id'),
                    location_longitude = form.sample_location_longitude.data,
                    location_altitude = form.sample_location_altitude.data,
                    location_latitude = form.sample_location_latitude.data,
                    location_datum = form.sample_location_datum.data)

        chromatographic_condition = Chromatographic_Condition(column_id = getattr(analytical_column,'id'),
                    sample_injection_volume = form.chrom_sample_injection_volume.data,
                    mobile_phase_a = form.chrom_mobile_phase_a.data,
                    mobile_phase_b = form.chrom_mobile_phase_b.data)

        db.session.add(ms_system)
        db.session.add(sample_location)
        db.session.add(chromatographic_condition)
        db.session.commit()

        chrom_time = Chrom_Time(acqui_chrom_id = getattr(chromatographic_condition,'id'),
                    event_stage = form.chrom_event_stage.data,
                    flow_rate = form.chrom_flow_rate.data,
                    gradient_a_or_b = form.chrom_gradient_a_or_b.data,
                    a_or_b = form.chrom_a_or_b.data,
                    oven_temp = form.chrom_oven_temperature.data)

        lc_system = Lc_System(acqui_chrom_id = getattr(chromatographic_condition,'id'),
                    user_id = getattr(current_user,'id'),
                    system_brand = form.chrom_system_brand.data,
                    system_model = form.chrom_system_model.data,
                    system_chrom_class = form.chrom_system_class.data,
                    system_component_type = form.chrom_component_type.data)

        db.session.add(chrom_time)
        db.session.add(lc_system)

        db.session.commit()

        return redirect(url_for('hams.saveJob', chosenMethod = chosenMethod))
    return render_template('acquisition.html', title = "HAMS", form = form, method = chosenMethod, icon = user_icon)

@hams.route("/saveJob/<chosenMethod>", methods=['GET', 'POST'])
@login_required
def saveJob(chosenMethod):
    form = AlgorithmnForm()
    user_icon = getUserIcon()
    j_id = getJobID()
    if form.validate_on_submit():
        url = DownloadFromS3ToLocal(current_user.user_email, "12f4096091b0f8cc")
        print(url)
        return redirect(url)
        #urlopen(url)
    return render_template('saveJob.html', title = "HAMS", form = form, icon = user_icon)

@hams.route("/download/<jobid>", methods=['GET', 'POST'])
def download(jobid):
    url = DownloadFromS3ToLocal(current_user.user_email, jobid)
    print(url)
    return redirect(url)

#def redirect_delete(URL):
#    redirect(URL)
#    command = 'sudo rm -R /home/ubuntu/deco3801/application/static/data/download/' + current_user.user_email
#    subprocess.call(command, shell=True)


import numpy as np
from flask import Flask,request,jsonify,render_template
import pickle

from google_trans_new import google_translator
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

translator = google_translator()

app= Flask(__name__)
@app.route('/',methods=['GET','POST'])

def main():
    if request.method=="POST":
        sentence=request.form.get("inp")
        translation = translator.translate(sentence.encode('unicode-escape').decode('ASCII'), lang_tgt='en')
        analyser = SentimentIntensityAnalyzer()
        result = analyser.polarity_scores(translation)
        if(result["compound"] < 0):
            return render_template("index.html",message="Negativeâ˜¹")
        if(result["compound"] == 0):
            return render_template("index.html",message="NeutralðŸ˜�")
        if(result["compound"] > 0):
            return render_template("index.html",message="PositiveðŸ˜Š  ")
    return render_template("index.html")


# coding: utf-8

# In 3c66b paper, equation for total residual (1). Try setting phi and theta_n equal to 0, and orbital inclination angle equal to 90', for edge-on. Try to find maximum amplitude? (Also need to calculate amplitudes A (9) and B (10). For now set eccentricity to 0, eventually add it in cuz life's not always circular.

import numpy as np
import scipy as sci
import astropy.constants as const

map = np.transpose(np.genfromtxt("11yr_skymap_v4.txt", skip_header=1))

#raj = input("Raj: ")
#dec = input("Dec: ")

ul_col = map[3]
theta_col = map[1]
phi_col = map[2]

"""
theta_dict = {}
for i in range(len(phi_col)):
    #print(theta_col[i])
    if not theta_col[i] in theta_dict:
        theta_dict[theta_col[i]] = {}
    theta_dict[theta_col[i]][phi_col[i]] = ul_col[i]
    #print(theta_dict[theta_col[i]][phi_col[i]])
print(theta_dict)
"""

theta_dict = {}
#for i in range(len(map[4])):
for i in range(10):
    theta_dict.setdefault(theta_col[i], {})[phi_col[i]] = ul_col[i]
print(theta_dict)

def convert_angles(raj, dec):
    # For now
    return raj, dec

def binary_search(array, value):
    mid = (len(array))/2
    if value == array[mid]:
        return mid
    elif len(array) < 2:
        return array
    elif value > array[mid]:
        return binary_search(array[mid:], value)
    elif value < array[mid]:
        return binary_search(array[:mid], value)

def find_ul(theta, phi):
    target_theta = theta
    target_phi = phi

    theta_array = np.array(list(theta_dict.keys()))
    #print(theta_array)

    theta_arg = np.argmin(abs(theta_array - target_theta))
    best_theta = theta_array[theta_arg]
    print("The closest theta value to {0}, is {1}".format(theta, best_theta))
    #print(theta_dict)
    #print(theta_dict[best_theta])

    phi_array = np.array(list(theta_dict[best_theta].keys()))
    phi_arg = np.argmin(abs(phi_array - target_phi))
    best_phi = phi_array[phi_arg]
    print("The closest phi value to {0}, given theta ~ {1}, is {2}".format(target_phi, best_theta, best_phi))

    print("Given the best values of theta and phi, the continuous source upper limit in the region given is: {0}".format(theta_dict[best_theta][best_phi]))
    #for theta, phi in theta_dict.items():
        #print(phi)
        #print(np.array(list(phi.keys())))
        #print(np.array(list(phi.values())))

    #for theta in theta_dict.items
    #nested = list(theta_dict.values())
    #print(nested)

    #phi_array = nested.keys()

    #ul_array = phi_array(list(phi_array.values()))
    #dist_theta = abs(ul_array - target_phi)
    #arg = np.argmin(dist_theta)
    #answer(k[arg])
    #return binary_search(theta_row, theta)

def strain_approx(Distance, Radius, Frequency, Mass):
    return (32*(np.pi**2)*const.G.value)/(Distance*const.c.value**4)* Mass*(Radius**2)*(Frequency**2)

print(strain_approx(4.5e20, 20, 1/(1.05*3.154e+7), 6e30))

find_ul(.2, .3)


"""functions for cleaning lfb data"""

import pandas as pd
from datetime import datetime

def convert_date_time(df):
    """convert time to time stamp object"""
    df['DDDateTimeOfCall'] = df['DDDateTimeOfCall'].apply(str)
    df['DDDateTimeOfCall'] = df["DDDateTimeOfCall"].apply(lambda x: datetime.strptime(x,"%Y-%m-%d %H:%M:%S"))
    return df

def drop_columns(df):
    """remove columns that are noisy or do not provide useful information"""
    df = df.drop('IncidentType', 1)
    df = df.drop('FireInvestigationAttendanceID', 1)
    df = df.drop('LocationOfFirestartOther', 1)
    df = df.drop('OrganisationName', 1)
    df = df.drop('maincausemodel', 1)
    df = df.drop('maincauseappliancemanufacturerother', 1)
    return df

def drop_rows(df):
    """remove rows that are not geolocated"""
    df = df[df['PropertyClass'].notnull()]
    df = df[df['PropertyCategory'].notnull()]
    df = df[df['ParentPropertyType'].notnull()]
    df = df[df['PropertyType'].notnull()]
    df = df[df['PostCode'].notnull()]
    df = df[df['IncGeo_BoroughCode'].notnull()]
    df = df[df['IncGeo_BoroughName'].notnull()]
    df = df[df['IncGeo_WardCode'].notnull()]
    df = df[df['IncGeo_WardName'].notnull()]
    return df

def fill_missing_values(df):
    """handle rows with incomplete or missing values"""
    df['IgnitionToDiscovery'] = df['IgnitionToDiscovery'].fillna('Not known')
    df['DiscoveryToCall'] = df['DiscoveryToCall'].fillna('Not known')
    df['Action'] = df['Action'].fillna('Not known')
    df['ActionParent'] = df['ActionParent'].fillna('None')
    df['FloorOrDeckOfOrigin'] = df['FloorOrDeckOfOrigin'].fillna(0)
    df['TenureDescription'] = df['TenureDescription'].fillna('Not \
                                                             applicable')
    df['PossibleMentalHealthIssues'] = df['PossibleMentalHealthIssues'].fillna(0)
    df['PossibleMobilityIssues'] = df['PossibleMobilityIssues'].fillna(0)
    df['EvidenceOfAlcoholConsumption'] = df['EvidenceOfAlcoholConsumption'].fillna(0)
    df['EvidenceOfPrescriptionDrugUse'] = df['EvidenceOfPrescriptionDrugUse'].fillna(0)
    df['EvidenceOfRecreationalDrugUse'] = df['EvidenceOfRecreationalDrugUse'].fillna(0)
    df['socialcarebeingreceived'] = df['socialcarebeingreceived'].fillna(0)
    df['OtherHumanFactor'] = df['OtherHumanFactor'].fillna('Not known')
    df['PropertyCoveredByRro'] = df['PropertyCoveredByRro'].fillna(0)
    df['NumPumps'] = df['NumPumps'].fillna(1)
    df['MainCause'] = df['MainCause'].fillna('Not applicable')
    df['ParentMainCause'] = df['ParentMainCause'].fillna('Motive: Not known')
    df['IgnitionSourcePower'] = df['IgnitionSourcePower'].fillna('Unknown')
    df['ParentIgnSource'] = df['ParentIgnSource'].fillna('Unknown')
    df['IgnitionSource'] = df['IgnitionSource'].fillna('Unknown')
    df['ItemFirstIgnited'] = df['ItemFirstIgnited'].fillna('Not known')
    df['ParentItemFirstIgnited'] = df['ParentItemFirstIgnited'].fillna('Other/Not known')
    df['LocationFireStarted'] = df['LocationFireStarted'].fillna('Not known')
    df['HouseholdOccupancy'] = df['HouseholdOccupancy'].fillna('Other')
    df['OccupationStatus'] = df['OccupationStatus'].fillna('Not known')
    df['CausedBy'] = df['CausedBy'].fillna('Not known')
    return df

def all_cleaning(df):
    """perform all cleaning operations"""
    df = convert_date_time(df)
    df = drop_columns(df)
    df = drop_rows(df)
    df = fill_missing_values(df)
    return df


import json
import demistomock as demisto
import pytest

def util_open_file(path):
    with open(path, mode='r') as f:
        return f.read()

def util_load_json(path):
    with open(path, mode='r') as f:
        return json.loads(f.read())

def test_set_email_reply():
    """Unit test
        Given
        - Email author, email recipients and email cc.
        When
        - Setting the email reply.
        Then
        - Validate that the email reply is in the correct format.
        """
    from PreprocessEmail import set_email_reply
    expected_result = util_open_file('test_data/email_reply.txt')
    result = set_email_reply('test@gmail.com', '["test1@gmail.com"]', 'test2@gmail.com', 'test',
                             [{'name': 'image.png'}])
    assert result in expected_result

EMAIL_HTML = """
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
image 1:

image 2:

On Thu, Oct 22, 2020 at 1:56 AM Avishai Brandeis <
avishai@demistodev.onmicrosoft.com> wrote:

please add multiple inline images

"""

EMAIL_HTML_NO_ALT = """

<meta http-equiv="Content-Type" content="text/html; charset=utf-8">

"""

EXPECTED_RESULT_1 = """
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
image 1:

image 2:

"""

EXPECTED_RESULT_2 = """
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
image 1:

image 2:

"""

EXPECTED_RESULT_3 = """
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
image 1:

image 2:

"""

EXPECTED_RESULT_NO_ALT = """

<meta http-equiv="Content-Type" content="text/html; charset=utf-8">

"""

@pytest.mark.parametrize(
    "email_html,entry_id_list,expected",
    [(EMAIL_HTML, [('image_1.png', '37@119')], EXPECTED_RESULT_1),
     (EMAIL_HTML, [], EXPECTED_RESULT_2),
     (EMAIL_HTML, [('image_1.png', '37@119'), ('image_2.png', '38@120')], EXPECTED_RESULT_3),
     (EMAIL_HTML_NO_ALT, [('image_1.png', '37@119'), ('image_2.png', '38@120')], EXPECTED_RESULT_NO_ALT),
     ]
)
def test_create_email_html(email_html, entry_id_list, expected):
    """
        Given
        - The email's Html representation.
        When
        1. Only one of the images were uploaded to the server hence only one entry id exists within the entry id list.
        2. None of the images were uploaded to the server -> an empty entry_id_list.
        3. All images were uploaded to the server.
        Then
        - The images' src attribute would be replaced as expected.
    """
    from PreprocessEmail import create_email_html
    result = create_email_html(email_html, entry_id_list)
    assert result == expected

def test_get_entry_id_list():
    """
        Given
        - List of the email's attachments, List of files of the email's related incident.
        When
        - building an entry id list in order to replace the email's attachments source path.
        Then
        - Ensures that only the email attachments entry id's were returned and not all files entries.
    """
    from PreprocessEmail import get_entry_id_list
    attachments = [
        {
            "description": "",
            "name": "image_1.png",
            "path": "131_dd98957a-d5c3-42e0-8a81-f3ce7fa68215",
            "showMediaFile": False,
            "type": ""
        },
        {
            "description": "",
            "name": "image_2.png",
            "path": "131_17545998-4b16-4e58-8e6c-2221ada856d4",
            "showMediaFile": False,
            "type": ""
        }
    ]
    files = [
        {
            "EntryID": "30@119",
            "Extension": "png",
            "Info": "image/png",
            "MD5": "md5",
            "Name": "attachment_1.png",
            "SHA1": "sha1",
            "SHA256": "sha256",
            "SHA512": "sha512",
            "SSDeep": "ssdeep",
            "Size": 63111,
            "Type": "PNG image data, 1209 x 398, 8-bit/color RGBA, non-interlaced"
        },
        {
            "EntryID": "34@119",
            "Extension": "png",
            "Info": "image/png",
            "MD5": "md5",
            "Name": "attachment_2.png",
            "SHA1": "4sha1",
            "SHA256": "sha256",
            "SHA512": "sha512",
            "SSDeep": "ssdeep",
            "Size": 9580,
            "Type": "PNG image data, 264 x 60, 8-bit/color RGBA, non-interlaced"
        },
        {
            "EntryID": "35@119",
            "Extension": "png",
            "Info": "image/png",
            "MD5": "md5",
            "Name": "image_1.png",
            "SHA1": "4sha1",
            "SHA256": "sha256",
            "SHA512": "sha512",
            "SSDeep": "ssdeep",
            "Size": 9580,
            "Type": "PNG image data, 264 x 60, 8-bit/color RGBA, non-interlaced"
        },
        {
            "EntryID": "36@119",
            "Extension": "png",
            "Info": "image/png",
            "MD5": "md5",
            "Name": "image_2.png",
            "SHA1": "4sha1",
            "SHA256": "sha256",
            "SHA512": "sha512",
            "SSDeep": "ssdeep",
            "Size": 9580,
            "Type": "PNG image data, 264 x 60, 8-bit/color RGBA, non-interlaced"
        }]
    expected = [('image_1.png', '35@119'), ('image_2.png', '36@119')]
    assert expected == get_entry_id_list(attachments, files)

FILES = [
    {
        "SHA256": "SHA256"
    },
    {
        "EntryID": "4@131",
        "Extension": "png",
        "Info": "image/png",
        "MD5": "605ebf7bc83a00840a3ea90c8ed56515",
        "Name": "image_1.png",
        "SHA1": "SHA1",
        "SHA256": "SHA256",
        "SHA512": "SHA512",
        "SSDeep": "SSDeep",
        "Size": 127884,
        "Type": "PNG image data, 275 x 184, 8-bit/color RGBA, non-interlaced"
    },
    {
        "SHA256": "SHA256"
    },
    {
        "EntryID": "5@131",
        "Extension": "png",
        "Info": "image/png",
        "MD5": "fe96373473915fea94980b588de5fbb6",
        "Name": "image_1.png",
        "SHA1": "SHA1",
        "SHA256": "SHA256",
        "SHA512": "SHA512",
        "SSDeep": "SSDeep",
        "Size": 107596,
        "Type": "PNG image data, 225 x 224, 8-bit/color RGBA, non-interlaced"
    }
]

def test_main(mocker):
    """
        Given
        - A new incident of type Email Communication
        When
        - An email reply to an existing incident was sent
        Then
        - Return False to drop the newly created incident and attach the relevant data to the existing
        email related incident.
    """
    import PreprocessEmail
    from PreprocessEmail import main
    incident = util_load_json('test_data/get_incident_details_result.json')
    mocker.patch.object(demisto, 'incident', return_value=incident)
    mocker.patch.object(PreprocessEmail, 'get_email_related_incident_id', return_value='123')
    mocker.patch.object(PreprocessEmail, 'get_incident_by_query',
                        return_value=[util_load_json('test_data/email_related_incident_response.json')])
    mocker.patch.object(PreprocessEmail, 'get_attachments_using_instance')
    mocker.patch.object(PreprocessEmail, 'get_incident_related_files', return_value=FILES)
    mocker.patch.object(demisto, 'results')
    main()
    assert not demisto.results.call_args[0][0]

def test_get_email_related_incident_id(mocker):
    """
        Given
        - Multiple incidents with the same identifying code
        When
        - Making a query for all incidents with a certain identifying code but with different subjects
        Then
        - Validate that the correct incident is returned (the one with the matching subject)
    """
    import PreprocessEmail
    from PreprocessEmail import get_email_related_incident_id
    mocker.patch.object(PreprocessEmail, 'get_incident_by_query',
                        return_value=[{'emailsubject': 'subject 1', 'id': '1'},
                                      {'emailsubject': 'subject 2', 'id': '2'}])
    id = get_email_related_incident_id('12345678', 'subject 2')
    assert id == '2'


import json
import concurrent.futures
import os
import sys

def create_schema(file_name):

    file_name_and_ext = os.path.basename(file_name)
    basename = os.path.splitext(file_name_and_ext)[0]

    strings = ['nvarchar',
               'char',
               'nchar',
               'varchar',
               'ntext']

    integers = ['mediumint',
                'smallint',
                'tinyint',
                'int']

    floats = ['decimal',
              'float',
              'real']

    json_schema = []
    with open(file_name, 'r') as sql_file:
        for line in sql_file:
            json_dict = {}
            if 'NULL' in line or 'NOT NULL' in line:
                if 'SET' not in line:
                    line_split = line.strip().split(']')

                    name = line_split[0]

                    if '[' in name:
                        name = name.strip('[')
                        if ']' in name:
                            name = name.strip(']')
                            if '(%)' in name:
                                name = name.strip('%')

                    if len(name.split()) > 1:
                        name = f'{name[0]}_{name[1]}'

                    data_type = line_split[1].split('[')[1]

                    if data_type in integers:
                        data_type = 'integer'
                        json_dict['name'] = name
                        json_dict['type'] = data_type
                        json_schema.append(json_dict)

                    if data_type in strings:
                        data_type = 'string'
                        json_dict['name'] = name
                        json_dict['type'] = data_type
                        json_schema.append(json_dict)

                    if data_type in floats:
                        data_type = 'float'
                        json_dict['name'] = name
                        json_dict['type'] = data_type
                        json_schema.append(json_dict)

                    if data_type == 'bit':
                        data_type = 'boolean'
                        json_dict['name'] = name
                        json_dict['type'] = data_type
                        json_schema.append(json_dict)

    with open(os.path.abspath(f'../P53_Database/P53_data_schema/{basename}.json'), 'w') as out_file:

        json.dump(json_schema,
                   out_file,
                   indent=4
                   )

def main():

    arg = input('Would you like to run this in parallel? (Y/n): ')

    file_names =  [
        '../P53_Database/P53/dbo.FUNCTION_ISHIOKA.Table.sql',
                # '../P53_Database/P53/dbo.AA_change.Table.sql',
                # '../P53_Database/P53/dbo.AA_codes.Table.sql',
                #    '../P53_Database/P53/dbo.GermlineRefView.Table.sql',
                #    '../P53_Database/P53/dbo.GermlineView.Table.sql',
                #    '../P53_Database/P53/dbo.SomaticView.Table.sql'
                ]

    abs_path = [os.path.abspath(a_file) for a_file in file_names]

    # Synchronous
    if arg.lower() == 'n' or arg.lower() == 'no':
        for a_file in abs_path:
            create_schema(a_file)

    # Parallelized
    if arg.lower() == 'y' or arg.lower() == 'yes':
        with concurrent.futures.ProcessPoolExecutor() as executor:
            executor.map(create_schema, abs_path)

if __name__ == '__main__':
    main()


import json
import logging
import os
import sys
from argparse import ArgumentParser

import torch
from nni.algorithms.nas.pytorch.proxylessnas import ProxylessNasTrainer
from torchvision import transforms

import datasets
from model import SearchMobileNet
from nni.algorithms.nas.pytorch.proxylessnas import ProxylessNasTrainer
from putils import LabelSmoothingLoss, accuracy, get_parameters
from retrain import Retrain

logger = logging.getLogger('nni_proxylessnas')

if __name__ == "__main__":
    parser = ArgumentParser("proxylessnas")
    # configurations of the model
    parser.add_argument("--n_cell_stages", default='4,4,4,4,4,1', type=str)
    parser.add_argument("--stride_stages", default='2,2,2,1,2,1', type=str)
    parser.add_argument("--width_stages", default='24,40,80,96,192,320', type=str)
    parser.add_argument("--bn_momentum", default=0.1, type=float)
    parser.add_argument("--bn_eps", default=1e-3, type=float)
    parser.add_argument("--dropout_rate", default=0, type=float)
    parser.add_argument("--no_decay_keys", default='bn', type=str, choices=[None, 'bn', 'bn#bias'])
    # configurations of imagenet dataset
    parser.add_argument("--data_path", default='/data/imagenet/', type=str)
    parser.add_argument("--train_batch_size", default=256, type=int)
    parser.add_argument("--test_batch_size", default=500, type=int)
    parser.add_argument("--n_worker", default=32, type=int)
    parser.add_argument("--resize_scale", default=0.08, type=float)
    parser.add_argument("--distort_color", default='normal', type=str, choices=['normal', 'strong', 'None'])
    # configurations for training mode
    parser.add_argument("--train_mode", default='search', type=str, choices=['search_v1', 'search', 'retrain'])
    # configurations for search
    parser.add_argument("--checkpoint_path", default='./search_mobile_net.pt', type=str)
    parser.add_argument("--arch_path", default='./arch_path.pt', type=str)
    parser.add_argument("--no-warmup", dest='warmup', action='store_false')
    # configurations for retrain
    parser.add_argument("--exported_arch_path", default=None, type=str)

    args = parser.parse_args()
    if args.train_mode == 'retrain' and args.exported_arch_path is None:
        logger.error('When --train_mode is retrain, --exported_arch_path must be specified.')
        sys.exit(-1)

    model = SearchMobileNet(width_stages=[int(i) for i in args.width_stages.split(',')],
                            n_cell_stages=[int(i) for i in args.n_cell_stages.split(',')],
                            stride_stages=[int(i) for i in args.stride_stages.split(',')],
                            n_classes=1000,
                            dropout_rate=args.dropout_rate,
                            bn_param=(args.bn_momentum, args.bn_eps))
    logger.info('SearchMobileNet model create done')
    model.init_model()
    logger.info('SearchMobileNet model init done')

    # move network to GPU if available
    if torch.cuda.is_available():
        device = torch.device('cuda')
    else:
        device = torch.device('cpu')

    logger.info('Creating data provider...')
    data_provider = datasets.ImagenetDataProvider(save_path=args.data_path,
                                                  train_batch_size=args.train_batch_size,
                                                  test_batch_size=args.test_batch_size,
                                                  valid_size=None,
                                                  n_worker=args.n_worker,
                                                  resize_scale=args.resize_scale,
                                                  distort_color=args.distort_color)
    logger.info('Creating data provider done')

    if args.no_decay_keys:
        keys = args.no_decay_keys
        momentum, nesterov = 0.9, True
        optimizer = torch.optim.SGD([
            {'params': get_parameters(model, keys, mode='exclude'), 'weight_decay': 4e-5},
            {'params': get_parameters(model, keys, mode='include'), 'weight_decay': 0},
        ], lr=0.05, momentum=momentum, nesterov=nesterov)
    else:
        optimizer = torch.optim.SGD(get_parameters(model), lr=0.05, momentum=momentum, nesterov=nesterov, weight_decay=4e-5)

    if args.train_mode == 'search':
        from nni.retiarii.trainer.pytorch import ProxylessTrainer
        from torchvision.datasets import ImageNet
        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                         std=[0.229, 0.224, 0.225])
        dataset = ImageNet(args.data_path, transform=transforms.Compose([
            transforms.RandomResizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))
        trainer = ProxylessTrainer(model,
                                   loss=LabelSmoothingLoss(),
                                   dataset=dataset,
                                   optimizer=optimizer,
                                   metrics=lambda output, target: accuracy(output, target, topk=(1, 5,)),
                                   num_epochs=120,
                                   log_frequency=10)
        trainer.fit()
        print('Final architecture:', trainer.export())
        json.dump(trainer.export(), open('checkpoint.json', 'w'))
    elif args.train_mode == 'search_v1':
        # this is architecture search
        logger.info('Creating ProxylessNasTrainer...')
        trainer = ProxylessNasTrainer(model,
                                      model_optim=optimizer,
                                      train_loader=data_provider.train,
                                      valid_loader=data_provider.valid,
                                      device=device,
                                      warmup=args.warmup,
                                      ckpt_path=args.checkpoint_path,
                                      arch_path=args.arch_path)

        logger.info('Start to train with ProxylessNasTrainer...')
        trainer.train()
        logger.info('Training done')
        trainer.export(args.arch_path)
        logger.info('Best architecture exported in %s', args.arch_path)
    elif args.train_mode == 'retrain':
        # this is retrain
        from nni.nas.pytorch.fixed import apply_fixed_architecture
        assert os.path.isfile(args.exported_arch_path), \
            "exported_arch_path {} should be a file.".format(args.exported_arch_path)
        apply_fixed_architecture(model, args.exported_arch_path)
        trainer = Retrain(model, optimizer, device, data_provider, n_epochs=300)
        trainer.run()


import scrapy
from scrapy.http import FormRequest, Request
import csv
import itertools
from scrapy.utils.response import open_in_browser
from urllib.parse import urlencode
from ..items import GoogleShoppingItem
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import itertools

class ShgSpider(scrapy.Spider):
    name = 'shg'
    start_urls = ['https://shopping.google.com/']
    gtin = ''
    sku = ''
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}

    def parse(self, response):
        sku = ['LSN120HEV2 lg']
        with open('/home/nisl/Downloads/Google Shopping SKUs - Sheet1.csv', 'r') as file:
            reader = csv.reader(file)
            for line in itertools.islice(reader, 10):
        # for i in sku:
                ved = response.xpath('//*[@id="kO001e"]/c-wiz/form/@data-ved').extract_first()
                params = {
                    'psb': '1',
                    'tbm': 'shop',
                    'q': str(line[0]) + ' ' + str(line[2]),
                    # 'q': str(line[0]),
                    'hl': 'en-US',
                    'ved': str(ved)
                }
                self.sku = str(line[0]) + ' ' + str(line[2])
                # self.sku = str(i)
                url = "https://www.google.com/search?" + urlencode(params)
                print(url)
                yield scrapy.Request(url, callback=self.parse2, meta={'dont_redirect': True,
                                                                      'handle_httpstatus_list': [302],
                                                                      'sku': str(line[0]) + ' ' + str(line[2])})

    def parse2(self, response):
        url = response.xpath('/html/body/div[6]/div[2]/div/a/@href').extract_first()
        if not url:
            url = response.xpath('/html/body/div[5]/div[2]/div/a/@href').extract_first()
        print(url)
        if url:
            link = 'https://www.google.com' + str(url)
            yield response.follow(link, callback=self.final_data, meta={'link': link, 'sku': response.meta['sku']})

    def final_data(self, response):
        item = GoogleShoppingItem()

        driver = webdriver.Chrome(executable_path='/home/nisl/WORKING_DIRECTORY/New folder/chromedriver')
        driver.get(response.meta['link'])
        time.sleep(8)
        try:
            gtin = driver.find_element(By.XPATH, '//*[@id="specs"]/div[2]/table/tbody/tr[4]/td[2]').text
            title = driver.find_element(By.XPATH, '//*[@id="sg-product__pdp-container"]/div/div[3]/div[1]/span').text
            google_part_numbers = driver.find_element(By.XPATH, '//*[@id="specs"]/div[2]/table/tbody/tr[3]/td[2]').text
            google_brand_name = driver.find_element(By.XPATH, '//*[@id="specs"]/div[2]/table/tbody/tr[2]/td[2]').text
            brand = driver.find_element(By.XPATH, '//*[@id="specs"]/div[2]/table/tbody/tr[2]/td[2]').text
            page_url = driver.current_url

            item['gtin'] = gtin
            item['title'] = title
            item['google_part_numbers'] = google_part_numbers
            item['google_brand_name'] = google_brand_name
            item['brand'] = brand
            item['page_url'] = page_url
            item['sku'] = response.meta['sku']
            yield item
        except Exception as e:
            gtin = ''

        all_data = response.xpath('//*[@id="online"]/div')
        i = 1
        for data in all_data:
            i = i + 1
            sold_by = data.xpath('./div[2]/a/text()').extract_first()
            item_price = data.xpath('./div[1]/div[1]/b/text()').extract_first()
            shipping_price = data.xpath('./div[1]/div[3]/text()').extract_first()
            tax_price = data.xpath('./div[1]/div[2]/text()').extract_first()
            url = data.xpath('./div[2]/a/@href').extract_first()
            if item_price:
                if '$' in item_price:
                    item_price = item_price.replace('$', '')
            if tax_price:
                if "+$ tax" in tax_price:
                    tax_price = str(tax_price).replace('+$', '')
                    tax_price = str(tax_price).replace('tax', '')
            # next_page = data.xpath('./div[1]/div[2]/a/text()').extract_first()

            # if next_page == "Next >":
            #     next_page_url = data.xpath('./div[1]/div[2]/a/@href').extract_first()
            #     link = 'https://www.google.com' + str(next_page_url)
            #     yield response.follow(link, callback=self.final_data, meta={'link': response.meta['link'],
            #                                                                 'sku': response.meta['sku']})
            if sold_by:
                item['sold_by'] = sold_by
                item['item_price'] = item_price
                if "Free shipping" in shipping_price:
                    item['shipping_price'] = 0.0
                else:
                    item['shipping_price'] = shipping_price
                item['tax_price'] = tax_price
                item['url'] = url

                yield item


from django.shortcuts import render
from rest_framework.views import APIView
from rest_framework import generics , status
from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.core.serializers import serialize
from django.http import JsonResponse

from aymenProject.pagination import PageNumberPaginationDataOnly
from .serializers import *
from django.contrib.auth import authenticate
from rest_framework_simplejwt.tokens import RefreshToken
from rest_framework.permissions import IsAuthenticated
from rest_framework.parsers import MultiPartParser
import random
import string
import json

class CreateFormView(generics.ListCreateAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = FormSerializer
    pagination_class = PageNumberPaginationDataOnly
    def get_queryset(self):
        query_set = Form.objects.all()
        return query_set

    def post(self,request):
        code = ''.join(random.choice(string.ascii_letters + string.digits) for x in range(30))
        choices = Choices(choice = "Option 1")
        choices.save()
        question = Questions(question_type = "multiple choice", question= "Untitled Question", required= False)
        question.save()
        question.choices.add(choices)
        category = Category.objects.get(pk=request.data["category"])
        question.save()
        form = Form(code = code, title = request.data['title'], creator=request.user)
        form.save()
        form.questions.add(question)
        form.category = category
        form.save()
        return Response(FormSerializer(form).data,status=status.HTTP_201_CREATED)

def get_client_ip(request):
    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_forwarded_for:
        ip = x_forwarded_for.split(',')[0]
    else:
        ip = request.META.get('REMOTE_ADDR')
    return ip

class SubmitFormView(APIView):
    permission_classes = (IsAuthenticated,)
    def post(self,request,code):
        answers = json.loads(request.body.decode('utf-8'))
        for a in answers:
            print(a,answers[a],dir(answers[a]))
        try:
            form = Form.objects.get(code = code)
            code = ''.join(random.choice(string.ascii_letters + string.digits) for x in range(20))
            if form.authenticated_responder:
                response = Responses(response_code = code, response_to = form, responder_ip = get_client_ip(request), responder = request.user)
                response.save()
            else:
                if not form.collect_email:
                    response = Responses(response_code = code, response_to = form, responder_ip = get_client_ip(request))
                    response.save()
                else:
                    response = Responses(response_code = code, response_to = form, responder_ip = get_client_ip(request), responder_email=request.POST["email-address"])
                    response.save()
            for i in answers:
                print('******',i)
                #Excluding csrf token
                if i == "csrfmiddlewaretoken" or i == "email-address":
                    continue
                question = form.questions.get(id = i)
                for j in answers[i]:
                    answer = Answer(answer=j, answer_to = question)
                    answer.save()
                    response.response.add(answer)
                    response.save()
            return Response(FormSerializer(form).data,status=status.HTTP_200_OK)

        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

    # def get(self,request,code):
    #     try:
    #         form = Form.objects.get(code = code)
    #         return Response(data=FormSerializer(form).data)
    #     except Form.DoesNotExist:
    #         return Response(status=status.HTTP_404_NOT_FOUND)
class ResponsesFormView(APIView):
    permission_classes = (IsAuthenticated,)
    def get(self,request,code):
        try:
            form = Form.objects.get(code = code)
            responsesSummary = []
            choiceAnswered = {}
            filteredResponsesSummary = {}
            for question in form.questions.all():
                answers = Answer.objects.filter(answer_to = question.id)
                if question.question_type == "multiple choice" or question.question_type == "checkbox":
                    choiceAnswered[question.question] = choiceAnswered.get(question.question, {})
                    for answer in answers:
                        choice = answer.answer_to.choices.get(choice = answer.answer).choice
                        choiceAnswered[question.question][choice] = choiceAnswered.get(question.question, {}).get(choice, 0) + 1
                responsesSummary.append({"question": QuestionSerilizer(question).data, "answers":json.loads(serialize("json",answers)) })
            for answr in choiceAnswered:
                filteredResponsesSummary[answr] = {}
                keys = choiceAnswered[answr].values()
                for choice in choiceAnswered[answr]:
                    filteredResponsesSummary[answr][choice] = choiceAnswered[answr][choice]
            resp = Responses.objects.filter(response_to = form)
            return Response(data={
                "form": FormSerializer(form).data,
                "responses": json.loads(serialize("json",resp)),
                "responsesSummary": responsesSummary,
                "filteredResponsesSummary": filteredResponsesSummary
            },status=status.HTTP_201_CREATED)
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

class ResponseFormView(APIView):
    permission_classes = (IsAuthenticated,)
    def get(self,request,code,response_code):
        try:
            formInfo = Form.objects.get(code = code)
            responseInfo = Responses.objects.filter(response_code = response_code)
            total_score = 0
            score = 0
            if responseInfo.count() == 0:
                return Response(status=status.HTTP_404_NOT_FOUND)
            else: responseInfo = responseInfo[0]
            if formInfo.is_quiz:
                for i in formInfo.questions.all():
                    total_score += i.score
                for i in responseInfo.response.all():
                    if i.answer_to.question_type == "short" or i.answer_to.question_type == "paragraph":
                        if i.answer == i.answer_to.answer_key: score += i.answer_to.score
                    elif i.answer_to.question_type == "multiple choice":
                        answerKey = None
                        for j in i.answer_to.choices.all():
                            if j.is_answer: answerKey = j.id
                        if answerKey is not None and int(answerKey) == int(i.answer):
                            score += i.answer_to.score
                _temp = []
                for i in responseInfo.response.all():
                    if i.answer_to.question_type == "checkbox" and i.answer_to.pk not in _temp:
                        answers = []
                        answer_keys = []
                        for j in responseInfo.response.filter(answer_to__pk = i.answer_to.pk):
                            answers.append(int(j.answer))
                            for k in j.answer_to.choices.all():
                                if k.is_answer and k.pk not in answer_keys: answer_keys.append(k.pk)
                            _temp.append(i.answer_to.pk)
                        if answers == answer_keys: score += i.answer_to.score
            return Response(data= {
            "form": FormSerializer(formInfo).data,
            "response": ResponseSerilizer(responseInfo).data,
            "score": score,
            "total_score": total_score
    },status=status.HTTP_200_OK)
        except Responses.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

class DeleteFormView(generics.DestroyAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = FormSerializer
    def delete(self,request,code):
        try:
            form = Form.objects.get(code = code)
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
        operation = form.delete()
        data = {}
        if operation:
            data["sucess"] = "deleted sucessfully"
        else:
            data["failure"] = "delete failed"
        return Response(data=data)

class UpdateFormView(APIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = FormSerializer
    def get(self,request,code):
        try:
            form = Form.objects.get(code = code)
            return Response(data=FormSerializer(form).data)
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
    def patch(self,request,code):
        try:
            form = Form.objects.get(code = code)
            print(form.title)
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
        data=request.data.copy()
        data['code'] = form.code
        data['creator'] = form.creator.pk
        data['category'] = request.data.get('category',form.category)
        data['questions'] = request.data.get('questions', form.questions.values_list(flat=True))
        data['title'] = request.data.get('title',form.title)
        serializer = FormSerializer(form,data=data)
        data = {}
        if serializer.is_valid():
            serializer.save()
            return Response(data=serializer.data)
        return Response(serializer._errors,status=status.HTTP_400_BAD_REQUEST)

class CreateQuestionView(generics.ListCreateAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = QuestionSerilizer
    def get_queryset(self):
        return Questions.objects.all()
    def post(self,request,code):
        data=request.data.copy()
        form = Form.objects.get(code=code)
        choices = Choices(choice = "Option 1")
        choices.save()
        question = Questions(question_type = "multiple choice", question= data['title'], required= data['is_manadatory'])
        question.save()
        question.choices.add(choices)
        question.save()
        form.questions.add(question)
        form.save()
        return Response(FormSerializer(form).data,status=status.HTTP_201_CREATED)

class CreateChoiceView(APIView):
    def post(self,request,code):
        question = Questions.objects.get(id=code)
        data=request.data.copy()

        choices = Choices(choice = data['title'])
        choices.save()
        question.choices.add(choices)
        question.save()
        return Response(QuestionChoice(question).data,status=status.HTTP_201_CREATED)

class UpdateQuestionView(APIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = QuestionSerilizer
    def get(self,request,code):
        try:
            question = Questions.objects.get(id = code)
            return Response(data={"question":QuestionChoice(question).data})
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

    def patch(self,request,code):
        try:
            question = Questions.objects.get(pk = code)
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
        print(request.data)
        data=request.data.copy()
        data['question'] = request.data.get('question',question.question)
        data['question_type'] = request.data.get('question_type',question.question_type)
        data['feedback'] = request.data.get('feedback',question.feedback)
        data['answer_key'] = request.data.get('answer_key',question.answer_key)
        data['choices'] = request.data.get('choices',question.choices.values_list(flat=True))

        serializer = QuestionSerilizer(question,data=data)
        data = {}
        if serializer.is_valid():
            serializer.save()
            data["success"] = "updated sucessfully"
            return Response(data=data)
        return Response(serializer._errors,status=status.HTTP_400_BAD_REQUEST)

class DeleteQuestionView(generics.DestroyAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = QuestionSerilizer
    def delete(self,request,code):
        try:
            question = Questions.objects.get(pk = code)
        except Form.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
        operation = question.delete()
        data = {}
        if operation:
            data["sucess"] = "deleted sucessfully"
        else:
            data["failure"] = "delete failed"
        return Response(data=data)

class DeleteChoiceView(generics.DestroyAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = ChoiceSerilizer
    def delete(self,request,code):
        try:
            choice = Choices.objects.get(pk = code)
        except Choices.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
        operation = choice.delete()
        data = {}
        if operation:
            data["sucess"] = "deleted sucessfully"
        else:
            data["failure"] = "delete failed"
        return Response(data=data)

class FormQuestionsView(APIView):
    permission_classes = (IsAuthenticated,)
    pagination_class = PageNumberPaginationDataOnly
    def get(self,request,code):
        query_set = Form.objects.get(code=code)
        query = Questions.objects.filter(id__in=query_set.questions.values_list('id',flat=True))
        serializer = QuestionChoice(query,many=True)
        result = serializer.data
        data =  {
            'questions' : result
        }
        return Response(data=data)


"""BERT and RNN model for sentence pair classification.

Author: Yixu GAO (yxgao19@fudan.edu.cn)

Used for SMP-CAIL2020-Argmine.
"""
import torch

from torch import nn
from torch.nn.utils.rnn import pack_padded_sequence, PackedSequence
from transformers.modeling_bert import BertModel

class BertForClassification(nn.Module):
    """BERT with simple linear model."""
    def __init__(self, config):
        """Initialize the model with config dict.

        Args:
            config: python dict must contains the attributes below:
                config.bert_model_path: pretrained model path or model type
                    e.g. 'bert-base-chinese'
                config.hidden_size: The same as BERT model, usually 768
                config.num_classes: int, e.g. 2
                config.dropout: float between 0 and 1
        """
        super().__init__()
        self.bert = BertModel.from_pretrained(config.bert_model_path)
        for param in self.bert.parameters():
            param.requires_grad = True
        self.linear = nn.Linear(config.hidden_size, config.num_classes)
        self.dropout = nn.Dropout(config.dropout)
        self.num_classes = config.num_classes

    def forward(self, input_ids, attention_mask, token_type_ids):
        """Forward inputs and get logits.

        Args:
            input_ids: (batch_size, max_seq_len)
            attention_mask: (batch_size, max_seq_len)
            token_type_ids: (batch_size, max_seq_len)

        Returns:
            logits: (batch_size, num_classes)
        """
        batch_size = input_ids.shape[0]
        bert_output = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            encoder_hidden_states=False)
        # bert_output[0]: (batch_size, sequence_length, hidden_size)
        # bert_output[1]: (batch_size, hidden_size)
        pooled_output = bert_output[1]
        pooled_output = self.dropout(pooled_output)
        logits = self.linear(pooled_output).view(batch_size, self.num_classes)
        logits = nn.functional.softmax(logits, dim=-1)
        # logits: (batch_size, num_classes)
        return logits

class RnnForSentencePairClassification(nn.Module):
    """Unidirectional GRU model for sentences pair classification.
    2 sentences use the same encoder and concat to a linear model.
    """
    def __init__(self, config):
        """Initialize the model with config dict.

        Args:
            config: python dict must contains the attributes below:
                config.vocab_size: vocab size
                config.hidden_size: RNN hidden size and embedding dim
                config.num_classes: int, e.g. 2
                config.dropout: float between 0 and 1
        """
        super().__init__()
        self.embedding = nn.Embedding(
            config.vocab_size, config.hidden_size, padding_idx=0)
        self.rnn = nn.GRU(
            config.hidden_size, hidden_size=config.hidden_size,
            bidirectional=False, batch_first=True)
        self.linear = nn.Linear(config.hidden_size * 2, config.num_classes)
        self.dropout = nn.Dropout(config.dropout)
        self.num_classes = config.num_classes

    def forward(self, s1_ids, s2_ids, s1_lengths, s2_lengths):
        """Forward inputs and get logits.

        Args:
            s1_ids: (batch_size, max_seq_len)
            s2_ids: (batch_size, max_seq_len)
            s1_lengths: (batch_size)
            s2_lengths: (batch_size)

        Returns:
            logits: (batch_size, num_classes)
        """
        batch_size = s1_ids.shape[0]
        # ids: (batch_size, max_seq_len)
        s1_embed = self.embedding(s1_ids)
        s2_embed = self.embedding(s2_ids)
        # embed: (batch_size, max_seq_len, hidden_size)
        s1_packed: PackedSequence = pack_padded_sequence(
            s1_embed, s1_lengths, batch_first=True, enforce_sorted=False)
        s2_packed: PackedSequence = pack_padded_sequence(
            s2_embed, s2_lengths, batch_first=True, enforce_sorted=False)
        # packed: (sum(lengths), hidden_size)
        self.rnn.flatten_parameters()
        _, s1_hidden = self.rnn(s1_packed)
        _, s2_hidden = self.rnn(s2_packed)
        s1_hidden = s1_hidden.view(batch_size, -1)
        s2_hidden = s2_hidden.view(batch_size, -1)
        hidden = torch.cat([s1_hidden, s2_hidden], dim=-1)
        hidden = self.linear(hidden).view(-1, self.num_classes)
        hidden = self.dropout(hidden)
        logits = nn.functional.softmax(hidden, dim=-1)
        # logits: (batch_size, num_classes)
        return logits


import pygame
import random

# 初始化
pygame.init()

# 創建視窗
WIDTH = 1000
HIGH = 1000
screen = pygame.display.set_mode((WIDTH, HIGH))

# 標題
pygame.display.set_caption('First Game By York')

# 時間管理
clock = pygame.time.Clock()

# 遊戲幀數
FPS = 60

# 各類顏色
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
ORANGE = (255, 97, 0)
Green = (255, 0, 255)
# 字體載入
font_name = pygame.font.match_font('arial')

def draw_text(surf, text, size, x, y):
    font = pygame.font.Font(font_name, size)
    text_surface = font.render(text, True, BLACK)
    text_rect = text_surface.get_rect()
    text_rect.centerx = x
    text_rect.top = y
    surf.blit(text_surface, text_rect)

def draw_health(surf, hp, x, y):
    if hp < 0:
        hp = 0
    BAR_LENGTH = 100
    BAR_HIGHT = 10
    fill = (hp/100)*BAR_LENGTH
    outline_recy = pygame.Rect(x, y, BAR_LENGTH, BAR_HIGHT)
    fill_rect = pygame.Rect(x, y, fill, BAR_HIGHT)
    pygame.draw.rect(surf, Green, fill_rect)
    pygame.draw.rect(surf, Green, outline_recy, 2)

# 物件
class Player(pygame.sprite.Sprite):
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.Surface((10, 10))
        self.image.fill(BLACK)
        self.rect = self.image.get_rect()
        self.rect.centerx = WIDTH / 2
        self.rect.bottom = HIGH - 10
        self.speed = 10
        self.helf = 100

    def update(self):
        key_pressed = pygame.key.get_pressed()
        if key_pressed[pygame.K_d]:
            self.rect.x += self.speed
        if key_pressed[pygame.K_a]:
            self.rect.x -= self.speed
        if key_pressed[pygame.K_w]:
            self.rect.y -= self.speed
        if key_pressed[pygame.K_s]:
            self.rect.y += self.speed

        if self.rect.right > WIDTH:
            self.rect.right = WIDTH
        if self.rect.left < 0:
            self.rect.left = 0
        if self.rect.top < 0:
            self.rect.top = 0
        if self.rect.bottom > HIGH:
            self.rect.bottom = HIGH

    def shoot(self):
        bullet = Bullet(self.rect.centerx, self.rect.top)
        all_sprites.add((bullet))
        bullets.add(bullet)

class Rock(pygame.sprite.Sprite):
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)
        self.n = 0
        self.width = 30
        self.high = 30
        self.image = pygame.Surface((self.width, self.high))
        self.image.fill(RED)
        self.rect = self.image.get_rect()
        self.rect.x = random.randrange(0, WIDTH - self.width)
        self.rect.y = random.randrange(-100, -40)
        self.speedy = random.randrange(2, 10)
        self.speedx = random.randrange(-2, 2)

    def update(self):
        self.rect.y += self.speedy
        self.rect.x += self.speedx
        if self.rect.top > HIGH:
            self.rect.x = random.randrange(0, WIDTH - self.width)
            self.rect.y = random.randrange(-100, -40)
            self.speedy = random.randrange(2, 10)
            self.speedx = random.randrange(-2, 2)

class Bullet(pygame.sprite.Sprite):
    def __init__(self, x, y):
        pygame.sprite.Sprite.__init__(self)
        self.n = 0
        self.width = 10
        self.high = 30
        self.image = pygame.Surface((self.width, self.high))
        self.image.fill(ORANGE)
        self.rect = self.image.get_rect()
        self.rect.centerx = x
        self.rect.bottom = y
        self.speed = -10

    def update(self):
        self.rect.y += self.speed
        if self.rect.bottom < 0:
            self.kill()

# 物件群組，所有物件聽命於此
all_sprites = pygame.sprite.Group()
rocks = pygame.sprite.Group()
bullets = pygame.sprite.Group()
players = pygame.sprite.Group()
player = Player()
all_sprites.add(player)
players.add(player)
for i in range(20):
    rock = Rock()
    all_sprites.add(rock)
    rocks.add(rock)

# 遊戲狀態
running = True
# 遊戲迴圈

score = 0

while running:
    clock.tick(FPS)  # 遊戲FPS
    # 取得輸入
    for event in pygame.event.get():  # 如果點擊右上角的叉叉則結束遊戲
        if event.type == pygame.QUIT:
            running = False
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_SPACE:
                player.shoot()
    # 更新遊戲
    all_sprites.update()
    hits = pygame.sprite.groupcollide(rocks, bullets, True, True)
    for hit in hits:
        score += 100
        r = Rock()
        all_sprites.add(r)
        rocks.add(r)

    hits = pygame.sprite.groupcollide(players, rocks, False, True)
    for hit in hits:
        player.helf -= 10
        if player.helf <= 0:
            running = False

    # 畫面顯示
    screen.fill(WHITE)
    all_sprites.draw(screen)
    draw_text(screen, str(score), 18, WIDTH / 2, 10)
    draw_health(screen, player.helf, 5, 10)
    pygame.display.update()

# 遊戲結束
pygame.quit()


from keras import layers as KLayers
from keras import models as KModels
from keras import optimizers as KOpts
from keras_datareaders import ClassificationReader as reader
from keras_callbacks import ProgressBarCallback as bar
import numpy as np
import pickle
from matplotlib import pyplot as plt

def builder_generator(randomDim):
    model = KModels.Sequential()

    model.add(KLayers.Dense(512*4*4,input_dim=randomDim)) # ?,100 => ?,512*4*4
    model.add(KLayers.Activation("relu"))
    model.add(KLayers.Reshape((4,4,512)))
    model.add(KLayers.BatchNormalization(momentum=0.8))
    model.add(KLayers.Dropout(rate=0.8))

    model.add(KLayers.UpSampling2D()) # ?,4,4,512 => ?,8,8,512

    model.add(KLayers.Conv2D(256, kernel_size=3, padding='same')) #?,8,8,512 => ?,8,8,256
    model.add(KLayers.Activation("relu"))
    model.add(KLayers.BatchNormalization(momentum=0.8))
    model.add(KLayers.Dropout(rate=0.8))

    model.add(KLayers.UpSampling2D())  #?,8,8,256 => ?,16,16,256

    model.add((KLayers.Conv2D(128, kernel_size=3, padding='same'))) #?,16,16,256 => ?,16,16,128
    model.add(KLayers.Activation("relu"))
    model.add(KLayers.BatchNormalization(momentum=0.8))
    model.add(KLayers.Dropout(rate=0.8))

    model.add(KLayers.UpSampling2D()) #?,16,16,128 => ?,32,32,128

    model.add(KLayers.Conv2D(3, kernel_size=3, padding='same')) #?,32,32,128 => ?,32,32,3
    model.add(KLayers.Activation("tanh"))

    return model

def builder_discriminator():
    model = KModels.Sequential()

    model.add(KLayers.Conv2D(128, kernel_size=3, strides=2, padding='same', input_shape=(32,32,3))) #?,32,32,3 => ?,16,16,128
    model.add(KLayers.LeakyReLU(alpha=0.2))
    model.add(KLayers.Dropout(rate=0.8))

    model.add(KLayers.Conv2D(256, kernel_size=3, strides=2, padding='same')) #?,16,16,128 => ?,8,8,256
    model.add(KLayers.LeakyReLU(alpha=0.2))
    model.add(KLayers.BatchNormalization(momentum=0.8))
    model.add(KLayers.Dropout(rate=0.8))

    model.add(KLayers.Conv2D(512, kernel_size=3, strides=2, padding='same')) #?,8,8,256 => ?,4,4,512
    model.add(KLayers.LeakyReLU(alpha=0.2))
    model.add(KLayers.BatchNormalization(momentum=0.8))
    model.add(KLayers.Dropout(rate=0.8))

    model.add(KLayers.Flatten())
    model.add(KLayers.Dense(1, activation="sigmoid"))

    return model

PHRASE = "TRAIN"

if PHRASE == "TRAIN":
    adam = KOpts.Adam(lr=0.0002, beta_1=0.5)
    reader = reader.ClassificationReader()
    imageList, labelList = reader.readData()

    discriminator = builder_discriminator()
    discriminator.compile(optimizer=adam, loss='binary_crossentropy')
    discriminator.trainable = False

    generator = builder_generator(100)
    ganInput = KLayers.Input(shape=(100,))
    ganOutput = discriminator(generator(ganInput))
    dcgan = KModels.Model(inputs=ganInput, outputs=ganOutput)
    dcgan.compile(loss='binary_crossentropy', optimizer=adam)
    dLosses = []
    gLosses = []
    batchSize = 32
    epochs = 20

    for epoch in range(1, (epochs + 1)):
        batchCount = imageList.shape[0] // batchSize
        print('Epochs:', epochs)
        print('Batch size:', batchSize)
        print('Batches per epoch:', batchCount)
        progBar = bar.ProgressBarGAN(epochs, batchCount, "D Loss:%.3f,G Loss:%.3f")
        samples_image = []

        for _ in range(batchCount):
            noise = np.random.normal(0, 1, size=[batchSize, 100])
            imageBatch = imageList[np.random.randint(0, imageList.shape[0], size=batchSize)]

            # Generate fake cifar images
            generatedImages = generator.predict(noise)
            # print np.shape(imageBatch), np.shape(generatedImages)
            discriminator.trainable = True

            d_loss_real = discriminator.train_on_batch(imageBatch, np.ones(batchSize))
            d_loss_fake = discriminator.train_on_batch(generatedImages, np.zeros(batchSize))
            dloss = np.add(d_loss_fake,d_loss_real) * 0.5
            # dloss = discriminator.train_on_batch(X, yDis)

            # Train generator
            noise = np.random.normal(0, 1, size=[batchSize, 100])
            yGen = np.ones(batchSize)
            discriminator.trainable = False
            gloss = dcgan.train_on_batch(noise, yGen)
            progBar.show(dloss, gloss)

        dLosses.append(dloss)
        gLosses.append(gloss)
        if epoch == 1 or epoch % 5 == 0:
            noise = np.random.normal(0, 1, size=[10, 100])
            generatedImages = generator.predict(noise)
            generatedImages = generatedImages.reshape(10, 32, 32,3)
            samples_image.append(generatedImages)
    with open('train_samples.pkl', 'wb') as f:
        pickle.dump(samples_image, f)
else:
    with open('train_samples.pkl', 'rb') as f:
        samples = pickle.load(f)
    # view_samples(-1, samples)

    show_imgs = samples

    # 指定图片形状
    rows, cols = len(show_imgs), 10
    fig, axes = plt.subplots(figsize=(30,12), nrows=rows, ncols=cols, sharex=True, sharey=True)

    for sample, ax_row in zip(show_imgs, axes):
        for img, ax in zip(sample[::int(len(sample)/cols)], ax_row):
            ax.imshow(img.reshape((32,32)), cmap='Greys_r')
            ax.xaxis.set_visible(False)
            ax.yaxis.set_visible(False)

    plt.show()


import torch
import numpy as np
from skimage import color

class HammingCoder():
    '''encodes and decodes hamming code
    I/O with tensors'''

    def __init__(self, total_bits=31, device=0):
        '''initializes encode/decode matrix'''
        self._total_bits = total_bits
        parity_bits = np.log2(total_bits+1)
        assert int(parity_bits) - parity_bits == 0, 'total_bits should be 2^n-1 form'
        self._parity_bits = int(parity_bits)
        self._data_bits = self._total_bits - self._parity_bits
        P = self._gen_P()
        self._G = torch.Tensor(np.hstack([np.eye(self._data_bits), P.T]).T).to(device)
        self._H = torch.Tensor(np.hstack([P, np.eye(self._parity_bits)])).to(device)

    def _gen_P(self):
        '''internal function to make parity-checking matrix and error-bit lookup table'''
        pre_p = np.zeros((self._parity_bits, self._total_bits+1)) #(5, 32)
        rem_cols = [0]
        for r_idx in range(self._parity_bits):
            block_size = 2**r_idx
            for c_idx in range(self._total_bits+1):
                pre_p[r_idx][c_idx] = int(c_idx%(2*block_size) >= block_size)
            rem_cols.append(block_size)
        P = np.delete(pre_p, rem_cols, axis=1)
        # not so pretty solution to finding error bit
        self._decode_lookup = [e for e in range(self._total_bits)
                               if e not in rem_cols] + rem_cols
        self._decode_lookup = [(e-1)%self._total_bits+1 for e in self._decode_lookup]
        self._decode_lookup = {e:i for i, e in enumerate(self._decode_lookup)}
        return P

    def encode(self, msg):
        '''wants to be fed floatTensors'''
        enc = torch.mv(self._G, msg)
        enc = enc%2
        return enc

    def decode(self, enc):
        '''wants to be fed byteTensors; returns floatTensor'''
        enc = enc.float()
        syndrome = torch.mv(self._H, enc)
        syndrome = syndrome % 2
        if torch.sum(syndrome) != 0:
            # error - performs single-bit correction
            syndrome = syndrome
            pre_error_bit = sum([v.item()*(2**i) for i, v in enumerate(syndrome)])
            error_bit = self._decode_lookup[pre_error_bit]
            if error_bit < self._data_bits:
                # worst case where error in data; otherwise on parity bit, ignore
                enc[error_bit] = 1-enc[error_bit]
        return enc[:self._data_bits]

def LAB_L2_dist(img1, img2, mode='max'):
    '''takes two rgb images and calculates the LAB L2 distance.
    Uses the CIE76 formula for perceptual distance calculation, which might
    lead to underestimations for saturated colors. In the CIE76 formula, color
    distance of 2.3 corresponds to a "just noticiable difference".

    INPUT
    img1, img2: assumed to be 3-dim tensors of the form (3, *, *)
    normalized to (-1, 1) range for convenience; please take note!
    mode: {"mean", "max"}, optional, the type of l2 dist aggregation desired.'''
    def _normalize(img):
        return ((img + 1) * 127.5).cpu().numpy().astype('uint8').transpose(1, 2, 0)

    img1 = _normalize(img1)
    img2 = _normalize(img2)
    lab_img1 = color.rgb2lab(img1)
    lab_img2 = color.rgb2lab(img2)
    deltaE_76 = np.sqrt(np.sum((lab_img1 - lab_img2)**2, axis=2))
    if mode == 'mean':
        return np.mean(deltaE_76)
    elif mode == 'max':
        return np.max(deltaE_76)
    else:
        raise ValueError(f'unrecognized mode {mode}')


#
# usage: python k71.py {file name}
#

import sys
from collections import Counter

def stopwords(sl, fr):
    cn = Counter([w for s in sl for w in list(set(s))])
    st = len(sl) * fr / 100
    return [k for k, v in cn.items() if v > st]

def is_stopword(w, ss):
    return w in ss

if __name__ == '__main__':
    fn = sys.argv[1]

    sl = [l.split()[1:] for l in open(fn)]
    ss = stopwords(sl, 10)

    for w in ['is', 'a', 'the', 'happy', 'sad', 'cry']:
        print(w, is_stopword(w, ss))


from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary

caps = webdriver.DesiredCapabilities().FIREFOX
caps["marionette"] = True

binary = FirefoxBinary(r'D:\tool\Fire\firefox.exe')

fp = webdriver.FirefoxProfile()
fp.set_preference("permissions.default.image",2)
fp.set_preference("javascript.enabled", False)

driver = webdriver.Firefox(firefox_binary=binary, firefox_profile=fp, capabilities=caps)
driver.get("http://you.ctrip.com/travels/shanghai2.html")


from django.shortcuts import render, get_object_or_404
from django.db.models.functions import Coalesce
from django.conf import settings
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.db.models import Q
from ustmenu.models import UstMenu
from .models import Urun
from kategori.models import Kategori, AltKategori
from marka.models import Marka

# Create your views here.
def magaza(request, takma_kategori_adi=None, takma_alt_kategori_adi=None):
    ust_menu = UstMenu.objects.all().order_by('menu_adi')
    ust_menu_alt_kategoriler = AltKategori.objects.all().filter(ust_kategori__in=ust_menu.values_list('menu_adi'))

    kategori_adi = None

    if takma_kategori_adi is not None and takma_alt_kategori_adi is None:
        kategoriler = get_object_or_404(Kategori, takma_kategori_adi=takma_kategori_adi)
        urunler = Urun.objects.all().filter(kategori=kategoriler, mevcut=True)
        kategori_adi = kategoriler.kategori_adi
    elif takma_kategori_adi is not None and takma_alt_kategori_adi is not None:
        kategoriler = get_object_or_404(Kategori, takma_kategori_adi=takma_kategori_adi)
        alt_kategoriler = get_object_or_404(AltKategori, takma_alt_kategori_adi=takma_alt_kategori_adi)
        urunler = Urun.objects.all().filter(kategori=kategoriler, alt_kategori=alt_kategoriler, mevcut=True)
        kategori_adi = kategoriler.kategori_adi + ' / ' + alt_kategoriler.alt_kategori_adi
    else:
        urunler = Urun.objects.all().filter(mevcut=True)

    arama = request.GET.get('arama')

    if arama:
        urunler = urunler.all().filter(Q(urun_adi__icontains=arama) | Q(etiketler__icontains=arama)
                                       | Q(stok_kodu__icontains=arama) | Q(kategori__kategori_adi__icontains=arama)
                                       | Q(alt_kategori__alt_kategori_adi__icontains=arama))

    siralama = request.GET.get('sirala', default='maxtarih')

    if siralama == 'maxtarih':
        urunler = urunler.all().order_by('-eklenme_tarihi')
    elif siralama == 'mintarih':
        urunler = urunler.all().order_by('eklenme_tarihi')
    elif siralama == 'maxfiyat':
        urunler = urunler.all().annotate(son_fiyat=Coalesce('indirimli_fiyat', 'fiyat')).order_by('-son_fiyat')
    elif siralama == 'minfiyat':
        urunler = urunler.all().annotate(son_fiyat=Coalesce('indirimli_fiyat', 'fiyat')).order_by('son_fiyat')

    urun_sayisi = urunler.count()

    paginator = Paginator(urunler, 9)
    sayfa = request.GET.get('sayfa')
    sayfa_urunleri = paginator.get_page(sayfa)

    icerik = {
        'ust_menu': ust_menu,
        'ust_menu_alt_kategoriler': ust_menu_alt_kategoriler,
        'urunler': sayfa_urunleri,
        'urun_sayisi': urun_sayisi,
        'siralama': siralama,
        'kategori_adi': kategori_adi,
        'arama': arama
    }
    icerik.update(settings.GLOBAL_SETTINGS)
    return render(request, 'magaza/magaza.html', icerik)

def urun_detayi(request, takma_kategori_adi=None, takma_alt_kategori_adi=None, takma_urun_adi=None):
    ust_menu = UstMenu.objects.all().order_by('menu_adi')
    ust_menu_alt_kategoriler = AltKategori.objects.all().filter(ust_kategori__in=ust_menu.values_list('menu_adi'))

    try:
        urun = Urun.objects.get(kategori__takma_kategori_adi=takma_kategori_adi, alt_kategori__takma_alt_kategori_adi=takma_alt_kategori_adi, takma_urun_adi=takma_urun_adi)
    except Exception as e:
        raise e

    icerik = {
        'ust_menu': ust_menu,
        'ust_menu_alt_kategoriler': ust_menu_alt_kategoriler,
        'takma_kategori_adi': takma_kategori_adi,
        'takma_alt_kategori_adi': takma_alt_kategori_adi,
        'urun': urun
    }
    return render(request, 'magaza/urun_detayi.html', icerik)


import pickle
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from model import *
from prep_data import *
import sys

#### preparing dataset
with open("../data/MNIST.pkl", 'rb') as file_handle:
    MNIST = pickle.load(file_handle)

data = MNIST_Dataset(MNIST['train_image'])
batch_size = 512 #### データセットを512ずつにわける
data_loader = DataLoader(data,
                         batch_size = batch_size,
                         shuffle = True)

#### build a TD-VAE model
input_size = 784 #### inputの大きさ
processed_x_size = 784 #### 入力xの大きさ
belief_state_size = 50 #### 信念状態の大きさ
state_size = 8 #### 状態は全部で8
tdvae = TD_VAE(input_size, processed_x_size, belief_state_size, state_size)
tdvae = tdvae.cuda() #### GPU使用

#### training
optimizer = optim.Adam(tdvae.parameters(), lr = 0.0005) #### 学習率の設定
num_epoch = 4000 #### 4000回学習する
log_file_handle = open("../log/loginfo_new.txt", 'w') #### log
for epoch in range(num_epoch):
    for idx, images in enumerate(data_loader): #### enumerateでindex番号も取ってこれる
        images = images.cuda()
        tdvae.forward(images)
        t_1 = np.random.choice(16)   ## t1ランダムで決めて、
        t_2 = t_1 + np.random.choice([1,2,3,4]) ## そこから1~4の間で飛ばして学習させてる
        loss = tdvae.calculate_loss(t_1, t_2) ##model.pyのcalculate_lossモジュール呼び出し
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        print("epoch: {:>4d}, idx: {:>4d}, loss: {:.2f}".format(epoch, idx, loss.item()),
              file = log_file_handle, flush = True)

        print("epoch: {:>4d}, idx: {:>4d}, loss: {:.2f}".format(epoch, idx, loss.item()))

    if (epoch + 1) % 50 == 0:
        torch.save({
            'epoch': epoch,
            'model_state_dict': tdvae.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'loss': loss
        }, "../output/model/new_model_epoch_{}.pt".format(epoch))


import json
from enum import Enum
from datetime import datetime
from venv import create

from typeguard import typechecked

from posterr.config import ConfigManager
from posterr.services.base import ServiceBase

class PostType(Enum):
    NORMAL = "normal"
    REPOSTED = "reposted"
    QUOTED = "quoted"

@typechecked
class Post(ServiceBase):
    _id: str
    type: Enum
    text: str
    user_id: str
    parent_id: str
    created_at: datetime

    entity_name: str = "post"

    def __init__(
        self,
        text: str = "",
        user_id: str = "",
        parent_id: str = "",
        type: str = PostType.NORMAL.name,
    ) -> None:
        maxium_size: int = ConfigManager().config.get_int("post.maximum_size")
        if len(text) > maxium_size:
            raise ValueError(f"Post has a maximum lenght of {maxium_size}")
        if parent_id:
            self.parent_id = parent_id
        if type:
            self.type = PostType[type].value

        self.text = text
        self.user_id = user_id
        self.created_at = datetime.now()

    def __str__(self) -> str:
        date_format: str = ConfigManager().config.get_string("post.date_format")
        post_dict: dict = self.__dict__

        post_type: PostType = post_dict.pop("type")
        created_at: str = post_dict.pop("created_at")

        if type(created_at) != str:
            created_at = created_at.strftime(date_format)  # ex May 24, 2021

        return json.dumps(
            {**self.__dict__, "type": post_type, "created_at": created_at}
        )


import json
import os
import ssl
from datetime import date
import pandas as pd
import geopandas as gpd
from util import get_data, normalize_text, get_company_names, company_rename, get_company_list, prepare_ids, apply_system_id, set_cwd_to_script, updated_month_year
import numpy as np
ssl._create_default_https_context = ssl._create_unverified_context
set_cwd_to_script()

# def get_sql(sql=False, query='projects_regdocs.sql'):
#     csv_name = query.replace(".sql", ".csv")
#     if sql:
#         print('reading sql '+query)
#         df = execute_sql(os.path.join(os.getcwd(), "queries"), query)
#         df.to_csv('raw_data/'+csv_name, index=False)
#     else:
#         print('reading local csv '+csv_name)
#         df = pd.read_csv('raw_data/'+csv_name)
#     return df

def import_simplified(replace, name='economic_regions.json'):
    read_path = os.path.join(os.getcwd(), "../data_output/conditions/base_maps/", name)
    df = gpd.read_file(read_path)
    df = df.set_geometry('geometry')
    fr_cols = ['PRNAME', 'ERNAME']
    for splitcols in fr_cols:
        df[splitcols] = [x.split('/') for x in df[splitcols]]
        df[splitcols+'_en'] = [x[0].strip() for x in df[splitcols]]
        df[splitcols+'_fr'] = [x[0].strip() if len(x) == 1 else x[-1].strip() for x in df[splitcols]]

    for delete in ['PRNAME', 'ERNAME', 'PRUID', 'ERUID', 'PRNAME_fr', 'ERNAME_fr']:
        del df[delete]

    df['region_id'] = [er.strip()+'/'+pr.strip() for er, pr in zip(df['ERNAME_en'],
                                                                   df['PRNAME_en'])]

    df['region_id'] = df['region_id'].replace(replace)
    df = df[df['region_id'].isin(replace.values())]
    del df['ERNAME_en']
    del df['PRNAME_en']
    return df

def export_files(df, folder, name):
    write_path = os.path.join(os.getcwd(), folder, name)
    df.to_file(write_path, driver='GeoJSON')
    print(folder+' done ', 'CRS: '+str(df.crs))

def conditions_on_map(df, shp):
    shp = pd.merge(shp,
                   df,
                   how='inner',
                   left_on=['region_id'],
                   right_on=['id'])

    del shp['region_id']
    shp = shp[~shp.geometry.is_empty]
    shp = shp[shp.geometry.notna()]
    for numeric_col in ['In Progress', 'Closed']:
        if numeric_col in shp.columns:
            shp[numeric_col] = shp[numeric_col].fillna(0)

    # df becomes map metadata
    df = df.fillna(0)
    for cols in [["Closed", "c"], ["In Progress", "i"]]:
        if cols[0] in df:
            df[cols[0]] = [int(x) for x in df[cols[0]]]
    return shp, df

def condition_meta_data(df, project_names):

    def convert_to_int(df):
        df = df.replace({np.nan: 0})
        for col in ['In Progress', 'Closed']:
            try:
                df[col] = [int(x) for x in df[col]]
            except:
                None
        return df

    def add_missing(df):
        if 'In Progress' not in df.columns:
            df['In Progress'] = 0
        if 'Closed' not in df.columns:
            df['Closed'] = 0
        return df

    # df contains the condition data for the spcecific company
    meta = {}
    # get the summary stats for the boxes above the map
    status = df[['condition id', 'Condition Status', 'Location']].copy().reset_index(drop=True)
    status['loc'] = ['no' if '-1' in x else 'yes' for x in status['Location']]
    status = status[status['loc'] != 'no']
    del status['Location']
    del status['loc']
    status = status.drop_duplicates(subset=['condition id'])
    status = status.groupby(['Condition Status']).size().reset_index()
    status = pd.pivot_table(status, values=0, columns="Condition Status")
    status = add_missing(status)
    status = status.to_dict(orient='records')[0]
    df['Location'] = df['Location'].astype("object")
    notInMap = 0
    noLoc = {}
    for location, statusloc in zip(df['Location'], df['Condition Status']):
        if "-1" in location:
            notInMap = notInMap+1
            if statusloc in noLoc:
                noLoc[statusloc] = noLoc[statusloc]+1
            else:
                noLoc[statusloc] = 1

    status['notOnMap'] = {"total": notInMap,
                          "status": noLoc}

    # get the date the data was pulled
    today = date.today()
    status['updated'] = [today.year, today.month-1, today.day]
    # status['updated'] = [2021, 2, 19]  # override todays date with custom date

    # get the current company name
    status['companyName'] = list(df['Company'])[0]
    meta['summary'] = status

    # once the status summary is calculated, blank locations and null locations can be removed
    df = df[df['id'] != -1].copy().reset_index(drop=True)

    # get the unique project names sorted by number of open conditions
    project = df[['condition id', 'Short Project Name', 'id', 'Condition Status', 'Regdocs']].copy().reset_index(drop=True)
    # add the project name lookup to metadata
    this_company_projects = list(set(project["Short Project Name"]))
    this_lookup = project_names[project_names['id'].isin(this_company_projects)]
    meta['projectLookup'] = prepare_ids(this_lookup)
    project['Regdocs'] = project['Regdocs'].astype('object')
    project['Regdocs'] = project['Regdocs'].fillna('noRegdocs')
    project = project.groupby(['Short Project Name', 'id', 'Condition Status', 'Regdocs']).size().reset_index()
    project = pd.pivot_table(project,
                             values=0,
                             index=['Short Project Name', 'id', 'Regdocs'],
                             columns='Condition Status').reset_index()

    project['Regdocs'] = project['Regdocs'].replace('noRegdocs', np.nan)
    project = add_missing(project)

    project = project.sort_values(by=['In Progress', 'id'], ascending=False)
    project['In Progress'] = pd.to_numeric(project['In Progress'])

    project = convert_to_int(project)
    project['Regdocs'] = [int(x) for x in project['Regdocs']]
    # optimize json size
    project = project.rename(columns={"Short Project Name": "n"})
    project['v'] = [[inProgress, closed, regdocs] for inProgress, closed, regdocs in zip(project['In Progress'],
                                                                                project['Closed'],
                                                                                project['Regdocs'])]
    for delete in ['In Progress', 'Closed', 'Regdocs']:
        del project[delete]
    project = project.to_dict(orient='records')
    meta['projects'] = project

    # get the unique project themes sorted by number of open conditions
    theme = df[['condition id', 'Theme(s)', 'id', 'Condition Status']].copy().reset_index(drop=True)
    theme['Theme(s)'] = [[str(i) for i in x] for x in theme['Theme(s)']]
    theme['Theme(s)'] = [', '.join(x) for x in theme['Theme(s)']]
    theme = theme.groupby(['Theme(s)', 'id', 'Condition Status']).size().reset_index()
    theme = pd.pivot_table(theme,
                           values=0,
                           index=['Theme(s)', 'id'],
                           columns='Condition Status').reset_index()
    theme = add_missing(theme)
    theme = theme.sort_values(by=['In Progress', 'id'], ascending=False)
    # optimize json size
    theme = convert_to_int(theme)
    theme = theme.rename(columns={"Theme(s)": "t"})
    theme['t'] = [[str(i.strip()) for i in x.split(',')] for x in theme['t']]
    theme["v"] = [[inProgress, closed] for inProgress, closed in zip(theme['In Progress'],
                                                                     theme['Closed'])]
    for delete in ['In Progress', 'Closed']:
        del theme[delete]
    theme = theme.to_dict(orient='records')
    meta['themes'] = theme

    df_all = df.copy()
    del df_all['Location']
    df_all = df_all.groupby(['id',
                             'Condition Status']).agg({
                                 'condition id': 'count',
                                 'Short Project Name': lambda x: list(x),
                                 'Theme(s)': lambda t: list(t)})

    for delete in ['Short Project Name', 'Theme(s)']:
        del df_all[delete]

    df_all = df_all.reset_index()
    df_all = pd.pivot_table(df_all,
                            values='condition id',
                            index=['id'],
                            columns='Condition Status').reset_index()

    # this fixes the map color index being thrown off by non map conditions
    df_all = df_all[df_all["id"] != "-1"].copy()
    return df_all, meta

def add_links(df, sql):
    df_links = get_data(os.getcwd(), "projects_regdocs.sql", db='tsql23cap', sql=sql)
    l = {}
    for name, folder in zip(df_links['EnglishProjectName'], df_links['CS10FolderId']):
        l[name] = folder

    regdocs = []
    for proj in df['Project Name']:
        try:
            regdocs.append(l[proj])
        except KeyError:
            regdocs.append(np.nan)
    df['Regdocs'] = regdocs
    # calculate warning for number of projects with no regdocs link
    df_no_link = df[~df['Regdocs'].notnull()].copy().reset_index(drop=True)
    no_link_proj = df_no_link['Short Project Name'].nunique()
    all_proj = df['Short Project Name'].nunique()
    pct = round((no_link_proj/all_proj)*100, 1)
    print("There are "+str(no_link_proj) +" projects without a regdocs link (" +str(pct) +"% of all projects)")
    return df

def idify_conditions(df, sql):

    def list_id(df, column, toReplace):
        new_themes = []
        for t in df[column]:
            if "," in t:
                t = [x.strip() for x in t.split(",")]
                t = [toReplace[x] for x in t]
                new_themes.append(t)
            else:
                try:
                    new_themes.append([toReplace[t.strip()]])
                except:
                    new_themes.append(["-1"])

        df[column] = new_themes
        return df

    projects = get_data(os.getcwd(), "conditionProjects.sql", "tsql23cap", sql)
    themes = get_data(os.getcwd(), "conditionThemes.sql", "tsql23cap", sql)
    regions = get_data(os.getcwd(), "conditionRegions.sql", "tsql23cap", sql)

    for ids in [projects, themes, regions]:
        ids['id'] = [str(x) for x in ids['id']]

    project_replace = {value: key for key, value in zip(projects['id'],
                                                       projects['e'])}
    theme_replace = {value: key for key, value in zip(themes['id'],
                                                     themes['e'])}
    region_replace = {value: key for key, value in zip(regions['id'],
                                                      regions['e'])}

    df['Short Project Name'] = df['Short Project Name'].replace(project_replace)

    df = list_id(df, "Theme(s)", theme_replace)
    df["Location"] = [x.replace(" /", "/").replace("/ ", "/") for x in df["Location"]]
    df = list_id(df, "Location", region_replace)

    # save themes and regions for import into front end
    files = [[themes, "themes"], [regions, "regions"]]
    for file in files:
        id_save = prepare_ids(file[0])
        with open('../data_output/conditions/metadata/'+file[1]+'.json', 'w') as fp:
            json.dump(id_save, fp)

    return df, region_replace, projects

def process_company(df, company, project_names, regions_map, test, save):
    this_company_data = {}
    folder_name = company.replace(' ', '').replace('.', '')
    df_c = df[df['Company'] == company].copy().reset_index(drop=True)
    if not df_c.empty:
        df_c['condition id'] = [str(ins)+'_'+str(cond) for ins, cond in zip(df_c['Instrument Number'], df_c['Condition Number'])]
        expanded_locations = []
        for unique in df_c['condition id']:
            row = df_c[df_c['condition id'] == unique].copy().reset_index(drop=True)
            for region in list(row['Location'])[0]:
                row['id'] = region
                expanded_locations.append(row.copy())

        df_all = pd.concat(expanded_locations, axis=0, sort=False, ignore_index=True)
        # calculate metadata here
        dfmeta, meta = condition_meta_data(df_all, project_names)
        meta["build"] = True
        this_company_data['meta'] = meta
        shp, map_meta = conditions_on_map(dfmeta, regions_map)

        this_company_data['regions'] = shp.to_json()
        this_company_data['mapMeta'] = map_meta.to_dict(orient='records')
        if not test and save:
            with open('../data_output/conditions/'+folder_name+'.json', 'w') as fp:
                json.dump(this_company_data, fp)
    else:
        meta = {"companyName": company}
        shp = None
        dfmeta = None
        this_company_data = {'meta': {"companyName": company,
                                      "build": False},
                             'regions': "{}",
                             'mapMeta': []}

        if not test and save:
            with open('../data_output/conditions/'+folder_name+'.json', 'w') as fp:
                json.dump(this_company_data, fp)
    return df_c, shp, dfmeta, meta

def process_conditions(remote=False,
                       sql=False,
                       non_standard=True,
                       company_names=False,
                       companies=False,
                       test=False,
                       save=True):
    if remote:
        print('downloading remote conditions file')
        link = 'http://www.cer-rec.gc.ca/open/conditions/conditions.csv'
        df = pd.read_csv(link,
                         # sep='\t',
                         # lineterminator='\r',
                         encoding="latin-1",
                         error_bad_lines=True)
        df = normalize_text(df, ['Location', 'Short Project Name', 'Theme(s)'])

    elif test:
        print('reading test conditions data')
        df = pd.read_csv('./raw_data/test_data/conditions.csv', encoding="UTF-16", sep='\t')
        df = normalize_text(df, ['Location', 'Short Project Name', 'Theme(s)'])
    else:
        print('reading local conditions data')
        df = pd.read_csv('./raw_data/conditions_en.csv', encoding="UTF-16", sep='\t')
        df = normalize_text(df, ['Location', 'Short Project Name', 'Theme(s)'])

    for date_col in ['Effective Date', 'Issuance Date', 'Sunset Date']:
        df[date_col] = pd.to_datetime(df[date_col])

    if not non_standard:
        # only include non-standard conditions
        df = df[df['Condition Type'] != 'Standard']

    delete_cols = ['Condition',
                   'Condition Phase',
                   'Instrument Activity',
                   'Condition Type',
                   'Condition Filing']

    for delete in delete_cols:
        del df[delete]

    for r in ['\n', '"']:
        df['Company'] = df['Company'].replace(r, '', regex=True)
    df['Company'] = [x.strip() for x in df['Company']]
    df['Condition Status'] = df['Condition Status'].astype('object')
    df['Condition Status'] = [str(x).strip() for x in df['Condition Status']]
    # preliminary processing
    df['Company'] = df['Company'].replace(company_rename())
    df = apply_system_id(df, "Company")

    df = df[df['Short Project Name'] != "SAM/COM"].copy().reset_index(drop=True)

    df = add_links(df, sql)
    if company_names:
        print(get_company_names(df['Company']))

    df, region_replace, project_names = idify_conditions(df, sql)
    regions_map = import_simplified(region_replace)

    if companies:
        company_files = companies
    else:
        company_files = get_company_list("all")

    for company in company_files:
        try:
            df_c, shp, dfmeta, meta = process_company(df, company, project_names, regions_map, test, save)
            print("completed: "+company)
        except:
            print("conditions error: "+company)
            raise

    updated_month_year("conditions")
    return df_c, shp, dfmeta, meta

if __name__ == "__main__":
    print('starting conditions...')
    df_, regions_, map_meta_, meta_ = process_conditions(remote=False, save=True, sql=True, companies=["NGTL"])
    # company_df, regions, mapMeta, meta = process_conditions(remote=False, companies=['NGTL'], test=True)
    print('completed conditions!')


import torch
import matplotlib.pyplot as plt
from data_handler import DataHandler

def display(config_path, model_path):

    data_handler = DataHandler(config_path)
    height, width = data_handler.raw_image.shape
    model = torch.load(model_path)
    predicted_image = model(data_handler.torch_data["features"]).detach().numpy().reshape(height, width)
    plt.imshow(predicted_image, cmap="gray")
    plt.show()

# # Display preprocessing:
#
# for count, element in enumerate(formatted):
#     print(f"int16 representation: {int(element, 2)}\n"
#           f"16 bits binary representation: {element}\n"
#           f"Hexadecimal representation: {hex(int(element, 2))}\n"
#           f"8 bits hexadecimal representation: {hex(int(element[:8], 2))}\n")
#     if count == 5:
#         break
#
# # Display histograms
#
# images = [clipped_image, normalized_image, equalized_image, adapt_equalized_image]
#
# fig = plt.figure()
# gs = fig.add_gridspec(2, 2, hspace=0, wspace=0)
# axs = gs.subplots(sharex='col', sharey='row')
# fig.suptitle('Sharing x per column, y per row')
#
# for counter, image in enumerate(images):
#     hist, bins = np.histogram(image, 2**8)
#     axs.ravel()[counter] = plt.subplot(2, 2, counter + 1)
#     axs.ravel()[counter].plot(bins[:-1], hist)
# plt.tight_layout()
# plt.show()
#
#
# # Display processed image
# images_figure, axs = plt.subplots(2, 2)
# axs[0, 0].imshow(clipped_image, cmap='gray', vmin=clipped_image.min(), vmax=clipped_image.max())
# axs[0, 1].imshow(normalized_image, cmap='gray')
# axs[1, 0].imshow(equalized_image, cmap='gray')
# axs[1, 1].imshow(adapt_equalized_image, cmap='gray')
# plt.show()