from django.urls import path
from . import views

urlpatterns = [
     path('', views.create_invoice, name='home'),
     path('create_product/', views.create_product, name='create_product'),
     path('view_product/', views.view_product, name='view_product'),
     path('edit_product/<int:pk>', views.edit_product, name='edit_product'),
     path('delete_product/<int:pk>/', views.delete_product, name='delete_product'),
     path('upload_product_excel', views.upload_product_from_excel,
         name='upload_product_excel'),
    # path('create_customer/', views.create_customer, name='create_customer'),
    # path('view_customer/', views.view_customer, name='view_customer'),
    # path('edit_customer/<int:pk>', views.edit_customer, name='edit_customer'),
    # path('delete_customer/<int:pk>/', views.delete_customer, name='delete_customer'),

    path('create_invoice/', views.create_invoice, name='create_invoice'),
    path('view_invoice/', views.view_invoice, name='view_invoice'),
    path('delete_invoice/<int:pk>/', views.delete_invoice, name='delete_invoice'),
    path('delete_all_invoice/', views.delete_all_invoice, name='delete_all_invoice'),
    path('download_all_invoice/', views.download_all, name='download_all_invoice'),
    path('view_invoice_detail/<int:pk>/', views.view_invoice_detail, name='view_invoice_detail'),
    path('invoice-detail/<id>/', views.view_invoice_pdf_detail, name='view_invoice_pdf_detail'),
    path('invoice-download/<id>/', views.generate_PDF, name='invoice-download'),
    path('view_all_service/', views.all_service_management, name='service-view-all'),
    path('view_pending_service/', views.pending_service_management, name='service-view-pending'),
    path('view_complate_service/', views.complate_service_management, name='service-view-complate'),
    path('service_status_change/<int:pk>', views.service_status_change, name="service_status_change"),

]


"""
Automatically encrypt and decrypt a field with a KMIP KMS provider.
"""
import configuration_queryable as configuration
from pprint import pprint
from bson.codec_options import CodecOptions
from pymongo import MongoClient
from pymongo.encryption import ClientEncryption
from pymongo.encryption_options import AutoEncryptionOpts

def configure_data_keys(kmip_configuration):
    db_name, coll_name = configuration.key_vault_namespace.split(".", 1)
    key_vault_client = MongoClient(configuration.connection_uri)
    key_vault_client[db_name][coll_name].create_index(
        [("keyAltNames", 1)],unique=True,partialFilterExpression={"keyAltNames": {"$exists": True}}
    )
    client_encryption = ClientEncryption(
        kmip_configuration["kms_providers"],
        configuration.key_vault_namespace,
        MongoClient(configuration.connection_uri),
        CodecOptions(),
        kms_tls_options=kmip_configuration["kms_tls_options"]
    )

    # Create a new data key and json schema for the encryptedField.
    data_key_id_1 = client_encryption.create_data_key(
        'kmip', key_alt_names=['pymongo_encryption_example_1'])

    data_key_id_2 = client_encryption.create_data_key(
        'kmip', key_alt_names=['pymongo_encryption_example_2'])

    data_key_id_3 = client_encryption.create_data_key(
        'kmip', key_alt_names=['pymongo_encryption_example_3'])

    encryption_data_keys = {"key1": data_key_id_1,"key2":data_key_id_2, "key3":data_key_id_3}

    return encryption_data_keys

def configure_queryable_session(encrypted_fields_schema):
    auto_encryption = AutoEncryptionOpts(
        configuration.kms_providers,
        configuration.key_vault_namespace,
        encrypted_fields_map=encrypted_fields_schema,
        kms_tls_options=configuration.kms_tls_options,
        schema_map=None

    )
    secure_client = MongoClient(configuration.connection_uri,auto_encryption_opts=auto_encryption)
    return secure_client;

def create_schema(data_keys):
    # We are creating a encrypted_fields_map that has an validation schema attached,
    # that uses the encrypt attribute to define which fields should be encrypted.
    encrypted_db_name, encrypted_coll_name = configuration.encrypted_namespace.split(".", 1)
    encrypted_fields_map = {
        f"{encrypted_db_name}.{encrypted_coll_name}": {
            "fields": [
                {
                    "keyId": data_keys["key1"],
                    "path": "contact.email",
                    "bsonType": "string",
                    "queries": {"queryType": "equality"},
                },
                {
                    "keyId": data_keys["key2"],
                    "path": "contact.mobile",
                    "bsonType": "string",
                    "queries": {"queryType": "equality"},
                },
                {
                    "keyId": data_keys["key3"],
                    "path": "ssn",
                    "bsonType": "string",
                    "queries": {"queryType": "equality"},
                }
            ],
        },
    }
    return encrypted_fields_map

def reset():
    db_name, coll_name = configuration.encrypted_namespace.split(".", 1)
    mongo_client = MongoClient(configuration.connection_uri)
    mongo_client.drop_database(db_name)

def create_user(secure_client):
    # Create Encrypted Collection
    db_name, coll_name = configuration.encrypted_namespace.split(".", 1)
    db=secure_client[db_name]
    db.create_collection(coll_name)

    # Insert encrypted record
    coll = secure_client[db_name][coll_name]
    coll.insert_one({
        "firstName": 'Alan',
        "lastName":  'Turing',
        "ssn":       '901-01-0001',
        "address": {
            "street": '123 Main',
            "city": 'Omaha',
            "zip": '90210'
        },
        "contact": {
            "mobile": '202-555-1212',
            "email":  'alan@example.com'
        }
    })
    print("Queryable Encryption: Decrypted document:")
    print("===================")
    pprint((coll.find_one({"ssn":"901-01-0001"})))
    unencrypted_coll = MongoClient(configuration.connection_uri)[db_name][coll_name]
    print("Queryable Encryption: Encrypted document:")
    print("===================")

    pprint((unencrypted_coll.find_one()))

def configure_kmip_provider():
    return {"kms_providers":configuration.kms_providers,"kms_tls_options":configuration.kms_tls_options}

def main():
    reset()
    #1,2 Configure your KMIP Provider and Certificates
    kmip_provider_config = configure_kmip_provider()
    #3 Configure Encryption Data Keys
    data_keys_config = configure_data_keys(kmip_provider_config)
    #4 Create Schema for Queryable Encryption, will be stored in database
    encrypted_fields_map = create_schema(data_keys_config)
    #5 Configure Encrypted Client
    secure_client = configure_queryable_session(encrypted_fields_map)
    #6 Run Query
    create_user(secure_client)
if __name__ == "__main__":
    main()


from django.http import JsonResponse
from datetime import datetime
import pandas as pd
import os
from .app import swat as app

param_names = {'':'', 'RCH':'Reach', 'GIS':'GIS',  'MO':'Month', 'DA':'Day', 'YR':'Year', 'AREAkm2':'Area (km2)',
               'FLOW_INcms':'Inflow (cms)', 'FLOW_OUTcms':'Outflow (cms)', 'EVAPcms':'Evaporation (cms)',
               'TLOSS':'Transpiration Loss (cms)', 'SED_INtons':'Sediment Inflow (tons)', 'SED_OUTtons':'Sediment Outflow (tons)',
               'SEDCONCmg/kg':'Sediment Concentration (mg/kg)', 'ORGN_INkg':'Organic Nitrogen Inflow (kg)',
               'ORGN_OUTkg':'Organic Nitrogen Outflow (kg)', 'ORGP_INkg':'Organic Phosphorus Inflow (kg)',
               'ORGP_OUTkg':'Organic Phosphorus Outflow (kg)', 'NO3_INkg':'Nitrate Inflow (kg)','NO3_OUTkg':'Nitrate Outflow (kg)',
               'NH4_INkg':'Ammonia Inflow (kg)', 'NH4_OUTkg': 'Ammonia Outflow (kg)', 'NO2_INkg': 'Nitrogen Dioxide Inflow (kg)',
               'NO2_OUTkg': 'Nitrogen Dioxide Outflow (kg)', 'MINP_INkg':'Mineral Phosphorus Inflow (kg)',
               'MINP_OUTkg':'Mineral Phosphorus Outflow (kg)', 'CHLA_INkg':'Chlorophyll-a Inflow (kg)', 'CHLA_OUTkg': 'Chlorophyll-a Outflow (kg)',
               'CBOD_INkg':'Carbonaceous BOD Inflow (kg)','CBOD_OUTkg': 'Carbonaceous BOD Outflow (kg)',
               'DISOX_INkg': 'Dissolved Oxygen Inflow (kg)', 'DISOX_OUTkg': 'Dissolved Oxygen Outflow (kg)',
               'SOLPST_INmg': 'Soluble Pesticide Inflow (mg)', 'SOLPST_OUTmg':'Soluble Pesticide Outflow (mg)',
               'SORPST_INmg':'Pesticide Sorbed to Sediment Transport Inflow (mg)',
               'SORPST_OUTmg':'Pesticide Sorbed to Sediment Transport Outflow (mg)', 'REACTPSTmg':'Loss of Pesticide from Water by Reaction (mg)',
               'VOLPSTmg':'Loss of Pesticide from Water by Volatilization (mg)', 'SETTLPSTmg':'Pesticide Transfer from Water to River Bed Sediment (mg)',
               'RESUSP_PSTmg':'Resuspension of Pesticide from River Bed to Water (mg)', 'DIFFUSEPSTmg':'Diffusion of Pesticide from Water to River Bed Sediment (mg)',
               'REACBEDPSTmg':'Loss of Pesticide from River Bed by Reaction (mg)', 'BURYPSTmg':'Loss of Pesticide from River Bed by Buryial (mg)',
               'BED_PSTmg':'Pesticide in River Bed Sediment (mg)', 'BACTP_OUTct':'Persistent Bacterial Outflow (count)',
               'BACTLP_OUT':'Less Persistent Bacterial Outflow (count)', 'CMETAL#1kg':'Conservative metal #1 Outflow (kg)',
               'CMETAL#2kg': 'Conservative Metal #2 Outflow (kg)', 'CMETAL#3kg':'Conservative Metal #3 Outflow (kg)', 'TOT Nkg':'Total Nitrogen (kg)',
               'TOT Pkg':'Total Phosphourus (kg)', 'NO3ConcMg/l':'Nitrate Concentration (mg/l)','WTEMPdegc':'Water Temperature (deg C)'
               }
def extract_monthly_rch(watershed, start, end, parameters, reachid):

    app_workspace = app.get_app_workspace()
    monthly_rch_path = os.path.join(app_workspace.path, 'rch_data', watershed, 'output_monthly.rch')
    param_vals = ['']
    with open(monthly_rch_path) as f:
        for line in f:
            if 'RCH' in line:
                paramstring = line.strip()
                for i in range(0, len(paramstring)-1):
                    if paramstring[i].islower() and paramstring[i+1].isupper() and paramstring[i] != 'c':
                        print(paramstring[i])
                        print(paramstring[i+1])
                        paramstring = paramstring[0:i+1] + ' ' + paramstring[i+1:]
                        print(paramstring)
                param_vals = param_vals + paramstring.split()
                for i in range(0,len(param_vals)-3):
                    if param_vals[i] == 'TOT':
                        new_val = param_vals[i]+param_vals[i+1]
                        param_vals[i] = new_val
                        param_vals.pop(i+1)
                    # if param_vals[i] == 'NO3Conc':
                    #     new_val = param_vals[i] + param_vals[i+1]
                    #     param_vals[i] = new_val
                    #     param_vals.pop(i+1)
                break
    print(param_vals)

    dt_start = datetime.strptime(start, '%B %Y')
    dt_end = datetime.strptime(end, '%B %Y')

    year_start = dt_start.year
    month_start = dt_start.month
    year_end = dt_end.year
    month_end = dt_end.month

    date_year = [2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015]
    start_year_index = date_year.index(year_start)
    end_year_index = date_year.index(year_end)
    start_index = start_year_index * 12 + month_start - 1
    end_index = end_year_index * 12 + month_end - 1

    daterange = pd.date_range(start, end, freq='1M')
    daterange = daterange.union([daterange[-1] + 1])
    daterange_str = [d.strftime('%b %y') for d in daterange]
    daterange_mil = [int(d.strftime('%s')) * 1000 for d in daterange]

    rchDict = {'Dates': daterange_str, 'ReachID': reachid, 'Parameters': parameters, 'Values':{}, 'Names': [], 'Timestep': 'Monthly'}
    for x in range(0,len(parameters)):
        param_index = param_vals.index(parameters[x])
        param_name = param_names[parameters[x]]
        print(param_name)
        data = []
        f = open(monthly_rch_path)

        header1 = f.readline()
        header2 = f.readline()
        header3 = f.readline()
        header4 = f.readline()
        header5 = f.readline()
        header6 = f.readline()
        header7 = f.readline()
        header8 = f.readline()
        header9 = f.readline()

        for num, line in enumerate(f,1):
            line = line.strip()
            columns = line.split()
            if columns[1] == reachid and 1 <= float(columns[3]) <= 12:
                data.append(float(columns[param_index]))

        f.close()
        ts = []
        data = data[start_index:end_index + 1]
        i = 0
        while i < len(data):
            ts.append([daterange_mil[i],data[i]])
            i += 1

        rchDict['Values'][x] = ts
        rchDict['Names'].append(param_name)

    return rchDict

def extract_daily_rch(watershed, start, end, parameters, reachid):

    app_workspace = app.get_app_workspace()
    daily_rch_path = os.path.join(app_workspace.path, 'rch_data', watershed, 'output_daily.rch')

    param_vals = ['']
    with open(daily_rch_path) as f:
        for line in f:
            if 'RCH' in line:
                param_vals = param_vals + line.strip().split()
                break

    print(param_vals)

    param_names = ['', 'Reach', 'GIS', 'Month', 'Day', 'Year', 'Area (km2)', 'Inflow (cms)', 'Outflow (cms)',
                   'Evaporation (cms)', 'Transpiration Loss (cms)', 'Sediment Inflow (tons)', 'Sediment Outflow (tons)',
                   'Sediment Concentration (mg/kg)', 'Organic Nitrogen Inflow (kg)', 'Organic Nitrogen Outflow (kg)',
                   'Organic Phosphorus Inflow (kg)', 'Organic Phosphorus Outflow (kg)', 'Nitrate Inflow (kg)',
                   'Nitrate Outflow (kg)'
                   ]

    dt_start = datetime.strptime(start, '%B %d, %Y')
    start_index = dt_start.timetuple().tm_yday
    dt_end = datetime.strptime(end, '%B %d, %Y')
    end_index = dt_end.timetuple().tm_yday

    year_start = str(dt_start.year)
    year_start_str = ' ' + year_start + ' '

    daterange = pd.date_range(start, end, freq='1d')
    daterange = daterange.union([daterange[-1]])
    daterange_str = [d.strftime('%b %d, %Y') for d in daterange]
    daterange_mil = [int(d.strftime('%s')) * 1000 for d in daterange]

    rchDict = {'Dates': daterange_str, 'ReachID': reachid, 'Parameters': parameters, 'Values': {}, 'Names': [], 'Timestep': 'Daily'}

    for x in range(0, len(parameters)):

        param_index = param_vals.index(parameters[x])
        param_name = param_names[param_index]

        data = []
        f = open(daily_rch_path)

        for skip_line in f:
            if year_start_str in skip_line:
                break

        for num, line in enumerate(f,1):
            line = line.strip()
            columns = line.split()
            date = datetime.strptime(columns[3] + '/' + columns [4] + '/' + columns[5], '%m/%d/%Y')
            if columns[1] == str(reachid) and dt_start <= date <= dt_end:
                data.append(float(columns[param_index]))
            elif date > dt_end:
                break

        f.close()
        ts = []
        i = 0
        while i < len(data):
            ts.append([daterange_mil[i],data[i]])
            i += 1

        rchDict['Values'][x] = ts
        rchDict['Names'].append(param_name)

    return rchDict


import fasttext
from fasttext import load_model

from nltk.tokenize import MWETokenizer, word_tokenize, RegexpTokenizer
import re
import nltk
import unicodedata

normalizer = {'òa': 'oà',
              'óa': 'oá',
              'ỏa': 'oả',
              'õa': 'oã',
              'ọa': 'oạ',
              'òe': 'oè',
              'óe': 'oé',
              'ỏe': 'oẻ',
              'õe': 'oẽ',
              'ọe': 'oẹ',
              'ùy': 'uỳ',
              'úy': 'uý',
              'ủy': 'uỷ',
              'ũy': 'uỹ',
              'ụy': 'uỵ',
              'Ủy': 'Uỷ'}

multiple_punctuation_pattern = re.compile(r"([\"\.\?\!\,\:\;\-])(?:[\"\.\?\!\,\:\;\-]){1,}")
word_tokenizer = MWETokenizer(separator='')
multiple_emoji_pattern = re.compile(u"(["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\u00a9"
        u"\u00ae"
        u"\u2000-\u3300"
        "]){1,}", flags= re.UNICODE )

def normalize_text(text):
  for absurd, normal in normalizer.items():
    text = text.replace(absurd, normal)
  return text

def preprocess(text):
  text = unicodedata.normalize("NFC", text)
  text = multiple_emoji_pattern.sub(r"\g<1> ", text)
  text = multiple_punctuation_pattern.sub(r" \g<1> ", text)
  text = normalize_text(text.lower())
  text = re.sub('(free|ship|toàn quốc|nhập mã)', '',text)
  text = re.sub('(\()[\s\w]*(\))', '', text)
  text = re.sub('(\[)[\s\w\%\d]+(\])', '', text)
  text = re.sub('[^\w]+', ' ', text)
  text = re.sub('(\d+\w+)', '', text)
  text = re.sub('\d+', '', text)
  text = word_tokenizer.tokenize(word_tokenize(text))
  return ' '.join(text)

class Prediction:
    def __init__(self):
        # Load fasttext model
        self.model = load_model('./model_sell_detection.bin')

    def process(self, string_line):
        # pre process sent
        preprocessed_string = preprocess(string_line)
        # return prediction
        return self.model.predict(preprocessed_string)[0][0]


#!/usr/bin/env python3

import sys
import json
import random
from itertools import permutations

vertex_mapping = {}
partial_vertex_encodings = set()
vertex_encodings = set()
edge_encodings = set()
possible_solutions = []
graph = {}
base_mapping = {
    "0": "1",   # A
    "1": "0",   # T
    "2": "3",   # C
    "3": "2"    # G
}

def usage():
    print("""
Usage: {} graph.json start_vertex end_vertex
    """.format(sys.argv[0]))
    sys.exit(0)

def get_all_vertex(graph: dict, vertex_mapping: dict) -> None:
    """ Generate a string encoding for each vertex and record each recording in the vertex_mapping dict """
    for vertex in graph:
        generate_vertex_encoding(vertex, vertex_mapping)

def generate_vertex_encoding(vertex: str, vertex_mapping: dict) -> None:
    """ Generate a string encoding of a vertex that is representative of a DNA strand """
    encoding = generate_random_string(20)
    while encoding[10:] in partial_vertex_encodings or encoding[:10] in partial_vertex_encodings:
        encoding = generate_random_string(20)
    partial_vertex_encodings.add(encoding[10:])
    partial_vertex_encodings.add(encoding[:10])
    vertex_encodings.add(encoding)
    vertex_mapping[vertex] = encoding

def generate_random_string(size: int) -> str:
    """ Generate a string of a given size consisting of characters 0,1,2, and 3 """
    return "".join([ str(random.randrange(4)) for _ in range(size) ])

def get_complement_encoding(s: str) -> str:
    """ Generate the Watson-Crick complement string encoding of a DNA strand string encoding """
    return "".join([ base_mapping[i] for i in s ])

def generate_edge(vertex_5: str, vertex_3: str) -> str:
    """ Generate the edge between two vertex DNA strands """
    return "".join([ get_complement_encoding(vertex_5[10:]), get_complement_encoding(vertex_3[:10]) ])

def generate_all_edges(graph: dict, edge_encodings: set) -> None:
    """ Generate string encoding representations of each edge in the directed graph, record this information in the edge_encodings set """
    for vertex in graph:
        for v in graph[vertex]:
            edge_encodings.add(generate_edge(vertex_mapping[vertex],vertex_mapping[v]))

def connected_edges(edge_a: str, edge_b: str) -> str:
    """ Determine if two edges can logically follow each other in a graph based on string encodings for those edges and the encodings of every vertex """
    vertex = get_complement_encoding("{}{}".format(edge_a[10:],edge_b[:10]))
    if vertex in vertex_encodings:
        return vertex
    return None

def is_valid_path(edge_list: list) -> list:
    """ Determine if a list of string encodings for edges is a valid path through the directed graph """
    path = []
    # add first vertex
    partial_vertex = get_complement_encoding(edge_list[0][:10])
    if partial_vertex not in partial_vertex_encodings:
        return None
    for vertex in vertex_encodings:
        if partial_vertex == vertex[10:]:
            path.append(vertex)
    # add middle vertices
    for i in range(len(edge_list)-1):
        vertex = connected_edges(edge_list[i],edge_list[i+1])
        if not vertex:
            return None
        path.append(vertex)
    # add last vertex
    partial_vertex = get_complement_encoding(edge_list[-1][10:])
    if partial_vertex not in partial_vertex_encodings:
        return None
    for vertex in vertex_encodings:
        if partial_vertex == vertex[:10]:
            path.append(vertex)
    return path

def generate_all_possible_solutions(edge_encodings: set, possible_solutions: list) -> None:
    """
    Generates many possible paths through the graph.
    Many more paths are generated than are required for a solution, this is done intentionally to mimic how this algorithm would be executed in a laboratory
    Every permutation of edge encodings is produced.
    Edges may be repeated and possible solutions of varying sizes are also produced.
    """
    edge_bank = []
    for edge in edge_encodings:
        edge_bank.append(edge)
        edge_bank.append(edge)

    min_size = 2
    max_size = len(edge_encodings) + 2
    for i in range(min_size, max_size):
        for perm in permutations(edge_bank, i):
            possible_solution = is_valid_path(perm)
            if possible_solution:
                possible_solutions.append(possible_solution)

def convert_edges_to_vertices(edge_list: list) -> list:
    """ Converts a list of edges into a list of vertices """
    vertices_list = []
    for i in range(len(edge_list)-1):
        vertices_list.append( get_complement_encoding("{}{}".format(edge_list[i][10:],edge_list[i+1][:10])))
    return vertices_list

def eliminate_invalid_endings(possible_solutions: list, start_vertex: str, end_vertex: str) -> None:
    """ Eliminates strands from the possible_solutions list that don't start and end at the provided vertices """
    solutions_to_remove = []
    for solution in possible_solutions:
        if solution[0] != start_vertex or solution[-1] != end_vertex:
            solutions_to_remove.append(solution)
    for bad_solution in solutions_to_remove:
        possible_solutions.remove(bad_solution)

def eliminate_wrong_size(possible_solutions: list, vertex_encodings: set)-> None:
    """ Eliminates strands that do not have the correct number of vertices """
    solutions_to_remove = []
    for solution in possible_solutions:
        if not len(solution) == len(vertex_encodings):
            solutions_to_remove.append(solution)
    for bad_solution in solutions_to_remove:
        possible_solutions.remove(bad_solution)

def eliminate_missing_vertex(possible_solutions: list, vertex_encodings: set)-> None:
    """ Eliminates strands that are missing a vertex """
    solutions_to_remove = []
    for solution in possible_solutions:
        for vertex in vertex_encodings:
            if not vertex in solution:
                solutions_to_remove.append(solution)
                break
    for bad_solution in solutions_to_remove:
        possible_solutions.remove(bad_solution)

def vertex_list_human_readable(vertex_list: list) -> str:
    """ Return a human readable path from a strand of vertices """
    human_readable = ""
    for v in vertex_list:
        for vm in vertex_mapping:
            if vertex_mapping[vm] == v:
                human_readable = "{}{}".format(human_readable,vm)
    return human_readable

if __name__ == '__main__':
    if(len(sys.argv) != 4):
        usage()
    start_vertex = sys.argv[2].strip()
    end_vertex = sys.argv[3].strip()

    with open(sys.argv[1]) as json_file:
        graph = json.load(json_file)

    print("Creating DNA strands for vertices and edges")
    get_all_vertex(graph, vertex_mapping)
    generate_all_edges(graph, edge_encodings)

    print("The following are vertex encodings")
    for v in vertex_mapping:
        print("{}: 5' {} 3'".format(v,vertex_mapping[v]))

    print("Generating strands encompassing all possible solutions")
    generate_all_possible_solutions(edge_encodings, possible_solutions)

    print("Eliminating strands that don't start and end with the specified vertices")
    eliminate_invalid_endings(possible_solutions,vertex_mapping[start_vertex],vertex_mapping[end_vertex])
    if not possible_solutions:
        print("There are no remaining strands that could be possible solutions")

    print("Eliminating strands that are not the correct size")
    eliminate_wrong_size(possible_solutions, vertex_encodings)
    if not possible_solutions:
        print("There are no remaining strands that could be possible solutions")

    print("Eliminating strands that don't include every vertex")
    eliminate_missing_vertex(possible_solutions, vertex_encodings)

    hamil_paths = set()
    if possible_solutions:
        print("YES, this is a valid Hamiltonian path")
        print("Valid Solutions:")
        for soln in possible_solutions:
            hamil_paths.add(tuple(soln))
        for hamil in hamil_paths:
            print("".join(hamil))
            print(vertex_list_human_readable(hamil))
    else:
        print("NO, this is not a valid Hamiltonian path")


from collections import namedtuple
import cStringIO as StringIO
import codecs
import json
from splunksdc import logging
from splunksdc.archive import ArchiveFactory

logger = logging.get_module_logger()

Metadata = namedtuple('Metadata', ['source', 'sourcetype'])

class Decoder(object):
    def __init__(self, **kwargs):
        self._af = ArchiveFactory.create_default_instance()

    def __call__(self, fileobj, source):
        raise NotImplementedError()

    def _open(self, fileobj, filename):
        return self._af.open(fileobj, filename)

    @staticmethod
    def _product_multiple_lines(sequence):
        lines = [line for line in sequence]
        lines.append('')
        return '\n'.join(lines)

class CloudTrailLogsDecoder(Decoder):
    def __call__(self, fileobj, source):
        for member, uri in self._open(fileobj, source):
            document = json.load(member)
            if self._is_digest(document):
                logger.info('Ignore CloudTail digest file.', source=source)
                continue
            records = document['Records']
            records = (json.dumps(item) for item in records)
            records = self._product_multiple_lines(records)
            yield records, Metadata(uri, 'aws:cloudtrail')

    @staticmethod
    def _is_digest(document):
        return 'Records' not in document

class ELBAccessLogsDecoder(Decoder):
    def __call__(self, fileobj, source):
        for member, uri in self._open(fileobj, source):
            yield UTFStreamDecoder.create(member), Metadata(uri, 'aws:elb:accesslogs')

class CloudFrontAccessLogsDecoder(Decoder):
    def __call__(self, fileobj, source):
        for member, uri in self._open(fileobj, source):
            yield UTFStreamDecoder.create(member), Metadata(uri, 'aws:cloudfront:accesslogs')

class S3AccessLogsDecoder(Decoder):
    def __call__(self, fileobj, source):
        for member, uri in self._open(fileobj, source):
            yield UTFStreamDecoder.create(member), Metadata(uri, 'aws:s3:accesslogs')

class ConfigDecoder(Decoder):
    def __call__(self, fileobj, source):
        for member, uri in self._open(fileobj, source):
            document = json.load(member)
            records = document['configurationItems']
            records = (json.dumps(item) for item in records)
            records = self._product_multiple_lines(records)
            yield records, Metadata(source, 'aws:config')

class CustomLogsDecoder(Decoder):
    def __init__(self, **kwargs):
        super(CustomLogsDecoder, self).__init__()
        self._sourcetype = kwargs.get('sourcetype', '')

    def __call__(self, fileobj, source):
        for member, uri in self._open(fileobj, source):
            yield UTFStreamDecoder.create(member), Metadata(uri, self._sourcetype)

class UTFStreamDecoder(object):
    # keep longer signature ahead
    _BOM_SIGNATURE = [
        (codecs.BOM_UTF32_LE, 'utf-32-le'),
        (codecs.BOM_UTF32_BE, 'utf-32-be'),
        (codecs.BOM_UTF8, 'utf-8-sig'),
        (codecs.BOM_UTF16_LE, 'utf-16-le'),
        (codecs.BOM_UTF16_BE, 'utf-16-be'),
    ]

    @classmethod
    def _create_decoder(cls, head):
        encoding = 'utf-8'
        for signature, name in cls._BOM_SIGNATURE:
            if head.startswith(signature):
                encoding = name
                break
        factory = codecs.getincrementaldecoder(encoding)
        decoder = factory(errors='replace')
        return decoder

    @classmethod
    def create(cls, fileobj):
        if isinstance(fileobj, str):
            fileobj = StringIO.StringIO(fileobj)
        head = fileobj.read(4096)
        decoder = cls._create_decoder(head)
        obj = cls(decoder, fileobj)
        obj._decode(head)
        return obj

    def __init__(self, decoder, fileobj):
        self._fileobj = fileobj
        self._decoder = decoder
        self._pending = ''
        self._exhausted = False
        self._chunk_size = 4 * 1024 * 1024

    def _next_chunk(self):
        return self._fileobj.read(self._chunk_size)

    def _decode(self, data=''):
        final = False if data else True
        self._pending += self._decoder.decode(data, final=final).encode('utf-8')

    def read(self, size):
        while len(self._pending) < size and not self._exhausted:
            chunk = self._next_chunk()
            if not chunk:
                self._decode()
                self._exhausted = True
                break
            self._decode(chunk)

        chunk = self._pending[:size]
        self._pending = self._pending[size:]
        return chunk

class DecoderFactory(object):
    @classmethod
    def create_default_instance(cls):
        factory = cls()
        factory.register('CustomLogs', CustomLogsDecoder)
        factory.register('CloudTrail', CloudTrailLogsDecoder)
        factory.register('ELBAccessLogs', ELBAccessLogsDecoder)
        factory.register('CloudFrontAccessLogs', CloudFrontAccessLogsDecoder)
        factory.register('S3AccessLogs', S3AccessLogsDecoder)
        factory.register('Config', ConfigDecoder)
        return factory

    def __init__(self):
        self._registry = dict()

    def create(self, name, **kwargs):
        name = name.lower()
        decoder_type = self._registry.get(name)
        return decoder_type(**kwargs)

    def register(self, name, decode_type):
        name = name.lower()
        self._registry[name] = decode_type


import cv2
import numpy as np

img = cv2.imread('IMG_3879.jpg')

img_reshape = cv2.resize(img,(700,700))

med_val = np.median(img)

threshold1 = int(max(0,0.7*med_val))

threshold2 = int(max(0,1.3*med_val))

blurred_img = cv2.blur(img_reshape,ksize=(8,8))

edges = cv2.Canny(blurred_img,threshold1,threshold2)

cv2.imshow('frame',img_reshape)
cv2.waitKey(0)

cv2.imshow('frame1',edges)
cv2.waitKey(0)


import json
import requests

'''stringOfJsonData = '{"name": "Zophie", "isCat": true, "miceCaught": 0,"felineIQ": null}'

jsondata = json.loads(stringOfJsonData)

file1=open('jsonfile.json','w')
json.dump(stringOfJsonData,file1)
'''
response = requests.get("https://jsonplaceholder.typicode.com/todos")
todos = json.loads(response.text)
todostring=json.dumps(todos)

print(todos)

file1=open('jsonfile2.txt','w')
file1.write(todostring)


from collections import defaultdict
from typing import Union, Set, List, Dict, Any
from pathlib import Path

import networkx
import yaml

from librep.config.type_definitions import ArrayLike, PathLike
from librep.base.data import Dataset, SimpleDataset
from librep.base.transform import Transform
from librep.base.estimator import Estimator
from librep.base.evaluators import SupervisedEvaluator
from librep.workflow.objects import ObjectDatabase

class Operator:
    def __init__(self, name: str = ""):
        self.name = name

    def run(self):
        raise NotImplementedError

    def __str__(self) -> str:
        return f"Operator: {self.name}"

class PlaceHolder(Operator):
    def run(self, X):
        return X

    def __str__(self) -> str:
        return f"op=PlaceHolder ({self.name})"

class TransformFitOp(Operator):
    def __init__(
        self, transform: Transform, fit_params: Dict[str, Any] = None, name: str = ""
    ):
        super().__init__(name)
        self.transform = transform
        self.fit_params: Dict[str, Any] = fit_params or dict()

    def run(self, dataset: Dataset) -> Transform:
        X, y = dataset[:][0], dataset[:][1]
        return self.transform.fit(X, y, **self.fit_params)

    def __str__(self) -> str:
        return f"op=TransformFit ({self.name})"

# TODO this should not return a SimpleDataset...
class TransformOp(Operator):
    def run(self, transform: Transform, dataset: Dataset) -> Dataset:
        X = dataset[:][0]
        return SimpleDataset(transform.transform(X), dataset[:][1])

    def __str__(self) -> str:
        return f"op=Transform ({self.name})"

class EstimatorFitOp(Operator):
    def __init__(
        self, estimator: Estimator, fit_params: Dict[str, Any] = None, name: str = ""
    ):
        super().__init__(name)
        self.estimator = estimator
        self.fit_params: Dict[str, Any] = fit_params or dict()

    def run(self, dataset: Dataset) -> Estimator:
        X, y = dataset[:][0], dataset[:][1]
        return self.estimator.fit(X, y, **self.fit_params)

    def __str__(self) -> str:
        return f"op=EstimatorFit ({self.name})"

class EstimatorPredictOp(Operator):
    def run(self, estimator: Estimator, dataset: Dataset) -> ArrayLike:
        X = dataset[:][0]
        return estimator.predict(X)

    def __str__(self) -> str:
        return f"op=EstimatorPredict ({self.name})"

class SupervisedEvaluatorOp(Operator):
    def __init__(self, evaluator: SupervisedEvaluator, name: str = ""):
        super().__init__(name)
        self.evaluator = evaluator

    def run(self, dataset: Dataset, y_pred: ArrayLike) -> Any:
        y_true = dataset[:][1]
        return self.evaluator.evaluate(y_true, y_pred)

    def __str__(self) -> str:
        return f"op=SupervisedEvaluator ({self.name})"

class GraphBuilder:
    def __init__(self, database: ObjectDatabase):
        self.database = database

    def create_node(self, values: dict, name: str = ""):
        if values["type"] == "transform":
            if values["operation"] == "fit":
                obj = self.database.instantiate(values["transform"])
                return TransformFitOp(obj, name=name)
            elif values["operation"] == "transform":
                return TransformOp(name=name)
            else:
                raise ValueError(f"Invalid operation '{values['operation']}'")

        elif values["type"] == "estimator":
            if values["operation"] == "fit":
                obj = self.database.instantiate(values["estimator"])
                return EstimatorFitOp(obj, name=name)
            elif values["operation"] == "predict":
                return EstimatorPredictOp(name=name)
            else:
                raise ValueError(f"Invalid operation '{values['operation']}'")

        elif values["type"] == "evaluator":
            if values["operation"] == "supervised":
                obj = self.database.instantiate(values["evaluator"])
                return SupervisedEvaluatorOp(obj, name=name)
            else:
                raise ValueError(f"Invalid operation '{values['operation']}'")

    def from_dict(
        self, nodes: Dict[str, Any], inputs: List[str] = None, name: str = ""
    ) -> networkx.DiGraph:
        inputs = inputs or []
        graph = networkx.DiGraph(flow=name)

        for i in inputs:
            graph.add_node(i, **{"__op__": PlaceHolder(name=i)})

        for node_name, node_values in nodes.items():
            if "input" in node_values:
                graph.add_edges_from((v, node_name) for v in node_values["input"])
                del node_values["input"]
            node_values["__op__"] = self.create_node(node_values, name=node_name)
            graph.add_node(node_name, **node_values)
        return graph

    def from_yaml(self, path: PathLike) -> Dict[str, networkx.DiGraph]:
        path = Path(path)
        with path.open("r") as f:
            values = yaml.load(f, Loader=yaml.FullLoader)
        results = dict()
        if "flow" in values:
            for flow_name, node_values in values["flow"].items():
                results[flow_name] = self.from_dict(
                    node_values["nodes"], node_values.get("inputs", {}),
                    name=flow_name
                )
        return results

class Executor:

    def run(self, graph: networkx.DiGraph, placeholders: Dict[str, Any]):
        cache = dict()
        cache.update(placeholders)
        result = None
        print("----- Building done -------\n")

        print(f"Cache: {cache}")
        print()

        topo_sort = list(networkx.topological_sort(graph))
        print(f"Topo sort: {topo_sort}")
        print()

        for node in topo_sort:
            op = graph.nodes[node]["__op__"]
            if isinstance(op, PlaceHolder):
                inputs = [cache[node]]
            else:
                inputs = [cache[u] for (u, v) in graph.in_edges(node)]
            print(f"({node}) executing ({op}): {inputs}")
            result = op.run(*inputs)
            print(f"({node}) result: {result}")
            print()
            cache[node] = result
        return result

# import numpy as np
# from pathlib import Path
# from librep.base.data import SimpleDataset
# from librep.workflow.objects import ObjectCreator, ObjectDatabase
# from librep.workflow.graph import GraphBuilder, Executor
# dataset = SimpleDataset(X=np.arange(16).reshape(2, 8), y=np.array([0, 1]))
# creator = ObjectCreator()
# database = ObjectDatabase.from_yaml(creator=creator, path=Path("objects.yaml"))
# graphs = GraphBuilder(database).from_yaml(Path("workflow.yaml"))
# g = graphs["simple_workflow"]
# h = graphs["simple_workflow_predict"]
# result = Executor().run(g, placeholders={"dataset": dataset})
# result2 = Executor().run(h, placeholders={"dataset": dataset, "trained_model": result})


from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
from django.urls import path, include
from rest_framework.documentation import include_docs_urls

from blackbox.core.views.landing import index

urlpatterns = [
    path('', index),
    path('api/', include('blackbox.core.urls')),
    path('api/', include('blackbox.client_config.urls')),
    path('api/', include('blackbox.cms.urls')),
    path('api/', include('blackbox.discuss.urls')),
    path('api/docs/', include_docs_urls(title='MiNi CMS API Docs')),
    path('api-auth/', include('rest_framework.urls')),
    path('admin/', admin.site.urls),
]

urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

if settings.FORCE_STATIC_FILE_SERVING and not settings.DEBUG:
    settings.DEBUG = True
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
    settings.DEBUG = False


# Import the required libraries
import nltk
import torch
import requests
import torchtext
import regex as re
import urllib.request
from PIL import Image
from tqdm import tqdm
from io import BytesIO
import os, json, argparse, csv
import torch.nn.functional as F
from nltk.corpus import stopwords
from typing import Union, Iterable
from torchtext.data.utils import get_tokenizer
from src.vocab import build_vocab_from_iterator

nltk.download('stopwords')

# Function to create preprocessed data dictionary
def preprocess(data_path, dataset_name = "coco", split = "all"):

    preprocessed_dict = {}

    # For MS coco dataset
    if dataset_name == "coco":
        data = json.load(open(data_path, "r"))

        if split == 'test':
            raise ValueError("'test' split has no annotations")

        for image_dict in tqdm(data["images"], position = 0, leave = True):
            Id = image_dict["id"]
            image_path = image_dict["file_name"]

            if "train" in image_path:
                dir_path = "./datasets/coco/train2014"
            elif "val" in image_path:
                dir_path = "./datasets/coco/val2014"
            preprocessed_dict[Id] = {"image_path": os.path.join(dir_path, image_path), "captions": []}

        for annotation_dict in data["annotations"]:
            image_id = annotation_dict["image_id"]
            caption = annotation_dict["caption"]
            if image_id in preprocessed_dict:
                preprocessed_dict[image_id]["captions"].append(caption)

    # For Flickr30k dataset
    elif dataset_name == "flickr30k":
        data = json.load(open(data_path, "r"))
        dir_path = "./datasets/flickr30k/flickr30k-images"

        for image_dict in tqdm(data["images"], position = 0, leave = True):

            Id = image_dict["imgid"]
            image_path = image_dict["filename"]

            if split in [image_dict["split"], "all"]:

                preprocessed_dict[Id] = {"image_path": os.path.join(dir_path, image_path), "captions": []}
                sentences_list = image_dict["sentences"]

                for token_sent_dict in sentences_list:
                    image_id = token_sent_dict["imgid"]
                    caption = token_sent_dict["raw"]
                    preprocessed_dict[image_id]["captions"].append(caption)

    # For Flickr8k dataset
    elif dataset_name == "flickr8k":

        with open('./datasets/flickr8k/Flickr_8k.trainImages.txt') as f:
            train_images = f.readlines()
            for i in range(len(train_images)):
                train_images[i] = train_images[i][:-1]
        with open('./datasets/flickr8k/Flickr_8k.testImages.txt') as f:
            test_images = f.readlines()
            for i in range(len(test_images)):
                test_images[i] = test_images[i][:-1]
        with open('./datasets/flickr8k/Flickr_8k.devImages.txt') as f:
            dev_images = f.readlines()
            for i in range(len(dev_images)):
                dev_images[i] = dev_images[i][:-1]

        with open(data_path) as f:
            lines = f.readlines()

        for i, line in enumerate(lines):

            token_list = line.split()
            Id = i//5 + 1
            image_file = token_list[0][:-2]

            if split == "train" and image_file not in train_images:
                continue
            if split == "test" and image_file not in test_images:
                continue
            if split == "val" and image_file not in dev_images:
                continue

            image_path = os.path.join("./datasets/flickr8k/Flicker8k_Dataset", image_file)
            caption = (" ").join(token_list[1:])
            if i % 5 == 0:
                preprocessed_dict[Id] = {"image_path": image_path, "captions": []}
            preprocessed_dict[Id]["captions"].append(caption)

    # For Toronto COCO-QA dataset
    elif dataset_name == "toronto-cocoqa":

        data = json.load(open(data_path, "r"))
        train_questions = open("datasets/toronto-cocoqa/train/questions.txt", "r").readlines()
        train_answers = open("datasets/toronto-cocoqa/train/types.txt", "r").readlines()
        train_image_ids = open("datasets/toronto-cocoqa/train/img_ids.txt", "r").readlines()

        val_questions = open("datasets/toronto-cocoqa/test/questions.txt", "r").readlines()
        val_answers = open("datasets/toronto-cocoqa/test/types.txt", "r").readlines()
        val_image_ids = open("datasets/toronto-cocoqa/test/img_ids.txt", "r").readlines()

        ques_ans_dict = {}

        if "train" or "all" in split:
            for i in range(len(val_image_ids)):
                ques_ans_dict[int(train_image_ids[i][:-1])] = {"question": preprocess_caption(train_questions[i][:-1]),
                                                                "answer": int(train_answers[i][:-1])}
        elif "val" in split:
            for i in range(len(val_image_ids)):
                ques_ans_dict[int(val_image_ids[i][:-1])] = {"question": preprocess_caption(val_questions[i][:-1]),
                                                             "answer": int(val_answers[i][:-1])}

        if split == 'test':
            raise ValueError("'test' split has no annotations")

        for image_dict in tqdm(data["images"], position = 0, leave = True):
            Id = image_dict["id"]
            image_path = image_dict["file_name"]

            if "train" in image_path:
                dir_path = "./datasets/coco/train2014"
            elif "val" in image_path:
                dir_path = "./datasets/coco/val2014"

            if Id not in ques_ans_dict.keys():
                continue

            preprocessed_dict[Id] = {"image_path": os.path.join(dir_path, image_path),
                                    "question": ques_ans_dict[Id]["question"],
                                    "answer": ques_ans_dict[Id]["answer"],
                                    "captions": []}

        for annotation_dict in data["annotations"]:
            image_id = annotation_dict["image_id"]
            caption = annotation_dict["caption"]
            if image_id in preprocessed_dict:
                preprocessed_dict[image_id]["captions"].append(caption)

    return preprocessed_dict

# Function to obtain the word frequency
def get_word_frequency(words):

    word_freq = dict()

    for word in words:
        if word in word_freq:
            word_freq[word] += 1
        else:
            word_freq[word] = 1

    return word_freq

# Function to preprocess caption
def preprocess_caption(caption):

    caption = re.sub('[^a-zA-Z0-9.?,]', ' ', caption)

    url = re.compile(r'https?://\S+|www\.\S+')
    caption = url.sub(r'', caption)

    html = re.compile(r'<.*?>')
    caption = html.sub(r'', caption)

    emoji_pattern = re.compile("["
                            u"\U0001F600-\U0001F64F"  # emoticons
                            u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                            u"\U0001F680-\U0001F6FF"  # transport & map symbols
                            u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                            u"\U00002702-\U000027B0"
                            u"\U000024C2-\U0001F251"
                            "]+", flags=re.UNICODE)
    caption = emoji_pattern.sub(r'', caption)

    return caption

# Function to preprocess for concept vocabulary
def preprocess_caption_for_concept(caption, tokenizer):

    caption = re.sub('[^a-zA-Z0-9.?,]', ' ', caption)

    url = re.compile(r'https?://\S+|www\.\S+')
    caption = url.sub(r'', caption)

    html = re.compile(r'<.*?>')
    caption = html.sub(r'', caption)

    emoji_pattern = re.compile("["
                            u"\U0001F600-\U0001F64F"  # emoticons
                            u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                            u"\U0001F680-\U0001F6FF"  # transport & map symbols
                            u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                            u"\U00002702-\U000027B0"
                            u"\U000024C2-\U0001F251"
                            "]+", flags=re.UNICODE)
    caption = emoji_pattern.sub(r'', caption)

    stop_words = set(stopwords.words('english'))

    tokens = tokenizer(caption)
    filtered_caption = []

    for w in tokens:
        if w not in stop_words:
            filtered_caption.append(w)

    filtered_caption = " ".join(filtered_caption)

    return filtered_caption

# Function to create the overall vocabulary
def build_vocabulary(dataset = "coco", split = "train"):

    preprocessed_dict = json.load(open(f'./datasets/{dataset}/{split}_image_captions.json', 'r'))

    corpus = []
    for image_id, path_caption_dict in preprocessed_dict.items():
        captions = preprocessed_dict[image_id]["captions"]
        for i in range(len(captions)):
            captions[i] = preprocess_caption(captions[i])
        corpus.extend(captions)

    # Tokenizer: Converts a string or a sentence into a list of words.
    tokenizer = get_tokenizer("basic_english")
    tokens = [tokenizer(caption) for caption in corpus]

    vocabulary = build_vocab_from_iterator(tokens, min_freq = 5, specials = ("<UNK>"), specials_first = True)

    torch.save(vocabulary, f'./datasets/{dataset}/{split}_vocabulary.pth')

# Function to build the concept vocabulary
def build_concept_vocabulary(dataset = "coco", split = "all", num_concepts = 512):

    preprocessed_dict = json.load(open(f'./datasets/{dataset}/{split}_image_captions.json', 'r'))
    tokenizer = get_tokenizer("basic_english")

    corpus = []
    for image_id, path_caption_dict in preprocessed_dict.items():
        captions = preprocessed_dict[image_id]["captions"]
        for i in range(len(captions)):
            captions[i] = preprocess_caption_for_concept(captions[i], tokenizer)
        corpus.extend(captions)

    tokens = [tokenizer(caption) for caption in corpus]

    vocabulary = build_vocab_from_iterator(tokens, max_tokens = num_concepts)
    torch.save(vocabulary, f'./datasets/{dataset}/{split}_concept_vocabulary.pth')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', default = 'toronto-cocoqa', type = str, choices=['coco', 'flickr8k', 'flickr30k', 'toronto-cocoqa'])
    parser.add_argument('--data_path', type = str)
    parser.add_argument('--split', default = 'train', type = str, choices=['train', 'val', 'test', 'all'])
    parser.add_argument('--preprocess', default = False, type = bool)
    parser.add_argument('--get_vocab', default = False, type = bool)
    parser.add_argument('--build_concept_vocab', default = False, type = bool)
    parser.add_argument('--num_concepts', default = 512, type = int)

    args = parser.parse_args()

    if args.preprocess:
        preprocessed_dict = preprocess(args.data_path, args.dataset, split = args.split)
        save_path = open(f'./datasets/{args.dataset}/{args.split}_image_captions.json', 'w')
        json.dump(preprocessed_dict, save_path)

    if args.get_vocab:
        build_vocabulary(dataset = args.dataset, split = args.split)

    if args.build_concept_vocab:
        build_concept_vocabulary(dataset = args.dataset, split = args.split)


import torch
import numpy as np
from torchvision import datasets
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from PIL import Image
from torchvision.utils import save_image

use_cuda = torch.cuda.is_available()
# convert data to a normalized torch.FloatTensor
transform = transforms.Compose([
    transforms.ToTensor(),])
# choose the training and test datasets
train_data = datasets.CIFAR10('data', train=True,
                              download=True, transform=transform)
test_data = datasets.CIFAR10('data', train=False,
                             download=True, transform=transform)

num_workers = 0
batch_size = 20
# prepare data loaders
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, num_workers=num_workers)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, num_workers=num_workers)

print(len(train_loader))
# specify the image classes
classes = ['airplane', 'automobile', 'bird', 'cat', 'deer','dog', 'frog', 'horse', 'ship', 'truck']

####
j=0
for i in range(len(train_loader)):
  noise_factor=0.1
  # obtain one batch of training images
  # dataiter = iter(train_loader)
  images, labels = next(iter(train_loader))
  # convert images to numpy for display

  noisy_imgs = images + noise_factor * torch.randn(*images.shape)
  noisy_imgs = np.clip(noisy_imgs, 0., 1.)
  for i in range(len(images)):
    save_image(images[i],f"data/train/clean/clean_{j}.png")
    save_image(noisy_imgs[i],f"data/train/noisy/noisy_{j}.png")
    j+=1


import torch
from torch import nn

class WavEncoder(nn.Module):
    def __init__(self):
        super(WavEncoder, self).__init__()
        self.feat_extractor = nn.Sequential(
            nn.Conv1d(1, 16, 15, stride=5, padding=1600),
            nn.BatchNorm1d(16),
            nn.LeakyReLU(0.3, inplace=True),
            nn.Conv1d(16, 32, 15, stride=6),
            nn.BatchNorm1d(32),
            nn.LeakyReLU(0.3, inplace=True),
            nn.Conv1d(32, 64, 15, stride=6),
            nn.BatchNorm1d(64),
            nn.LeakyReLU(0.3, inplace=True),
            nn.Conv1d(64, 32, 15, stride=6),
        )

    def forward(self, wav_data):
        # wav_data: one channle and 16k sample rate
        wav_data = wav_data.unsqueeze(1)  # add channel dim
        out = self.feat_extractor(wav_data)
        return out.transpose(1, 2)  # to (batch x seq x dim)

if __name__ == '__main__':
    net = WavEncoder()
    inp = torch.ones(32, 6 * 16000)
    out = net(inp)
    print(out.shape)


import numpy as np
from baselines.common.runners import AbstractEnvRunner
from gym import spaces
import pickle
from baselines import logger
from common.util import DataRecorder
import os
from copy import deepcopy

class Runner(AbstractEnvRunner):
    def __init__(self, env, model, nsteps, save_interval):
        super().__init__(env=env, model=model, nsteps=nsteps)
        assert isinstance(env.action_space,
                          spaces.Discrete), 'This ACER implementation works only with discrete action spaces!'

        self.nact = env.action_space.n
        nenv = self.nenv
        self.nbatch = nenv * nsteps
        self.batch_ob_shape = (nenv * (nsteps + 1),) + env.observation_space.shape

        # self.obs = env.reset()
        self.obs_dtype = env.observation_space.dtype
        self.obs_shape = env.observation_space.shape
        self.ac_dtype = env.action_space.dtype

        self.recoder = DataRecorder(os.path.join(logger.get_dir(), "runner_data"))
        self.save_interval = save_interval

        self.size = [int(x) for x in self.env.spec.id.split("-")[2].split("x")]
        self.desired_pos = np.asarray(self.size) - 1
        logger.info("-" * 50)
        logger.info("-" * 15, "desired_pos:", self.desired_pos, "-" * 15)
        logger.info("-" * 50)

        self.goals, self.goal_infos = self.get_goal(self.nenv)
        self.episode_step = np.zeros(self.nenv, dtype=np.int32)
        self.episode = np.zeros(self.nenv, dtype=np.int32)

    def run(self):
        # enc_obs = np.split(self.obs, self.nstack, axis=3)  # so now list of obs steps
        mb_obs, mb_actions, mb_mus, mb_dones, mb_rewards, mb_goals = [], [], [], [], [], [],
        episode_info = {}
        for _ in range(self.nsteps):
            actions, mus, states = self.model.step(self.obs, S=self.states, M=self.dones, goals=self.goals)
            mb_obs.append(np.copy(self.obs))
            mb_actions.append(actions)
            mb_mus.append(mus)
            mb_dones.append(self.dones)
            mb_goals.append(np.copy(self.goals))
            obs, rewards, dones, infos = self.env.step(actions)
            self.episode_step += 1
            for env_idx in range(self.nenv):
                if dones[env_idx]:
                    self.episode_step[env_idx] = 0
                    self.episode[env_idx] += 1
                    episode_info["episode"] = infos[env_idx]["episode"]
            # states information for statefull models like LSTM
            self.states = states
            self.dones = dones
            self.obs = obs
            mb_rewards.append(rewards)
        mb_goals.append(np.copy(self.goals))
        mb_obs.append(np.copy(self.obs))
        mb_dones.append(self.dones)

        mb_obs = np.asarray(mb_obs, dtype=self.obs_dtype).swapaxes(1, 0)
        mb_actions = np.asarray(mb_actions, dtype=self.ac_dtype).swapaxes(1, 0)
        mb_rewards = np.asarray(mb_rewards, dtype=np.float32).swapaxes(1, 0)
        mb_mus = np.asarray(mb_mus, dtype=np.float32).swapaxes(1, 0)
        mb_goals = np.asarray(mb_goals, dtype=self.goals.dtype).swapaxes(1, 0)
        mb_dones = np.asarray(mb_dones, dtype=np.bool).swapaxes(1, 0)

        mb_masks = mb_dones  # Used for statefull models like LSTM's to mask state when done
        mb_dones = mb_dones[:, 1:]  # Used for calculating returns. The dones array is now aligned with rewards

        # shapes are now [nenv, nsteps, []]
        # When pulling from buffer, arrays will now be reshaped in place, preventing a deep copy.

        results = dict(
            obs=mb_obs,
            actions=mb_actions,
            rewards=mb_rewards,
            mus=mb_mus,
            dones=mb_dones,
            masks=mb_masks,
            goal_obs=mb_goals,
            episode_info=episode_info,
        )
        return results

    def get_goal(self, nb_goal):
        return np.array([self.desired_pos for _ in range(nb_goal)]), [{} for _ in range(nb_goal)]

    def initialize(self):
        pass

    @staticmethod
    def check_goal_reached_v2(obs_info, goal_info):
        pass


#!/usr/bin/env python3

# running on python 3 - open cv 4.2.0
import numpy as np
import cv2 as cv
import csv
import time

import rospy
from std_msgs.msg import Float64
from marvelmind_nav.msg import hedge_pos_ang

# change variables here
frame_width = 640
frame_height = 480
fps = 30

class FalconVideo(object):
	def __init__(self):
		self.cap = cv.VideoCapture(0)
		self.width = self.cap.set(3,frame_width)
		self.height = self.cap.set(4,frame_height)
		# Define the codec and create VideoWriter object
		self.codec = cv.VideoWriter_fourcc(*'H264')
		self.out = cv.VideoWriter('/home/ubuntu/Videos/falcon/output.avi', self.codec, fps, (frame_width,  frame_height))

		self.log_file = open('/home/ubuntu/Videos/falcon/output.csv', 'w')
		self.file_writer = csv.writer(self.log_file)
		self.is_recording = False
		self.x = 0
		self.y = 0
		self.yaw = 0

		self.robot_pose_sub = rospy.Subscriber("hedge_pos_ang", hedge_pos_ang, self.robot_pose_cb, queue_size=1)
		self.robot_pose_sub = rospy.Subscriber("robot_yaw", Float64, self.robot_yaw_cb, queue_size=1)

		self.frame_count = 0

	def recording(self):
		if not self.is_recording:
			rospy.loginfo_throttle(5,"Waiting for recording trigger...")
		else:
			rospy.loginfo("Starting video recording!")
			self.start_time = time.time()
			while self.cap.isOpened():
				try:
					self.ret, self.frame = self.cap.read()
					if not self.ret:
							print("Can't receive frame (stream end?). Exiting ...")
							break
					self.frame = cv.flip(self.frame, -1)
					# write the flipped frame
					self.out.write(self.frame)
					# write to csv file
					self.current_time = time.time()
					self.timestamp = self.current_time - self.start_time
					self.file_writer.writerow([self.timestamp, self.x, self.y, self.yaw])
					#print("frame "+str(self.frame_count)+" received")
					self.frame_count+=1
					# show image feed
					#cv.imshow('frame', self.frame)
					if cv.waitKey(1) == ord('q') or rospy.is_shutdown():
							break
				except KeyboardInterrupt or rospy.ROSInterruptException:
					break

	def robot_pose_cb(self, msg):
		self.x = msg.x_m
		self.y = msg.y_m

	def robot_yaw_cb(self, msg):
		self.yaw = msg.data

if __name__ == "__main__":
	rospy.init_node("falcon_cam_node")
	rospy.loginfo("Initiating falcon cam node")
	rate = rospy.Rate(10)

	rospy.set_param("/is_recording", False)
	video = FalconVideo()

	while not rospy.is_shutdown():
		rate.sleep()
		video.is_recording = rospy.get_param("/is_recording")
		video.recording()
	# release all when ros is stopped
	video.cap.release()
	video.out.release()
	cv.destroyAllWindows()
	video.log_file.close()


from urlextract import URLExtract
from wordcloud import WordCloud
from collections import Counter
import pandas as pd
import emoji

extractor=URLExtract()

def fetch_stats(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]
    # fetch the number of messages
    num_messages = df.shape[0]
    # fetch the total number of words
    words = []
    for message in df['message']:
        words.extend(message.split())
    # fetch the total number of media messages
    num_media_msg=df[df['message']=='<Media omitted>\n'].shape[0]

    links=[]
    for message in df['message']:

        links.extend(extractor.find_urls(message))
    return num_messages, len(words),num_media_msg,len(links)

def busy_user(df):
    x=df['user'].value_counts().head()
    new_df = round(df['user'].value_counts() / df.shape[0] * 100, 2).reset_index().rename(columns={'index': 'user', 'user': 'percent'})

    return x,new_df

def generate_wordcloud(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    word_cloud = WordCloud(width=500, height=300,min_font_size=10, max_words=100,background_color="white")
    df_wc=word_cloud.generate(df['message'].str.cat(sep=" "))
    return df_wc

def most_com_word(selected_user,df):
    f=open('stop_hinglish.txt','r')
    stop_word=f.read()

    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    temp=df[df['user'] != 'group_notification']
    temp=temp[temp['message'] != '<Media omitted>\n']
    words=[]

    for message in temp['message']:
        for word in message.lower().split():
            if word not in stop_word and len(word)>5:
               words.append(word)

    most_com=pd.DataFrame(Counter(words).most_common(20))
    return most_com

def emoji(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    emojis=[]
    for message in df['message']:
        emojis.extend([c for c in message if c in emoji.EMOJI_DATA])
    emoji_df=pd.DataFrame(Counter(emojis).most_common(len(Counter(emojis))))
    return emoji_df

#time analysis
def time_analysis(selected_user,df):
     if selected_user != 'Overall':
        df = df[df['user'] == selected_user]
     time_analysis=df.groupby(['year','month_num','month']).count()['message'].reset_index()
     time=[]
     for i in range(time_analysis.shape[0]):
         time.append(time_analysis['month'][i]+'-'+str(time_analysis['year'][i]))

     time_analysis['time']=time

     return time_analysis

#Daily_Timeline
def daily_analysis(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    daily_timeline=df.groupby(['only_date']).count()['message'].reset_index()
    return daily_timeline

def busy_day(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    busy_day=df['day_name'].value_counts()
    return busy_day

def monthly_analysis(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    monthly_analysis=df['month'].value_counts()
    return monthly_analysis

def het_map(selected_user,df):
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    activiy_heat_map=df.pivot_table(index='day_name',columns='period',values='message',aggfunc='count').fillna(0)
    return activiy_heat_map


from django.shortcuts import render
from products.models import Product, Comment

def index(request):
    """
    A view to return the index page
    """

    products = Product.objects.all()
    featured_products = list(Product.objects.filter(featured=True).order_by(
        'featured'))[-9:-1][::-1]  # show recent 8 featured products
    # show recently added 8 products
    recently_added = products.order_by('-id')[:8]

    comments = Comment.objects.all().order_by('-id')[:10]
    context = {
        'products': products,
        'featured_products': featured_products,
        'recently_added': recently_added,
        'comments': comments
    }

    return render(request, 'home/index.html', context)


import datetime
import hashlib
from dateutil.parser import parse

def fix_datetime(datetime_str, local_dt_str = None):
	'''
	Given a HH:MM:SS string from a telemetry sentence, produce a complete timestamp, using the current system time as a guide for the date.
	'''

	if local_dt_str is None:
		_now = datetime.datetime.utcnow()
	else:
		_now = parse(local_dt_str)

	# Are we in the rollover window?
	if _now.hour == 23 or _now.hour == 0:
		_outside_window = False
	else:
		_outside_window = True

	# Parsing just a HH:MM:SS will return a datetime object with the year, month and day replaced by values in the 'default'
	# argument.
	_imet_dt = parse(datetime_str, default=_now)

	if _outside_window:
		# We are outside the day-rollover window, and can safely use the current zulu date.
		return _imet_dt
	else:
		# We are within the window, and need to adjust the day backwards or forwards based on the sonde time.
		if _imet_dt.hour == 23 and _now.hour == 0:
			# Assume system clock running slightly fast, and subtract a day from the telemetry date.
			_imet_dt = _imet_dt - datetime.timedelta(days=1)

		elif _imet_dt.hour == 00 and _now.hour == 23:
			# System clock running slow. Add a day.
			_imet_dt = _imet_dt + datetime.timedelta(days=1)

		return _imet_dt

#
#	iMet Radiosonde Functions
#

def imet_unique_id(telemetry, custom=""):
	'''
	Generate a 'unique' imet radiosonde ID based on the power-on time, frequency, and an optional location code.
	This requires the following fields be present in the telemetry dict:
		datetime_dt (datetime)  (will need to be generated above)
		frame (int) - Frame number
		freq (str) - Frequency, as "%.3f MHz"
	'''

	_imet_dt = telemetry['datetime_dt']

	# Determine power on time: Current time -  number of frames (one frame per second)
	_power_on_time = _imet_dt - datetime.timedelta(seconds=telemetry['frame'])

	# Now we generate a string to hash based on the power-on time, the frequency, and the custom field.
	_temp_str = _power_on_time.strftime("%Y-%m-%dT%H:%M:%SZ") + telemetry['freq'] + custom

	# Calculate a SHA256 hash of the
	_hash = hashlib.sha256(_temp_str.encode('ascii')).hexdigest().upper()

	return "IMET-" + _hash[-8:]

if __name__ == "__main__":

	# Testing scripts for the above.

	test_data = [
		{'datetime':'23:59:58', 'frame': 50, 'freq': '402.000 MHz', 'local_dt': "2019-03-01T23:59:58Z"},
		{'datetime':'23:59:58', 'frame': 50, 'freq': '402.000 MHz', 'local_dt': "2019-03-01T23:59:57Z"},
		{'datetime':'23:59:58', 'frame': 50, 'freq': '402.000 MHz', 'local_dt': "2019-03-02T00:00:03Z"},
		{'datetime':'00:00:00', 'frame': 52, 'freq': '402.000 MHz', 'local_dt': "2019-03-01T23:59:57Z"},
		{'datetime':'00:00:00', 'frame': 52, 'freq': '402.000 MHz', 'local_dt': "2019-03-02T00:00:03Z"},
		{'datetime':'00:00:01', 'frame': 53, 'freq': '402.000 MHz', 'local_dt': "2019-03-01T23:59:57Z"},
		{'datetime':'00:00:01', 'frame': 53, 'freq': '402.000 MHz', 'local_dt': "2019-03-02T00:00:03Z"},
		{'datetime':'11:59:58', 'frame': 42, 'freq': '402.000 MHz', 'local_dt': "2019-03-01T12:00:03Z"},
		{'datetime':'12:00:02', 'frame': 46, 'freq': '402.000 MHz', 'local_dt': "2019-03-01T12:00:03Z"},
		]

	for _test in test_data:

		_test['datetime_dt'] = imet_fix_datetime(_test['datetime'], local_dt_str = _test['local_dt'])
		print("Input Time: %s, Local Time: %s, Output Time: %s" % (_test['datetime'], _test['local_dt'], _test['datetime_dt'].strftime("%Y-%m-%dT%H:%M:%SZ")))
		_test['id'] = imet_unique_id(_test)
		print("Generated ID: %s" % _test['id'])
		print(" ")


import os
import random
import numpy as np
import torch
import torch.nn as nn
import math
import tqdm
import pytorch_warmup as warmup
from torch.optim.lr_scheduler import ReduceLROnPlateau
from data_loader import DataLoaderRegression
from pathlib import Path
import argparse
import glob
from utils import utils_calculate_psnr_ssim as util
import PIL.Image

class PixelWeightingNetwork(nn.Module):
    def __init__(self, in_channels=6, out_channels=1):
        super(PixelWeightingNetwork, self).__init__()

        self.conv1 = nn.Conv2d(in_channels, 64, kernel_size=3, padding=1)
        self.relu1 = nn.ReLU()

        self.conv2 = nn.Conv2d(64, 64, kernel_size=3, padding=1)
        self.relu2 = nn.ReLU()

        self.conv3 = nn.Conv2d(64, out_channels, kernel_size=3, padding=1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        x = self.relu1(self.conv1(input))
        x = self.relu2(self.conv2(x))
        x = self.sigmoid(self.conv3(x))

        return x

class ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size=3, padding=1):
        super(ResidualBlock, self).__init__()

        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.relu1 = nn.ReLU()

        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size, padding=padding)
        self.bn2 = nn.BatchNorm2d(out_channels)

    def forward(self, x):
        identity = x

        out = self.relu1(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))

        return out + identity

class PixelWeightingNetworkResNet(nn.Module):
    def __init__(self, in_channels=6, out_channels=1):
        super(PixelWeightingNetworkResNet, self).__init__()

        self.conv1 = nn.Conv2d(in_channels, 64, kernel_size=3, padding=1)
        self.relu1 = nn.ReLU()

        self.res1 = ResidualBlock(64, 64)
        self.res2 = ResidualBlock(64, 64)
        self.res3 = ResidualBlock(64, 64)
        self.res4 = ResidualBlock(64, 64)

        self.conv2 = nn.Conv2d(64, out_channels, kernel_size=3, padding=1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x = self.relu1(self.conv1(x))

        x = self.res1(x)
        x = self.res2(x)
        x = self.res3(x)
        x = self.res4(x)

        x = self.sigmoid(self.conv2(x))

        return x

def get_args_parser():
    parser = argparse.ArgumentParser('Training the binary classifier', add_help=False)
    parser.add_argument('--num_images', default=10, type=int)
    parser.add_argument('--epochs', default=10, type=int)
    parser.add_argument('--train_hr_dir', type=str, help='hq location')
    parser.add_argument('--train_lr_dir', type=str, help='lr location')
    parser.add_argument('--train_ttt_dir', type=str, help='ttt location')
    parser.add_argument('--train_pretrain_dir', type=str, help='pretrain location')
    parser.add_argument('--test_hr_dir', type=str, help='hq location')
    parser.add_argument('--test_lr_dir', type=str, help='lr location')
    parser.add_argument('--test_ttt_dir', type=str, help='ttt location')
    parser.add_argument('--test_pretrain_dir', type=str, help='pretrain location')
    parser.add_argument('--output_dir', type=str, help="location for new model checkpoints")
    parser.add_argument('--batch_size', type=int, default=32, help="Batch size")
    parser.add_argument('--num_workers', default=10, type=int)
    parser.add_argument('--lr', default=0.00001, type=float, help='Learning rate')
    parser.add_argument('--min_lr', default=0.000001, type=float, help='Min learning rate')
    parser.add_argument('--seed', default=2023, type=int, help='training seed')
    parser.add_argument('--img_size', default=48, type=int, help='training seed')
    parser.add_argument('--pin_mem', action='store_true',
                        help='Pin CPU memory in DataLoader for more efficient (sometimes) transfer to GPU.')
    parser.add_argument('--no_pin_mem', action='store_false', dest='pin_mem')
    parser.add_argument('--warmup_epochs', type=int, default=10, metavar='N',
                        help='epochs to warmup LR')
    parser.add_argument('--window_size', type=int, default=8, help='SwinIR model window size')
    parser.add_argument('--opt_type', type=str, default='Adam', help='Optimizer for SwinIR model (SGD/Adam)')
    parser.set_defaults(pin_mem=True)
    return parser

def train_model(args):

    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    model = PixelWeightingNetworkResNet()
    model.to(device)
    model = torch.nn.DataParallel(model)

    if args.opt_type == 'Adam':
        optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
    elif args.opt_type == 'SGD':
        optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9)
    optimizer_to(optimizer, device)

    criterion = torch.nn.MSELoss().to(device)
    lr_scheduler = ReduceLROnPlateau(optimizer, 'min')
    warmup_scheduler = warmup.BaseWarmup(optimizer, warmup_params=['momentum'])

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)

    save_dir = args.output_dir
    if not os.path.exists(args.output_dir):
         os.mkdir(args.output_dir)
    print(f'Saving to {save_dir}')

    train_dataset = DataLoaderRegression(args.train_hr_dir, args.train_lr_dir,
                                         args.train_pretrain_dir, args.train_ttt_dir,
                                         split='train', img_size=args.img_size)

    test_dataset = DataLoaderRegression(args.test_hr_dir, args.test_lr_dir,
                                         args.test_pretrain_dir, args.test_ttt_dir,
                                         split='test', img_size=args.img_size)
    sampler_train = torch.utils.data.RandomSampler(train_dataset)
    sampler_val = torch.utils.data.SequentialSampler(test_dataset)

    train_data_loader = torch.utils.data.DataLoader(train_dataset,
        sampler=sampler_train,
        batch_size=args.batch_size,
        num_workers=args.num_workers,
        pin_memory=args.pin_mem,
        drop_last=True
    )

    test_data_loader = torch.utils.data.DataLoader(
        test_dataset, sampler=sampler_val,
        batch_size=args.batch_size,
        num_workers=args.num_workers,
        pin_memory=args.pin_mem,
        drop_last=False
    )

    # Training loop
    print("Training Batches:", len(train_data_loader))
    best_prec1 = 0
    for epoch in range(args.epochs):
        print(f"Starting epoch {epoch}...")
        prec1 = train(args, train_data_loader, model, optimizer, criterion, epoch, device)
        print(f'Done training. Epoch [{epoch:05}]: Loss={prec1}')
        test_prec1 = test(args, test_data_loader, model, criterion, epoch, device)
        best_prec1 = max(prec1, best_prec1)
        state_dict = {
            'epoch': epoch + 1,
            'arch': 'swinir',
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
            'test_prec1': test_prec1
        }
        print(f'Done testing. Epoch [{epoch:05}]: Loss={test_prec1}')
        with warmup_scheduler.dampening():
            lr_scheduler.step(test_prec1)
        if (epoch + 1) % 2 == 0:
            torch.save(state_dict['state_dict'], f'{save_dir}/weighting_s{args.img_size}_ep{epoch+1}_win{args.window_size}_{prec1:.3f}_{test_prec1:.3f}.pth')

def combined_image(swinir_img, fine_tuned_img, weights):
    return weights * swinir_img + (1 - weights) * fine_tuned_img

def mse_loss(y_true, y_pred):
    return torch.mean((y_true - y_pred)**2)

def train(args, data_loader, model, optimizer, criterion, epoch, device):
    loss_vals, mse_vals= [], []
    model.train()

    for idx, (inputs, targets) in enumerate(tqdm.tqdm(data_loader)):
        image_orig, image_ttt = inputs
        image_orig = image_orig.to(device)
        image_ttt = image_ttt.to(device)
        targets = targets.to(device)

        inputs = torch.cat((image_orig, image_ttt), dim=1)
        inputs = inputs.to(device)

        weights = model(inputs)
        outputs = combined_image(image_orig, image_ttt, weights).to(device)

        loss = criterion(outputs, targets)

        optimizer.zero_grad()

        loss.backward()

        optimizer.step()

        with torch.no_grad():
            step_loss = loss.item()
            step_mse = mse_loss(targets, outputs)

        loss_vals.append(step_loss)
        mse_vals.append(step_mse.item())

    avg_loss = np.mean(loss_vals)
    avg_mae = np.mean(mse_vals)

    print(f"Epoch [{epoch + 1}], Loss: {avg_loss:.4f}, MSE: {avg_mae:.4f}")

    return avg_loss

@torch.no_grad()
def test(args, data_loader, model, criterion, epoch, device):
    model.eval()
    loss_vals, mse_vals = [], []
    for idx, (inputs, targets) in enumerate(tqdm.tqdm(data_loader)):
        image_orig, image_ttt = inputs
        image_orig = image_orig.to(device, non_blocking=True)
        image_ttt = image_ttt.to(device, non_blocking=True)
        targets = targets.to(device, non_blocking=True)
        weights = model(torch.cat((image_orig, image_ttt), dim=1))
        outputs = combined_image(image_orig, image_ttt, weights).to(device)
        loss = criterion(outputs, targets)
        with torch.no_grad():
            test_loss = loss.item()
            test_mse = mse_loss(targets, outputs)

        loss_vals.append(test_loss)
        mse_vals.append(test_mse.item())

    avg_loss = np.mean(loss_vals)
    avg_mse = np.mean(mse_vals)

    print(f"Epoch [{epoch + 1}], Loss: {avg_loss:.4f}, MSE: {avg_mse:.4f}")

    return avg_loss

def prep_image(path):
    image = PIL.Image.open(path)
    image = np.array(image) / 255.
    image = image.transpose(2, 0, 1)
    image = image[np.newaxis,...]
    return torch.from_numpy(image).float()

@torch.no_grad()
def weight_and_psnr(model, ttt_path, orig_path, lr_path, gt_path, device, switch_after=1, scale=4):
    model.eval()
    weighted_psnrs = []
    weighted_ssims = []
    ttt_psnrs = []
    ttt_ssims = []
    orig_psnrs = []
    orig_ssims = []
    for idx, gt_image_path in enumerate(tqdm.tqdm(glob.glob(os.path.join(gt_path, '*.png')))):
        # if idx == switch_after:
        #     device = torch.device("cuda:1")
        #     model.to(device)
        image_name = os.path.split(gt_image_path)[-1]
        gt_image = prep_image(gt_image_path)
        lr_image = prep_image(os.path.join(lr_path, image_name))
        _, _, h_old, w_old = lr_image.size()
        gt_image = gt_image[..., :h_old * 4, :w_old * 4]  # crop gt
        gt_image.to(device)
        lr_image.to(device)
        ttt_image = prep_image(os.path.join(ttt_path, image_name)).to(device)
        orig_image = prep_image(os.path.join(orig_path, image_name)).to(device)
        model_output = model(torch.cat((orig_image, ttt_image), dim=1))
        output = combined_image(orig_image, ttt_image, model_output).to(device)
        gt_image = gt_image[:, :, :output.shape[2], :output.shape[3]]
        assert output.shape == gt_image.shape
        gt_image = (gt_image[0].detach().cpu().numpy().transpose(1, 2, 0) * 255.0).round().astype(np.uint8)
        output = (output[0].detach().cpu().numpy().transpose(1, 2, 0) * 255.0).round().astype(np.uint8)
        ttt_image = (ttt_image[0].detach().cpu().numpy().transpose(1, 2, 0) * 255.0).round().astype(np.uint8)
        orig_image = (orig_image[0].detach().cpu().numpy().transpose(1, 2, 0) * 255.0).round().astype(np.uint8)
        # Weighted:
        psnr_y = util.calculate_psnr(output, gt_image, crop_border=scale, test_y_channel=True)
        ssim_y = util.calculate_ssim(output, gt_image, crop_border=scale, test_y_channel=True)
        weighted_psnrs.append(psnr_y)
        weighted_ssims.append(ssim_y)
        # TTT
        psnr_y = util.calculate_psnr(ttt_image, gt_image, crop_border=scale, test_y_channel=True)
        ssim_y = util.calculate_ssim(ttt_image, gt_image, crop_border=scale, test_y_channel=True)
        ttt_psnrs.append(psnr_y)
        ttt_ssims.append(ssim_y)
        # ORIGINAL
        psnr_y = util.calculate_psnr(orig_image, gt_image, crop_border=scale, test_y_channel=True)
        ssim_y = util.calculate_ssim(orig_image, gt_image, crop_border=scale, test_y_channel=True)
        orig_psnrs.append(psnr_y)
        orig_ssims.append(ssim_y)

    print('Mean orig psnr:', np.mean(orig_psnrs))
    print('Mean orig ssim:', np.mean(orig_ssims))
    print('Mean ttt psnr:', np.mean(ttt_psnrs))
    print('Mean ttt ssim:', np.mean(ttt_ssims))
    print('Mean weighting psnr:', np.mean(weighted_psnrs))
    print('Mean weighting ssim:', np.mean(weighted_ssims))

def adjust_learning_rate(optimizer, epoch, args):
    """Decay the learning rate with half-cycle cosine after warmup"""
    if epoch < args.warmup_epochs:
        lr = args.lr * epoch / args.warmup_epochs
    else:
        lr = args.min_lr + (args.lr - args.min_lr) * 0.5 * \
            (1. + math.cos(math.pi * (epoch - args.warmup_epochs) / (args.epochs - args.warmup_epochs)))
    for param_group in optimizer.param_groups:
        if "lr_scale" in param_group:
            param_group["lr"] = lr * param_group["lr_scale"]
        else:
            param_group["lr"] = lr
    return lr

def optimizer_to(optim, device):
    """
    Move pretrained optimizer to device used by PyTorch
    """
    for param in optim.state.values():
        if isinstance(param, torch.Tensor):
            param.data = param.data.to(device)
            if param._grad is not None:
                param._grad.data = param._grad.data.to(device)
        elif isinstance(param, dict):
            for subparam in param.values():
                if isinstance(subparam, torch.Tensor):
                    subparam.data = subparam.data.to(device)
                    if subparam._grad is not None:
                        subparam._grad.data = subparam._grad.data.to(device)

def main():
    # Argument parser
    parser = get_args_parser()
    args = parser.parse_args()
    if args.output_dir:
        Path(args.output_dir).mkdir(parents=True, exist_ok=True)
    train_model(args)

if __name__ == '__main__':
    main()


"""Simple pyton library for nure-cist API"""

__version__ = '1.7.0'

import requests
import time
from typing import Literal

def convert_time(time_toconvert: str | int, pattern: str) -> int:
    if type(time_toconvert) is int:
        try:
            time.strptime(time.ctime(time_toconvert))
            return time_toconvert
        except ValueError:
            raise ValueError(f"Invalid timestamp was given: {time_toconvert}")
    elif type(time_toconvert) is str:
        try:
            timestamp = int(time.mktime(time.strptime(time_toconvert, pattern)))
            return timestamp
        except ValueError:
            raise ValueError(f"Unable to parse the input time string.\nBe sure to write time in format like {pattern}")
    else:
        raise ValueError("Invalid input type was given.")

def get_groups() -> list:
    groups_respond = requests.get('https://api.mindenit.tech/groups')

    if groups_respond.status_code == 200:
        respond = []
        for group in groups_respond.json():
            group["id"] = int(group["id"])
            respond.append(group)
        return respond
    else:
        groups_respond.raise_for_status()

def get_teachers() -> list:
    teachers_respond = requests.get('https://api.mindenit.tech/teachers')

    if teachers_respond.status_code == 200:
        respond = []
        for teacher in teachers_respond.json():
            teacher["id"] = int(teacher["id"])
            respond.append(teacher)
        return respond
    else:
        teachers_respond.raise_for_status()

def get_auditoriums() -> list:
    auditoriums_respond = requests.get('https://api.mindenit.tech/auditories')

    if auditoriums_respond.status_code == 200:
        respond = []
        for auditorium in auditoriums_respond.json():
            auditorium["id"] = int(auditorium["id"])
            respond.append(auditorium)
        return respond
    else:
        auditoriums_respond.raise_for_status()

def get_schedule(request_type: Literal['group', 'teacher', 'auditory'],
                 request_id: int,
                 start_time: str,
                 end_time: str,
                 pattern: str = "%Y-%m-%d %H:%M"
                 ) -> list:
    start_time = convert_time(start_time, pattern)
    end_time = convert_time(end_time, pattern)

    if request_type not in ['group', 'teacher', 'auditory']:
        raise ValueError('Invalid request type')

    schedule_respond = requests.get(
        f'https://api.mindenit.tech/schedule?type={request_type}&id={request_id}&start_time={start_time}&end_time={end_time}')

    if schedule_respond.status_code == 200:
        respond = []
        for lesson in schedule_respond.json():
            lesson["startTime"] = int(lesson["startTime"])
            lesson["endTime"] = int(lesson["endTime"])
            lesson["subject"]["id"] = int(lesson["subject"]["id"])

            teachers = []
            for teacher in lesson["teachers"]:
                teacher["id"] = int(teacher["id"])
                teachers.append(teacher)
            lesson["teachers"] = teachers

            groups = []
            for group in lesson["groups"]:
                group["id"] = int(group["id"])
                groups.append(group)
            lesson["groups"] = groups

            respond.append(lesson)

        return respond
    else:
        schedule_respond.raise_for_status()

def find_group(group_name: str) -> list:
    groups = get_groups()
    for group in groups:
        if group["name"].lower() == group_name.lower():
            return group
    raise ValueError(f"Couldn\'t find group \"{group_name}\"")

def find_teacher(teacher_name: str) -> list:

    teacher_name = teacher_name.lower()
    teachers = get_teachers()

    result = []
    for teacher in teachers:
        if teacher_name[-1] == '.':
            if teacher["shortName"].lower() == teacher_name:
                result.append(teacher)
                return result
        else:
            if teacher_name in teacher["shortName"].lower():
                result.append(teacher)

    if len(result) != 0:
        return result

    raise ValueError(f"Couldn\'t find teacher \"{teacher_name}\", make shour that you wrote name correctly (for example \"Саманцов О. О.\" or \"Саманцов\")")

def find_auditorium(auditorium_name: str) -> list:
    auditoriums = get_auditoriums()
    for auditorium in auditoriums:
        if auditorium["name"] == auditorium_name:
            return auditorium
    raise ValueError(f"Couldn\'t find group \"{auditorium_name}\"")


"""authentik API Modelviewset tests"""
from typing import Callable

from django.test import TestCase
from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet

from authentik.api.v3.urls import router

class TestModelViewSets(TestCase):
    """Test Viewset"""

def viewset_tester_factory(test_viewset: type[ModelViewSet]) -> Callable:
    """Test Viewset"""

    def tester(self: TestModelViewSets):
        self.assertIsNotNone(getattr(test_viewset, "search_fields", None))
        self.assertIsNotNone(getattr(test_viewset, "ordering", None))
        filterset_class = getattr(test_viewset, "filterset_class", None)
        if not filterset_class:
            self.assertIsNotNone(getattr(test_viewset, "filterset_fields", None))

    return tester

for _, viewset, _ in router.registry:
    if not issubclass(viewset, (ModelViewSet, ReadOnlyModelViewSet)):
        continue
    setattr(TestModelViewSets, f"test_viewset_{viewset.__name__}", viewset_tester_factory(viewset))


import argparse
import configparser
import os

from get_robot_status import get_robot_position

from util import get_dir_path

PI=3.1415926
TOLERANCE = 0.1

parser = argparse.ArgumentParser()
# model 型号
parser.add_argument('-model', '--model', help='Model No.')
# step 步骤序号
parser.add_argument('-step', '--step', help='Step No.')
# read 读取对应步骤的坐标
parser.add_argument('-read', '--read', action='store_true', help='Read Config')
# write 保存当前步骤对应的机械臂坐标
parser.add_argument('-write', '--write', action='store_true', help='Write Config')

args = parser.parse_args()
read_flag = False
write_flag = False
step_no = ''
model_no = ''
config_dir = 'config'
config_file = ''
current_directory = get_dir_path()
socket_config_path = os.path.join(current_directory, config_dir, 'config.ini')
socket_config_name = 'Current_Model'

if args.read:
    # print(f'read: {args.read}')
    read_flag = args.read

if args.write:
    # print(f'read: {args.write}')
    write_flag = args.write

if args.step:
    # print(f'step: {args.step}')
    step_no = args.step

if args.model:
    # print(f'step: {args.step}')
    model_no = args.model

def set_current_model():
    config = configparser.ConfigParser()
    config.read(socket_config_path)  # 读取现有的配置文件

    if not config.has_section(socket_config_name):
        config.add_section(socket_config_name)
    config[socket_config_name] = {'model': model_no}

    with open(socket_config_path, 'w') as configfile:
        config.write(configfile)

def get_location():
    # 检查当前目录下是否存在配置文件
    if os.path.exists(config_file):
        # 创建配置解析器对象
        config = configparser.ConfigParser()
        config.read(config_file)

        # 检查步骤号对应的节是否存在于配置中
        section_name = f'{step_no}'
        if config.has_section(section_name):
            location = config.get(section_name, 'location')
            # print(location)
            return location
        else:
            print("Step does not exist")
            return ''
    else:
        print("Config file does not exist")
        return ''

def save_params():
    ensure_config_dir()
    location = get_robot_position()
    if len(location) != 6 :
        print("error")
        return

    location = list(location)  # 将元组转换为列表
    location[3] = location[3] * 180 / PI
    location[4] = location[4] * 180 / PI
    location[5] = location[5] * 180 / PI
    location = tuple(location)

    final_location = [round(x, 3) for x in location]

    # 检测是否水平
    # if check_position(final_location) == False:
        # print('Please check the horizontal state of the robotic arm.')
    # 创建配置解析器对象
    config = configparser.ConfigParser()

    # 检查当前目录下是否存在配置文件
    if os.path.exists(config_file):
        # 如果文件存在，则读取其中的数据
        config.read(config_file)

    # 将步骤号和位置信息添加到配置对象中
    section_name = f'{step_no}'
    config[section_name] = {'location': ','.join(str(coord) for coord in final_location)}

    # 将配置对象写入配置文件
    with open(config_file, 'w') as file:
        config.write(file)
        print(f"Configuration saved to{{aaaa}} {config_file}.")

def read_params():
    location = get_location()
    # print(f"Location for Step {step_no}: {location}")
    print(f"{location}")

def ensure_config_dir():
    if not os.path.exists(config_dir):
        os.makedirs(config_dir)

def check_position(position):
    x_error = round(abs(position[3] - 180.00), 4)
    y_error = round(abs(position[4]), 4)

    if x_error <= TOLERANCE and y_error <= TOLERANCE:
        return True
    else:
        return False

# step_no = 1
# model_no = 'CalcConfig'
# write_flag = True
if model_no != '' and step_no == '':

    print('set model')
    set_current_model()
# elif step_no != '' and model_no != '':
elif step_no != '' and model_no == '':
    if read_flag == False and write_flag == False:
        read_flag = True

    # config_file = os.path.join(os.path.join(current_directory, config_dir), f'{model_no}.ini')
    print(socket_config_path)
    config = configparser.ConfigParser()
    if config.read(socket_config_path):
        if config.has_section(socket_config_name):
            model_no = config.get(socket_config_name, 'model')
    if model_no == '':
        print('pls set current model first!')

    config_file = os.path.join(model_no, 'CalcConfig.ini')
    print(config_file)
    if read_flag:
        read_params()
    if write_flag:
        save_params()
else:
    print("Please input the correct params")

# 先初始化项目路径
# py .\location.py -model D:\Image\25mm

# 读取配置文件中对应step的location
# py .\location.py -step {step名字} -read

# 将当前机械臂坐标信息，写入对应step中
# py .\location.py -step {step名字} -write