import openpyxl
import datetime
from openpyxl.utils.cell import get_column_letter

workbook = openpyxl.load_workbook(filename="testMKTCAPCashBalanceCPJAN18.xlsx")
sheet = workbook["CashBalanceJAN18"]
number_rows = sheet.max_row
number_columns = sheet.max_column
CURRENTDATE = "03/09/2021"
#######################

def findClosePrice():

    for i in range(1,number_rows):
        print("---------------finding ClosePrice----------------")
        try:
            tickerName = str(sheet[("A"+str(i+1))].value)
            endDateValue = str(sheet[("D"+str(i+1))].value)[:10]
            endDate = datetime.datetime.strptime(endDateValue, '%Y-%m-%d')
            # endDateM5 = endDate + datetime.timedelta(days=-5)

            strEndDate = endDate.strftime('%d/%m/%Y')
            # strEndDateM5 = endDateM5.strftime('%d/%m/%Y')
            currentDateOBJ = datetime.datetime.strptime(CURRENTDATE, "%d/%m/%Y")
            endDateOBJ = datetime.datetime.strptime(strEndDate, "%d/%m/%Y")
            print(tickerName + " cashBalance endDate " + strEndDate)

            tickerFileOpen = "cleanDate" + tickerName + ".xlsx"
            print(".....Opening '" + tickerFileOpen + "'....")

            tickerWorkBook = openpyxl.load_workbook(filename=tickerFileOpen)
            tickerSheet = tickerWorkBook.active
            tickerNumber_rows = tickerSheet.max_row
            # tickerNumber_columns = tickerSheet.max_column
            loopIndex = 0
            foundDate = False
            while foundDate == False:
                #if endDate is in the future, break the loop || loop more than 6 times
                if foundDate == True or currentDateOBJ < endDateOBJ or loopIndex > 6:
                    break

                k = 0
                while k < tickerNumber_rows:
                    tickerDate = str(tickerSheet["A"+str(k+1)].value)
                    strEndDateValue = strEndDate
                    if loopIndex > 0:
                        tempDate = datetime.datetime.strptime(strEndDate, '%d/%m/%Y') + datetime.timedelta(days=-loopIndex)
                        strEndDateValue = tempDate.strftime('%d/%m/%Y')

                        # strEndDate = str(tickerSheet["A"+str(k+1)].value)[:10]
                        # tickerDateDateTime = datetime.datetime.strptime(tickerDate, '%Y-%m-%d')
                        # tickerDateMinus1 = tickerDateDateTime + datetime.timedelta(days=-1)
                        # tickerDate = tickerDateMinus1.strftime('%d/%m/%Y')

                    # tickerCell = "A" + str(k+1)
                    # print('this is tickerdate', tickerDate, ' strEndDate', strEndDate)
                    if tickerDate == strEndDateValue:
                        foundDate = True

                        # print('Found EndDate at Cell Number'+tickerCell)
                        tickerDateM7 = str(tickerSheet["A"+str(k-6)].value)
                        tickerDateM6 = str(tickerSheet["A"+str(k-5)].value)
                        tickerDateM5 = str(tickerSheet["A"+str(k-4)].value)
                        tickerDateM4 = str(tickerSheet["A"+str(k-3)].value)
                        tickerDateM3 = str(tickerSheet["A"+str(k-2)].value)
                        tickerDateM2 = str(tickerSheet["A"+str(k-1)].value)
                        tickerDateM1 = str(tickerSheet["A"+str(k)].value)
                        tickerDate0 = str(tickerSheet["A"+str(k+1)].value)
                        tickerDateP1 = str(tickerSheet["A"+str(k+2)].value)
                        tickerDateP2 = str(tickerSheet["A"+str(k+3)].value)
                        tickerDateP3 = str(tickerSheet["A"+str(k+4)].value)
                        tickerDateP4 = str(tickerSheet["A"+str(k+5)].value)
                        tickerDateP5 = str(tickerSheet["A"+str(k+6)].value)

                        tickerCPEndDateM7 = str(tickerSheet["E"+str(k-6)].value)
                        tickerCPEndDateM6 = str(tickerSheet["E"+str(k-5)].value)
                        tickerCPEndDateM5 = str(tickerSheet["E"+str(k-4)].value)
                        tickerCPEndDateM4 = str(tickerSheet["E"+str(k-3)].value)
                        tickerCPEndDateM3 = str(tickerSheet["E"+str(k-2)].value)
                        tickerCPEndDateM2 = str(tickerSheet["E"+str(k-1)].value)
                        tickerCPEndDateM1 = str(tickerSheet["E"+str(k)].value)
                        tickerCPEndDate = str(tickerSheet["E"+str(k+1)].value)
                        tickerCPEndDateP1 = str(tickerSheet["E"+str(k+2)].value)
                        tickerCPEndDateP2 = str(tickerSheet["E"+str(k+3)].value)
                        tickerCPEndDateP3 = str(tickerSheet["E"+str(k+4)].value)
                        tickerCPEndDateP4 = str(tickerSheet["E"+str(k+5)].value)
                        tickerCPEndDateP5 = str(tickerSheet["E"+str(k+6)].value)

                        sheet["F"+str(i+1)] = float(tickerCPEndDateM7) if tickerCPEndDateM7 != "None" else "DNE"
                        sheet["G"+str(i+1)] = float(tickerCPEndDateM6) if tickerCPEndDateM6 != "None" else "DNE"
                        sheet["H"+str(i+1)] = float(tickerCPEndDateM5) if tickerCPEndDateM5 != "None" else "DNE"
                        sheet["I"+str(i+1)] = float(tickerCPEndDateM4) if tickerCPEndDateM4 != "None" else "DNE"
                        sheet["J"+str(i+1)] = float(tickerCPEndDateM3) if tickerCPEndDateM3 != "None" else "DNE"
                        sheet["K"+str(i+1)] = float(tickerCPEndDateM2) if tickerCPEndDateM2 != "None" else "DNE"
                        sheet["L"+str(i+1)] = float(tickerCPEndDateM1) if tickerCPEndDateM1 != "None" else "DNE"
                        sheet["M"+str(i+1)] = float(tickerCPEndDate) if tickerCPEndDate != "None" else "DNE"
                        sheet["N"+str(i+1)] = float(tickerCPEndDateP1) if tickerCPEndDateP1 != "None" else "DNE"
                        sheet["O"+str(i+1)] = float(tickerCPEndDateP2) if tickerCPEndDateP2 != "None" else "DNE"
                        sheet["P"+str(i+1)] = float(tickerCPEndDateP3) if tickerCPEndDateP3 != "None" else "DNE"
                        sheet["Q"+str(i+1)] = float(tickerCPEndDateP4) if tickerCPEndDateP4 != "None" else "DNE"
                        sheet["R"+str(i+1)] = float(tickerCPEndDateP5) if tickerCPEndDateP5 != "None" else "DNE"

                        print(tickerName + " " + tickerDateM7 + " " + tickerCPEndDateM7)
                        print(tickerName + " " + tickerDateM6 + " " + tickerCPEndDateM6)
                        print(tickerName + " " + tickerDateM5 + " " + tickerCPEndDateM5)
                        print(tickerName + " " + tickerDateM4 + " " + tickerCPEndDateM4)
                        print(tickerName + " " + tickerDateM3 + " " + tickerCPEndDateM3)
                        print(tickerName + " " + tickerDateM2 + " " + tickerCPEndDateM2)
                        print(tickerName + " " + tickerDateM1 + " " + tickerCPEndDateM1)
                        print("###")
                        print("END CashBalance " + tickerName + " " + tickerDate0 + " " + tickerCPEndDate)
                        print("###")
                        print(tickerName + " " + tickerDateP1 + " " + tickerCPEndDateP1)
                        print(tickerName + " " + tickerDateP2 + " " + tickerCPEndDateP2)
                        print(tickerName + " " + tickerDateP3 + " " + tickerCPEndDateP3)
                        print(tickerName + " " + tickerDateP4 + " " + tickerCPEndDateP4)
                        print(tickerName + " " + tickerDateP5 + " " + tickerCPEndDateP5)

                        workbook.save("testMKTCAPCashBalanceCPJAN18.xlsx")

                        break
                    k += 1

                loopIndex += 1
                print("-------------------------------")

        except Exception as e:
            print(e)
            print("Not a DATE")
            break

    workbook.save("testMKTCAPCashBalanceCPJAN18.xlsx")

################

def findOpenPrice():

    for i in range(1, number_rows):
        print("---------------finding OpenPrice----------------")
        try:
            tickerName = str(sheet[("A"+str(i+1))].value)
            endDateValue = str(sheet[("D"+str(i+1))].value)[:10]
            endDate = datetime.datetime.strptime(endDateValue, '%Y-%m-%d')

            strEndDate = endDate.strftime('%d/%m/%Y')

            currentDateOBJ = datetime.datetime.strptime(CURRENTDATE, "%d/%m/%Y")
            endDateOBJ = datetime.datetime.strptime(strEndDate, "%d/%m/%Y")
            print(tickerName + " cashBalance endDate " + strEndDate)

            tickerFileOpen = "cleanDate" + tickerName + ".xlsx"
            print(".....Opening '" + tickerFileOpen + "'....")

            tickerWorkBook = openpyxl.load_workbook(filename=tickerFileOpen)
            tickerSheet = tickerWorkBook.active
            tickerNumber_rows = tickerSheet.max_row

            loopIndex = 0
            foundDate = False
            while foundDate == False:
                #if endDate is in the future, break the loop || loop more than 6 times

                if foundDate == True or currentDateOBJ < endDateOBJ or loopIndex > 6:
                    break

                k = 0
                while k < tickerNumber_rows:
                    tickerDate = str(tickerSheet["A"+str(k+1)].value)
                    strEndDateValue = strEndDate
                    if loopIndex > 0:
                        tempDate = datetime.datetime.strptime(strEndDate, "%d/%m/%Y") + datetime.timedelta(days=-loopIndex)
                        strEndDateValue = tempDate.strftime('%d/%m/%Y')

                    if tickerDate == strEndDateValue:
                        foundDate = True
                        tickerDateM7 = str(tickerSheet["A"+str(k-6)].value)
                        tickerDateM6 = str(tickerSheet["A"+str(k-5)].value)
                        tickerDateM5 = str(tickerSheet["A"+str(k-4)].value)
                        tickerDateM4 = str(tickerSheet["A"+str(k-3)].value)
                        tickerDateM3 = str(tickerSheet["A"+str(k-2)].value)
                        tickerDateM2 = str(tickerSheet["A"+str(k-1)].value)
                        tickerDateM1 = str(tickerSheet["A"+str(k)].value)
                        tickerDate0 = str(tickerSheet["A"+str(k+1)].value)
                        tickerDateP1 = str(tickerSheet["A"+str(k+2)].value)
                        tickerDateP2 = str(tickerSheet["A"+str(k+3)].value)
                        tickerDateP3 = str(tickerSheet["A"+str(k+4)].value)
                        tickerDateP4 = str(tickerSheet["A"+str(k+5)].value)
                        tickerDateP5 = str(tickerSheet["A"+str(k+6)].value)

                        tickerOPEndDateM7 = str(tickerSheet["B"+str(k-6)].value)
                        tickerOPEndDateM6 = str(tickerSheet["B"+str(k-5)].value)
                        tickerOPEndDateM5 = str(tickerSheet["B"+str(k-4)].value)
                        tickerOPEndDateM4 = str(tickerSheet["B"+str(k-3)].value)
                        tickerOPEndDateM3 = str(tickerSheet["B"+str(k-2)].value)
                        tickerOPEndDateM2 = str(tickerSheet["B"+str(k-1)].value)
                        tickerOPEndDateM1 = str(tickerSheet["B"+str(k)].value)
                        tickerOPEndDate = str(tickerSheet["B"+str(k+1)].value)
                        tickerOPEndDateP1 = str(tickerSheet["B"+str(k+2)].value)
                        tickerOPEndDateP2 = str(tickerSheet["B"+str(k+3)].value)
                        tickerOPEndDateP3 = str(tickerSheet["B"+str(k+4)].value)
                        tickerOPEndDateP4 = str(tickerSheet["B"+str(k+5)].value)
                        tickerOPEndDateP5 = str(tickerSheet["B"+str(k+6)].value)

                        sheet["F"+str(i+1)] = float(tickerOPEndDateM7) if tickerOPEndDateM7 != "None" else "DNE"
                        sheet["G"+str(i+1)] = float(tickerOPEndDateM6) if tickerOPEndDateM6 != "None" else "DNE"
                        sheet["H"+str(i+1)] = float(tickerOPEndDateM5) if tickerOPEndDateM5 != "None" else "DNE"
                        sheet["I"+str(i+1)] = float(tickerOPEndDateM4) if tickerOPEndDateM4 != "None" else "DNE"
                        sheet["J"+str(i+1)] = float(tickerOPEndDateM3) if tickerOPEndDateM3 != "None" else "DNE"
                        sheet["K"+str(i+1)] = float(tickerOPEndDateM2) if tickerOPEndDateM2 != "None" else "DNE"
                        sheet["L"+str(i+1)] = float(tickerOPEndDateM1) if tickerOPEndDateM1 != "None" else "DNE"
                        sheet["M"+str(i+1)] = float(tickerOPEndDate) if tickerOPEndDate != "None" else "DNE"
                        sheet["N"+str(i+1)] = float(tickerOPEndDateP1) if tickerOPEndDateP1 != "None" else "DNE"
                        sheet["O"+str(i+1)] = float(tickerOPEndDateP2) if tickerOPEndDateP2 != "None" else "DNE"
                        sheet["P"+str(i+1)] = float(tickerOPEndDateP3) if tickerOPEndDateP3 != "None" else "DNE"
                        sheet["Q"+str(i+1)] = float(tickerOPEndDateP4) if tickerOPEndDateP4 != "None" else "DNE"
                        sheet["R"+str(i+1)] = float(tickerOPEndDateP5) if tickerOPEndDateP5 != "None" else "DNE"

                        print(tickerName + " " + tickerDateM7 + " " + tickerOPEndDateM7)
                        print(tickerName + " " + tickerDateM6 + " " + tickerOPEndDateM6)
                        print(tickerName + " " + tickerDateM5 + " " + tickerOPEndDateM5)
                        print(tickerName + " " + tickerDateM4 + " " + tickerOPEndDateM4)
                        print(tickerName + " " + tickerDateM3 + " " + tickerOPEndDateM3)
                        print(tickerName + " " + tickerDateM2 + " " + tickerOPEndDateM2)
                        print(tickerName + " " + tickerDateM1 + " " + tickerOPEndDateM1)
                        print("###")
                        print("END CashBalance " + tickerName + " " + tickerDate0 + " " + tickerOPEndDate)
                        print("###")
                        print(tickerName + " " + tickerDateP1 + " " + tickerOPEndDateP1)
                        print(tickerName + " " + tickerDateP2 + " " + tickerOPEndDateP2)
                        print(tickerName + " " + tickerDateP3 + " " + tickerOPEndDateP3)
                        print(tickerName + " " + tickerDateP4 + " " + tickerOPEndDateP4)
                        print(tickerName + " " + tickerDateP5 + " " + tickerOPEndDateP5)
                        workbook.save("CashBalanceOPJAN18.xlsx")
                        break
                    k += 1

                loopIndex += 1
                print("-------------------------------")
        except Exception as e:
            print(e)
            print("Not a DATE")
            break
    workbook.save("CashBalanceOPJAN18.xlsx")

def addMktCap():

    for i in range(1, number_rows):
        print("--------------adding Market Capitalization--------------")
        try:
            tickerName = str(sheet[("A"+str(i+1))].value)
            endDateValue = str(sheet[("D"+str(i+1))].value)[:10]
            endDate = datetime.datetime.strptime(endDateValue, '%Y-%m-%d')
            # endDateM5 = endDate + datetime.timedelta(days=-5)

            strEndDate = endDate.strftime('%d/%m/%Y')
            # strEndDateM5 = endDateM5.strftime('%d/%m/%Y')
            currentDateOBJ = datetime.datetime.strptime(CURRENTDATE, "%d/%m/%Y")
            endDateOBJ = datetime.datetime.strptime(strEndDate, "%d/%m/%Y")
            print(tickerName + " cashBalance endDate " + strEndDate)

            tickerFileOpen = "cleanDate" + tickerName + ".xlsx"
            print(".....Opening '" + tickerFileOpen + "'....")

            tickerWorkBook = openpyxl.load_workbook(filename=tickerFileOpen)
            tickerSheet = tickerWorkBook.active
            tickerNumber_rows = tickerSheet.max_row
            # tickerNumber_columns = tickerSheet.max_column
            loopIndex = 0
            foundDate = False
            while foundDate == False:
                #if endDate is in the future, break the loop || loop more than 6 times
                if foundDate == True or currentDateOBJ < endDateOBJ or loopIndex > 6:
                    break

                k = 0
                while k < tickerNumber_rows:
                    tickerDate = str(tickerSheet["A"+str(k+1)].value)
                    strEndDateValue = strEndDate
                    if loopIndex > 0:
                        tempDate = datetime.datetime.strptime(strEndDate, '%d/%m/%Y') + datetime.timedelta(days=-loopIndex)
                        strEndDateValue = tempDate.strftime('%d/%m/%Y')

                        # strEndDate = str(tickerSheet["A"+str(k+1)].value)[:10]
                        # tickerDateDateTime = datetime.datetime.strptime(tickerDate, '%Y-%m-%d')
                        # tickerDateMinus1 = tickerDateDateTime + datetime.timedelta(days=-1)
                        # tickerDate = tickerDateMinus1.strftime('%d/%m/%Y')

                    # tickerCell = "A" + str(k+1)
                    # print('this is tickerdate', tickerDate, ' strEndDate', strEndDate)
                    if tickerDate == strEndDateValue:
                        foundDate = True

                        # print('Found EndDate at Cell Number'+tickerCell)

                        tickerDate0 = str(tickerSheet["A"+str(k+1)].value)

                        tickerMktCap = str(tickerSheet["H"+str(k+1)].value)

                        sheet["AQ"+str(i+1)] = float(tickerMktCap) if tickerMktCap != "NaN" else "DNE"

                        print("###")
                        print("Market Cap of " + tickerName + " on " + tickerDate0 + " is " + tickerMktCap)
                        print("###")

                        workbook.save("testMKTCAPCashBalanceCPJAN18.xlsx")
                        break
                    k += 1

                loopIndex += 1
                print("-------------------------------")

        except Exception as e:
            print(e)
            print("An error occured")
            break

    workbook.save("testMKTCAPCashBalanceCPJAN18.xlsx")

def addFreeFloat():

    for i in range(1, number_rows):
        print("--------------adding Free Float Percentage--------------")
        try:
            tickerName = str(sheet[("A"+str(i+1))].value)
            endDateValue = str(sheet[("D"+str(i+1))].value)[:10]
            endDate = datetime.datetime.strptime(endDateValue, '%Y-%m-%d')
            # endDateM5 = endDate + datetime.timedelta(days=-5)

            strEndDate = endDate.strftime('%d/%m/%Y')
            # strEndDateM5 = endDateM5.strftime('%d/%m/%Y')
            currentDateOBJ = datetime.datetime.strptime(CURRENTDATE, "%d/%m/%Y")
            endDateOBJ = datetime.datetime.strptime(strEndDate, "%d/%m/%Y")
            print(tickerName + " cashBalance endDate " + strEndDate)

            tickerFileOpen = "cleanDate" + tickerName + ".xlsx"
            print(".....Opening '" + tickerFileOpen + "'....")

            tickerWorkBook = openpyxl.load_workbook(filename=tickerFileOpen)
            tickerSheet = tickerWorkBook.active
            tickerNumber_rows = tickerSheet.max_row
            # tickerNumber_columns = tickerSheet.max_column
            loopIndex = 0
            foundDate = False
            while foundDate == False:
                #if endDate is in the future, break the loop || loop more than 6 times
                if foundDate == True or currentDateOBJ < endDateOBJ or loopIndex > 6:
                    break

                k = 0
                while k < tickerNumber_rows:
                    tickerDate = str(tickerSheet["A"+str(k+1)].value)
                    strEndDateValue = strEndDate
                    if loopIndex > 0:
                        tempDate = datetime.datetime.strptime(strEndDate, '%d/%m/%Y') + datetime.timedelta(days=-loopIndex)
                        strEndDateValue = tempDate.strftime('%d/%m/%Y')

                        # strEndDate = str(tickerSheet["A"+str(k+1)].value)[:10]
                        # tickerDateDateTime = datetime.datetime.strptime(tickerDate, '%Y-%m-%d')
                        # tickerDateMinus1 = tickerDateDateTime + datetime.timedelta(days=-1)
                        # tickerDate = tickerDateMinus1.strftime('%d/%m/%Y')

                    # tickerCell = "A" + str(k+1)
                    # print('this is tickerdate', tickerDate, ' strEndDate', strEndDate)
                    if tickerDate == strEndDateValue:
                        foundDate = True

                        # print('Found EndDate at Cell Number'+tickerCell)

                        tickerDate0 = str(tickerSheet["A"+str(k+1)].value)
                        tickerFloatSharesOutstanding = float(tickerSheet["J" + str(k+1)].value)
                        tickerCommonSharesOutstanding = float(tickerSheet["K"+str(k+1)].value)
                        tickerFreeFloatPercentage = "NaN"
                        if tickerFloatSharesOutstanding != "NaN" and tickerCommonSharesOutstanding != "NaN":
                            tickerFreeFloatPercentage = tickerFloatSharesOutstanding/tickerCommonSharesOutstanding

                        sheet["AR"+str(i+1)] = float(tickerFreeFloatPercentage) if tickerFreeFloatPercentage != "NaN" else "DNE"

                        print("###")
                        print("Free Float Percentage of " + tickerName + " on " + tickerDate0 + " is " + str(tickerFreeFloatPercentage))
                        print("###")

                        workbook.save("testMKTCAPCashBalanceCPJAN18.xlsx")
                        break
                    k += 1

                loopIndex += 1
                print("-------------------------------")

        except Exception as e:
            print(e)
            print("An error occured")
            break

    workbook.save("testMKTCAPCashBalanceCPJAN18.xlsx")

#########

getUserInput = False
while not getUserInput:
    userInput = input("cp / op / mktcap / ff / ccmktcapff ? ")
    if userInput == "cp":
        getUserInput = True
        findClosePrice()
    elif userInput == "op":
        getUserInput = True
        findOpenPrice()
    elif userInput =="mktcap":
        getUserInput = True
        addMktCap()
    elif userInput == "ff":
        getUserInput = True
        addFreeFloat()
    elif userInput == "exit":
        getUserInput = True
    elif userInput == "ccmktcapff":
        getUserInput = True
        findClosePrice()
        addMktCap()
        addFreeFloat()


import os
import logging
import string
import platform
import csv
from io import TextIOWrapper

from glycan_profiling import serialize
from glycan_profiling.serialize import (
    Protein, Glycopeptide, IdentifiedGlycopeptide,
    func, MSScan, GlycopeptideSpectrumMatch)

from glycan_profiling.task import TaskBase
from glycan_profiling.serialize import DatabaseBoundOperation

from glycan_profiling.chromatogram_tree import Unmodified
from glycan_profiling.tandem.ref import SpectrumReference
from glycan_profiling.tandem.glycopeptide.scoring import CoverageWeightedBinomialModelTree

from glycan_profiling.plotting import figure
from glycan_profiling.plotting.sequence_fragment_logo import glycopeptide_match_logo
from glycan_profiling.plotting.spectral_annotation import TidySpectrumMatchAnnotator

from ms_deisotope.output import ProcessedMSFileLoader

from matplotlib import pyplot as plt, style
from matplotlib import rcParams as mpl_params

status_logger = logging.getLogger("glycresoft.status")

def format_filename(s):
    """Take a string and return a valid filename constructed from the string.
    Uses a whitelist approach: any characters not present in valid_chars are
    removed. Also spaces are replaced with underscores.
    """
    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
    filename = ''.join(c for c in s if c in valid_chars)
    filename = filename.replace(' ', '_')
    return filename

class SpectrumAnnotatorExport(TaskBase, DatabaseBoundOperation):
    def __init__(self, database_connection, analysis_id, output_path, mzml_path=None):
        DatabaseBoundOperation.__init__(self, database_connection)
        self.analysis_id = analysis_id
        self.mzml_path = mzml_path
        self.output_path = output_path
        self.analysis = self.session.query(serialize.Analysis).get(self.analysis_id)
        self.scan_loader = None
        self._mpl_style = {
            'figure.facecolor': 'white',
            'figure.edgecolor': 'white',
            'font.size': 10,
            'savefig.dpi': 72,
            'figure.subplot.bottom': .125
        }

    def _make_scan_loader(self):
        if self.mzml_path is not None:
            if not os.path.exists(self.mzml_path):
                raise IOError("No such file {}".format(self.mzml_path))
            self.scan_loader = ProcessedMSFileLoader(self.mzml_path)
        else:
            self.mzml_path = self.analysis.parameters['sample_path']
            if not os.path.exists(self.mzml_path):
                raise IOError((
                    "No such file {}. If {} was relocated, you may need to explicily pass the"
                    " corrected file path.").format(
                    self.mzml_path,
                    self.database_connection._original_connection))
            self.scan_loader = ProcessedMSFileLoader(self.mzml_path)
        return self.scan_loader

    def _load_spectrum_matches(self):
        query = self.query(GlycopeptideSpectrumMatch).join(
            GlycopeptideSpectrumMatch.scan).filter(
            GlycopeptideSpectrumMatch.analysis_id == self.analysis_id).order_by(
            MSScan.index)
        return query.all()

    def run(self):
        scan_loader = self._make_scan_loader()
        gpsms = self._load_spectrum_matches()
        if not os.path.exists(self.output_path):
            os.makedirs(self.output_path)
        n = len(gpsms)
        self.log("%d Spectrum Matches" % (n,))
        for i, gpsm in enumerate(gpsms):
            scan = scan_loader.get_scan_by_id(gpsm.scan.scan_id)
            gpep = gpsm.structure.convert()
            if i % 10 == 0:
                self.log("... %0.2f%%: %s @ %s" % (((i + 1) / float(n) * 100.0), gpep, scan.id))
            with style.context(self._mpl_style):
                fig = figure()
                grid = plt.GridSpec(nrows=5, ncols=1)
                ax1 = fig.add_subplot(grid[1, 0])
                ax2 = fig.add_subplot(grid[2:, 0])
                ax3 = fig.add_subplot(grid[0, 0])
                match = CoverageWeightedBinomialModelTree.evaluate(scan, gpep)
                ax3.text(0, 0.5, (
                    str(match.target) + '\n' + scan.id +
                    '\nscore=%0.3f    q value=%0.3g' % (gpsm.score, gpsm.q_value)), va='center')
                ax3.axis('off')
                match.plot(ax=ax2)
                glycopeptide_match_logo(match, ax=ax1)
                fname = format_filename("%s_%s.pdf" % (scan.id, gpep))
                path = os.path.join(self.output_path, fname)
                abspath = os.path.abspath(path)
                if len(abspath) > 259 and platform.system().lower() == 'windows':
                    abspath = '\\\\?\\' + abspath
                fig.savefig(abspath, bbox_inches='tight')
                plt.close(fig)

class CSVSpectrumAnnotatorExport(SpectrumAnnotatorExport):
    def __init__(self, database_connection, analysis_id, outstream, mzml_path=None, fdr_threshold=0.05):
        super(CSVSpectrumAnnotatorExport, self).__init__(
            database_connection, analysis_id, None, mzml_path)
        self.outstream = outstream
        try:
            self.is_binary = 'b' in self.outstream.mode
        except AttributeError:
            self.is_binary = True
        if self.is_binary:
            try:
                self.outstream = TextIOWrapper(outstream, 'utf8', newline="")
            except AttributeError:
                # must be Py2
                pass
        self.fdr_threshold = fdr_threshold
        self.writer = csv.writer(self.outstream, delimiter=',')

    def get_header(self):
        return [
            "glycopeptide",
            "scan_id",
            "fragment_name",
            "peak_mass",
            "peak_charge",
            "peak_intensity",
            "mass_accuracy_ppm",
        ]

    def _load_spectrum_matches(self):
        query = self.query(GlycopeptideSpectrumMatch).join(
            GlycopeptideSpectrumMatch.scan).filter(
            GlycopeptideSpectrumMatch.analysis_id == self.analysis_id,
            GlycopeptideSpectrumMatch.is_best_match,
            GlycopeptideSpectrumMatch.q_value <= self.fdr_threshold).order_by(
                GlycopeptideSpectrumMatch.score.desc(), MSScan.index)
        return query.all()

    def convert_object(self, obj):
        records = []
        for pfp in sorted(obj.solution_map, key=lambda x: x.fragment.mass):
            peak, fragment = pfp
            rec = [
                str(obj.target),
                str(obj.scan.scan_id),
                fragment.name,
                peak.neutral_mass,
                peak.charge,
                peak.intensity,
                pfp.mass_accuracy() * 1e6
            ]
            records.append(rec)
        return records

    def status_update(self, message):
        status_logger.info(message)

    def writerows(self, iterable):
        self.writer.writerows(iterable)

    def writerow(self, row):
        self.writer.writerow(row)

    def run(self):
        header = self.get_header()
        self.writerow(header)

        scan_loader = self._make_scan_loader()
        gpsms = self._load_spectrum_matches()

        n = len(gpsms)
        for i, gpsm in enumerate(gpsms):
            scan = scan_loader.get_scan_by_id(gpsm.scan.scan_id)
            gpep = gpsm.structure.convert()
            match = CoverageWeightedBinomialModelTree.evaluate(scan, gpep)
            self.writerows(self.convert_object(match))
            if i % 100 == 0 and i:
                self.status_update("%d Spectrum Matches Handled (%0.2f%%)" % (i, i * 100.0 / n))


import logging
from logging.handlers import RotatingFileHandler

import groupy
import forecastio
import giphypop
import requests
import bs4
from geopy.geocoders import Nominatim
from flask import Flask, request

application = Flask(__name__)
application.config.from_object('settings')

WEATHER_ICONS = {
  'clear-day': '☀',
  'clear-night': '🌑',
  'rain': '☔',
  'snow': '☃❄',
  'sleet': '☃❄',
  'wind': '🌬',
  'fog': '≈',
  'cloudy': '☁',
  'partly-cloudy-day': '⛅',
  'partly-cloudy-night': '☁🌑'
}

commands = {}

def command(cmd):
  def wrapped_command(function):
    def wrapper(*args, **kwargs):
      function(*args, **kwargs)
    if cmd not in commands:
      commands[cmd] = []
    commands[cmd].append(wrapper)
  return wrapped_command

@command('!weather')
def weather(bot, message, author=None, debug=False):
  api_key = application.config['FORECASTIO_API_KEY']

  if api_key is None:
    print('FORECASTIO_API_KEY not configured, not displaying weather information!')

  geolocator = Nominatim()
  location = geolocator.geocode(message or 'Austin, TX')
  forecast = forecastio.load_forecast(api_key, location.latitude, location.longitude)
  current = forecast.currently()
  icon = WEATHER_ICONS[current.icon] or ''

  post = '{0}  {1} {2}° (feels like {3}°)'.format(icon, current.summary, round(current.temperature), round(current.apparentTemperature))

  if debug:
    print(post)
  else:
    bot.post(post)

@command('!gif')
def gif(bot, message, author=None, debug=False):
  img = giphypop.translate(phrase=message, strict=True)

  if debug:
    print(img.media_url)
  else:
    bot.post(img.media_url)

@command('!slap')
def slap(bot, message, author=None, debug=False):
  if author is None:
    return

  slap = '{0} slaps {1} around a bit with a large trout'.format(author, message)

  if debug:
    print(slap)
  else:
    bot.post(slap)

@command('!h')
def horoscope(bot, message, author=None, debug=False):
  sign = message.lower()

  signs = [
    'aries',
    'taurus',
    'gemini',
    'cancer',
    'leo',
    'virgo',
    'libra',
    'scorpio',
    'sagittarius',
    'capricorn',
    'aquarius',
    'pisces'
  ]

  if sign not in signs:
    error = '{0} is not a star sign, zodiac thing, or whatever.'.format(message)
    if debug:
      print(error)
    else:
      bot.post()
    return

  url = 'http://www.horoscope.com/us/horoscopes/general/horoscope-general-daily-today.aspx?sign={0}'.format(signs.index(sign) + 1)
  page = requests.get(url)

  soup = bs4.BeautifulSoup(page.content, 'lxml')
  elements = soup.select('.block-horoscope-text')
  if len(elements) == 0:
    print('Unable to find horoscope for: {0}'.format(message))
    return
  horoscope = '{0}: {1}'.format(message, elements[0].getText().strip())
  if debug:
    print(horoscope)
  else:
    bot.post(horoscope)

@application.route('/pancakebot', methods=['POST'])
def hello():
  data = request.get_json()
  bot = groupy.Bot.list().first

  user = data['name']
  message = data['text']

  print('Received message: {0}'.format(data))

  for command in commands:
    if not message.startswith(command + ' ') and message != command:
      continue

    print('Executing command: {0}'.format(command))

    for callback in commands[command]:
      try:
        callback(bot, message[len(command):].strip(), author=user, debug=application.config['DEBUG'])
      except Exception as e:
        print('Error executing command: {0}'.format(command))
        print(e)

  return 'OK'

if __name__ == '__main__':
  application.run(host='0.0.0.0', port=5555)


#!/usr/bin/env python
# encoding: utf-8
'''
#-------------------------------------------------------------------
#                   CONFIDENTIAL --- CUSTOM STUDIOS
#-------------------------------------------------------------------
#
#                   @Project Name : 高级图片转换特效之毛玻璃
#
#                   @File Name    : effects_1.py
#
#                   @Programmer   : autofelix
#
#                   @Start Date   : 2022/01/14 13:14
#
#                   @Last Update  : 2022/01/14 13:14
#
#-------------------------------------------------------------------
'''
import cv2
import numpy as np

class picture:
    '''
     This is a main Class, the file contains all documents.
     One document contains paragraphs that have several sentences
     It loads the original file and converts the original file to new content
     Then the new content will be saved by this class
    '''
    def __init__(self):
        self.path = 'assets/picture.jpeg'

    def hello(self):
        '''
        This is a welcome speech
        :return: self
        '''
        print('*' * 50)
        print(' ' * 20 + '高级图片转换特效之毛玻璃')
        print(' ' * 5 + '作者: autofelix  Date: 2022-01-16 13:14')
        print(' ' * 5 + '主页: https://autofelix.blog.csdn.net')
        print('*' * 50)
        return self

    def run(self):
        # 读取原始图像
        src = cv2.imread(self.path)
        # 新建目标图像
        dst = np.zeros_like(src)
        # 获取图像行和列
        rows, cols = src.shape[:2]
        # 定义偏移量和随机数
        offsets = 5
        random_num = 0
        # 毛玻璃效果: 像素点邻域内随机像素点的颜色替代当前像素点的颜色
        for y in range(rows - offsets):
            for x in range(cols - offsets):
                random_num = np.random.randint(0, offsets)
                dst[y, x] = src[y + random_num, x + random_num]
        # 显示图像
        cv2.imshow('src', src)
        cv2.imshow('dst', dst)
        cv2.waitKey()
        cv2.destroyAllWindows()

if __name__ == '__main__':
    picture().hello().run()


import wx
from mdbutton import MDButton

class DirPicker(wx.Panel):
    def __init__ (self, parent,
                  style = wx.TAB_TRAVERSAL,
                  labelText = 'Folder Path:',
                  initValue = '',
                  buttonText = 'Browse',
                  dialogTitle = 'Choose a folder',
                  pathCallback = None,
                  name = 'dirpicker'):
        super().__init__(parent, style=style, name=name)
        self.labelText = labelText
        self.initValue = initValue
        self.buttonText = buttonText
        self.dialogTitle = dialogTitle
        self.callback = pathCallback
        self.Font = parent.Font
        self.BackgroundColour = parent.BackgroundColour
        self.ForegroundColour = parent.ForegroundColour
        self.createPicker()

    def createPicker(self):
        vsizer = wx.BoxSizer(wx.VERTICAL)
        self.label = self.makeLabel()
        vsizer.Add(self.label, 0, wx.TOP, 16)
        hsizer = wx.BoxSizer(wx.HORIZONTAL)
        hsizer.Add(self.makeField(), 1, wx.ALIGN_BOTTOM|wx.RIGHT, 16)
        self.button = self.makeButton()
        hsizer.Add(self.button, 0)
        vsizer.Add(hsizer, 0, wx.EXPAND|wx.TOP, 8)
        self.SetAutoLayout(True)
        self.SetSizerAndFit(vsizer)
        self.Layout()

    def makeLabel(self):
        label = wx.StaticText(self, -1, self.labelText, style=wx.ALIGN_LEFT)
        label.BackgroundColour = self.BackgroundColour
        self.sizeText(label, self.labelText)
        return label

    def makeField(self):
        self.field = wx.TextCtrl(self, value=self.initValue,
            style=wx.TE_READONLY|wx.BORDER_NONE)
        self.field.BackgroundColour = self.BackgroundColour
        self.field.ForegroundColour = self.ForegroundColour
        self.field.Font = self.Font
        self.field.SetCanFocus(False)
        self.field.ToolTip = f'Path to: {self.labelText}'
        self.sizeText(self.field, self.initValue)
        line = wx.Panel(self, size=(-1, 1))
        line.BackgroundColour = self.ForegroundColour
        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.field, 1, wx.EXPAND)
        sizer.Add(line, 0, wx.EXPAND)
        return sizer

    def makeButton(self):
        button = MDButton(self, self.buttonText)
        button.ToolTip = f'Click to browse to: {self.labelText}'
        button.Bind(wx.EVT_BUTTON, self.onBrowse)
        return button

    def setCallback(self, callbackFunc):
        self.callback = callbackFunc

    def onBrowse(self, ev=None):
        dirPath = wx.DirSelector(self.dialogTitle,
            style=wx.DD_DEFAULT_STYLE|wx.DD_DIR_MUST_EXIST)
        if dirPath:
            self.field.Value = dirPath
            if self.callback:
                self.callback(self.Name, dirPath)

    def sizeText(self, elem, value):
        width = self.CharWidth * (len(value) * 1.2 if value else 60) # (320, -1)
        elem.MinSize = (round(width), self.CharHeight * 1.2)


import requests
import bs4 #beautiful soup
#요청
url  = 'https://www.naver.com/'

# response = requests.get(url)

# html = response.text
# # url을 텍스트로 변환
# soup = bs4.BeautifulSoup(html, 'html.parser')
# keywords = soup.select('.ah_l .ah_item .ah_a .ah_k')
# # soup중에서 불러오기/parser했기에 가능
# print(keywords)

selector = '.ah_l .ah_item .ah_a .ah_k'
html = requests.get(url).text
soup = bs4.BeautifulSoup(html, 'html.parser')
ranks = soup.select(selector)

for rank in ranks:
    print(rank.text)
# 왜 텍스트 두 번?


import argparse
import gc
import os
import time

import numpy as np
import torch
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.nn as nn
import torch.optim as optim
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader

from data_loader.pose_check_dataset import MultiTaskDatasetV2
from models.uninet_mt import UniNet_MT_V2
from utils.utils import get_step_schedule_with_warmup, dict2cuda, add_summary, \
	DictAverageMeter, calc_stat

cudnn.benchmark = True

parser = argparse.ArgumentParser(description='Deep stereo using adaptive cost volume.')
parser.add_argument('--root_path', type=str, help='path to root directory.')
parser.add_argument('--train_list', type=str, help='train scene list.', default='')
parser.add_argument('--val_list', type=str, help='val scene list.', default='')
parser.add_argument('--save_path', type=str, help='path to save checkpoints.')
parser.add_argument('--restore_path', type=str, default='')

parser.add_argument('--epochs', type=int, default=20)
parser.add_argument('--lr', type=float, default=0.001)
parser.add_argument('--lr_idx', type=str, default="50,100,160:0.5")
parser.add_argument('--wd', type=float, default=0.0, help='weight decay')
parser.add_argument('--batch_size', type=int, default=32)

parser.add_argument('--log_freq', type=int, default=100, help='print and summary frequency')
parser.add_argument('--save_freq', type=int, default=10000, help='save checkpoint frequency.')
parser.add_argument('--eval_freq', type=int, default=10000, help='evaluate frequency.')

parser.add_argument('--sync_bn', action='store_true',help='Sync BN.')
parser.add_argument('--opt_level', type=str, default="O0")
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--local_rank', type=int, default=0)
parser.add_argument('--num_workers', type=int, default=4)
parser.add_argument('--distributed', action='store_true')

args = parser.parse_args()
# num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
is_distributed = args.distributed

torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
device = torch.device("cuda")

if args.sync_bn:
	import apex
	import apex.amp as amp

def print_func(data: dict, prefix: str= ''):
	for k, v in data.items():
		if isinstance(v, dict):
			print_func(v, prefix + '.' + k)
		elif isinstance(v, list):
			print(prefix+'.'+k, v)
		else:
			print(prefix+'.'+k, v.shape)

def main(args, model, optimizer, scheduler, train_loader, val_loader, train_sampler, start_step=0):

	train_step = start_step
	start_ep = start_step // len(train_loader)

	model.train()
	for ep in range(start_ep, args.epochs):
		np.random.seed()
		train_scores = DictAverageMeter()
		if train_sampler is not None:
			train_sampler.set_epoch(ep)

		for batch_idx, sample in enumerate(train_loader):
			tic = time.time()

			sample_cuda = dict2cuda(sample)

			# print_func(sample_cuda)
			optimizer.zero_grad()
			ret = model(sample_cuda)
			loss = ret['loss'].mean()
			preds = ret['preds']
			loss_items = [l.mean() for l in ret['loss_items']]

			# print_func(outputs)
			if is_distributed and args.sync_bn:
				with amp.scale_loss(loss, optimizer) as scaled_loss:
					scaled_loss.backward()
			else:
				loss.backward()

			optimizer.step()
			scheduler.step()

			if train_step % args.log_freq == 0:
				train_scores.update({'loss': float(loss),
				                     'contact_loss': float(loss_items[0]),
				                     'stable_loss': float(loss_items[1]),
				                     'offset_loss': float(loss_items[2]),
				                     'variance_loss': float(loss_items[3])
				                     })

				calc_stat(sample_cuda, preds[0], train_scores, label_type='stable')
				calc_stat(sample_cuda, preds[1], train_scores, label_type='contact')

				avg_stat = train_scores.mean()
				print("[Rank: {}] time={:.2f} Epoch {}/{}, Iter {}/{}, lr {:.6f}, stats: {}".format(
					args.local_rank, time.time() - tic,
					ep, args.epochs, batch_idx, len(train_loader),
					optimizer.param_groups[0]["lr"],
					avg_stat))
				if on_main:
					add_summary([{'type': 'scalars', 'tags': list(avg_stat.keys()),
					              'vals': list(avg_stat.values())}],
					            logger=logger, step=train_step, flag='train')

				del sample_cuda
				del avg_stat
				gc.collect()

			if on_main and train_step % args.save_freq == 0:
				torch.save({"step": train_step,
			                "model": model.module.state_dict(),
			                "optimizer": optimizer.state_dict(),
				            "scheduler": scheduler.state_dict(),
				            },
			                "{}/model_{:08d}.ckpt".format(args.save_path, train_step))

			if train_step % args.eval_freq == 0:
				print('evaluating model_{:08d}.ckpt ...'.format(train_step))
				with torch.no_grad():
					test(args, model, val_loader, train_step)
				model.train()

			train_step += 1

		del train_scores
		gc.collect()

def test(args, model, test_loader, train_step):
	model.eval()
	val_scores = DictAverageMeter()
	for batch_idx, sample in enumerate(test_loader):
		sample_cuda = dict2cuda(sample)
		ret = model(sample_cuda)
		preds = ret['preds']
		calc_stat(sample_cuda, preds[0], val_scores, label_type='stable')
		calc_stat(sample_cuda, preds[1], val_scores, label_type='contact')

	avg_stat = val_scores.mean()
	print("[Rank: {}] step {:06d}, stats: {}".format(args.local_rank, train_step, avg_stat))
	if on_main:
		add_summary([{'type': 'scalars', 'tags': list(avg_stat.keys()),
		              'vals': list(avg_stat.values())}],
		            logger=logger, step=train_step, flag='val')

	del sample_cuda
	del avg_stat
	del val_scores
	gc.collect()

def distribute_model(args):
	def sync():
		if not dist.is_available():
			return
		if not dist.is_initialized():
			return
		if dist.get_world_size() == 1:
			return
		dist.barrier()
	if is_distributed:
		torch.cuda.set_device(args.local_rank)
		torch.distributed.init_process_group(
			backend="nccl", init_method="env://"
		)
		sync()

	start_step = 0

	model: torch.nn.Module = UniNet_MT_V2(mask_channel=True, bootle_neck=256)
	if args.restore_path:
		checkpoint = torch.load(args.restore_path, map_location=torch.device("cpu"))
		model.load_state_dict(checkpoint['model'], strict=True)

	model.to(device)

	optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, betas=(0.9, 0.999),
	                       weight_decay=args.wd)

	train_set = MultiTaskDatasetV2(root_dir=args.root_path, list_path=args.train_list,
	                               use_aug=True, )
	print('train set ready.')
	val_set = MultiTaskDatasetV2(root_dir=args.root_path, list_path=args.val_list,
	                             use_aug=False,)
	print('val set ready.')
	if is_distributed:
		if args.sync_bn:
			model = apex.parallel.convert_syncbn_model(model)
			model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, )
			print('Convert BN to Sync_BN successful.')

		model = nn.parallel.DistributedDataParallel(
			model, device_ids=[args.local_rank], output_device=args.local_rank,)

		train_sampler = torch.utils.data.DistributedSampler(train_set, num_replicas=dist.get_world_size(),
		                                                    rank=dist.get_rank())
		val_sampler = torch.utils.data.DistributedSampler(val_set, num_replicas=dist.get_world_size(),
		                                                   rank=dist.get_rank())
	else:
		model = nn.DataParallel(model)
		train_sampler, val_sampler = None, None

	def worker_init_fn(worker_id):
		np.random.seed(np.random.get_state()[1][0] + worker_id)

	train_loader = DataLoader(train_set, args.batch_size, sampler=train_sampler,
	                          num_workers=args.num_workers, pin_memory=True,
	                          drop_last=True, shuffle=not is_distributed, worker_init_fn=worker_init_fn)
	val_loader = DataLoader(val_set, 64, sampler=val_sampler,
	                        num_workers=1, pin_memory=True,
	                        drop_last=False, shuffle=False, worker_init_fn=worker_init_fn)

	milestones = list(map(float, args.lr_idx.split(':')[0].split(',')))
	assert np.max(milestones) <= 1.0, milestones
	milestones = list(map(lambda x: int(float(x) * float(len(train_loader) * args.epochs)), milestones))
	gamma = float(args.lr_idx.split(':')[1])
	warpup_iters = min(500, int(0.05*len(train_loader)))

	scheduler = get_step_schedule_with_warmup(optimizer=optimizer, milestones=milestones,
	                                          gamma=gamma, warmup_iters=warpup_iters)

	if args.restore_path:
		optimizer.load_state_dict(checkpoint['optimizer'])
		scheduler.load_state_dict(checkpoint['scheduler'])
		start_step = checkpoint['step']
		print("Restoring checkpoint {} ...".format(args.restore_path))

	return model, optimizer, scheduler, train_loader, val_loader, train_sampler, start_step

if __name__ == '__main__':
	model, optimizer, scheduler, train_loader, val_loader, train_sampler, start_step = distribute_model(args)
	on_main = (not is_distributed) or (dist.get_rank() == 0)
	if on_main:
		os.makedirs(args.save_path, exist_ok=True)
		logger = SummaryWriter(args.save_path)
		print(args)

	main(args=args, model=model, optimizer=optimizer, scheduler=scheduler,
	     train_loader=train_loader, val_loader=val_loader, train_sampler=train_sampler, start_step=start_step)


from django.contrib.auth import get_user_model
from django.db.models import Count
from django import forms
from django.utils.encoding import force_text

from ads.models import LineItem

User = get_user_model()

COUNTRIES_CHOICES = [(x[0], x[1]) for x in User.objects.exclude(country_code__isnull=True).values_list('country_code', 'country_name').annotate(count=Count('country_code')).order_by('-count')]
CITIES_CHOICES = [(x[0], x[0]) for x in User.objects.exclude(city__isnull=True).values_list('city').annotate(count=Count('city')).order_by('-count')]

class ArrayMultipleSelected(forms.SelectMultiple):
    def format_value(self, value):
        if value is None and self.allow_multiple_selected:
            return []
        if not isinstance(value, (tuple, list)):
            # This means it should be a comma delimited list of items so parse it
            value = value.split(',')
        return [force_text(v) if v is not None else '' for v in value]

class GeoAdder(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super(GeoAdder, self).__init__(*args, **kwargs)
        self.fields['targeting_countries'].widget = ArrayMultipleSelected(
            attrs={'class': 'multi-select-input'}, choices=COUNTRIES_CHOICES
        )
        self.fields['targeting_cities'].widget = ArrayMultipleSelected(
            attrs={'class': 'multi-select-input'}, choices=CITIES_CHOICES
        )

    class Meta:
        model = LineItem
        exclude = []


"""empty message

Revision ID: da5815fb47ef
Revises: 462e5f0b6353
Create Date: 2020-04-28 12:37:28.619048

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = 'da5815fb47ef'
down_revision = '462e5f0b6353'
branch_labels = None
depends_on = None

def upgrade():
    pass

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('snow_child',
    sa.Column('id', mysql.BIGINT(display_width=20), autoincrement=False, nullable=False, comment='关联的公司用户id'),
    sa.Column('name', mysql.VARCHAR(length=20), nullable=True, comment='测试名字'),
    sa.PrimaryKeyConstraint('id'),
    comment='雪花算法子表',
    mysql_comment='雪花算法子表',
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.drop_table('Snow_Child')
    # ### end Alembic commands ###


import yfinance as yf
import sys, os

file = open("stonks.txt",'r')
list_of_stonks = []
ticker_objs = {}
i = 0

for line in file:
    if len(line)>1:
        list_of_stonks.append(line.strip('\n'))

for ticker in list_of_stonks:
    i = i + 1
    try:
        ticker_objs[ticker] = yf.Ticker(ticker)
        print(str(i) + "," + ticker + "," + ticker_objs[ticker].info['sector'] + "," + ticker_objs[ticker].info['industry'])
    except Exception as e:
        print(e.args)

##############################################

## More random code
# import yfinance as yf
#
# list_of_stonks = ['AAPL','MSFT','TSLA','AMD','GME']
# stock_tickers = {}
#
# for ticker in list_of_stonks:
#     print("Getting ", ticker)
#     stock_tickers[ticker] = yf.Ticker(ticker)
#
# print(stock_tickers['AAPL'].balancesheet)   # pandas dataframe


from __future__ import unicode_literals

import codecs
import json
import os
import sys
from textwrap import dedent

from compat import unittest

from distlib import __version__
from distlib.compat import StringIO
from distlib.metadata import (LegacyMetadata, Metadata, METADATA_FILENAME,
                              LEGACY_METADATA_FILENAME, PKG_INFO_PREFERRED_VERSION,
                              MetadataMissingError, MetadataUnrecognizedVersionError,
                              MetadataInvalidError, _ATTR2FIELD)

from support import LoggingCatcher, TempdirManager, DistlibTestCase, in_github_workflow

HERE = os.path.abspath(os.path.dirname(__file__))
IN_GITHUB_WORKFLOW = in_github_workflow()

class LegacyMetadataTestCase(LoggingCatcher, TempdirManager,
                             DistlibTestCase):

    maxDiff = None
    restore_environ = ['HOME']

    def setUp(self):
        super(LegacyMetadataTestCase, self).setUp()
        self.argv = sys.argv, sys.argv[:]

    def tearDown(self):
        sys.argv = self.argv[0]
        sys.argv[:] = self.argv[1]
        super(LegacyMetadataTestCase, self).tearDown()

    ####  Test various methods of the LegacyMetadata class

    def get_file_contents(self, name):
        name = os.path.join(HERE, name)
        f = codecs.open(name, 'r', encoding='utf-8')
        try:
            contents = f.read() % sys.platform
        finally:
            f.close()
        return contents

    def test_instantiation(self):
        PKG_INFO = os.path.join(HERE, 'PKG-INFO')
        f = codecs.open(PKG_INFO, 'r', encoding='utf-8')
        try:
            contents = f.read()
        finally:
            f.close()

        fp = StringIO(contents)

        m = LegacyMetadata()
        self.assertRaises(MetadataUnrecognizedVersionError, m.items)

        m = LegacyMetadata(PKG_INFO)
        self.assertEqual(len(m.items()), 22)

        m = LegacyMetadata(fileobj=fp)
        self.assertEqual(len(m.items()), 22)

        m = LegacyMetadata(mapping=dict(name='Test', version='1.0'))
        self.assertEqual(len(m.items()), 17)

        d = dict(m.items())
        self.assertRaises(TypeError, LegacyMetadata,
                          PKG_INFO, fileobj=fp)
        self.assertRaises(TypeError, LegacyMetadata,
                          PKG_INFO, mapping=d)
        self.assertRaises(TypeError, LegacyMetadata,
                          fileobj=fp, mapping=d)
        self.assertRaises(TypeError, LegacyMetadata,
                          PKG_INFO, mapping=m, fileobj=fp)

    def test_mapping_api(self):
        content = self.get_file_contents('PKG-INFO')
        metadata = LegacyMetadata(fileobj=StringIO(content))
        self.assertIn('Version', metadata.keys())
        self.assertIn('0.5', metadata.values())
        self.assertIn(('Version', '0.5'), metadata.items())

        metadata.update({'version': '0.6'})
        self.assertEqual(metadata['Version'], '0.6')
        metadata.update([('version', '0.7')])
        self.assertEqual(metadata['Version'], '0.7')
        # use a kwarg to update
        metadata.update(version='0.6')
        self.assertEqual(metadata['Version'], '0.6')

        # make sure update method checks values like the set method does
        metadata.update({'version': '1--2'})
        self.assertEqual(len(self.get_logs()), 1)

        self.assertEqual(list(metadata), metadata.keys())

    def test_attribute_access(self):
        content = self.get_file_contents('PKG-INFO')
        metadata = LegacyMetadata(fileobj=StringIO(content))
        for attr in _ATTR2FIELD:
            self.assertEqual(getattr(metadata, attr), metadata[attr])

    def test_read_metadata(self):
        fields = {'name': 'project',
                  'version': '1.0',
                  'description': 'desc',
                  'summary': 'xxx',
                  'download_url': 'http://example.com',
                  'keywords': ['one', 'two'],
                  'requires_dist': ['foo']}

        metadata = LegacyMetadata(mapping=fields)
        PKG_INFO = StringIO()
        metadata.write_file(PKG_INFO)
        PKG_INFO.seek(0)

        metadata = LegacyMetadata(fileobj=PKG_INFO)

        self.assertEqual(metadata['name'], 'project')
        self.assertEqual(metadata['version'], '1.0')
        self.assertEqual(metadata['summary'], 'xxx')
        self.assertEqual(metadata['download_url'], 'http://example.com')
        self.assertEqual(metadata['keywords'], ['one', 'two'])
        self.assertEqual(metadata['platform'], [])
        self.assertEqual(metadata['obsoletes'], [])
        self.assertEqual(metadata['requires-dist'], ['foo'])

    def test_write_metadata(self):
        # check support of non-ASCII values
        tmp_dir = self.mkdtemp()
        my_file = os.path.join(tmp_dir, 'f')

        metadata = LegacyMetadata(mapping={
                                     'name': 'my.project',
                                     'author': 'Café Junior',
                                     'summary': 'Café torréfié',
                                     'description': 'Héhéhé',
                                     'keywords': ['café', 'coffee']
                                  })
        metadata.write(my_file)

        # the file should use UTF-8
        metadata2 = LegacyMetadata()
        fp = codecs.open(my_file, encoding='utf-8')
        try:
            metadata2.read_file(fp)
        finally:
            fp.close()

        # XXX when keywords are not defined, metadata will have
        # 'Keywords': [] but metadata2 will have 'Keywords': ['']
        # because of a value.split(',') in LegacyMetadata.get
        self.assertEqual(metadata.items(), metadata2.items())

        # ASCII also works, it's a subset of UTF-8
        metadata = LegacyMetadata(mapping={'author': 'Mister Cafe',
                                     'name': 'my.project',
                                     'author': 'Cafe Junior',
                                     'summary': 'Cafe torrefie',
                                     'description': 'Hehehe'
                                  })
        metadata.write(my_file)

        metadata2 = LegacyMetadata()
        fp = codecs.open(my_file, encoding='utf-8')
        try:
            metadata2.read_file(fp)
        finally:
            fp.close()

    def test_metadata_read_write(self):
        PKG_INFO = os.path.join(HERE, 'PKG-INFO')
        metadata = LegacyMetadata(PKG_INFO)
        out = StringIO()
        metadata.write_file(out)

        out.seek(0)
        res = LegacyMetadata()
        res.read_file(out)
        self.assertEqual(metadata.values(), res.values())

    ####  Test checks

    def test_check_version(self):
        metadata = LegacyMetadata()
        metadata['Name'] = 'vimpdb'
        metadata['Home-page'] = 'http://pypi.org'
        metadata['Author'] = 'Monty Python'
        missing, warnings = metadata.check()
        self.assertEqual(missing, ['Version'])

    def test_check_version_strict(self):
        metadata = LegacyMetadata()
        metadata['Name'] = 'vimpdb'
        metadata['Home-page'] = 'http://pypi.org'
        metadata['Author'] = 'Monty Python'
        self.assertRaises(MetadataMissingError, metadata.check, strict=True)

    def test_check_name(self):
        metadata = LegacyMetadata()
        metadata['Version'] = '1.0'
        metadata['Home-page'] = 'http://pypi.org'
        metadata['Author'] = 'Monty Python'
        missing, warnings = metadata.check()
        self.assertEqual(missing, ['Name'])

    def test_check_name_strict(self):
        metadata = LegacyMetadata()
        metadata['Version'] = '1.0'
        metadata['Home-page'] = 'http://pypi.org'
        metadata['Author'] = 'Monty Python'
        self.assertRaises(MetadataMissingError, metadata.check, strict=True)

    def test_check_author(self):
        metadata = LegacyMetadata()
        metadata['Version'] = '1.0'
        metadata['Name'] = 'vimpdb'
        metadata['Home-page'] = 'http://pypi.org'
        missing, warnings = metadata.check()
        self.assertEqual(missing, ['Author'])

    def test_check_homepage(self):
        metadata = LegacyMetadata()
        metadata['Version'] = '1.0'
        metadata['Name'] = 'vimpdb'
        metadata['Author'] = 'Monty Python'
        missing, warnings = metadata.check()
        self.assertEqual(missing, ['Home-page'])

    def test_check_matchers(self):
        metadata = LegacyMetadata()
        metadata['Version'] = 'rr'
        metadata['Name'] = 'vimpdb'
        metadata['Home-page'] = 'http://pypi.org'
        metadata['Author'] = 'Monty Python'
        metadata['Requires-dist'] = ['Foo (a)']
        metadata['Obsoletes-dist'] = ['Foo (a)']
        metadata['Provides-dist'] = ['Foo (a)']
        missing, warnings = metadata.check()
        self.assertEqual(len(warnings), 4)

    ####  Test fields and metadata versions

    def test_metadata_versions(self):
        metadata = LegacyMetadata(mapping={'name': 'project',
                                           'version': '1.0'})
        self.assertEqual(metadata['Metadata-Version'],
                         PKG_INFO_PREFERRED_VERSION)
        self.assertNotIn('Provides', metadata)
        self.assertNotIn('Requires', metadata)
        self.assertNotIn('Obsoletes', metadata)

        metadata['Classifier'] = ['ok']
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.1')

        metadata = LegacyMetadata()
        metadata['Download-URL'] = 'ok'
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.1')

        metadata = LegacyMetadata()
        metadata['Obsoletes'] = 'ok'
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.1')

        del metadata['Obsoletes']
        metadata['Obsoletes-Dist'] = 'ok'
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.2')
        metadata.set('Obsoletes', 'ok')
        # See issue #140. Relaxed checking on Obsoletes
        # self.assertRaises(MetadataConflictError,
                          # metadata.set_metadata_version)
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '2.2')

        del metadata['Obsoletes']
        del metadata['Obsoletes-Dist']
        metadata.set_metadata_version()
        metadata['Version'] = '1'
        self.assertEqual(metadata['Metadata-Version'], '1.1')

        # make sure the _best_version function works okay with
        # non-conflicting fields from 1.1 and 1.2 (i.e. we want only the
        # requires/requires-dist and co. pairs to cause a conflict, not all
        # fields in _314_MARKERS)
        metadata = LegacyMetadata()
        metadata['Requires-Python'] = '3'
        metadata['Classifier'] = ['Programming language :: Python :: 3']
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.2')

        PKG_INFO = os.path.join(HERE, 'SETUPTOOLS-PKG-INFO')
        metadata = LegacyMetadata(PKG_INFO)
        self.assertEqual(metadata['Metadata-Version'], '1.0')

        PKG_INFO = os.path.join(HERE, 'SETUPTOOLS-PKG-INFO2')
        metadata = LegacyMetadata(PKG_INFO)
        self.assertEqual(metadata['Metadata-Version'], '1.1')

        # make sure an empty list for Obsoletes and Requires-dist gets ignored
        metadata['Obsoletes'] = []
        metadata['Requires-dist'] = []
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.1')

        # Update the _fields dict directly to prevent 'Metadata-Version'
        # from being updated by the _set_best_version() method.
        metadata._fields['Metadata-Version'] = '1.618'
        self.assertRaises(MetadataUnrecognizedVersionError, metadata.keys)

        # add a test for 2.1
        metadata = LegacyMetadata()
        metadata['Description-Content-Type'] = 'text/markdown; charset=UTF-8; variant=CommonMark'
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '2.1')

    def test_version(self):
        LegacyMetadata(mapping={'author': 'xxx',
                          'name': 'xxx',
                          'version': 'xxx',
                          'home_page': 'xxxx'
                       })
        logs = self.get_logs()
        self.assertEqual(1, len(logs))
        self.assertIn('not a valid version', logs[0])

    @unittest.skipIf(IN_GITHUB_WORKFLOW, 'This test is end-of-line dependent')
    def test_description(self):
        content = self.get_file_contents('PKG-INFO')
        metadata = LegacyMetadata()
        metadata.read_file(StringIO(content))

        # see if we can read the description now
        DESC = os.path.join(HERE, 'LONG_DESC.txt')
        f = open(DESC)
        try:
            wanted = f.read()
        finally:
            f.close()
        self.assertEqual(wanted, metadata['Description'])

        # save the file somewhere and make sure we can read it back
        out = StringIO()
        metadata.write_file(out)
        out.seek(0)

        out.seek(0)
        metadata = LegacyMetadata()
        metadata.read_file(out)
        self.assertEqual(wanted, metadata['Description'])

    def test_description_folding(self):
        # make sure the indentation is preserved
        out = StringIO()
        desc = dedent("""\
        example::
              We start here
            and continue here
          and end here.
        """)

        metadata = LegacyMetadata()
        metadata['description'] = desc
        metadata.write_file(out)

        # folded_desc = desc.replace('\n', '\n' + (7 * ' ') + '|')
        folded_desc = desc.replace('\n', '\n' + (8 * ' '))
        self.assertIn(folded_desc, out.getvalue())

    def test_project_url(self):
        metadata = LegacyMetadata()
        metadata['Project-URL'] = [('one', 'http://ok')]
        self.assertEqual(metadata['Project-URL'], [('one', 'http://ok')])
        metadata.set_metadata_version()
        self.assertEqual(metadata['Metadata-Version'], '1.2')

        # make sure this particular field is handled properly when written
        fp = StringIO()
        metadata.write_file(fp)
        self.assertIn('Project-URL: one,http://ok', fp.getvalue().split('\n'))

        fp.seek(0)
        metadata = LegacyMetadata()
        metadata.read_file(fp)
        self.assertEqual(metadata['Project-Url'], [('one', 'http://ok')])

    # TODO copy tests for v1.1 requires, obsoletes and provides from distutils
    # (they're useless but we support them so we should test them anyway)

    def test_provides_dist(self):
        fields = {'name': 'project',
                  'version': '1.0',
                  'provides_dist': ['project', 'my.project']}
        metadata = LegacyMetadata(mapping=fields)
        self.assertEqual(metadata['Provides-Dist'],
                         ['project', 'my.project'])
        self.assertEqual(metadata['Metadata-Version'], '1.2', metadata)
        self.assertNotIn('Requires', metadata)
        self.assertNotIn('Obsoletes', metadata)

    def test_requires_dist(self):
        fields = {'name': 'project',
                  'version': '1.0',
                  'requires_dist': ['other', 'another (==1.0)']}
        metadata = LegacyMetadata(mapping=fields)
        self.assertEqual(metadata['Requires-Dist'],
                         ['other', 'another (==1.0)'])
        self.assertEqual(metadata['Metadata-Version'], '1.2')
        self.assertNotIn('Provides', metadata)
        self.assertEqual(metadata['Requires-Dist'],
                         ['other', 'another (==1.0)'])
        self.assertNotIn('Obsoletes', metadata)

        # make sure write_file uses one RFC 822 header per item
        fp = StringIO()
        metadata.write_file(fp)
        lines = fp.getvalue().split('\n')
        self.assertIn('Requires-Dist: other', lines)
        self.assertIn('Requires-Dist: another (==1.0)', lines)

        # test warnings for invalid version constraints
        # XXX this would cause no warnings if we used update (or the mapping
        # argument of the constructor), see comment in LegacyMetadata.update
        metadata = LegacyMetadata()
        metadata['Requires-Dist'] = 'Funky (Groovie)'
        metadata['Requires-Python'] = '1a-4'
        self.assertEqual(len(self.get_logs()), 2)

        # test multiple version matches
        metadata = LegacyMetadata()

        # XXX check PEP and see if 3 == 3.0
        metadata['Requires-Python'] = '>=2.6, <3.0'
        metadata['Requires-Dist'] = ['Foo (>=2.6, <3.0)']
        self.assertEqual(self.get_logs(), [])

    def test_obsoletes_dist(self):
        fields = {'name': 'project',
                  'version': '1.0',
                  'obsoletes_dist': ['other', 'another (<1.0)']}
        metadata = LegacyMetadata(mapping=fields)
        self.assertEqual(metadata['Obsoletes-Dist'],
                         ['other', 'another (<1.0)'])
        self.assertEqual(metadata['Metadata-Version'], '1.2')
        self.assertNotIn('Provides', metadata)
        self.assertNotIn('Requires', metadata)
        self.assertEqual(metadata['Obsoletes-Dist'],
                         ['other', 'another (<1.0)'])

    def test_fullname(self):
        md = LegacyMetadata()
        md['Name'] = 'a b c'
        md['Version'] = '1 0 0'
        s = md.get_fullname()
        self.assertEqual(s, 'a b c-1 0 0')
        s = md.get_fullname(True)
        self.assertEqual(s, 'a-b-c-1.0.0')

    def test_fields(self):
        md = LegacyMetadata()
        self.assertTrue(md.is_multi_field('Requires-Dist'))
        self.assertFalse(md.is_multi_field('Name'))
        self.assertTrue(md.is_field('Obsoleted-By'))
        self.assertFalse(md.is_field('Frobozz'))

class MetadataTestCase(LoggingCatcher, TempdirManager,
                       DistlibTestCase):
    def test_init(self):
        "Test initialisation"
        md = Metadata()
        self.assertIsNone(md._legacy)
        self.assertRaises(MetadataMissingError, md.validate)
        md.name = 'dummy'
        self.assertRaises(MetadataMissingError, md.validate)
        md.version = '0.1'
        self.assertRaises(MetadataMissingError, md.validate)
        md.summary = 'Summary'
        md.validate()
        self.assertEqual(md.name, 'dummy')
        self.assertEqual(md.version, '0.1')

        # Initialise from mapping
        md = Metadata(mapping={
                        'metadata_version': '2.0',
                        'name': 'foo',
                        'version': '0.3.4',
                        'summary': 'Summary',
                      })
        md.validate()
        self.assertEqual(md.name, 'foo')
        self.assertEqual(md.version, '0.3.4')
        self.assertEqual(md.run_requires, [])
        self.assertEqual(md.meta_requires, [])
        self.assertEqual(md.provides, ['foo (0.3.4)'])

        # Initialise from legacy metadata
        fn = os.path.join(HERE, 'fake_dists', 'choxie-2.0.0.9.dist-info',
                          LEGACY_METADATA_FILENAME)
        md = Metadata(path=fn)
        md.validate()
        self.assertIsNotNone(md._legacy)
        self.assertEqual(set(md.run_requires), set(['towel-stuff (0.1)', 'nut']))
        self.assertEqual(md.metadata_version, '1.2')
        self.assertEqual(md.version, '2.0.0.9')
        self.assertEqual(md.meta_requires, [])
        self.assertEqual(set(md.provides),
                         set(['choxie (2.0.0.9)', 'truffles (1.0)']))

        # Initialise from new metadata
        fn = os.path.join(HERE, METADATA_FILENAME)
        md = Metadata(path=fn)
        md.validate()
        self.assertIsNone(md._legacy)
        self.assertEqual(md.metadata_version, '2.0')
        self.assertEqual(md.name, 'foobar')
        self.assertEqual(md.version, '0.1')
        self.assertEqual(md.provides, ['foobar (0.1)'])

    def test_add_requirements(self):
        md = Metadata()
        md.name = 'bar'
        md.version = '0.5'
        md.add_requirements(['foo (0.1.2)'])
        self.assertEqual(md.run_requires, [{ 'requires': ['foo (0.1.2)']}])

        fn = os.path.join(HERE, 'fake_dists', 'choxie-2.0.0.9.dist-info',
                          LEGACY_METADATA_FILENAME)
        md = Metadata(path=fn)
        md.add_requirements(['foo (0.1.2)'])
        self.assertEqual(set(md.run_requires),
                         set(['towel-stuff (0.1)', 'nut', 'foo (0.1.2)']))

    def test_requirements(self):
        fn = os.path.join(HERE, METADATA_FILENAME)
        md = Metadata(path=fn)
        self.assertEqual(md.meta_requires, [{'requires': ['bar (1.0)']}])
        r = md.get_requirements(md.run_requires)
        self.assertEqual(r, ['foo'])
        r = md.get_requirements(md.run_requires, extras=['certs'])
        self.assertEqual(r, ['foo', 'certifi (0.0.8)'])
        r = md.get_requirements(md.run_requires, extras=['certs', 'ssl'])
        if sys.platform != 'win32':
            self.assertEqual(r, ['foo', 'certifi (0.0.8)'])
        else:
            self.assertEqual(set(r), set(['foo', 'certifi (0.0.8)',
                                          'wincertstore (0.1)']))
        for ver in ('2.5', '2.4'):
            env = {'python_version': ver}
            r = md.get_requirements(md.run_requires,
                                    extras=['certs', 'ssl'], env=env)
            if sys.platform != 'win32':
                self.assertEqual(set(r), set(['foo', 'certifi (0.0.8)',
                                              'ssl (1.16)']))
            elif ver == '2.4':
                self.assertEqual(set(r), set(['certifi (0.0.8)', 'ssl (1.16)',
                                              'wincertstore (0.1)', 'foo',
                                              'ctypes (1.0.2)']))
            else:
                self.assertEqual(set(r), set(['certifi (0.0.8)', 'ssl (1.16)',
                                              'wincertstore (0.1)', 'foo']))
        env['sys_platform'] = 'win32'
        r = md.get_requirements(md.run_requires,
                                extras=['certs', 'ssl'], env=env)
        self.assertEqual(set(r), set(['foo', 'certifi (0.0.8)', 'ssl (1.16)',
                                      'ctypes (1.0.2)', 'wincertstore (0.1)']))
        env['python_version'] = '2.5'
        r = md.get_requirements(md.run_requires,
                                extras=['certs', 'ssl'], env=env)
        self.assertEqual(set(r), set(['foo', 'certifi (0.0.8)', 'ssl (1.16)',
                                      'wincertstore (0.1)']))
        r = md.get_requirements(md.run_requires, extras=[':test:'])
        self.assertEqual(r, ['foo', 'nose'])
        r = md.get_requirements(md.run_requires, extras=[':test:', 'udp'])
        self.assertEqual(set(r), set(['foo', 'nose', 'nose-udp']))
        self.assertEqual(md.dependencies, {
            'provides': ['foobar (0.1)'],
            'meta_requires': [
                {
                    'requires': ['bar (1.0)']
                }
            ],
            'extras': ['ssl', 'certs'],
            'build_requires': [],
            'test_requires': [
                {
                    'requires': ['nose'],
                },
                {
                    'requires': ['nose-udp'],
                    'extra': 'udp',
                }
            ],
            'run_requires': [
                {
                    'requires': ['foo']
                },
                {
                    'requires': ['certifi (0.0.8)'],
                    'extra': 'certs',
                },
                {
                    'requires': ['wincertstore (0.1)'],
                    'extra': 'ssl',
                    'environment': "sys_platform=='win32'",
                },
                {
                    'requires': ['ctypes (1.0.2)'],
                    'extra': 'ssl',
                    'environment': "sys_platform=='win32' and "
                                   "python_version=='2.4'",
                },
                {
                    'requires': ['ssl (1.16)'],
                    'extra': 'ssl',
                    'environment': "python_version in '2.4, 2.5'",
                }
            ]
        })

    def test_write(self):
        dfn = self.temp_filename()
        # Read legacy, write new
        sfn = os.path.join(HERE, 'fake_dists', 'choxie-2.0.0.9.dist-info',
                           LEGACY_METADATA_FILENAME)
        md = Metadata(path=sfn)
        md.write(path=dfn)
        with codecs.open(dfn, 'r', 'utf-8') as f:
            data = json.load(f)
        self.assertEqual(data, {
            'metadata_version': '2.0',
            'generator': 'distlib (%s)' % __version__,
            'name': 'choxie',
            'version': '2.0.0.9',
            'license': 'BSD',
            'summary': 'Chocolate with a kick!',
            'description': 'Chocolate with a longer kick!',
            'provides': ['truffles (1.0)', 'choxie (2.0.0.9)'],
            'run_requires': [{'requires': ['towel-stuff (0.1)', 'nut']}],
            'keywords': [],
        })
        # Write legacy, compare with original
        md.write(path=dfn, legacy=True)
        nmd = Metadata(path=dfn)
        d1 = md.todict()
        d2 = nmd.todict()
        self.assertEqual(d1, d2)

    def test_valid(self):
        """
        Tests to check that missing and invalid metadata is caught.
        """
        md = Metadata()
        self.assertRaises(MetadataMissingError, md.validate)
        try:
            md.name = 'Foo Bar'
        except MetadataInvalidError:
            pass
        md.name = 'foo_bar'
        # Name now OK, but version and summary to be checked
        self.assertRaises(MetadataMissingError, md.validate)
        try:
            md.version = '1.0a'
        except MetadataInvalidError:
            pass
        md.version = '1.0'
        # Name and version now OK, but summary to be checked
        self.assertRaises(MetadataMissingError, md.validate)
        try:
            md.summary = ''
        except MetadataInvalidError:
            pass
        try:
            md.summary = ' ' * 2048
        except MetadataInvalidError:
            pass
        md.summary = ' ' * 2047
        md.validate()
        md.summary = ' '
        md.validate()

if __name__ == '__main__':  # pragma: no cover
    unittest.main()


from collections import namedtuple
from enum import Enum

MyStruct = namedtuple("MyStruct", ("field1", "field2"))
m = MyStruct("foo", "bar")

class Room(Enum):
    Description = 1
    North = 2
    West = 3
    South = 4
    East = 5
    Up = 6
    Down = 7

def LydiaRoom(op):
    return {
        Room.Description: "You are in Lydia's room. Everything is picked up and looks like except for the candy wrapper on the floor.",
        Room.West: "You walk into the call. There is a scratching post. It is less interesting than the rug."
    }.get(op, "Can't go that way")

commands = {
    "R":"Description",
    "N":"North",
    "S":"South",
    "E":"East",
    "W":"West",
    "U":"Up",
    "D":"Down"
}

print("Allowed commands:")
print(commands)


from allennlp.commands.elmo import ElmoEmbedder
from argparse import ArgumentParser
from collections import Counter
from joblib import Parallel, delayed
import json
from multiprocessing import Pool
import numpy as np
import os
import pickle
import random
import re
from tqdm import tqdm

from utils.spacy import bulk_tokenize
from utils.constants import VOCAB_PREFIX, UNK_ID, UNK, FOLLOWUP_TO_ID, YESNO_TO_ID

def parsed_to_word_tuple(question, answer):
    f = lambda w: UNK if w is None else w
    tpl = parsed_to_tuple(question, answer)
    return (f(tpl[1]), f(tpl[3]), f(tpl[5]), f(tpl[7]))

def split_symbol(sent, offsets, symbol, test=lambda splitted, w: True):
    res = []
    reso = []
    for w, o in zip(sent, offsets):
        splitted = [w1 for w1 in w.split(symbol)]
        if len(splitted) > 1 and test(splitted, w):
            st = o[0]
            if len(splitted[0]) > 0:
                res.append(splitted[0])
                reso.append((st, st+len(splitted[0])))
                st += len(splitted[0])
            for w1 in splitted[1:]:
                res.append(symbol)
                reso.append((st, st+1))
                st += 1
                if len(w1) > 0:
                    res.append(w1)
                    reso.append((st, st+len(w1)))
                    st += len(w1)
        else:
            res.append(w)
            reso.append(o)
    return res, reso

def split_common_symbols(sent, offsets):
    sent, offsets = split_symbol(sent, offsets, '/', lambda splitted, w: not w.startswith('</'))
    sent, offsets = split_symbol(sent, offsets, '.', lambda splitted, w: all(len(x) > 3 or len(x) == 0 for x in splitted) and not re.match('^[0-9\.]+$', w))
    sent, offsets = split_symbol(sent, offsets, '-')
    return sent, offsets

def tokenize_one(item):
    strings_to_tokenize = [item['title'], item['section_title'], item['paragraphs'][0]['context'], item['background']]
    for qa in item['paragraphs'][0]['qas']:
        strings_to_tokenize.append(qa['question'])
        strings_to_tokenize.append(qa['orig_answer']['text'])

    tokenized, offsets = bulk_tokenize(strings_to_tokenize, return_offsets=True)

    #tokenized, offsets = list(map(list, zip(*[split_common_symbols(sent, o) for sent, o in zip(tokenized, offsets)])))

    retval = {'title': tokenized[0], 'section_title': tokenized[1], 'context': tokenized[2], 'background': tokenized[3] }
    tokenized = tokenized[4:]
    ctx_offsets = [(st-offsets[2][0][0], en-offsets[2][0][0]) for st, en in offsets[2]]

    qas = []
    parsed_idx = 0
    for qa in item['paragraphs'][0]['qas']:
        ans = tokenized[1]
        if qa['yesno'] == 'y':
            ans = ['Yes', ','] + tokenized[1]
        elif qa['yesno'] == 'n':
            ans = ['No', ','] + tokenized[1]

        char_st = qa['orig_answer']['answer_start']
        char_en = char_st + len(qa['orig_answer']['text'])
        ans_st = -1
        ans_en = -1
        for idx, (st, en) in enumerate(ctx_offsets):
            if en > char_st and ans_st < 0:
                ans_st = idx
            if st >= char_en and ans_en < 0:
                ans_en = idx
        if ans_en < 0:
            ans_en = len(ctx_offsets)
        assert ''.join(tokenized[1]) in ''.join(retval['context'][ans_st:ans_en]), '{} {}'.format(str(retval['context'][ans_st:ans_en]), str(tokenized[1]))
        qas.append({'question': tokenized[0], 'answer': ans, 'id': qa['id'],
            'start': ans_st, 'end': ans_en, 'yesno': qa['yesno'], 'followup': qa['followup']})
        tokenized = tokenized[2:]
        offsets = offsets[2:]
        parsed_idx += 2

    retval['qas'] = qas

    return retval

pool = Pool()

def tokenize_data(data):
    print('Tokenizing...')
    return list(tqdm(pool.imap(tokenize_one, data), total=len(data)))

def prepare_vocab(tokenized_data, vocab_file, wordvec_file, wordvec_dim, min_freq=3):
    print('Loading word vectors...')
    words = [] + VOCAB_PREFIX
    vecs = [np.random.randn(wordvec_dim) for _ in VOCAB_PREFIX]
    vecs[0] *= 0
    word2id = {w:i for i, w in enumerate(words)}

    print('Counting word frequency in the training set...')
    doc_freq = Counter()
    data_words = Counter()
    for item in tokenized_data:
        doc_words = set([x.lower() for x in item['title'] + item['section_title'] + item['background'] + item['context']])
        data_words.update([x for x in item['title']])
        data_words.update([x for x in item['section_title']])
        data_words.update([x for x in item['background']])
        data_words.update([x for x in item['context']])
        for qa in item['qas']:
            data_words.update([x.lower() for x in qa['question']])
            data_words.update([x for x in qa['answer']])
            doc_words.update([x.lower() for x in qa['question'] + qa['answer']])

        doc_freq.update(doc_words)

    data_chars = Counter(c for w in data_words for c in w)
    data_words_final = Counter()
    for w in data_words:
        data_words_final[w.lower()] += data_words[w]
    data_words = data_words_final
    for w in list(data_words.keys()):
        if w in word2id or data_words[w] < min_freq:
            del data_words[w]

    if os.path.exists(wordvec_file + '.pkl'):
        with open(wordvec_file + '.pkl', 'rb') as f:
            pretrained_words0 = pickle.load(f)
            pretrained_vecs0 = pickle.load(f)

        pretrained_words = []
        pretrained_vecs = []
        for word, vec in zip(pretrained_words0, pretrained_vecs0):
            if word.lower() in data_words:
                pretrained_words.append(word)
                pretrained_vecs.append(vec)

        pretrained_words_set = set(pretrained_words)
    else:
        pretrained_words = []
        pretrained_vecs = []
        pretrained_words_set = set()
        with open(wordvec_file) as f:
            processed_lines = 0
            for line in f:
                line = line.rstrip().split(' ')
                vec = [float(x) for x in line[-wordvec_dim:]]
                word = ' '.join(line[:-wordvec_dim])

                if word == '<unk>':
                    vecs[UNK_ID] = vec
                elif word.lower() not in pretrained_words_set and word.lower() in data_words:
                    pretrained_words.append(word.lower())
                    pretrained_words_set.add(word.lower())
                    pretrained_vecs.append(vec)

                processed_lines += 1

        with open(wordvec_file + '.pkl', 'wb') as f:
            pickle.dump(pretrained_words, f)
            pickle.dump(pretrained_vecs, f)

    print(f'{len(pretrained_words)} words loaded from the word vectors file.')

    #for w in data_words:
    #    if w in pretrained_words_set: continue
    #    words.append(w)
    #    vecs.append(np.random.randn(wordvec_dim))
    #    word2id[w] = len(word2id)
    words.append('cannotanswer')
    vecs.append(np.random.randn(wordvec_dim))
    word2id['cannotanswer'] = len(word2id)

    for w, vec in zip(pretrained_words, pretrained_vecs):
        if w not in data_words: continue
        words.append(w)
        vecs.append(vec)
        word2id[w] = len(word2id)

    vecs = np.array(vecs, dtype=np.float32)

    id2char = [] + VOCAB_PREFIX
    char2id = {c:i for i, c in enumerate(id2char)}
    for i, c in enumerate(data_chars.keys()):
        char2id[c] = len(id2char)
        id2char.append(c)

    assert len(word2id) == len(words) == vecs.shape[0]
    vocab = {'word2id': word2id, 'id2word': words, 'vecs': vecs, 'char2id': char2id, 'id2char': id2char}

    wordid2chars = []
    for i, w in enumerate(words):
        if w in VOCAB_PREFIX:
            wordid2chars.append([char2id[w]])
        else:
            wordid2chars.append([char2id[c] for c in w])
    vocab['wordid2chars'] = wordid2chars

    wordid2docfreq = []
    for i, w in enumerate(words):
        if w in VOCAB_PREFIX:
            if w != UNK:
                wordid2docfreq.append(np.inf)
            else:
                wordid2docfreq.append(1)
        else:
            wordid2docfreq.append(doc_freq[w])
    vocab['wordid2docfreq'] = wordid2docfreq

    print(f'{len(word2id) - len(pretrained_words) - len(VOCAB_PREFIX)} words added from the training set. Total vocab size: {len(word2id)}')

    return vocab

def map_data(tokenized_data, vocab):
    def map_field(field):
        return [vocab['word2id'].get(x.lower(), UNK_ID) for x in field]

    def map_char(field, do_lower=False):
        if do_lower:
            return [[vocab['char2id'].get(c, UNK_ID) for c in w.lower()] if w not in VOCAB_PREFIX else [vocab['char2id'][w]] for w in field]
        else:
            return [[vocab['char2id'].get(c, UNK_ID) for c in w] if w not in VOCAB_PREFIX else [vocab['char2id'][w]] for w in field]

    def map_idf(field):
        return [1 / vocab['wordid2docfreq'][vocab['word2id'].get(x.lower(), UNK_ID)] for x in field]

    def copy_mask(src, dst):
        return [[1 if w1.lower() == w2.lower() else 0 for w2 in dst] for w1 in src]

    def map_one(item):
        retval = {'title': map_field(item['title']),
                'title_char': map_char(item['title']),
                'section_title': map_field(item['section_title']),
                'section_title_idf': map_idf(item['section_title']),
                'section_title_char': map_char(item['section_title']),
                'background': map_field(item['background']),
                'background_char': map_char(item['background']),
                'context': map_field(item['context']),
                'context_char': map_char(item['context']),
                'qas': [{'question': map_field(x['question']), 'answer': map_field(x['answer']),
                    'question_char': map_char(x['question'], do_lower=True), 'answer_char': map_char(x['answer']),
                    'question_idf': map_idf(x['question']), 'answer_idf': map_idf(x['answer']),
                    'start': x['start'],
                    'end': x['end'],
                    'followup': FOLLOWUP_TO_ID[x['followup']],
                    'yesno': YESNO_TO_ID[x['yesno']]} for x in item['qas']]}
        return retval

    #elmo = ElmoEmbedder(cuda_device=0)
    #print('Computing ELMo features...')
    #elmo_features = [x[2] for x in tqdm(elmo.embed_sentences([item['context'] for item in tokenized_data], 20), total=len(tokenized_data))]

    print('Mapping tokenized data to indices...')
    return Parallel(n_jobs=-1, backend="threading")(delayed(map_one)(item) for item in tqdm(tokenized_data))

def split_train_data(quac_dir, file_name, val_count, tokenized_data, mapped_data):
    print('splitting training data into train_train and train_val...')

    # count wikipedia title occurrences
    title_counter = Counter([' '.join(x['title']) for x in tokenized_data])
    total_sections = sum(title_counter.values())
    train_sections = total_sections - val_count

    # split data by wikipedia articles to minimize leakage
    titles = list(title_counter.keys())
    random.seed(31415) # make sure we always get the same split
    random.shuffle(titles)
    train_sec = 0
    train_titles = set()
    for t in titles:
        train_titles.add(t)
        train_sec += title_counter[t]
        if train_sec >= train_sections:
            break

    train_tok = []
    train_idx = []
    val_tok = []
    val_idx = []

    print(f'Split into {train_sec} training sections and {total_sections - train_sec} validation sections.')

    for tok, idx in zip(tokenized_data, mapped_data):
        if ' '.join(tok['title']) in train_titles:
            train_tok.append(tok)
            train_idx.append(idx)
        else:
            val_tok.append(tok)
            val_idx.append(idx)

    def write_to_file(split, tok, idx):
        tok_file = os.path.join(args.quac_dir, f'_{split}.tokenized'.join(os.path.splitext(args.file_name)))
        idx_file = os.path.join(args.quac_dir, f'_{split}.idx'.join(os.path.splitext(args.file_name)))
        with open(tok_file, 'w') as f:
            json.dump(tok, f)
        with open(idx_file, 'w') as f:
            json.dump(idx, f)

    print('writing to file...')
    write_to_file('train', train_tok, train_idx)
    write_to_file('val', val_tok, val_idx)

    print('counting question frequency in the training set...')
    train_questions = [(tuple(iqa['question']), tuple(tuple(w) for w in iqa['question_char']), tuple([w.lower() for w in tqa['question']]), tuple(iqa['question_idf'])) for tpara, ipara in zip(train_tok, train_idx) for tqa, iqa in zip(tpara['qas'], ipara['qas'])]
    q_counter = Counter(train_questions)
    print('saving training question frequency to file...')
    with open(os.path.join(args.quac_dir, '_train_question_freq'.join(os.path.splitext(args.file_name))), 'w') as f:
        json.dump([[k, q_counter[k]] for k in q_counter], f)

if __name__ == "__main__":
    parser = ArgumentParser()

    parser.add_argument('--quac_dir', default='data/quac', help="Data directory for QuAC, should contain train_v0.2.json and val_v0.2.json")
    parser.add_argument('--file_name', default='train_v0.2.json', help="Data file to process")
    parser.add_argument('--wordvec_file', default='data/glove/glove.6B.100d.txt', help="File containing pretrained word embeddings")
    parser.add_argument('--vocab_file', default='vocab.pkl', help="Vocab file to save or load vocab from")
    parser.add_argument('--eval', action='store_true', help="Whether we are processing the dev/test split")
    parser.add_argument('--wordvec_dim', default=100, type=int, help="Dimension of word embeddings")
    parser.add_argument('--min_freq', default=3, type=int, help="Words in the training set occurring less than this many times will not get its own embedding")

    args = parser.parse_args()

    with open(os.path.join(args.quac_dir, args.file_name)) as f:
        data = json.load(f)

    tokenized_file = os.path.join(args.quac_dir, '.tokenized'.join(os.path.splitext(args.file_name)))
    if os.path.exists(tokenized_file):
        with open(tokenized_file) as f:
            tokenized_data = json.load(f)
    else:
        tokenized_data = tokenize_data(data['data'])
        with open(tokenized_file, 'w') as f:
            json.dump(tokenized_data, f)

    vocab_file = os.path.join(args.quac_dir, args.vocab_file)
    if not args.eval and not os.path.exists(vocab_file):
        vocab = prepare_vocab(tokenized_data, vocab_file, args.wordvec_file, args.wordvec_dim, min_freq=args.min_freq)
        with open(vocab_file, 'wb') as f:
            pickle.dump(vocab, f)
    else:
        if not os.path.exists(vocab_file):
            print('[ERROR] To preprocess evaluation data, you need to preprocess the training data first to obtain the vocabulary.')
            exit()
        with open(vocab_file, 'rb') as f:
            vocab = pickle.load(f)

    mapped_file = os.path.join(args.quac_dir, '.idx'.join(os.path.splitext(args.file_name)))
    if not os.path.exists(mapped_file):
        mapped_data = map_data(tokenized_data, vocab)
        with open(mapped_file, 'w') as f:
            json.dump(mapped_data, f)
    else:
        with open(mapped_file, 'r') as f:
            mapped_data = json.load(f)

    if not args.eval:
        split_train_data(args.quac_dir, args.file_name, 1000, tokenized_data, mapped_data)