import matplotlib as plt
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from sklearn import svm
from sklearn.covariance import EllipticEnvelope
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.neighbors import LocalOutlierFactor
from torch.utils.data.dataset import Dataset

import utils.custom_dataset
from utils.Plot import plot_confusion_matrix
from utils.Plot import plot_outliers

num_iterations = 50
num_features = 3

class MLP(nn.Module):
    def __init__(self, input_size=num_features, hidden_size=20, output_size=5):
        super(MLP, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.soft = nn.Softmax()

    def forward(self, x):
        output = self.fc1(x)
        output = F.relu(output)
        output = self.out(output)
        return self.soft(output)

    def loss(self, y_pred, target):
        criterion = nn.CrossEntropyLoss()
        return criterion(y_pred, target.long())

seed = 1200
annotation_path = "../Data/data/preprocessed_annotation_global.csv"
y = pd.read_csv(annotation_path)["label"]
names = y.astype('category').cat.categories
y = y.astype('category').cat.codes
meth_path = "../Data/data/preprocessed_Matrix_meth.csv"
mRNA_path = "../Data/data/preprocessed_Matrix_miRNA_deseq_correct.csv"
mRNA_normalized_path = "../Data/data/preprocessed_Matrix_mRNA_deseq_normalized_prot_coding_correct.csv"
files = [meth_path, mRNA_path, mRNA_normalized_path]
outliers = [LocalOutlierFactor(novelty=True), IsolationForest(), EllipticEnvelope(random_state=0), svm.OneClassSVM()]
filenames = ["meth", "mrna", "micro mrna"]
modelnames = ["mlp-local-outlier", "mlp-isolation-forest", "mlp-elliptic", "mlp-one-class"]
for modelname, outlier in zip(modelnames, outliers):
    for file, filename in zip(files, filenames):
        with open('../Data/outputs/' + filename + '-bnn-output.txt', 'w') as f:
            X = pd.read_csv(file, index_col=False, header=None)
            if (filename == "mrna"):
                X = pd.DataFrame(X[X.std().sort_values(ascending=False).head(1200).index].values.tolist())
            X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=seed, stratify=y)
            outlier_detector = outlier
            pca = PCA(n_components=num_features)
            X_train_transformed = pca.fit_transform(X_train)
            X_test_transformed = pca.transform(X_test)
            model = MLP()
            outlier_detector.fit(X_train_transformed)
            Not_Outliers = outlier_detector.predict(X_train_transformed)
            X_train_transformed = X_train_transformed[Not_Outliers == 1]
            y_train = y_train[Not_Outliers == 1]
            Not_Outliers = outlier_detector.predict(X_test_transformed)
            plot_outliers(X_test_transformed, Not_Outliers, X_train_transformed, modelname + filename + "pca")
            X_test_transformed = X_test_transformed[Not_Outliers == 1]
            y_test = y_test[Not_Outliers == 1]
            optimizer = optim.Adam(model.parameters())

            dataset = utils.custom_dataset.CustomDataset(X_train_transformed, y_train.to_numpy(),
                                                         transform=utils.custom_dataset.ToTensor())
            loader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)

            model.train()
            loss = 0
            for j in range(num_iterations):
                loss = 0
                for batch_id, data in enumerate(loader):
                    # calculate the loss and take a gradient step
                    pred = model(data["X"].view(-1, data["X"].shape[1]))
                    loss = model.loss(pred, data["y"])
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
                normalizer_train = len(loader.dataset)
                total_epoch_loss_train = loss.item() / normalizer_train

                print("Epoch ", j, " Loss ", total_epoch_loss_train)

            print(filename)

            num_samples = 10

            model.eval()
            print('Prediction when network is forced to predict')
            correct = 0
            total = 0
            dataset = utils.custom_dataset.CustomDataset(X_test_transformed, y_test.to_numpy(),
                                                         transform=utils.custom_dataset.ToTensor())
            loader = torch.utils.data.DataLoader(dataset, batch_size=32)
            probabilities = np.ndarray(shape=(0, 5))
            true_labels = np.ndarray([])
            for j, data in enumerate(loader):
                images = data["X"]
                labels = data["y"]
                pred = model(data["X"].view(-1, data["X"].shape[1]))
                probabilities = np.append(probabilities, pred.detach().numpy(), axis=0)
                true_labels = np.append(true_labels, labels)
                total += labels.size(0)
                correct += (torch.from_numpy(np.argmax(pred.detach().numpy(), axis=1)) == labels).sum().item()
            print("accuracy: %d %%" % (100 * correct / total))
            import pandas as pd

            pd.DataFrame(probabilities).to_csv("../Data/outputs/pred-" + modelname + filename + ".csv")
            pd.DataFrame(true_labels).to_csv("../Data/outputs/true-labels.csv")
            cnf_matrix = confusion_matrix(true_labels[1:], np.argmax(probabilities, axis=1))
            print()
            np.set_printoptions(precision=2)
            # PlotDir non-normalized confusion matrix
            plt.figure.Figure(figsize=(10, 10))

            plot_confusion_matrix(cnf_matrix, title=modelname + filename, classes=names)

            X2 = pd.read_csv("../Data/data/anomalies_preprocessed_Matrix_" + filename + ".csv", index_col=False,
                             header=None)
            y2 = pd.read_csv("../Data/data/anomalies_preprocessed_annotation_global.csv")["label"]
            if filename == "mrna":
                X2 = pd.DataFrame(X2[X2.std().sort_values(ascending=False).head(1200).index].values.tolist())
            X_transformed = pca.transform(X2)
            y_pred = outlier_detector.predict(X_transformed)
            plot_outliers(X_transformed, y_pred, X_train_transformed, "newdata-" + modelname + "-" + filename + "pca")
            y_pred[y_pred == -1] = 0
            true_label = y2.astype('category').cat.codes.to_numpy().copy()
            true_label[true_label == 0] = 1
            cnf_matrix = confusion_matrix(true_label, y_pred)

            print()
            np.set_printoptions(precision=2)
            # PlotDir non-normalized confusion matrix
            plt.figure.Figure(figsize=(10, 10))
            print("tot predicted")
            print((y_pred == 1).sum())
            plot_confusion_matrix(cnf_matrix,
                                  title=modelname + "-anomalies-" + filename,
                                  classes=["predicted", "Unknown"])


"""
Contains the Repository class, the main way to interact with a configfiles repo

Internally, repositories are stored as a folder with the following structure

(repo root)
|
=- index.json
=- scripts/
  =- (hash).py
  =- (hash2).py
  ...
=- locks/
  =- write_lock
  =- read_lock_0  (auto incremented using listdir)

index.json contains various information, such as script names and sources (often auto-generated)
scripts/ contains all of the scripts, named by their sha1 hashes
locks/ contains the lockfiles.

write_lock's presence indicates another configfiles instance is writing or modifying the repo and no reads nor changes should occur
read_lock's presence indicates another configfiles instance is reading the data, so no modification may occur.
Other reads can occur just fine, with autoincreasing numbers to be unique

The repository's index contains a few fields:

- version: currently 1, following fields are for this version
- scripts: dictionary of script objects
- start: start of script objects
- revision: increments with every additional script
- end

so:

- name
- files: list of files modified by the script (filenames)
- next: next script in chain
- prev: previous script in chain
"""

from socket import socket
from paramiko.transport import Transport
from .locks import RepoReadLock, RepoWriteLock
from ..auth import authenticate_transport, interpret_urlish
from hashlib import sha512
import json

class Repository:
    def __init__(self, url):
        self.url = url
        self.index = {}

        self.read_lock = RepoReadLock(url)
        self.write_lock = RepoWriteLock(url)

        self.client = None
        self.transport = None
        self.socket = socket()

        self.opened = False

    def open(self):
        """
        Opens the connection
        """

        self.socket.connect((interpret_urlish(self.url)[1], 22))
        self.transport = Transport(self.socket)
        self.transport.start_client()
        authenticate_transport(self.transport)
        self.client = self.transport.open_sftp_client()
        target_path = interpret_urlish(self.url)[2]
        try:
            self.client.stat(target_path)
        except IOError:
            self.client.mkdir(target_path)
        self.client.chdir(target_path)
        self.opened = True

    def close(self):
        """
        Closes the connection
        """

        if not self.opened:
            return
        self.client.close()
        self.transport.close()
        self.socket.close()
        self.opened = False

    def update(self):
        """
        Update the information in this class to match that of the remote.

        Raises exceptions if remote is invalid or in an invalid state
        """

        if not self.opened:
            self.open()

        with self.read_lock:
            with self.client.open("index.json") as f:
                self.index = json.load(f)

    def get_script(self, hname=None):
        """
        Get the script object
        """
        if hname is None:
            hname = self.index["start"]
        return self.index["scripts"][hname]

    def download_script(self, hname=None):
        if hname is None:
            hname = self.index["start"]
        with self.read_lock:
            with self.client.open("scripts/" + hname + ".py", "r") as f:
                return f.read()

    def get_revision(self):
        return self.index["revision"]

    def append_script(self, script_obj, script_contents):
        h = sha512()
        h.update(script_contents.encode("utf-7"))
        hname = h.hexdigest()
        script_obj["prev"] = self.index["end"]

        with self.write_lock:
            self.index["revision"] += 1
            if self.index["end"]: self.index["scripts"][self.index["end"]]["next"] = hname
            self.index["end"] = hname
            self.index["scripts"][hname] = script_obj

            if self.index["start"] == "":
                self.index["start"] = hname

            self._write()
            with self.client.open("scripts/" + hname + ".py", "w") as f:
                f.write(script_contents)

    def _write(self):
        with self.client.open("index.json", "w") as f:
            json.dump(self.index, f)

    def write(self):
        with self.write_lock:
            self._write()

    def __iter__(self):
        return FollowChainIterator(self, self.index["start"])

    def iterate_from(self, pos):
        return FollowChainIterator(self, pos)

    def new(self):
        if not self.opened:
            self.open()

        try:
            self.client.stat("index.json")
            raise RuntimeError("already init-ed, manually delete the folder to re-init")
        except IOError:
            pass

        # create the skeleton fs
        self.client.mkdir("locks")
        self.client.mkdir("scripts")

        # create a basic index.json
        self.index = {
                "version": 1,
                "revision": 0,
                "start": "",
                "end": "",
                "scripts": {}
        }

        self.write()

class FollowChainIterator:
    def __init__(self, repo, start):
        self.repo = repo
        self.pos = start

    def __next__(self):
        if not self.pos:
            raise StopIteration
        n = self.pos
        result = self.repo.get_script(self.pos)
        self.pos = result["next"]
        return n

    def __iter__(self):
        return self


from django.urls import path

from . import views

urlpatterns = [
    path("", views.SerialAPIView.as_view(), name='list'),
    path("<int:pk>/", views.SerialDetailAPIView.as_view(), name='detail'),
    path("review/", views.SerialReviewAPIView.as_view(), name='review'),
    path("rating/", views.SerialRatingAPIView.as_view(), name='rating'),
    path("video/<int:pk>/", views.VideoWatcher.as_view(), name='video')
]


import os
import platform

from PyQt5.QtCore import Qt

from PyQt5.QtWidgets import (QCheckBox, QDialog, QDialogButtonBox,
    QGridLayout, QLabel, QTabWidget, QTextEdit, QWidget)

import app
import lilychooser
import userguide
import util
import widgets
import job

class ToLyDialog(QDialog):
    """Dialog base class for all file import jobs.

    Basically the whole dialog is identical for all file types, the only
    part of the dialog that is specific is the widget with *input* options.
    These are created in a subclass's __init__ function and added to the
    self.impChecks list. Only then super() is called.
    """

    def __init__(self,
                 parent=None,
                 job_class=job.Job,
                 imp_prgm='',
                 input=None,
                 userg=''):
        super().__init__(parent)
        self._info = None
        self._imp_prgm = imp_prgm
        self._userg = userg
        self._input = input
        self._path = None
        self._job_class = job_class
        self._job = None

        self.addAction(parent.actionCollection.help_whatsthis)
        self.setWindowModality(Qt.WindowModal)
        mainLayout = QGridLayout()
        self.setLayout(mainLayout)

        tabs = QTabWidget()

        import_tab = QWidget()
        post_tab = QWidget()

        itabLayout = QGridLayout(import_tab)
        ptabLayout = QGridLayout(post_tab)

        tabs.addTab(import_tab, self._imp_prgm)
        tabs.addTab(post_tab, _("After Import"))

        self.formatCheck = QCheckBox()
        self.trimDurCheck = QCheckBox()
        self.removeScalesCheck = QCheckBox()
        self.runEngraverCheck = QCheckBox()

        self.postChecks = [self.formatCheck,
                           self.trimDurCheck,
                           self.removeScalesCheck,
                           self.runEngraverCheck]

        self.versionLabel = QLabel()
        self.lilyChooser = lilychooser.LilyChooser(toolcommand=self._imp_prgm)

        self.formatCheck.setObjectName("reformat")
        self.trimDurCheck.setObjectName("trim-durations")
        self.removeScalesCheck.setObjectName("remove-scaling")
        self.runEngraverCheck.setObjectName("engrave-directly")

        self.buttons = QDialogButtonBox(
            QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        userguide.addButton(self.buttons, self._userg)

        row = 0
        for r, w in enumerate(self.impChecks):
            row += r
            itabLayout.addWidget(w, row, 0, 1, 2)
        row += 1
        for r, w in enumerate(self.impExtra):
            row += r
            itabLayout.addWidget(w, row, 0, 1, 2)

        itabLayout.addWidget(widgets.Separator(), row + 1, 0, 1, 2)
        itabLayout.addWidget(self.versionLabel, row + 2, 0, 1, 0)
        itabLayout.addWidget(self.lilyChooser, row + 3, 0, 1, 2)
        itabLayout.setRowStretch(row + 4, 10)

        ptabLayout.addWidget(self.formatCheck, 0, 0, 1, 2)
        ptabLayout.addWidget(self.trimDurCheck, 1, 0, 1, 2)
        ptabLayout.addWidget(self.removeScalesCheck, 2, 0, 1, 2)
        ptabLayout.addWidget(self.runEngraverCheck, 3, 0, 1, 2)
        ptabLayout.setRowStretch(4, 6)

        mainLayout.addWidget(tabs, 0, 0, 6, 2)
        mainLayout.addWidget(self.buttons, 7, 0, 1, 2)

        self.buttons.accepted.connect(self.about_to_accept)
        self.buttons.rejected.connect(self.reject)

        self.lilyChooser.currentIndexChanged.connect(self.slot_lilypond_version_changed)
        self.slot_lilypond_version_changed()

    def translateUI(self):
        self.versionLabel.setText(_("LilyPond version:"))
        self.formatCheck.setText(_("Reformat source"))
        self.trimDurCheck.setText(_("Trim durations (Make implicit per line)"))
        self.removeScalesCheck.setText(_("Remove fraction duration scaling"))
        self.runEngraverCheck.setText(_("Engrave directly"))

    def about_to_accept(self):
        """Configure the job and close the dialog."""
        self.configure_job()
        self.accept()

    def configure_job(self):
        """Create and configure the job to be run.
        Has to be completed by the subclasses."""
        output = os.path.splitext(
            os.path.join(util.tempdir(), os.path.basename(self._input))
            )[0] + '.ly'
        self._job = j = self._job_class(
            command=self._info.toolcommand(self._imp_prgm),
            input=self._input,
            output=f'--output={output}',
            directory=os.path.dirname(self._input),
            encoding='utf-8')
        j._output_file = output
        if platform.system() == "Darwin":
            import macos
            if macos.inside_app_bundle():
                j.environment['PYTHONPATH'] = None
                j.environment['PYTHONHOME'] = None

    def get_post_settings(self):
        """Returns settings in the post import tab."""
        post = []
        for p in self.postChecks:
            post.append(p.isChecked())
        return post

    def job(self):
        """Return the current job."""
        if not self._job:
            self.configure_job()
        return self._job

    def set_input(self, path):
        """Set the full path to the input document."""
        self._input = path

    def slot_lilypond_version_changed(self):
        self._info = self.lilyChooser.lilyPondInfo()

    def loadSettings(self):
        """Get users previous settings."""
        post_default = [True, False, False, True]
        for i, d in zip(self.impChecks, self.imp_default):
            i.setChecked(self.settings.value(i.objectName(), d, bool))
        for p, f in zip(self.postChecks, post_default):
            p.setChecked(self.settings.value(p.objectName(), f, bool))

    def saveSettings(self):
        """Save users last settings."""
        for i in self.impChecks:
            self.settings.setValue(i.objectName(), i.isChecked())
        for p in self.postChecks:
            self.settings.setValue(p.objectName(), p.isChecked())


import requests
from bs4 import BeautifulSoup
from geojson import FeatureCollection, Feature, Point

from extractors import Extractor
from utils import extract_address_from_text, user_agent
from utils.opening_dates import create_from_freetext

class PoloMarketExtractor(Extractor):
    BASE_URL = 'https://www.polomarket.pl'

    def fetch_locations(self) -> FeatureCollection:
        response = requests.get(f'{self.BASE_URL}/pl/nasze-sklepy.html', headers={
            'User-Agent': user_agent()
        })
        soup = BeautifulSoup(response.content, 'html.parser')
        regions_links = ['/'.join([self.BASE_URL, x["href"]]) for x in soup.find_all('area')]

        features = []
        for region_link in regions_links:
            response = requests.get(region_link, headers={
                'User-Agent': user_agent()
            })

            soup = BeautifulSoup(response.content, 'html.parser')
            shops = soup.select('.region')
            for shop in shops:
                city = shop.find('span', string='Miasto:').nextSibling.text.strip()
                street = shop.find('span', string='Ulica:').nextSibling.text.strip()
                opening_hours_raw = shop.find('span', string='Godziny otwarcia:').nextSibling.text
                navigate_url = shop.find('a', string='Dojazd')
                address_tags = extract_address_from_text(street)

                features.append(Feature(
                    geometry=Point((float(navigate_url['data-lng']), float(navigate_url['data-lat']))),
                    properties=address_tags | {
                        'name': 'POLOMarket',
                        'brand': 'POLOMarket',
                        'brand:wikidata': 'Q11821937',
                        'shop': 'supermarket',
                        'addr:full': street,
                        'addr:city': city,
                        'opening_hours': create_from_freetext(opening_hours_raw)
                    }
                ))

        return FeatureCollection(features)


import ujson

from app.utils import api_models
from app.utils.decorators import something_might_go_wrong, get_todo_ids, load_model
from app.utils.models import TodoModel
from app.utils.responses import success

@something_might_go_wrong
@load_model(api_models.Todo, with_identity=True)
def create(todo: api_models.Todo, identity_id):
    """Create a new to do in the database"""
    new_todo = TodoModel(identity_id, **todo.dict())
    new_todo.save()
    todo_db = api_models.TodoInDB.parse_obj(new_todo.attribute_values)
    todo_out = api_models.TodoOut.parse_obj(todo_db)
    return success({"message": "Todo Created", "item": todo_out.dict()}, 201)

@something_might_go_wrong
@get_todo_ids
def read_single(user_id, todo_id, _):
    """Get a single to do item"""
    todo = TodoModel.get(user_id, todo_id)
    todo_db = api_models.TodoInDB.parse_obj(todo.attribute_values)
    todo_out = api_models.TodoOut.parse_obj(todo_db)
    return success(todo_out.dict())

@something_might_go_wrong
def read_active(event, _context):
    """Get all uncompleted to do items"""
    user_id = event["requestContext"]["identity"]["cognitoIdentityId"]
    items = TodoModel.query(user_id, TodoModel.completed == False)
    response_body = api_models.process_todo_db_list(items)
    return success(response_body)

@something_might_go_wrong
def read_completed(event, _context):
    """Get all completed to do items"""
    user_id = event["requestContext"]["identity"]["cognitoIdentityId"]
    items = TodoModel.query(user_id, TodoModel.completed == True)
    response_body = api_models.process_todo_db_list(items)
    return success(response_body)

def _modify_completed(user_id, todo_id, set_to):
    item = TodoModel.get(user_id, todo_id)
    item.completed = set_to
    item.save()
    item.refresh()
    return success({"message": "Updated", "item": item.attribute_values})

@something_might_go_wrong
@get_todo_ids
def mark_completed(user_id, todo_id, _):
    """Mark an item as completed"""
    return _modify_completed(user_id, todo_id, True)

@something_might_go_wrong
@get_todo_ids
def unmark_completed(user_id, todo_id, _):
    """Mark an to do item as not complete"""
    return _modify_completed(user_id, todo_id, False)

@something_might_go_wrong
@get_todo_ids
def update(user_id, todo_id, event):
    """Update the content or instrument of a to do item"""
    data = ujson.loads(event["body"])
    item = TodoModel.get(user_id, todo_id)
    if data.get("content"):
        item.content = data["content"]
    if data.get("relevantInstrument"):
        item.relevantInstrument = data["relevantInstrument"]
    item.save()
    item.refresh()
    todo_db = api_models.TodoInDB.parse_obj(item.attribute_values)
    todo_out = api_models.TodoOut.parse_obj(todo_db)
    return success({"message": "Updated", "item": todo_out.dict()})

@something_might_go_wrong
@get_todo_ids
def delete(user_id, todo_id, _):
    """Delete a to do item"""
    item = TodoModel.get(user_id, todo_id)
    item.delete()
    return success({"message": "deleted"}, 204)


#!/usr/bin/env python3
import sys
import re
from urllib.request import urlopen
from bs4 import BeautifulSoup

def stockQuote(watchlist):
    # default url for a stock quote search
    url="https://markets.businessinsider.com/stocks/"
    data=[] # create a temp list to stuff each url content into
    # iterate over our watchlist extracting html data
    for stock in watchlist:
        temp=urlopen(url+stock+"-stock")
        bs = BeautifulSoup(temp.read(), 'html.parser')
        data.append(bs)

    counter=0
    # iterate over our stock html data to pull out the closing price
    for stock in data:
        nameList=stock.findAll('div', {'class':'col-md-3 col-xs-6 text-right bold black'})
        price=nameList[0].get_text()
        print(watchlist[counter] + " " + price.strip())
        counter+=1

stockQuote(["MSFT","GOOGL","T","BRKB","AAPL","FB","SBUX","MS","SCHW","CSCO","PG","DAL","JNJ","PFE"])


import base64
import datetime

from dataclasses import dataclass
from pydantic import BaseModel

@dataclass
class AcfgParams:
    creator: str
    decimals: int
    total: int
    round: int
    url: str = None
    url_b64: str = None
    name: str = None
    name_b64: str = None
    unit_name: str = None
    unit_name_b64: str = None
    default_frozen: bool = None
    metadata_hash: str = None
    manager: str = None
    freeze: str = None
    reserve: str = None
    clawback: str = None

class Txn:
    def __init__(self, txninfo):
        self.data = txninfo

    @property
    def round_time(self):
        return self.data["round-time"]

    @property
    def note(self):
        if "note" in self.data:
            return base64.b64decode(self.data["note"])
        return None

    @property
    def type(self):
        return self.data["tx-type"]

class ACfgTxn(Txn):
    def __init__(self, txninfo):
        Txn.__init__(self, txninfo)
        params = self.data["asset-config-transaction"]["params"]
        for key in list(params.keys()):
            if "-" in key:
                params[key.replace("-", "_")] = params.pop(key)
        params["round"] = self.data["confirmed-round"]
        self.params = AcfgParams(**params)
        self.txn_id = self.data["id"]

    @property
    def created_asset_id(self):
        self.data.get("created-asset-index", None)

    def __repr__(self):
        rep = f"ACfgTxn({self.txn_id})"
        return rep

class AssetBaseSchema(BaseModel):
    asset_id: int
    name: str
    description: str = None
    asset_info: dict
    asset_metadata: dict = None
    is_destroyed: bool
    media: dict = None
    collection_asset: list = []
    network: str
    blockchain_updated_at: datetime.datetime


from backend.db.base_class import Base
from sqlalchemy import Column, Integer, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.orm import backref

class CartProduct(Base):
    __tablename__ = 'cart_products'
    cart_product_id = Column(Integer, primary_key=True)
    product_id = Column(Integer, ForeignKey('products.id'), nullable=False)
    product = relationship(
        "Product", foreign_keys=[product_id],
        backref=backref(
            "cart_products", cascade="all,delete"
        )
    )
    user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
    quantity = Column(Integer, nullable=False)

    @property
    def is_available(self):
        try:
            return sum([availability.quantity for availability in self.product.available]) > 0
        except Exception as e:
            print(e)

class CartStock(Base):
    __tablename__ = 'cart_stocks'
    cart_stock_id = Column(Integer, primary_key=True)
    stock_id = Column(Integer, ForeignKey('stocks.id'), nullable=False)
    stock = relationship(
        "Stock", foreign_keys=[stock_id],
        backref=backref(
            "cart_stocks", cascade="all,delete"
        )
    )
    user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
    quantity = Column(Integer, nullable=False)

    @property
    def is_available(self):
        try:
            stock_products = self.stock.products
            stock_availability = 0
            for stock_product in stock_products:
                if len([availability for availability in stock_product.product.available if availability.quantity > 0]) > 0:
                    stock_availability += 1
            return stock_availability == len(stock_products)
        except Exception as e:
            print(e)


""" Components
"""
import logging
import types
from zope.component import queryAdapter
from eea.relations.interfaces import IContentType
from eea.relations.component.interfaces import IContentTypeLookUp
from eea.relations.component.interfaces import IRelationsLookUp

logger = logging.getLogger('eea.relations.queryContentType')

def queryContentType(context):
    """ Lookup for context related content-type in portal_relations
    """
    connecter = queryAdapter(context, IContentTypeLookUp)
    if not connecter:
        if not isinstance(context, types.MethodType):
            logger.exception('No IContentTypeLookUp adapter found for '
                             '%s', context)
        return None
    return connecter()

def queryForwardRelations(context):
    """ Lookup for context possible forward relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return
    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return
    for relation in connecter.forward():
        yield relation

def queryBackwardRelations(context):
    """ Lookup for context possible backward relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return
    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return
    for relation in connecter.backward():
        yield relation

def getForwardRelationWith(context, ctype):
    """ Get forward relation with ctype

    Returns None if I can't find possible relation or
    possible relation object from portal_relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return None
    new_ctype = ctype
    if not IContentType.providedBy(ctype):
        new_ctype = queryContentType(ctype)
    if not new_ctype:
        return None

    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return None
    return connecter.forward_with(new_ctype)

def getBackwardRelationWith(context, ctype):
    """ Get backward relation with ctype

    Returns None if I can't find possible relation or
    possible relation object from portal_relations
    """
    if not IContentType.providedBy(context):
        context = queryContentType(context)
    if not context:
        return None

    if not IContentType.providedBy(ctype):
        ctype = queryContentType(ctype)
    if not ctype:
        return None

    connecter = queryAdapter(context, IRelationsLookUp)
    if not connecter:
        logger.exception('No IRelationsLookUp adapter found for '
                         '%s', context)
        return None
    return connecter.backward_with(ctype)


import numpy as np
import matplotlib.pyplot as plt
import numpy as np

def get_evasions_falues(path):
    vals = []
    with open(path, "r") as f:
        for line in f:
            vals.append(float(line.strip().split(",")[0]))
    return np.array(vals)

if __name__ == "__main__":
    # names = ["okreuk_sorel_malconv", "okreukvmi_sorel_malconv", "kreukmi_sorel_malconv", "nkreuk2_sorel_malconv", "kreuk_sorel_malconv"]
    # labels =["FGSM", "FGSM-VMI", "FGSM-MI", "FGSM(2)", "FGSM (og-t)"]
    # names = ["kreuk_sorel_malconv", "kreukmomentum_sorel_malconv", "kreukvmi_sorel_malconv"]
    # names = ["okreuk_sorel_malconv", "okreukvmi_sorel_malconv", "kreukmi_sorel_malconv"]
    # labels =["FGSM", "FGSM-MI", "FGSM-VMI"]
    #names = ["dosextend_malconv", "dosextend_sorel", "dosextend_gbt", "dosextend_malconv_gbtall_sorel_all", "dosextend_malconv_gbt", "dosextend_malconv_sorel", "dosextend_sorelall", "dosextend_gbt_all", "dosextend_malconv_gbtall_sorel", "dosextend_malconv_gbt_sorel_all"]
    #labels = ["Malconv", "SOREL", "GBT", "MalConv_GBTALL_SORELALL", "Malconv_GBT", "Malconv_SOREL", "SORELALL", "GBTALL", "Malconv_GBTALL_SOREL", "Mallconv_GBT_SORELALL"]
    names = ["dosextend_malconv_gbtall_sorel_all"]
    labels = ["i-Ensemble"]
    vals = [get_evasions_falues(f"./vt_outputs/{name}.txt") for name in names]
    # Plot histogram
    for i, val in enumerate(vals):
        plt.hist(val, bins=[1, 5, 10, 15, 20, 25, 30, 35], label=labels[i], alpha=0.5)
        # print("Average # of evasions for {}: {}".format(labels[i], val.mean()))
        print("Average % of evasions for {} (>= 15 evasions): {}".format(labels[i], 100 * (val >= 15).mean()))
        print("Average % of evasions for {} (>= 30 evasions): {}".format(labels[i], 100 * (val >= 30).mean()))
    # Save plot
    plt.xlabel("Number of classifiers evaded (out of 73)")
    plt.savefig(f"histogram.png")


import click
from string import Template
import ast
from os.path import isfile
import inflect
import pkg_resources

from crafter.commands import create_model

p = inflect.engine()

def create_relationship(relationship_type, model_1, model_2, assoc = None):
    assoc_name = ''
    if (relationship_type == 'many_many'):
        assoc_name = assoc

    statements = {
        "one_one" : [
            f"{model_2} = db.relationship('{model_2.capitalize()}', backref='{model_1.lower()}', lazy=True, uselist=False)",
            f"{model_1}_id = db.Column(db.Integer, db.ForeignKey('{model_1}.id'), nullable=False)",
        ],
        "one_many": [
            f"{p.plural(model_2)} = db.relationship('{model_2.capitalize()}', backref='{model_1.lower()}', lazy=True)",
            f"{model_1}_id = db.Column(db.Integer, db.ForeignKey('{model_1}.id'), nullable=False)",
        ],
        "many_one": [
            f"{model_2}_id = db.Column(db.Integer, db.ForeignKey('{model_2}.id'), nullable=False)",
            f"{p.plural(model_1)} = db.relationship('{model_1.capitalize()}', backref='{model_2.lower()}', lazy=True)",
        ],
        "many_many": [
            f"{p.plural(assoc_name)} = db.relationship('{model_2.capitalize()}', secondary={assoc_name}, backref=db.backref('{assoc_name}', lazy=True))",
            None,
        ],

    }

    insert_statement(statements[relationship_type][0], model_1)
    insert_statement(statements[relationship_type][1], model_2)

    if (relationship_type == "many_many"):
        create_assoc_table(assoc_name, model_1, model_2)
        add_import(assoc_name, model_1)

    click.echo(f"{model_1} and {model_2} Relationship created successfully ✅")

def add_import(assoc_name, model):
    with open(f"app/models/{model}.py", "r") as f:
        root = ast.parse(f.read())
    import_node = ast.parse(f"import {assoc_name}")
    root.body.insert(1, import_node)
    ast.fix_missing_locations(root)

    unparsed = ast.unparse(root)

    with open(f"app/models/{model}.py", "w") as f:
        f.write(unparsed)

def create_assoc_table(assoc_name, model_1, model_2):

    d = {
        'assoc_name': assoc_name,
        'model_1': model_1,
        'model_2': model_2,

    }

    with open(pkg_resources.resource_filename("crafter", "templates/assoc_table.tpl"), "r") as f:
        src = Template(f.read())
    result_table = src.substitute(d)

    with open(f"app/models/{assoc_name}.py", "w") as f:
        f.write(result_table)

    click.echo(f"{assoc_name} Table created successfully ✅")

def insert_statement(statement, model):
    if statement is None:
        if not isfile(f"app/models/{model}.py"):
            create_model(model)
        return
    if not isfile(f"app/models/{model}.py"):
        create_model(model)

    with open(f"app/models/{model}.py", "r") as f:
        root = ast.parse(f.read())

    classdef = root.body[-1]
    assign_node = ast.parse(statement)
    classdef.body.insert(2, assign_node)
    ast.fix_missing_locations(root)

    unparsed = ast.unparse(root)

    with open(f"app/models/{model}.py", "w") as f:
        f.write(unparsed)


# self supervised multimodal multi-task learning network
import os
import sys
import collections

import pdb
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd.function import Function
from torch.nn.utils.rnn import pad_sequence, pack_padded_sequence, pad_packed_sequence

from models.subNets.BertTextEncoder import BertTextEncoder

__all__ = ['SELF_MM']

class SELF_MM(nn.Module):
    def __init__(self, args):
        super(SELF_MM, self).__init__()
        # text subnets
        self.aligned = args.need_data_aligned
        self.text_model = BertTextEncoder(language=args.language, use_finetune=args.use_finetune)

        # audio-vision subnets
        audio_in, video_in = args.feature_dims[1:]
        self.audio_model = AuViSubNet(audio_in, args.a_lstm_hidden_size, args.audio_out, \
                            num_layers=args.a_lstm_layers, dropout=args.a_lstm_dropout)
        # change 32 -> 64 args.video_out = 64 2022年 04月 02日 星期六 16:47:28 CST
        # in conig/config_regression.py:266
        # args.text_out = 768
        # args.audio_out = 16
        # args.video_out = 64 changed 32->64
        # args.post_video_dim = 64 changed 32->64
        self.video_model = VideoSubNet(video_in, args.v_lstm_hidden_size, args.video_out, \
                            num_layers=args.v_lstm_layers, dropout=args.v_lstm_dropout)

        # the post_fusion layers
        self.post_fusion_dropout = nn.Dropout(p=0.0)
        self.post_fusion_layer_1 = nn.Linear(args.text_out + args.video_out + args.audio_out , args.post_fusion_dim)
        self.post_fusion_layer_2 = nn.Linear(args.post_fusion_dim, args.post_fusion_dim)
        self.post_fusion_layer_3 = nn.Linear(args.post_fusion_dim, 1)

        # second level fusion modulle
        # args.post_fusion_dim = 128
        self.post_fusion_layer_4 = nn.Linear(args.post_fusion_dim + args.text_out + args.video_out + args.audio_out , args.post_fusion_dim)
        self.post_fusion_layer_5 = nn.Linear(64,64)
        self.post_fusion_layer_5_1 = nn.Linear(64,64)
        # layer 6 is same for 3 , layer 3 is useless
        self.post_fusion_layer_6 = nn.Linear(64, 1)

        # the classify layer for text
        self.post_text_dropout = nn.Dropout(p=args.post_text_dropout)
        self.post_text_layer_1 = nn.Linear(args.text_out, args.post_text_dim)
        self.post_text_layer_2 = nn.Linear(args.post_text_dim, args.post_text_dim)
        self.post_text_layer_3 = nn.Linear(args.post_text_dim, 1)

        # the classify layer for audio
        self.post_audio_dropout = nn.Dropout(p=args.post_audio_dropout)
        self.post_audio_layer_1 = nn.Linear(args.audio_out, args.post_audio_dim)
        self.post_audio_layer_2 = nn.Linear(args.post_audio_dim, args.post_audio_dim)
        self.post_audio_layer_3 = nn.Linear(args.post_audio_dim, 1)

        # the classify layer for video
        self.post_video_dropout = nn.Dropout(p=args.post_video_dropout)
        self.post_video_layer_1 = nn.Linear(args.video_out, args.post_video_dim)
        self.post_video_layer_2 = nn.Linear(args.post_video_dim, args.post_video_dim)
        self.post_video_layer_3 = nn.Linear(args.post_video_dim, 1)

        '''
        # 0405 attention inspired by ../mmsa/models/singleTask/MFN.py
        attInShape = 64
        h_att1 = 64
        att1_dropout = 0.7
        self.att1_fc1 = nn.Linear(attInShape, h_att1)
        self.att1_fc2 = nn.Linear(h_att1, attInShape)
        self.att1_dropout = nn.Dropout(att1_dropout)
        '''

        '''
        0413 early_fusion lstm

        '''
        # self.norm = nn.BatchNorm1d(126) # 126 is text len with context  :
        self.norm = nn.BatchNorm1d(45-1) # 45 is text len : torch.Size([32, 45, 768])
        # self.lstm = nn.LSTM(33 + 768 + 709, 64, num_layers=1, dropout=0.0, bidirectional=False, batch_first=True)
        self.lstm = nn.LSTM(33 + 768 + 2048, 64, num_layers=1, dropout=0.0, bidirectional=False, batch_first=True)
        self.dropout = nn.Dropout(0.1)
        self.linear = nn.Linear(64,64)
        self.out = nn.Linear(64,1)

    def forward(self, text, audio, video):
        audio, audio_lengths = audio
        video, video_lengths = video

        mask_len = torch.sum(text[:,1,:], dim=1, keepdim=True)
        text_lengths = mask_len.squeeze().int().detach().cpu()

        # 0412 align subnet
        text_x = self.text_model(text)[:,1:,:]
        self.dst_len = text_x.size(1) # 45
        audio_x = audio
        video_x = video

        def align(x):
            raw_seq_len = x.size(1)
            if raw_seq_len == self.dst_len:
                return x
            if raw_seq_len // self.dst_len == raw_seq_len / self.dst_len:
                pad_len = 0
                pool_size = raw_seq_len // self.dst_len
            else:
                pad_len = self.dst_len - raw_seq_len % self.dst_len
                pool_size = raw_seq_len // self.dst_len + 1
            pad_x = x[:, -1, :].unsqueeze(1).expand([x.size(0), pad_len, x.size(-1)])
            x = torch.cat([x, pad_x], dim=1).view(x.size(0), pool_size, self.dst_len, -1)
            x = x.mean(dim=1)
            return x
        text_x = align(text_x)
        audio_x = align(audio_x)
        video_x = align(video_x)

        # 0413 Early_fusion LSTM
        x = torch.cat([text_x, audio_x, video_x], dim=-1)
        # import ipdb;ipdb.set_trace()
        x = self.norm(x)
        _, final_states = self.lstm(x)
        x = self.dropout(final_states[0][-1].squeeze(dim=0))
        x = F.relu(self.linear(x), inplace=True)
        x = self.dropout(x)
        ef_lstm_h = x
        ef_lstm_output = self.out(x)

        text = self.text_model(text)[:,0,:]

        if self.aligned:
            audio = self.audio_model(audio, text_lengths)
            video = self.video_model(video, text_lengths)
        else:
            audio = self.audio_model(audio, audio_lengths)
            video = self.video_model(video, video_lengths)

        '''
        0411 delete 64->64 self-attention
        # 0405 attention inspired by ../mmsa/models/singleTask/MFN.py
        attention = F.softmax(self.att1_fc2(self.att1_dropout(F.relu(self.att1_fc1(video)))),dim=1)
        cStar = video
        attended = attention*cStar
        # residual
        video = attended + video
        '''

        # fusion 0415 放弃后期融合
        fusion_h = torch.cat([text, audio, video], dim=-1)
        fusion_h = self.post_fusion_dropout(fusion_h)
        fusion_h = F.relu(self.post_fusion_layer_1(fusion_h), inplace=False)

        # text
        text_h = self.post_text_dropout(text)
        text_h = F.relu(self.post_text_layer_1(text_h), inplace=False)
        # audio
        audio_h = self.post_audio_dropout(audio)
        audio_h = F.relu(self.post_audio_layer_1(audio_h), inplace=False)
        # vision
        video_h = self.post_video_dropout(video)
        video_h = F.relu(self.post_video_layer_1(video_h), inplace=False)

        # classifier-fusion
        # 0415 give up late fusion
        '''
        x_f = F.relu(self.post_fusion_layer_2(fusion_h), inplace=False)
        x_f = torch.cat([x_f, text, audio, video], dim=-1)
        # layer_3 is useless
        # output_fusion = self.post_fusion_layer_3(x_f)
        x_f = F.relu(self.post_fusion_layer_4(x_f), inplace=False)
        '''
        # after layer_4 save x_f as fusion_h for unimodal label generation 0403
        # 0414
        # 只是用text模态，考虑bert模型的性能。
        # x_f = text
        # fusion_h = x_f
        # x_f = torch.cat((ef_lstm_h,text),dim=-1)
        # 0415 修改为只使用ef-lstm的情况。
        x_f = ef_lstm_h
        # 0415 ulgm
        # fusion_h = x_f

        x_f = self.post_fusion_dropout(x_f)
        x_f = F.relu(self.post_fusion_layer_5(x_f), inplace=False)
        x_f = F.relu(self.post_fusion_layer_5_1(x_f), inplace=False)
        output_fusion = self.post_fusion_layer_6(x_f)

        # classifier-text
        x_t = F.relu(self.post_text_layer_2(text_h), inplace=False)
        output_text = self.post_text_layer_3(x_t)

        # classifier-audio
        x_a = F.relu(self.post_audio_layer_2(audio_h), inplace=False)
        output_audio = self.post_audio_layer_3(x_a)

        # classifier-vision
        x_v = F.relu(self.post_video_layer_2(video_h), inplace=False)
        output_video = self.post_video_layer_3(x_v)

        # output_fusion出现nan情况,查看原因
        if output_fusion.isnan().any():
            pdb.set_trace()

        res = {
            'M': output_fusion,
            'T': output_text,
            'A': output_audio,
            'V': output_video,
            'Feature_t': text_h,
            'Feature_a': audio_h,
            'Feature_v': video_h,
            'Feature_f': x_f,
        }
        return res

class AuViSubNet(nn.Module):
    def __init__(self, in_size, hidden_size, out_size, num_layers=1, dropout=0.2, bidirectional=False):
        '''
        Args:
            in_size: input dimension
            hidden_size: hidden layer dimension
            num_layers: specify the number of layers of LSTMs.
            dropout: dropout probability
            bidirectional: specify usage of bidirectional LSTM
        Output:
            (return value in forward) a tensor of shape (batch_size, out_size)
        '''
        super(AuViSubNet, self).__init__()
        self.rnn = nn.LSTM(in_size, hidden_size, num_layers=num_layers, dropout=dropout, bidirectional=bidirectional, batch_first=True)
        self.dropout = nn.Dropout(dropout)
        self.linear_1 = nn.Linear(hidden_size, out_size)

    def forward(self, x, lengths):
        '''
        x: (batch_size, sequence_len, in_size)
        '''
        # pdb.set_trace()
        # mustard数据集样本长短差别极大,mean+3*std依然有
        # 被截断的样本存在,因此需要更新送入rnn的长度.否则
        # 会运行错误
        '''
        在预处理阶段进行了bug的修复,更新了长度
        max_length = x.shape[1]
        batch_size = x.shape[0]
        condition = lengths <= max_length
        lengths = torch.where(condition,lengths,torch.ones(batch_size,).to(x.device).long()*max_length)
        '''

        lengths = lengths.cpu()
        packed_sequence = pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)
        try:
            _, final_states = self.rnn(packed_sequence)
        except RuntimeError:
            pdb.set_trace()
            print('Error')
        h = self.dropout(final_states[0].squeeze())
        y_1 = self.linear_1(h)
        return y_1

class VideoSubNet(nn.Module):
    def __init__(self, in_size, hidden_size, out_size, num_layers=1, dropout=0.2, bidirectional=False):
        '''
        0409 dingning add attention in to VideoSubNet
        2048->128
        '''
        super(VideoSubNet, self).__init__()
        self.rnn1 = nn.LSTM(in_size, hidden_size, num_layers=num_layers, dropout=dropout, bidirectional=bidirectional, batch_first=True)
        self.rnn2 = nn.LSTM(64, 64, num_layers=1, dropout=dropout, bidirectional=bidirectional, batch_first=True)
        self.dropout1 = nn.Dropout(0.7)
        self.dropout2 = nn.Dropout(dropout)
        self.dropout3 = nn.Dropout(dropout)
        # self.linear1 = nn.Linear(709, 64)
        self.linear1 = nn.Linear(2048,64)
        self.linear2 = nn.Linear(64,64)
        self.linear3 = nn.Linear(hidden_size, out_size)

    def forward(self, x, lengths):
        '''
        x: (batch_size, sequence_len, in_size)
        '''
        lengths = lengths.cpu()
        packed_sequence = pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)
        try:
            _, final_states = self.rnn1(packed_sequence)
        except RuntimeError:
            pdb.set_trace()
            print('Error')
        #  0409 add attention
        x1 = self.linear1(x)
        attention = F.softmax(self.linear2(self.dropout1(F.relu(x1))),dim=-1)
        # attended:(batch_size,sequence_len,64)
        attended = attention * x1
        h0 = final_states[0].squeeze()
        # h1:(batch_size,1,64)
        h1 = torch.unsqueeze(h0,1)
        # h2:(batch_size,sequence_len+1,64)
        h2 = torch.cat((h1,attended),1)
        # this line need debug add 1
        lengths1 = lengths + 1
        packed_sequence1 = pack_padded_sequence(h2, lengths1, batch_first=True, enforce_sorted=False)
        _, final_states1 = self.rnn2(packed_sequence1)
        h3 = self.dropout2(final_states1[0].squeeze())

        h4 = self.dropout2(final_states[0].squeeze())
        h5 = h3 + h4
        # h:(batch_size,64)
        y_1 = self.linear3(h5)
        return y_1


# coding: utf-8
__author__ = 'ZFTurbo: https://kaggle.com/zfturbo, Weimin: https://kaggle.com/weimin'

if __name__ == '__main__':
    import os

    gpu_use = (0,1)
    print('GPU use: {}'.format(gpu_use))
    os.environ["KERAS_BACKEND"] = "tensorflow"
    os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

from a01_neural_nets import *
from a02_common_training_structures import *
from keras.applications.xception import preprocess_input as xception_preprocess_input
from keras.applications.inception_resnet_v2 import preprocess_input as inception_resnet_preprocess_input
import sys

PRECISION = 6
EPS = 0.00001

def readLines(f):
    with open(f, 'r') as f:
        return f.read().split('\n')

def process_tst_images(model_path, box_size, cache_path_test, preproc, is_weights=False, model_name=None):
    from keras.models import load_model
    from keras.optimizers import Adam
    from keras.utils import multi_gpu_model

    if not os.path.isdir(cache_path_test):
        os.mkdir(cache_path_test)

    restore_from_cache = True
    if is_weights:
        if model_name == 'zfturbo_inception_resnet':
            model = get_model_inception_resnet_v2()
            model.load_weights(model_path)

        elif model_name == 'zfturbo_resnet':
            model = get_model_resnet50_336()
            model.load_weights(model_path)

        elif model_name == 'weimin_inception_resnet':
            model = get_model_inception_resnet_v2()
            optim = Adam(lr=0.001)
            model = multi_gpu_model(model, gpus=2)
            model.compile(optimizer=optim, loss='binary_crossentropy', metrics=[f2beta_loss, fbeta])
            model.load_weights(model_path)

        elif model_name == 'weimin_xception':
            model=get_model_xception()
            optim = Adam(lr=0.001)
            model = multi_gpu_model(model, gpus=2)
            model.compile(optimizer=optim, loss='binary_crossentropy', metrics=[f2beta_loss, fbeta])
            model.load_weights(model_path)

        else:
            print("Unknown model_name: %s"%model_name)
            return None
        print("\nUsing weights to load model: \n%s loaded."%model_name)
    else:
        model = load_model(model_path, custom_objects={'f2beta_loss': f2beta_loss, 'fbeta': fbeta})
        print("\nUsing model architecture .h5 to load model: \n%s loaded."%model_name)

    test_files = glob.glob(TEST_IMAGES_PATH + '*.jpg')
    print('Number of images to predict:', len(test_files))

    for i, f in enumerate(sorted(test_files)):
        id = os.path.basename(f)[:-4]
        path = f

        cache_path = cache_path_test + id + '.pkl'
        if not os.path.isfile(cache_path) or restore_from_cache is False:
            im_full_big = read_single_image(path)
            im_full_big = cv2.resize(im_full_big, (box_size, box_size), cv2.INTER_LANCZOS4)

            batch_images = []
            batch_images.append(im_full_big.copy())
            batch_images.append(im_full_big[:, ::-1, :].copy())
            batch_images = np.array(batch_images, dtype=np.float32)
            batch_images = preproc(batch_images)

            preds = model.predict(batch_images)
            preds[preds < EPS] = 0
            preds = np.round(preds, PRECISION)
            save_in_file(preds, cache_path)

def create_submission(folder_path, thr_arr, out_file):
    index_arr_forward, index_arr_backward = get_classes_to_index_dicts()
    out = open(out_file, 'w')
    out.write('image_id,labels\n')
    test_files = glob.glob(TEST_IMAGES_PATH + '*.jpg')
    #print('Test valid:', len(test_files))
    for i, f in enumerate(test_files):
        id = os.path.basename(f)[:-4]
        cache_path = folder_path + id + '.pkl'
        preds = load_from_file(cache_path)
        preds = preds.mean(axis=0)
        total = 0
        out.write(id + ',')
        for j in range(len(preds)):
            if preds[j] >= thr_arr[j]:
                out.write(index_arr_backward[j] + ' ')
                total += 1
        out.write('\n')
        # print('Go for {} {}. Stored: {}'.format(i, id, total))
    out.close()

def voting_ensemble(subm_arr, out_path):
    s = pd.read_csv(subm_arr[0])
    s['labels_0'] = s['labels']
    #print(len(s))
    for i in range(1, len(subm_arr)):
        s1 = pd.read_csv(subm_arr[i])
        s1['labels_{}'.format(i)] = s1['labels']
        s = s.merge(s1[['image_id', 'labels_{}'.format(i)]], on='image_id', how='left')
        #print(s1.shape)
    #print(len(s))

    ## majority vote
    limit = len(subm_arr) // 2

    print('\nmajority vote num threshold: {}\n'.format(limit))
    merged_labels = []
    for index, row in s.iterrows():
        res = dict()
        for i in range(len(subm_arr)):
            line = row['labels_{}'.format(i)]
            if str(line) == 'nan':
                continue
            arr = line.strip().split(' ')
            for a in arr:
                if a in res:
                    res[a] += 1
                else:
                    res[a] = 1

        out_str = ''
        for el in res:
            if res[el] > limit:
                out_str += el + ' '

        # union for empty rows
        if out_str == '':
            temp_res = []
            for i in range(len(subm_arr)):
                if str(row['labels_{}'.format(i)]) != 'nan':
                    for i in row['labels_{}'.format(i)].strip().split():
                        temp_res.append(i)
            temp_res = list(set(temp_res))
            out_str = ' '.join(temp_res)

        merged_labels.append(out_str)
    s['labels'] = merged_labels

    s[['image_id', 'labels']].to_csv(out_path, index=False)

    return s[['image_id', 'labels']]

def voting_ensemble_mark_union(subm_arr, out_path):
    s = pd.read_csv(subm_arr[0])
    s['labels_0'] = s['labels']
    #print(len(s))
    for i in range(1, len(subm_arr)):
        s1 = pd.read_csv(subm_arr[i])
        s1['labels_{}'.format(i)] = s1['labels']
        s = s.merge(s1[['image_id', 'labels_{}'.format(i)]], on='image_id', how='left')
        #print(s1.shape)
    #print(len(s))

    ## majority vote
    limit = len(subm_arr) // 2

    print('\nmajority vote num threshold: {}\n'.format(limit))
    merged_labels = []
    use_union = []
    for index, row in s.iterrows():
        res = dict()
        for i in range(len(subm_arr)):
            line = row['labels_{}'.format(i)]
            if str(line) == 'nan':
                continue
            arr = line.strip().split(' ')
            for a in arr:
                if a in res:
                    res[a] += 1
                else:
                    res[a] = 1

        out_str = ''
        for el in res:
            if res[el] > limit:
                out_str += el + ' '

        # union for empty rows
        if out_str == '':
            use_union.append(1)
            temp_res = []
            for i in range(len(subm_arr)):
                if str(row['labels_{}'.format(i)]) != 'nan':
                    for i in row['labels_{}'.format(i)].strip().split():
                        temp_res.append(i)
            temp_res = list(set(temp_res))
            out_str = ' '.join(temp_res)
        else:
            use_union.append(0)

        merged_labels.append(out_str)
    s['labels'] = merged_labels
    s['use_union'] = use_union

    s[['image_id', 'labels', 'use_union']].to_csv(out_path, index=False)

    return s[['image_id', 'labels', 'use_union']]

def fix_empty_rows(in_subm, out_subm):
    s1 = pd.read_csv(in_subm)

    s1_ids = s1['image_id'].values
    s1_lbls = s1['labels'].values

    new_ids = []
    new_lbls = []
    total_empty = 0
    for i in range(len(s1_ids)):
        id1 = s1_ids[i]
        lbl = s1_lbls[i]
        if str(lbl) == 'nan':
            total_empty += 1
            lbl = '/m/01g317 /m/05s2s /m/07j7r'

        new_ids.append(id1)
        new_lbls.append(lbl)

    t = pd.DataFrame({'image_id': new_ids, 'labels': new_lbls})
    t.to_csv(out_subm, index=False)
    print("Total empty rows: {}".format(total_empty))

if __name__ == '__main__':
    start_time = time.time()

    ## params to search in the for loop below
    ## model / model weights file name,                  cache folder,                              threshold file name prefix,                   threshold points,     output file names,                img_dim, is_weights, preprocessing_fun,           model architecture string
    params = [
        ['inception_resnet_v2_temp_320_weights.h5',       'cache_inception_resnet_v2_test/',        'thr_arr_inception_resnet_v2',                (0.01, 0.99, 1, 0.99),  'inception_resnet_v2_299.csv',    (299, True,  preprocess_input,                  'zfturbo_inception_resnet')],
        ['resnet50_336_temp_488_weights.h5',              'cache_resnet50_test/',                   'thr_arr_resnet50_sh_336',                    (0.01, 0.99, 1, 0.99),  'resnet50_336.csv',               (336, True,  preprocess_input,                   'zfturbo_resnet')],
        ['weights_new_xception_tlatest.h5',               'cache_xception_test/',                   'thr_arr_xception',                           (0.01, 0.99, 1, 0.99),  'xception.csv',                   (299, True,  xception_preprocess_input,          'weimin_xception')],
        ['inception_resnet_model_oct_19_0.001_48_0.1.h5', 'cache_inception_resnet_version_1_test/', 'thr_arr_inception_resnet_version_1',         (0.01, 0.99, 1, 0.99),  'inception_resnet_version_1.csv', (299, True,  inception_resnet_preprocess_input,  'weimin_inception_resnet')],
        ['inception_resnet_model_5.h5',                   'cache_inception_resnet_version_2_test/', 'thr_arr_inception_resnet_version_2',         (0.01, 0.99, 1, 0.99),  'inception_resnet_version_2.csv', (299, True,  inception_resnet_preprocess_input,  'weimin_inception_resnet')],
        ['inception_resnet_model_Oct_31_v3_weights.h5',   'cache_inception_resnet_version_3_test/', 'thr_arr_inception_resnet_weimin_version_3',  (0.01, 0.99, 1, 0.99),  'inception_resnet_version_3.csv', (299, True,  inception_resnet_preprocess_input,  'weimin_inception_resnet')],
        ['inception_resnet_weimin_version_4_weights.h5',  'cache_inception_resnet_version_4_test/', 'thr_arr_inception_resnet_weimin_version_4',  (0.01, 0.99, 1, 0.99),  'inception_resnet_version_4.csv', (299, True,  inception_resnet_preprocess_input,  'weimin_inception_resnet')],
    ]

    ## loop through each model to generate prediction on one set of threshold - this has to be run at once (just to generate those image preds cache for each model), in order to proceed for random search later on
    ## get tuning label set for evaludation
    #test_image_classes = get_classes_for_tst_images_dict()
    #index_arr_forward, index_arr_backward = get_classes_to_index_dicts()

    ## get the index for special classes that don't exist in tuning label set
    thr_0_9999 = load_from_file(OUTPUT_PATH+'thr_arr_xception_sp_0.01_ep_0.99_min_1_def_0.9999.pklz')
    col_index_9999 = np.where(thr_0_9999==0.9999)[0]

    subm_arr = []
    for ind, model_param in enumerate(params):
        temp_models_path, temp_output_path, thre_prefix, thre_points, subm_path, process_params = model_param

        model_path = MODELS_PATH + temp_models_path
        cache_path = OUTPUT_PATH + temp_output_path
        start_point, end_point, min_number_of_entries, default_value = thre_points
        thr_path = OUTPUT_PATH + thre_prefix + '_sp_{}_ep_{}_min_{}_def_{}.pklz'.format(start_point, end_point,
                                                                                min_number_of_entries,
                                                                                default_value)
        # for the three special models to be in range of 0.1-0.9
        if temp_models_path == 'resnet50_336_temp_488_weights.h5':
            thr_path = OUTPUT_PATH + thre_prefix + '_sp_{}_ep_{}_min_{}_def_{}.pklz'.format(0.1, 0.9,
                                                                                1,
                                                                                0.9) # 0.8
        if temp_models_path == 'inception_resnet_model_Oct_31_v3_weights.h5':
            thr_path = OUTPUT_PATH + thre_prefix + '_sp_{}_ep_{}_min_{}_def_{}.pklz'.format(0.1, 0.9,
                                                                                1,
                                                                                0.9) # 0.8
        if temp_models_path == 'inception_resnet_weimin_version_4_weights.h5':
            thr_path = OUTPUT_PATH + thre_prefix + '_sp_{}_ep_{}_min_{}_def_{}.pklz'.format(0.1, 0.9,
                                                                                3,
                                                                                0.9) # 0.9
        #print('\n\n\n', thr_path, '\n\n\n')

        submit_path = SUBM_PATH + subm_path[:-4] + "_thre_{}_{}_{}_{}.csv".format(start_point, end_point, min_number_of_entries, default_value)
        if temp_models_path == 'resnet50_336_temp_488_weights.h5':
            submit_path = SUBM_PATH + subm_path[:-4] + "_thre_{}_{}_{}_{}.csv".format(0.1, 0.9, 1, 0.8)
        if temp_models_path == 'inception_resnet_model_Oct_31_v3_weights.h5':
            submit_path = SUBM_PATH + subm_path[:-4] + "_thre_{}_{}_{}_{}.csv".format(0.1, 0.9, 1, 0.8)
        if temp_models_path == 'inception_resnet_weimin_version_4_weights.h5':
            submit_path = SUBM_PATH + subm_path[:-4] + "_thre_{}_{}_{}_{}.csv".format(0.1, 0.9, 3, 0.9)

        subm_arr.append(submit_path)

        #if os.path.exists(submit_path):
        #    print("File {} already exists. Skipping creating submission for this model. ".format(submit_path))
            ## evaluate LS for this model
        #    s = pd.read_csv(submit_path)
        #    score, median_counts, mean_counts = evalute_local(s, 'labels', test_image_classes, index_arr_forward)
        #    print("Model {} scores {} median & mean counts of labels: {} {} for tuning label set. \n".format(temp_models_path, score, median_counts, mean_counts))
        #    continue

        process_tst_images(model_path, process_params[0], cache_path, process_params[2], is_weights=process_params[1], model_name=process_params[3])

        thr_arr = load_from_file(thr_path)

        ## convert special classes to certain default values
        if temp_models_path == 'resnet50_336_temp_488_weights.h5':
            thr_arr[col_index_9999] = 0.8
        if temp_models_path == 'inception_resnet_model_Oct_31_v3_weights.h5':
            thr_arr[col_index_9999] = 0.8
        if temp_models_path == 'inception_resnet_weimin_version_4_weights.h5':
            thr_arr[col_index_9999] = 0.9

        create_submission(cache_path, thr_arr, submit_path)

        ## evaluate LS for this model
        #s = pd.read_csv(submit_path)
        #score, median_counts, mean_counts = evalute_local(s, 'labels', test_image_classes, index_arr_forward)
        #print("Model {} scores {} median & mean counts of labels: {} {} for tuning label set. \n".format(temp_models_path, score, median_counts, mean_counts))
        #print("Done for model {}\n".format(ind+1))

    # Majority voting part
    majority_voting_submission_path = SUBM_PATH + 'majority_voting_{}_models_v2.2.csv'.format(len(subm_arr))
    s = voting_ensemble(subm_arr, majority_voting_submission_path)

    ## evaludate the result locally
    #score, median_counts, mean_counts = evalute_local(s, 'labels', test_image_classes, index_arr_forward)
    #print("Score {} median & mean counts of labels: {} {} for tuning label set. ".format(score, median_counts, mean_counts))

    # Fix empty rows
    final_subm_path = SUBM_PATH + 'final_subm_stage_2_{}_models.csv'.format(len(subm_arr))
    fix_empty_rows(majority_voting_submission_path, final_subm_path)

    print('Time: {:.0f} sec'.format(time.time() - start_time))


import re
import snscrape.modules.twitter as sntwitter
import pandas as pd
from tqdm import tqdm

def scraperV1(query, maxTweets):
    scraper = sntwitter.TwitterSearchScraper(query)
    data = scraper.get_items()

    tweets = []

    counter = 0

    for tweet in tqdm(data, total=maxTweets):
        if counter > maxTweets:
            break
        if tweet.lang == 'en':
            text = tweet.renderedContent
            text = re.sub(',','\\,',text)
            text = re.sub('\n',' ',text)
            l = []
            l.append(tweet.id)
            l.append(tweet.date)
            l.append(text)
            l.append(tweet.likeCount)
            l.append(tweet.retweetCount)
            l.append(tweet.replyCount)
            l.append(tweet.hashtags)
            tweets.append(l)
            counter +=1

    df = pd.DataFrame(tweets, columns=['id', 'date', 'renderedContent', 'likeCount', 'retweetCount', 'replyCount', 'hashtags'])

    return df


"""add blogview without timestamp

Revision ID: b4893dd631cc
Revises: 8553a96593c0
Create Date: 2017-11-18 00:15:21.615630

"""

# revision identifiers, used by Alembic.
revision = 'b4893dd631cc'
down_revision = '8553a96593c0'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('ix_blog_view_timestamp', table_name='blog_view')
    op.drop_column('blog_view', 'timestamp')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('blog_view', sa.Column('timestamp', mysql.DATETIME(), nullable=True))
    op.create_index('ix_blog_view_timestamp', 'blog_view', ['timestamp'], unique=False)
    # ### end Alembic commands ###


import ipaddress

def check_ip(ip):
    try:
        ipaddress.ip_address(ip)
        return True
    except ValueError as err:
        return False

if __name__ == "__main__":
    result = check_ip('10.1.1.1')
    print('Function result:', result)


# # Prototyping on mortgages

# Let's prototype a potentially useful ML app here.
# We'll abstract and test and expand later, but we can try to fail fast.

# ## Imports

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import roc_auc_score, precision_recall_fscore_support
from sklearn.preprocessing import label_binarize

# ## Read data

# We'll read the data and keep only those complaints with narratives.
# I don't like stateful manipulation of dataframes in the global scope,
# so we're going to wrap it in a function.

def read_complaints(filename):
    """
    Reads the consumer complaints data (in it's standard downloaded csv form)
    from the file `filename', relative to project root directory.
    """
    complaints = pd.read_csv(filename)
    complaints = complaints[complaints['Consumer complaint narrative'].notnull()]
    return complaints

complaints = read_complaints('data/raw/consumer_complaints.csv')

# Let's try to build an "issue classifier" from the free text.
# This could be used as an assistive system for the administrators that
# need to respond to the free text:
# complaints could be automatically routed to the right person
# (with the caveat that they need a method for correcting misclassifications!)

# As described in the field reference, the "Issue" identified for each
# complaint can take certain values depending on the product
# (which is specified by the user).

# Let's at least start simple, and focus on just one product.
# I'm choosing mortgages because I like the cut of it's jib.

# How many issues exist within the mortgage product category?

(
    complaints
    [complaints['Product'] == 'Mortgage'] # filter
    .groupby('Issue')
    ['Complaint ID'] # select column
    .count()
    .sort_values(ascending=False)
)

# Owing to the imbalanced number of entries for each issue, I suggest
# we take only the top 9 (arbitrarily) categories here,
# then introduce an "other" category.
# It's still an imbalanced dataset, but nonetheless, let's go ahead.
# Let's also encode the issue categories for ease of reference.

issues_abbreviations = {
    "Loan servicing, payments, escrow account": "loan_servicing",
    "Loan modification,collection,foreclosure": "loan_modification",
    "Trouble during payment process": "payment_process",
    "Struggling to pay mortgage": "struggling_to_pay",
    "Application, originator, mortgage broker": "application",
    "Settlement process and costs": "settlement",
    "Applying for a mortgage or refinancing an existing mortgage": "applying",
    "Closing on a mortgage": "closing",
    "Credit decision / Underwriting": "underwriting",
    "Incorrect information on your report": "other",
    "Applying for a mortgage": "applying",
    "Problem with a credit reporting company's investigation into an existing problem": "other",
    "Improper use of your report": "other",
    "Credit monitoring or identity theft protection services": "other",
    "Unable to get your credit report or credit score": "other",
    "Problem with fraud alerts or security freezes": "other"
}

# Our `issues_abbreviations` dict is not 1:1.
# Inverting won't work perfectly because we've mapped several categories to
# "other".
# We'll "fix" that by replacing the categories mapping to "other" with
# "Unknown" in the reverse mapping, since we can't be sure which issue they
# should be attributed to.
# Again, I'm going to encapsulate the stateful append to the dict with a
# function.

def create_abbreviations_issues():
    """
    Reverse the issues_abbreviations dict, encapsulating state
    """
    abbreviations_issues = {
        v:k for (k,v) in issues_abbreviations.items()
    }
    abbreviations_issues['other'] = 'Unknown'
    return abbreviations_issues

abbreviations_issues = create_abbreviations_issues()

# Now we can clean up the data.

def create_mortgages_frame():
    mortgages = (
        complaints
            [complaints['Product'] == 'Mortgage']
            [['Consumer complaint narrative', 'Issue']]
            .rename({
                'Consumer complaint narrative': 'complaint',
                'Issue': 'issue'
                },
                axis='columns')
            .reset_index(drop=True)
    )
    mortgages['issue'] = mortgages.issue.apply(
      lambda x: issues_abbreviations[x]
    )
    return mortgages

mortgages = create_mortgages_frame()

# Let's take a look.

mortgages.head()

def plot_mortgage_complaint_lengths():
    # This is the second time I've written this code, so maybe
    # it'd be a good thing to abstract!
    ax = mortgages.complaint.apply(len).plot(kind='hist', bins = 100)
    ax.set_xlabel('Character length of text')
    return ax

plot_mortgage_complaint_lengths()

# Still a wide range of lengths.

complaint_lengths = mortgages.complaint.apply(len)
complaint_lengths.describe()

# I'm suspicious of the extremely long messages.
# Let's look at some of them.

long_messages = (
    mortgages
    .where(mortgages.complaint.apply(lambda c: len(c) > 10000))
    .dropna()
)

long_messages.sample().complaint.values[0]

# Yup, they just seem like really long complaints ¯\\\_(ツ)_/¯

# How imbalanced is the dataset?

def plot_class_balance():
    ax = pd.value_counts(mortgages.issue).plot(kind='bar')
    ax.set_xlabel('Issue')
    ax.set_ylabel('Count')
    return ax

plot_class_balance()

# ## Classifying complaints

# So we have some handle on the data.
# Let's train a classifier to separate complaints into the issues identified.

# First, split into train and test sets, stratified across the classes.
# If we were going to do model selection too, we'd want to split into an
# additional set to give us some measure of expected real world performance
# (ie stop us overfitting the test set).
# We'll do that eventually, but the results of this experiment aren't going
# be used for anything except testing whether this problem is easily amenable
# to ML, so I'm not too concerned about holding out validation data right now.

train_X, test_X, train_y, test_y = train_test_split(
    mortgages.complaint,
    mortgages.issue,
    stratify=mortgages.issue,
    test_size=0.2
)

# We need a computable representation of text.
# For topic classification (which is what we're doing here),
# keywords usually work great, at least as a baseline.
# We'll use scikit's tf-idf.

vectorizer = TfidfVectorizer()
vectorized_train_X = vectorizer.fit_transform(train_X)
vectorized_test_X = vectorizer.transform(test_X)

# Yes, using vectorizer like this has broken my enforced rigour about
# state transforms in the global scope (vectorizer is now different to
# when it was created). I'll deal with it.

# We'll try a simple multinomial naive bayes classifier.
# Technically this should use integer counts (because that's what a multinomial
# distribution represents), but it works with td-idf in practice too.

clf = MultinomialNB()
clf.fit(vectorized_train_X, train_y)

# Predict on train and test using the fitted classifier.

train_y_pred = clf.predict(vectorized_train_X)
test_y_pred = clf.predict(vectorized_test_X)

# Calculate a measure of goodness.
# We'll use the area under the ROC curve (true positive vs false positive
# rate), with a weighted average over the multiple classes.
# To do this we must binarize the labels (because that's what the sklearn
# method wants).

labels = [x for x in set(issues_abbreviations.values())]

binary_true_train_y = label_binarize(train_y, classes=labels)
binary_pred_train_y = label_binarize(train_y_pred, classes=labels)
binary_true_test_y = label_binarize(test_y, classes=labels)
binary_pred_test_y = label_binarize(test_y_pred, classes=labels)

train_roc = roc_auc_score(
    binary_true_train_y,
    binary_pred_train_y,
    average='weighted'
)

test_roc = roc_auc_score(
    binary_true_test_y,
    binary_pred_test_y,
    average='weighted'
)

print(
  "The weighted average ROC AUC for the train set: {}"
  .format(train_roc)
)

print(
  "The weighted average ROC AUC for the test set: {}"
  .format(test_roc)
)

# OK. That's not a great score, but we used a simple
# method with all default params (for both td-idf and naive bayes).

# Let's also take a look at the precision, recall, f-score and support,
# again weighted by class imbalances.

train_prfs = precision_recall_fscore_support(
    binary_true_train_y,
    binary_pred_train_y,
    average='weighted'
)

test_prfs = precision_recall_fscore_support(
    binary_true_test_y,
    binary_pred_test_y,
    average='weighted'
)

# When calling the `precision...` function, we get an UndefinedMetricWarning
# telling us that there are labels with no predicted samples.
# That's likely because the classes in our training set are so imbalanced.
# There are multiple ways of dealing with that which we can investigate later.


import re
from pathlib import Path

import pytest
import spacy
from confection import Config
from spacy.util import make_tempdir

from spacy_llm.pipeline import LLMWrapper
from spacy_llm.registry import fewshot_reader, file_reader
from spacy_llm.ty import LLMTask
from spacy_llm.util import assemble_from_config

from ...tasks import make_summarization_task
from ..compat import has_openai_key

EXAMPLES_DIR = Path(__file__).parent / "examples"
TEMPLATES_DIR = Path(__file__).parent / "templates"

@pytest.fixture
def zeroshot_cfg_string():
    return """
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]

    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"
    max_n_words = 20

    [components.llm.model]
    @llm_models = "spacy.GPT-3-5.v2"
    """

@pytest.fixture
def fewshot_cfg_string():
    return f"""
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]

    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"
    max_n_words = 20

    [components.llm.task.examples]
    @misc = "spacy.FewShotReader.v1"
    path = {str((Path(__file__).parent / "examples" / "summarization.yml"))}

    [components.llm.model]
    @llm_models = "spacy.GPT-3-5.v2"
    """

@pytest.fixture
def ext_template_cfg_string():
    """Simple zero-shot config with an external template"""

    return f"""
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]
    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"
    max_n_words = 20

    [components.llm.task.template]
    @misc = "spacy.FileReader.v1"
    path = {str((Path(__file__).parent / "templates" / "summarization.jinja2"))}

    [components.llm.model]
    @llm_models = "spacy.GPT-3-5.v2"
    """

@pytest.fixture
def noop_config():
    return """
    [nlp]
    lang = "en"
    pipeline = ["llm"]
    batch_size = 128

    [components]

    [components.llm]
    factory = "llm"

    [components.llm.task]
    @llm_tasks = "spacy.Summarization.v1"

    [components.llm.model]
    @llm_models = "test.NoOpModel.v1"
    """

@pytest.fixture
def example_text() -> str:
    """Returns string to be used as example in tests."""
    return (
        "The atmosphere of Earth is the layer of gases, known collectively as air, retained by Earth's gravity "
        "that surrounds the planet and forms its planetary atmosphere. The atmosphere of Earth creates pressure, "
        "absorbs most meteoroids and ultraviolet solar radiation, warms the surface through heat retention "
        "(greenhouse effect), allowing life and liquid water to exist on the Earth's surface, and reduces "
        "temperature extremes between day and night (the diurnal temperature variation)."
    )

@pytest.mark.external
@pytest.mark.skipif(has_openai_key is False, reason="OpenAI API key not available")
@pytest.mark.parametrize(
    "cfg_string",
    [
        "zeroshot_cfg_string",
        "fewshot_cfg_string",
        "ext_template_cfg_string",
    ],
)
def test_summarization_config(cfg_string, request):
    cfg_string = request.getfixturevalue(cfg_string)
    orig_config = Config().from_str(cfg_string)
    nlp = spacy.util.load_model_from_config(orig_config, auto_fill=True)
    assert nlp.pipe_names == ["llm"]

    # also test nlp config from a dict in add_pipe
    component_cfg = dict(orig_config["components"]["llm"])
    component_cfg.pop("factory")

    nlp2 = spacy.blank("en")
    nlp2.add_pipe("llm", config=component_cfg)
    assert nlp2.pipe_names == ["llm"]

    pipe = nlp.get_pipe("llm")
    assert isinstance(pipe, LLMWrapper)
    assert isinstance(pipe.task, LLMTask)

@pytest.mark.external
@pytest.mark.skipif(has_openai_key is False, reason="OpenAI API key not available")
@pytest.mark.parametrize(
    "cfg_string",
    [
        "zeroshot_cfg_string",
        "fewshot_cfg_string",
        "ext_template_cfg_string",
    ],
)
def test_summarization_predict(cfg_string, example_text, request):
    """Use OpenAI to get summarize text.
    Note that this test may fail randomly, as the LLM's output is unguaranteed to be consistent/predictable
    """
    orig_cfg_string = cfg_string
    cfg_string = request.getfixturevalue(cfg_string)
    orig_config = Config().from_str(cfg_string)
    nlp = spacy.util.load_model_from_config(orig_config, auto_fill=True)

    # One of the examples exceeds the set max_n_words, so we expect a warning to be emitted.
    if orig_cfg_string == "fewshot_cfg_string":
        with pytest.warns(
            UserWarning,
            match=re.escape(
                "The provided example 'Life is a quality th...' has a summary of length 28, but `max_n_words` == 20."
            ),
        ):
            doc = nlp(example_text)
    else:
        doc = nlp(example_text)

    # Check whether a non-empty summary was written and we are somewhat close to the desired upper length limit.
    assert 0 < len(doc._.summary)
    if "ext" not in orig_cfg_string:
        nlp.select_pipes(disable=["llm"])
        assert (
            len(nlp(doc._.summary))
            <= orig_config["components"]["llm"]["task"]["max_n_words"] * 1.5
        )

@pytest.mark.external
@pytest.mark.skipif(has_openai_key is False, reason="OpenAI API key not available")
@pytest.mark.parametrize(
    "cfg_string_and_field",
    [
        ("zeroshot_cfg_string", None),
        ("fewshot_cfg_string", None),
        ("ext_template_cfg_string", None),
        ("zeroshot_cfg_string", "summary_x"),
    ],
)
def test_summarization_io(cfg_string_and_field, example_text, request):
    cfg_string, field = cfg_string_and_field
    orig_cfg_string = cfg_string
    cfg_string = request.getfixturevalue(cfg_string)
    orig_config = Config().from_str(cfg_string)
    if field:
        orig_config["components"]["llm"]["task"]["field"] = field
    nlp = spacy.util.load_model_from_config(orig_config, auto_fill=True)
    assert nlp.pipe_names == ["llm"]
    # ensure you can save a pipeline to disk and run it after loading
    with make_tempdir() as tmpdir:
        nlp.to_disk(tmpdir)
        nlp2 = spacy.load(tmpdir)
    assert nlp2.pipe_names == ["llm"]

    if orig_cfg_string == "fewshot_cfg_string":
        with pytest.warns(
            UserWarning,
            match=re.escape(
                "The provided example 'Life is a quality th...' has a summary of length 28, but `max_n_words` == 20."
            ),
        ):
            doc = nlp2(example_text)
    else:
        doc = nlp2(example_text)

    field = "summary" if field is None else field
    nlp2.select_pipes(disable=["llm"])
    assert 0 < len(nlp2(getattr(doc._, field)))
    if "ext" not in orig_cfg_string:
        assert (
            len(nlp2(getattr(doc._, field)))
            <= orig_config["components"]["llm"]["task"]["max_n_words"] * 1.5
        )

def test_jinja_template_rendering_without_examples(example_text):
    """Test if jinja template renders as we expected

    We apply the .strip() method for each prompt so that we don't have to deal
    with annoying newlines and spaces at the edge of the text.
    """
    nlp = spacy.blank("en")
    doc = nlp.make_doc(example_text)

    llm_ner = make_summarization_task(examples=None, max_n_words=10)
    prompt = list(llm_ner.generate_prompts([doc]))[0]

    assert (
        prompt.strip()
        == f"""
You are an expert summarization system. Your task is to accept Text as input and summarize the Text in a concise way.
The summary must not, under any circumstances, contain more than 10 words.
Here is the Text that needs to be summarized:
'''
{example_text}
'''
Summary:""".strip()
    )

@pytest.mark.parametrize(
    "examples_path",
    [
        str(EXAMPLES_DIR / "summarization.json"),
        str(EXAMPLES_DIR / "summarization.yml"),
        str(EXAMPLES_DIR / "summarization.jsonl"),
    ],
)
def test_jinja_template_rendering_with_examples(examples_path, example_text):
    """Test if jinja2 template renders as expected

    We apply the .strip() method for each prompt so that we don't have to deal
    with annoying newlines and spaces at the edge of the text.
    """
    nlp = spacy.blank("en")
    doc = nlp.make_doc(example_text)

    prompt_examples = fewshot_reader(examples_path)
    llm_ner = make_summarization_task(examples=prompt_examples, max_n_words=20)

    with pytest.warns(
        UserWarning,
        match=re.escape(
            "The provided example 'Life is a quality th...' has a summary of length 28, but `max_n_words` == 20."
        ),
    ):
        prompt = list(llm_ner.generate_prompts([doc]))[0]

    assert (
        prompt.strip()
        == f"""
You are an expert summarization system. Your task is to accept Text as input and summarize the Text in a concise way.
The summary must not, under any circumstances, contain more than 20 words.
Below are some examples (only use these as a guide):

Text:
'''
The United Nations, referred to informally as the UN, is an intergovernmental organization whose stated purposes are to maintain international peace and security, develop friendly relations among nations, achieve international cooperation, and serve as a centre for harmonizing the actions of nations. It is the world's largest international organization. The UN is headquartered on international territory in New York City, and the organization has other offices in Geneva, Nairobi, Vienna, and The Hague, where the International Court of Justice is headquartered.

The UN was established after World War II with the aim of preventing future world wars, and succeeded the League of Nations, which was characterized as ineffective. On 25 April 1945, 50 nations met in San Francisco, California for a conference and started drafting the UN Charter, which was adopted on 25 June 1945. The charter took effect on 24 October 1945, when the UN began operations. The organization's objectives, as defined by its charter, include maintaining international peace and security, protecting human rights, delivering humanitarian aid, promoting sustainable development, and upholding international law. At its founding, the UN had 51 member states; as of 2023, it has 193 – almost all of the world's sovereign states.
'''
Summary:
'''
UN is an intergovernmental organization to foster international peace, security, and cooperation. Established after WW2 with 51 members, now 193.
'''

Text:
'''
Life is a quality that distinguishes matter that has biological processes, such as signaling and self-sustaining processes, from matter that does not, and is defined by the capacity for growth, reaction to stimuli, metabolism, energy transformation, and reproduction. Various forms of life exist, such as plants, animals, fungi, protists, archaea, and bacteria. Biology is the science that studies life.

The gene is the unit of heredity, whereas the cell is the structural and functional unit of life. There are two kinds of cells, prokaryotic and eukaryotic, both of which consist of cytoplasm enclosed within a membrane and contain many biomolecules such as proteins and nucleic acids. Cells reproduce through a process of cell division, in which the parent cell divides into two or more daughter cells and passes its genes onto a new generation, sometimes producing genetic variation.

Organisms, or the individual entities of life, are generally thought to be open systems that maintain homeostasis, are composed of cells, have a life cycle, undergo metabolism, can grow, adapt to their environment, respond to stimuli, reproduce and evolve over multiple generations. Other definitions sometimes include non-cellular life forms such as viruses and viroids, but they are usually excluded because they do not function on their own; rather, they exploit the biological processes of hosts.
'''
Summary:
'''
Life is a quality defined by biological processes, including reproduction, genetics, and metabolism. There are two types of cells and organisms that can grow, respond, reproduce, and evolve.
'''

Here is the Text that needs to be summarized:
'''
{example_text}
'''
Summary:
""".strip()
    )

def test_external_template_actually_loads(example_text):
    template_path = str(TEMPLATES_DIR / "summarization.jinja2")
    template = file_reader(template_path)
    nlp = spacy.blank("en")
    doc = nlp.make_doc(example_text)

    llm_ner = make_summarization_task(template=template)
    prompt = list(llm_ner.generate_prompts([doc]))[0]
    assert (
        prompt.strip()
        == """
This is a test summarization template.
Here is the text: The atmosphere of Earth is the layer of gases, known collectively as air, retained by Earth's gravity that surrounds the planet and forms its planetary atmosphere. The atmosphere of Earth creates pressure, absorbs most meteoroids and ultraviolet solar radiation, warms the surface through heat retention (greenhouse effect), allowing life and liquid water to exist on the Earth's surface, and reduces temperature extremes between day and night (the diurnal temperature variation).
""".strip()
    )

def test_ner_serde(noop_config):
    config = Config().from_str(noop_config)
    nlp1 = assemble_from_config(config)
    nlp2 = assemble_from_config(config)
    nlp2.from_bytes(nlp1.to_bytes())

def test_ner_to_disk(noop_config, tmp_path: Path):
    config = Config().from_str(noop_config)
    nlp1 = assemble_from_config(config)
    nlp2 = assemble_from_config(config)

    path = tmp_path / "model"
    nlp1.to_disk(path)

    cfgs = list(path.rglob("cfg"))
    assert len(cfgs) == 1

    nlp2.from_disk(path)


import logging

logging.basicConfig(filename='demo.log',level=logging.DEBUG)

# x=10
# y=20
# z=x+y
# logging.info(z)

try:
    n1=int(input("enter the number:"))
    n2=int(input("enter the number:"))
    result=n1/n2
    logging.info(result)
except:
    logging.error("an unknown error happened")

# logging.critical("critical error happened")

# logging.error("an unknown error happened")

# logging.warning("expected value is an integer")

# logging.info("normal message")

# logging.debug("for developers")


import functools

@functools.lru_cache(maxsize=None)
def binomial(n, k):
    if k == n or k ==0:                         #n == k 일때 헷갈림..
        return 1
    return binomial(n-1, k-1) + binomial(n-1, k)

l=[[0 for value in range(2)] for value in range(1000)]

for i in range(1000):
    l[i]=list(map(int, input().split()))
    if l[i][0]==0 and l[i][1]==0: break

print(l)

for i in range(1000):
    if l[i][0]==0 and l[i][1]==0: break
    print(binomial(l[i][0], l[i][1]))

이항계수/이항 계수3.py


#!/usr/bin/python3
from fabric import SerialGroup
from fabric.exceptions import GroupException
from fabric.runners import Result
from termcolor import cprint
import sys

def format_proc(line):
    if len(line) < 4 or '-' * 5 in line or '=' * 5 in line or "/usr/" in line:
        return
    else:
        line_args = line.split()
        if "@QD" in line_args[4]:
            name, rtime = line_args[4].split('#')
            rdays = rtime.split("d")[0]
            if rdays == "first_epoch":
                color = "white"
            elif int(rdays) > 6:
                color = 'red'
            elif int(rdays) > 4:
                color = 'yellow'
            elif int(rdays) >= 1:
                color = 'blue'
            else: # = 0
                color = 'green'
            return [line_args[1], name[4:] + ":" + rtime, color, ['bold'], "QD"]
        else:
            return [line_args[1], line_args[4], None, ['dark'], "other"]

def retrieve_smis(server_list, out_filename=".smi_saves"):
    out_file = open(out_filename, "w", encoding="utf-8")
    server_group = SerialGroup(*server_list)
    try:
        return server_group.run('echo "new connection:" && hostname && \
                                 nvidia-smi', out_stream=out_file, hide=True)
    except GroupException as e:
        problem_list = []
        for key, val in e.result.items():
            if type(val) is not Result or val.failed:
                problem_list.append(key.original_host)
        return 'Err', problem_list

def format_smis(filename=".smi_saves"):
    smi_content = open(filename, "r", encoding="utf-8").read()
    all_processes_dict = {}
    smis = smi_content.split("new connection:")[1:]
    for smi in smis:
        smi = smi[1:]
        hostname = smi.partition('\n')[0]
        if "-------------------------------------------------" not in smi:
            all_processes_dict[hostname] = {}
            all_processes_dict[hostname]["ERR"] = ["Not able to retrieve processes"]
            continue
        processes = smi.split("="*77)[1].split("\n")[1:]
        gpu_set = False
        if "mlstudent" in hostname:
            gpu_num = hostname[9]
            hostname = "mlstudentpool"
            gpu_set = True
        if hostname not in all_processes_dict:
            all_processes_dict[hostname] = {}
        for process in processes:
            results = format_proc(process)
            if results is None:
                continue
            if not gpu_set:
                gpu_num = results.pop(0)
            else:
                results.pop(0)
            if gpu_num not in all_processes_dict[hostname]:
                all_processes_dict[hostname][gpu_num] = []
            all_processes_dict[hostname][gpu_num].append(results)
    return all_processes_dict

if __name__ == '__main__':
    server_list = ["mlstudentpool", "mlstudentpool2", "mlstudentpool3",
                   "mlstudentpool4",  "dl1", "stevens", "chichis",
                   "dgx-a", "dgx-b"]
    # server_list = ["dl1", "stevens"]
    # list = ["dl1", "chichis", "stevens", "dgx-a", "dgx-b"]
    retrieve_smis(server_list)
    if "-term" in sys.argv:
        formated_dict = format_smis()
        for hostname, gpus in formated_dict.items():
            print(hostname)
            for gpu, process_list in gpus.items():
                print(gpu, end=": ")
                for process in process_list:
                    process_name = process[0]
                    process_name += " " * (42 - len(process_name))
                    if len(process) == 4:
                        cprint(process_name, process[1], attrs=process[2], end="")
                    else:
                        print(process, end="")

                print()


import pickle
from datetime import datetime, timedelta
from NotificationCenter import debug, info, warning, error, critical
import ast

# 監視している通貨リスト
def get_monitoring_currency_cache():
    try:
        f = open('save/monitoring_currency_cache.bin', 'rb')
        r = pickle.load(f)
        return r
    except:
        data = {}
        set_monitoring_currency_cache(data)
        return data

def set_monitoring_currency_cache(data):
    f = open('save/monitoring_currency_cache.bin', 'wb')
    pickle.dump(data, f)
    f.close()

# 今取引しているコインについての情報
# 取引してないときはNULL？
# userは1or2
def get_position_cache(user):
    user = str(user)
    if user != '1' and user != '2':
        return
    try:
        path = 'save/position_cache' + str(user) + '.bin'
        with open(path, 'rb') as web:
            r = pickle.load(web)
            return r
    except:
        data = {
            'user': 0,
            'status': False,
            'pair': None,
            'amount': 0,
            'buy_time': None,
            'sell_time': None,
            'buy_coin': 0,
            'sell_coin': 0,
            'profit': 0,
            'mode': 0,
        }
        set_position_cache(user, data)
        warning("[CacheManager]例外：ファイルがないので初期ファイルを作成します")
        return data

def set_position_cache(user, data):
    user = str(user)
    if user != '1' and user != '2':
        return
    path = 'save/position_cache' + str(user) + '.bin'
    with open(path, 'wb') as web:
        pickle.dump(data, web)

def get_binance_api():
    f = open('binance_api.txt', 'r', encoding='UTF-8')
    api_txt = f.read()
    api_key = ast.literal_eval(api_txt)
    return api_key

class CacheManagerClass:
    def __init__(self):
        pass

if __name__ == "__main__":
    dict = {
        'status': False,
        'dt_now': None,
        'price': 0,
        'usecoin': None,
        'amount': 0,
        'wasOverbuy': False,
        'wasOversold': False,
        'crossoverbuy': False,
        'crossoversold': False,
        'buy_coin': 0,
        'sell_coin': 0,
        'mode': 0,
    }
    data = {'XEMUSDT': datetime.now() + timedelta(hours=1),
            'BATUSDT': datetime.now() + timedelta(hours=1)
            }
    # set_position_cache(1, dict)
    # set_monitoring_currency_cache(data)
    print(type(get_binance_api()))