import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import time
from scipy.io import loadmat

def load_data():
    data = loadmat('E:\\download_backup\\train_32x32.mat')
    datax, datay = data['X'].transpose((3, 0, 1, 2)), data['y'][:,0]  # shape of (73257, 32, 32, 3) and (73257,)
    pos_class=1
    neg_class=2
    dataIndx = (datay == pos_class) + (datay == neg_class)
    Data = datax[dataIndx] / 255.
    Target = datay[dataIndx].reshape(-1, 1)

    Target[Target == pos_class] = 1
    Target[Target == neg_class] = 0
    np.random.seed(421)
    randIndx = np.arange(len(Data))

    np.random.shuffle(randIndx)
    Data, Target = Data[randIndx], Target[randIndx]
    print(Data.shape)
    training_data, training_label = Data[:6000] , Target[:6000]
    valid_data, valid_label = Data[6000:7000], Target[6000:7000]
    test_data, test_label = Data[7000:], Target[7000:]

    return training_data, training_label,valid_data, valid_label,test_data, test_label

def rgb2gray(images):
    return np.expand_dims(np.dot(images, [0.2990, 0.5870, 0.1140]), axis=3)

def svm_learning(trainx, trainy,batch_size = 50):
    sess = tf.Session()
    x_data = tf.placeholder(shape=[None, 1024], dtype=tf.float32)
    y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)
    prediction_grid = tf.placeholder(shape=[None, 1024], dtype=tf.float32)

    b = tf.Variable(tf.random_normal(shape=[1, batch_size]))

    kernel = tf.matmul(x_data, tf.transpose(x_data))

    first_sum = tf.reduce_sum(b)
    b_mul= tf.matmul(tf.transpose(b), b)
    y_mul = tf.matmul(y_target, tf.transpose(y_target))
    second_sum = tf.reduce_sum(tf.multiply(kernel, tf.multiply(b_mul, y_mul)))
    loss = tf.negative(tf.subtract(first_sum, second_sum))

    kernel_pre = tf.matmul(x_data, tf.transpose(prediction_grid))

    pre_output = tf.matmul(tf.multiply(tf.transpose(y_target), b), kernel_pre)
    mean=tf.reduce_mean(pre_output)
    pred = tf.sign(pre_output - tf.reduce_mean(pre_output))
    accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.squeeze(tf.transpose(pred)), tf.squeeze(y_target)), tf.float32))

    #optimizer=tf.train.GradientDescentOptimizer(0.001).minimize(loss)
    optimizer = tf.train.AdamOptimizer(0.001).minimize(loss)

    init = tf.global_variables_initializer()
    sess.run(init)

    # Training loop
    loss_vec = []
    batch_accuracy = []

    for i in range(3000):
        rand_index = np.random.choice(len(d1), size=batch_size)
        rand_x = trainx[rand_index]
        rand_y = trainy[rand_index]
        sess.run(optimizer, feed_dict={x_data: rand_x, y_target: rand_y})
        pre=sess.run(mean, feed_dict={x_data: rand_x, y_target: rand_y,prediction_grid: rand_x})
        print(pre)

        if (i) % 100 == 0:
            pre = sess.run(pred, feed_dict={x_data: rand_x, y_target: rand_y, prediction_grid: rand_x})
            print(pre)
            temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y})
            loss_vec.append(temp_loss)

            acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x,
                                                     y_target: rand_y,
                                                     prediction_grid: rand_x})
            batch_accuracy.append(acc_temp)
            print('run:' + str(i + 1))
            print('Loss: ' + str(temp_loss))
            print('accracy: ' + str(acc_temp))
    plt.figure(1)
    plt.plot(loss_vec, color='RED', label="Train Data")
    plt.figure(2)
    plt.plot(batch_accuracy, color='RED', label="acc")
    plt.show()

d1,l1,d2,l2,d3,l3=load_data()
grey_test=np.reshape(rgb2gray(d3),[-1,1024])
grey_valid=np.reshape(rgb2gray(d2),[-1,1024])
grey_train=np.reshape(rgb2gray(d1),[-1,1024])
label_2=np.ones([6000,1])
label_2[l1==0]=-1

train_labels=np.concatenate((l1,label_2),axis=1)
print(label_2[:10])
valid_label_2=np.ones([1000,1])
valid_label_2[l2==1]=0
valid_labels=np.concatenate((l2,valid_label_2),axis=1)

test_label_2=np.ones([int(len(l3)),1])
test_label_2[l3==1]=0
test_labels=np.concatenate((l3,test_label_2),axis=1)
svm_learning(grey_train,label_2)


import sys
from collections import deque

N, M = map(int, sys.stdin.readline().split())
r, c, d = map(int, sys.stdin.readline().split())
# d => 0북 1동 2남 3서
dy = [-1, 0, 1, 0]
dx = [0, 1, 0, -1]
matrix = [list(map(int, sys.stdin.readline().split())) for _ in range(N)]

# 방향 전환 함수
def change(d):
    if d == 0:
        return 3
    elif d == 1:
        return 0
    elif d == 2:
        return 1
    elif d == 3:
        return 2

# 후진
def back(d):
    if d == 0:
        return 2
    elif d == 1:
        return 3
    elif d == 2:
        return 0
    elif d == 3:
        return 1

def bfs(r, c, d):
    res = 0
    q = deque()
    q.append((r, c, d))
    while q:
        r, c, d = q.popleft()
        # 1
        if matrix[r][c] == 0:
            res += 1
            matrix[r][c] = 2  # 청소 완료
        _d = d
        for i in range(4):
            _d = change(_d)
            _r, _c = r + dy[_d], c + dx[_d]
            if 0 <= _r < N and 0 <= _c < M and matrix[_r][_c] == 0:
                q.append((_r, _c, _d))
                break
            elif i == 3:
                _r, _c = r + dy[back(d)], c + dx[back(d)]
                q.append((_r, _c, d))
                if matrix[_r][_c] == 1:
                    return res

print(bfs(r, c, d))


import pyrealsense2 as rs

class RealsenseCam(object):

    def __init__(self):
        self.running = True
        self.pipe = rs.pipeline()
        cfg = rs.config()
        cfg.enable_stream(rs.stream.pose)
        self.pipe.start(cfg)
        self.frames = None

    def update(self):
        while self.running:
            self.frames = self.pipe.wait_for_frames()

    def run_threaded(self):
        if self.frames is not None:
            pose = self.frames.get_pose_frame()
            data = pose.get_pose_data()
            return data.translation

    def shutdown(self):
        self.running = False
        self.pipe.stop()


import sqlite3
from typing import Callable, List, Union

from common import _L

MigrationFunc = Callable[[sqlite3.Connection, bool], bool]
migrationFunctions: dict[int, MigrationFunc] = {}

def register(version: int):
    from functools import wraps
    def wrap(func: MigrationFunc):
        @wraps(func)
        def wrapper(conn: sqlite3.Connection, silent: bool):
            try:
                if func(conn, silent):
                    conn.cursor().execute(f"PRAGMA user_version = {version}")
                    conn.commit()
                    return True
                return False
            except sqlite3.OperationalError:
                _L.error(f"There was an error while migrating database to version {version}."
                        "Please contact the developer @ github.com/fabbi")
                return False
        migrationFunctions[version] = wrapper
        return wrapper
    return wrap

@register(1)
def update_1(conn: sqlite3.Connection, silent=False):
    c = conn.cursor()
    if not silent:
        _L.warn("Some games and platforms have been renamed! "
                "Check `--help` or the README.md for further info.")

    steps: List[Union[str, tuple]] = [
        """
        ALTER TABLE keys
        RENAME COLUMN description TO reward;
        """
        ,
        """
        ALTER TABLE keys
        RENAME COLUMN key TO code;
        """
        , # renamed "ps" to "psn"
        """
        UPDATE keys
        SET platform="psn"
        WHERE platform="ps";
        """
        , # renamed "xbox" to "xboxlive"
        """
        UPDATE keys
        SET platform="xboxlive"
        WHERE platform="xbox";
        """
        , # renamed "bl" to "bl1"
        """
        UPDATE keys
        SET game="bl1"
        WHERE game="bl";
        """
        , # "pc" is now "epic" and "steam"
        """
        CREATE VIEW tmp AS
        SELECT * FROM keys
        WHERE platform="pc";
        -- AND redeemed=0;

        INSERT INTO keys(reward, code, platform, game, redeemed)
        SELECT reward, code, "epic", game, redeemed
        FROM tmp;

        UPDATE keys
        SET platform="steam"
        WHERE id in (select id from tmp);

        DROP VIEW tmp;
        """
        ,
        """
        CREATE TABLE IF NOT EXISTS seen_platforms
        (key TEXT primary key, name TEXT)
        """
        ,
        """
        CREATE TABLE IF NOT EXISTS seen_games
        (key TEXT primary key, name TEXT)
        """
    ]

    from query import known_games, known_platforms
    for game in known_games.items():
        steps.append(("""
        INSERT INTO seen_games(key, name)
        VALUES(?, ?)
        """, game))
    for platform in known_platforms.items():
        steps.append(("""
        INSERT INTO seen_platforms(key, name)
        VALUES(?, ?)
        """, platform))

    for i, step in enumerate(steps):
        _L.debug(f"executing step {i} of update_1")
        try:
            if isinstance(step, tuple):
                c.execute(*step)
            else:
                c.executescript(step)
        except sqlite3.OperationalError as e:
            _L.error(f"'{e}' in\n{step}")
            return False
    return True


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Train a model using my PriceEstimatorTrainer class
    then evaluates on valid or test. Return results by route.

    Args:
        - model : {'rf', 'log_reg', 'knn', 'svm', 'xgb'}, default='rf'
                Model to train the data with (It must be
                in the model selector dictionary)

        - n_iter : int, default=0
                Number of iterations for random search params

        - -m, comment : str
                Explanatory notes about current experiment, doesn't affect
                training process

        - testset : {'valid', 'test'}, default='valid'
                Whether to use the validation set or the test set
                for evaluate the trained model.

        - -o, output : str
                Name to store the model. If no passed, model will not be
                stored.

    @author: Adrián Cervero - May 2021
    @github: https://github.com/adriancervero/flight-prices-prediction
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import argparse
import pickle
import logging
import os, sys

# My imports
import config as cfg
import model_selector
from PriceEstimatorTrainer import PriceEstimatorTrainer

def generate_model(param_grid):
    """ Returns given model with random params from param_grid parameter """
    n_estimators = np.random.choice(param_grid['n_estimators'])
    max_depth = np.random.choice(param_grid['max_depth'])
    criterion = np.random.choice(param_grid['criterion'])
    max_features = np.random.choice(param_grid['max_features'])
    min_samples_split = np.random.choice(param_grid['min_samples_split'])
    min_samples_leaf = np.random.choice(param_grid['min_samples_leaf'])

    model = RandomForestClassifier(n_estimators=n_estimators,
                                    max_depth=max_depth,
                                    criterion=criterion,
                                    max_features=max_features,
                                    min_samples_split=min_samples_split,
                                    random_state=cfg.RANDOM_STATE)
    return model

def run(model, n_iter, comment, testset, output):
    """ Performs training process """

    print("\n----- 04 - Training Model -----\n\n")

    # load training data
    os.chdir(sys.path[0])
    df = pd.read_csv(cfg.TRAIN_PATH)

    # Init model and get params for hypertuning
    m = model_selector.models[model]
    if model == 'rf':
        param_grid = model_selector.param_grid[model]

    #threshold_values = [0.4, 0.42, 0.44,0.46, 0.48, 0.5, 0.52, 0.54, 0.56, 0.58, 0.6]
    #for threshold in threshold_values:

    if n_iter == 0: # no hypertunning
        trainer = PriceEstimatorTrainer(model=m, pred_threshold=.5, n=10000)

        # fit model
        trainer.fit()

        # print results on validation
        acc, mean_savings = trainer.evaluate(on=testset, plot=True)

        logging.info(f"Precision: {acc}, Mean Savings %: {mean_savings}, Notes:{comment}, Model:{m}")
        logging.info("---------------------------------------------------------------------------------------")

        if output != '':
            output_path = cfg.MODEL_OUTPUT + output
            pickle.dump(trainer.model, open(output_path, 'wb'))
            print('\n\nmodel stored: ', output_path)

    else: # hypertunning

        for _ in range(0, n_iter):
            m = generate_model(param_grid)
            trainer = PriceEstimatorTrainer(model=m)

            # fit model
            trainer.fit()

            # print results on validation
            acc, mean_savings = trainer.evaluate(on=testset)

            logging.info(f"Precision: {acc}, Mean Savings %: {mean_savings}, Notes:{comment}, Model:{m}")
            logging.info("---------------------------------------------------------------------------------------")

if __name__ == '__main__':
    # ArgumentParser
    parser = argparse.ArgumentParser()
    parser.add_argument('--model', type=str, default='rf') # model to use
    parser.add_argument('-t', type=int, default=0) # random search hypertuning iterations
    parser.add_argument('-m', type=str, default='') # notes
    parser.add_argument('--on', type=str, default='valid') # evaluate on validation or test
    parser.add_argument('-o', type=str, default='') # model save name

    # read arguments from command line
    args = parser.parse_args()
    # logger
    logging.basicConfig(level=logging.DEBUG, filename="training_logs", filemode="a+",
                        format="%(asctime)-15s %(levelname)-8s %(message)s")

    run(
        model=args.model,
        n_iter = args.t,
        comment=args.m,
        testset=args.on,
        output=args.o,
    )


"""
Scripts makes various plots for the report using log data saved to file.
"""
import matplotlib.pyplot as plt
from sklearn.externals import joblib
import numpy as np
import os
from utils.ImageFolderWithPaths import ImageFolderWithPaths
from torchvision import transforms
import models
import torch
from pathlib import Path
import torch.nn.functional as F

def smooth_data(data, window_width):
    cumsum_vec = np.cumsum(np.insert(data, 0, 0))
    ma_vec = (cumsum_vec[window_width:] - cumsum_vec[:-window_width]) / window_width
    return ma_vec

def plot_learning_curves(file_path):
    print('Plot learning curves -')
    print('Loading logs saved at: ', file_path)
    (train_logs, valid_logs) = joblib.load(file_path)

    train_logs = np.array(train_logs)
    valid_logs = np.array(valid_logs)
    print('train logs length ', len(train_logs))
    print('valid logs length ', len(valid_logs))

    plt.figure()

    # smooth accuracy
    window_width = 50
    train_steps = train_logs[:, 0]
    valid_steps = valid_logs[:, 0]
    train_accs_smooth = smooth_data(train_logs[:, 1], window_width)
    valid_accs_smooth = smooth_data(valid_logs[:,1], window_width)
    print('length of smooth train acc ', len(train_accs_smooth))
    print('length of smooth valid acc ', len(valid_accs_smooth))

    # plot accuracy
    plt.subplot(1,2,1)
    plt.plot(train_steps[:len(train_accs_smooth)], train_accs_smooth, label='training')
    plt.plot(valid_steps[:len(valid_accs_smooth)], valid_accs_smooth, label='validation')
    # plt.plot(train_logs[:, 0], train_logs[:, 1], label='training')
    # plt.plot(valid_logs[:,0], valid_logs[:,1], label='validation')
    plt.ylabel('accuracy(%)', fontsize=13)
    plt.xlabel('steps', fontsize=13)
    plt.title('(a)', fontsize=13)
    plt.grid()
    plt.legend(fontsize=13)

    # plot loss
    train_loss_smooth = smooth_data(train_logs[:, 2], window_width)
    valid_loss_smooth = smooth_data(valid_logs[:, 2], window_width)
    plt.subplot(1, 2, 2)
    plt.plot(train_steps[:len(train_loss_smooth)], train_loss_smooth, label='training')
    plt.plot(valid_steps[:len(valid_loss_smooth)], valid_loss_smooth, label='validation')
    plt.ylabel('loss', fontsize=13)
    plt.xlabel('steps', fontsize=13)
    plt.title('(b)', fontsize=13)
    plt.grid()
    plt.legend(fontsize=13)

    plt.show()

def print_misclassified_uncertain_examples(data_dir, model_name, model_path ):
    """
    This function runs tests a model on the validation data and outputs a list of images that:
    1. were misclassified
    2. had predicted probabilities around 50% for both classes.
    """

    # load validation data
    valdir = os.path.join(data_dir, 'valset')
    normalize = transforms.Normalize(mean=[0.490, 0.455, 0.416],
                                     std=[0.252, 0.245, 0.247])
    val_loader = torch.utils.data.DataLoader(
        ImageFolderWithPaths(valdir, transforms.Compose([
            transforms.ToTensor(),
            normalize,
        ])),
        batch_size=1000, shuffle=False)

    # load saved model
    device = torch.device("cuda:0")
    print('Loading saved model {} at {}'.format(model_name, model_path))
    model_class = models.find_model(model_name)
    model = model_class().to(device)
    model.load(model_path)

    # test model.
    model.eval()
    test_loss = 0
    test_correct = 0
    predictions = []
    targets = []
    img_paths = []  # get names of each image without full path or extension. (we need names to order result)
    outputs = []

    with torch.no_grad():
        for batch_idx, (data, target, paths) in enumerate(val_loader):
            data, target = data.to(device), target.to(device)
            output = model(data)
            outputs.append(output)
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            predictions.append(pred)
            img_paths += [Path(path).name for path in paths]
            targets.append(target)
            test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss
            test_correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(val_loader.dataset)
    test_acc = 100. * test_correct / len(val_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, test_correct, len(val_loader.dataset), test_acc))

    predictions = torch.cat(predictions, 0).cpu().numpy().flatten()
    targets = torch.cat(targets, 0).cpu().numpy().flatten()
    img_paths = np.array(img_paths)
    misclassified_idx = targets != predictions
    print('Paths of {} misclassified examples:\n{}'.format(sum(misclassified_idx), img_paths[misclassified_idx]))

    outputs = torch.cat(outputs, 0).cpu().numpy()   # concatenate outputs from all batches
    exp_outputs = np.exp(outputs)                       # return all log estimates to probabilities
    exp_outputs = np.round(exp_outputs, 1)
    uncertain_idx = exp_outputs[:,0] == 0.5
    print('Paths of {} uncertain examples:\n{}'.format(sum(uncertain_idx), img_paths[uncertain_idx]))

def main():
    data_dir = r'C:\Users\Charles\Dropbox (NRP)\black-faces-db\ift6135data'
    output_dir = r'C:\Users\Charles\Dropbox (NRP)\black-faces-db\ift6135output'

    plot_learning_curves(output_dir + '\logs.pkl')
    print_misclassified_uncertain_examples(data_dir, 'BaseConvNet2', output_dir + '\saved_model.pt')

if __name__=='__main__':
    main()


import json
import os
import re
import sys
from urllib.parse import urlsplit

import requests
from django.conf import settings
from django.http import HttpResponse
from django.shortcuts import redirect, render
from opencivicdata.core.models import Division, Membership

from reports.models import Report
from reports.utils import get_offices, get_personal_url, module_name_to_metadata, remove_suffix_re, scrape_configuration, scrape_people

def home(request):
    sys.path.append(os.path.abspath('scrapers'))

    data = json.loads(requests.get('https://represent.opennorth.ca/representative-sets/?limit=0', verify=settings.SSL_VERIFY).text)

    names = {}
    for obj in data['objects']:
        # The `ca` scraper has "Parliament of Canada" as the root organization.
        if obj['name'] == 'House of Commons':
            names['Parliament of Canada'] = obj['data_url']
        else:
            names[obj['name']] = obj['data_url']

    reports = Report.objects.order_by('module').all()
    for report in reports:
        try:
            name = module_name_to_metadata(report.module)['name']
            if not report.exception:
                if name in names:
                    if names[name].startswith('https://scrapers.herokuapp.com/represent/'):
                        report.icon = 'noop'
                    else:
                        report.icon = 'replace'
                else:
                    report.icon = 'add'
        except ImportError:
            report.delete()  # delete reports for old modules

    return render(request, 'index.html', {
        'exceptions': Report.objects.exclude(exception='').count(),
        'reports': reports,
    })

def warnings(request):
    return render(request, 'warnings.html', {
        'reports': Report.objects.order_by('module').all(),
    })

def report(request, module_name):
    return HttpResponse(json.dumps(Report.objects.get(module=module_name).report), content_type='application/json')

def run(request, module_name):
    sys.path.append(os.path.abspath('scrapers'))
    parser, subcommand, handler = scrape_configuration()
    scrape_people(module_name, parser, subcommand, handler)
    return redirect('home')

def represent(request, module_name):
    sys.path.append(os.path.abspath('scrapers'))

    metadata = module_name_to_metadata(module_name)

    representatives = []

    # Exclude party memberships.
    queryset = Membership.objects.filter(organization__jurisdiction_id=metadata['jurisdiction_id'])

    if module_name.endswith('_candidates'):
        # Include only candidates.
        queryset.filter(role='candidate')
    else:
        # Exclude candidates and party memberships.
        queryset.exclude(role__in=('member', 'candidate'))

    for membership in queryset.prefetch_related('contact_details', 'person', 'person__links', 'person__sources', 'post'):
        person = membership.person

        # Not sure why this is necessary.
        if not isinstance(membership.extras, dict):
            membership.extras = json.loads(membership.extras)
        if not isinstance(person.extras, dict):
            person.extras = json.loads(person.extras)

        try:
            party_name = Membership.objects.select_related('organization').get(organization__classification='party', role='member', person=person).organization.name
        except Membership.DoesNotExist:
            party_name = None

        if person.gender == 'male':
            gender = 'M'
        elif person.gender == 'female':
            gender = 'F'
        else:
            gender = None

        # Candidates only.
        incumbent = person.extras.pop('incumbent', None)

        # @see https://represent.opennorth.ca/api/#fields
        representative = {
            'name': person.name,
            'elected_office': membership.role,
            'party_name': party_name,
            'email': next((contact_detail.value for contact_detail in membership.contact_details.all() if contact_detail.type == 'email'), None),
            'photo_url': person.image or None,
            'personal_url': get_personal_url(person),
            'gender': gender,
            'offices': json.dumps(get_offices(membership)),
            'extra': json.dumps(get_extra(person)),
        }

        sources = list(person.sources.all())

        # The first URL ought to be the most generic source.
        representative['source_url'] = sources[0].url

        if len(sources) > 1:
            # The last URL ought to be the most specific source.
            representative['url'] = sources[-1].url

        if incumbent:
            representative['incumbent'] = True

        match = re.search(r'^(\S+) (Ward \d+)$', membership.post.label)

        # If the person is part of Peel Regional Council.
        if match:
            parent = Division.objects.get(subtype1='csd', subtype2='', name=match.group(1))
            division = Division.objects.get(subtype1='csd', subid1=parent.subid1, name=match.group(2))
            boundary_set_slug = '{}-wards'.format(parent.name.lower())
            representative['district_name'] = membership.post.label
            representative['boundary_url'] = '/boundaries/{}/ward-{}/'.format(boundary_set_slug, division.subid2)
            representatives.append(representative)

        # If the person is associated to multiple boundaries.
        elif re.search(r'^Wards\b', membership.post.label):
            for district_id in re.findall(r'\d+', membership.post.label):
                representative = representative.copy()
                representative['district_id'] = district_id
                representative['district_name'] = 'Ward {}'.format(district_id)
                representatives.append(representative)

        else:
            division_id = metadata['division_id']

            if re.search('^ocd-division/country:ca/csd:(\d{7})\Z', division_id):
                geographic_code = division_id[-7:]
            elif re.search('^ocd-division/country:ca/cd:(\d{4})\Z', division_id):
                geographic_code = division_id[-4:]
            else:
                geographic_code = None

            post_label = remove_suffix_re.sub('', membership.post.label)

            # If the post label is numeric.
            if re.search(r'^\d+\Z', post_label):
                representative['district_id'] = post_label

            # If the person has a boundary URL.
            elif 'boundary_url' in membership.extras:
                representative['district_name'] = post_label
                representative['boundary_url'] = membership.extras['boundary_url']

            # If the post label is a Census geographic name.
            elif post_label == metadata['division_name'] and geographic_code:
                representative['district_name'] = post_label
                if len(geographic_code) == 7:
                    representative['boundary_url'] = '/boundaries/census-subdivisions/{}/'.format(geographic_code)
                elif len(geographic_code) == 4:
                    representative['boundary_url'] = '/boundaries/census-divisions/{}/'.format(geographic_code)

            else:
                representative['district_name'] = post_label
                district_id = re.search(r'^(?:District|Division|Ward) (\d+)\Z', post_label)
                if district_id:
                    representative['district_id'] = district_id.group(1)

            representatives.append(representative)

    return HttpResponse(json.dumps(representatives), content_type='application/json')

def get_extra(record):
    extra = record.extras
    for link in record.links.all():
        domain = '.'.join(urlsplit(link.url).netloc.split('.')[-2:])
        if domain in ('facebook.com', 'fb.com'):
            extra['facebook'] = link.url
        elif domain == 'instagram.com':
            extra['instagram'] = link.url
        elif domain == 'linkedin.com':
            extra['linkedin'] = link.url
        elif domain == 'twitter.com':
            extra['twitter'] = link.url
        elif domain == 'youtube.com':
            extra['youtube'] = link.url
    return extra


from datetime import timedelta

import requests
from django.conf import settings
from django.core.exceptions import ValidationError
from django.http import JsonResponse

class VantageRequestError(Exception):
    pass

session = requests.Session()

def vantage_request(params: dict):
    base_url = 'https://www.alphavantage.co/query'
    r = session.request('GET', base_url, params={'apikey': settings.VANTAGE_API_KEY, **params})
    if r.status_code != 200:
        raise VantageRequestError('Problem accessing URL', r.content.decode())
    data = r.json()
    if data.get('Note'):
        raise VantageRequestError("We've used up all our API calls. Try again in a few minutes.")
    errors = data.get('Error Message')
    if errors:
        raise VantageRequestError('Error accessing Vantage (%s): %r' % (r.url, errors))
    return data

REGION_ORDER = {'United Kingdom': 1, 'United States': 2}

def search_company_symbols(request):
    params = {'function': 'SYMBOL_SEARCH', 'keywords': request.GET.get('q')}
    r = vantage_request(params)
    data = [
        {'symbol': d['1. symbol'], 'name': d['2. name'], 'region': d['4. region'], 'currency': d['8. currency']}
        for d in r['bestMatches']
    ]
    data = sorted(data, key=lambda x: REGION_ORDER.get(x['region'], 10))
    return JsonResponse(data, safe=False)

def get_stock_data(date, symbol):
    params = {'function': 'TIME_SERIES_DAILY', 'symbol': symbol, 'outputsize': 'compact'}
    data = vantage_request(params)
    stocks_data = data['Time Series (Daily)']
    days = 0
    stock_data = None
    while not stock_data:
        # The market is closed on bank hols and public holidays
        date = (date - timedelta(days=days)).strftime('%Y-%m-%d')
        stock_data = stocks_data.get(date)
        if stock_data:
            return stock_data
        if days == 7:
            dates = list(stocks_data.keys())
            raise ValidationError(
                f"As we're only testing, you can only choose from the first 100 records."
                f"We have figures between {dates[-1]} - {dates[0]}"
            )
        days += 1
    return stock_data


import logging
import os
import shutil
import sys
import threading
import time

from daemoniker import Daemonizer

from gfl.api.listener import HttpListener
from gfl.conf import GflConf
from gfl.core.lfs import Lfs
from gfl.core.manager.node import GflNode
from gfl.core.manager.holder import ManagerHolder
from gfl.core.manager.manager import NodeManager
from gfl.shell import Shell
from gfl.utils import PathUtils

class Application(object):

    logger = None

    def __init__(self):
        super(Application, self).__init__()

    @classmethod
    def init(cls, force):
        if os.path.exists(GflConf.home_dir):
            if force:
                logging.shutdown()
                shutil.rmtree(GflConf.home_dir)
            else:
                raise ValueError("homedir not empty.")
        # create home dir
        os.makedirs(GflConf.home_dir)

        # generate config file
        GflConf.generate_config(PathUtils.join(GflConf.home_dir, "config.yaml"))
        # generate node address and key
        GflNode.init_node()
        # create data directories
        Lfs.init()

    @classmethod
    def run(cls, role, console, **kwargs):
        sys.stderr = open(os.devnull, "w")
        cls.logger = logging.getLogger("gfl")
        with Daemonizer() as (is_setup, daemonizer):
            main_pid = None
            if is_setup:
                main_pid = os.getpid()
            pid_file = PathUtils.join(GflConf.home_dir, "proc.lock")
            stdout_file = PathUtils.join(GflConf.logs_dir, "console_out")
            stderr_file = PathUtils.join(GflConf.logs_dir, "console_err")
            is_parent = daemonizer(pid_file, stdout_goto=stdout_file, stderr_goto=stderr_file)
            if is_parent:
                if console and main_pid == os.getpid():
                    Shell.startup()

        GflNode.load_node()

        if GflConf.get_property("net.mode") == "standalone":
            client_number = GflConf.get_property("net.standalone.client_number")
            for _ in range(len(GflNode.standalone_nodes), client_number):
                GflNode.add_standalone_node()

            ManagerHolder.default_manager = NodeManager(node=GflNode.default_node, role="server")

            for i in range(client_number):
                client_manager = NodeManager(node=GflNode.standalone_nodes[i], role="client")
                ManagerHolder.standalone_managers.append(client_manager)
        else:
            ManagerHolder.default_manager = NodeManager(node=GflNode.default_node, role=role)

        # cls.__startup_node_managers()
        HttpListener.start()

        while HttpListener.is_alive():
            time.sleep(2)

    @classmethod
    def __startup_node_managers(cls):
        # 遍历所有的NodeManager
        threading.Thread(target=ManagerHolder.default_manager.run).start()
        for nm in ManagerHolder.standalone_managers:
            t = threading.Thread(target=nm.run)
            t.start()

    @classmethod
    def __stop_node_managers(cls):
        if ManagerHolder.default_manager is not None:
            ManagerHolder.default_manager.stop()
        for nm in ManagerHolder.standalone_managers:
            nm.stop()


import pandas as pd
import mbuild as mb
import foyer
import unyt as u
import numpy as np
import math
import time
import parmed as pmd
import mosdef_cassandra as mc

T = 298.15 * u.K

mu_ideal_100box = -48.860310174012 * u.Unit('kJ/mol')

beta = 1/(u.kb*T)

N_goal = 40

cbrt_N_goal = N_goal ** (1.0/3.0)

mu_gas_5 = np.array([-49.04, -39.4, -36.4, -35.2, -35.0]) * u.Unit('kJ/mol')

P_gas = np.linspace(0.5, 15.0, 30, endpoint=True) * u.bar

P_n60 = 1.9121951616833333 * u.bar

mu_n60 = mu_gas_5[1]

mu_gas = mu_n60 + u.kb*T*np.log(P_gas/P_n60)

L = np.exp((-beta/3*(mu_gas - mu_ideal_100box)).to(''))*(100 * u.angstrom)*cbrt_N_goal

r32_struct = pmd.load_file('topology/r32.top', xyz='topology/r32.gro')
species_list = [r32_struct]

ensemble = 'gcmc'
moveset = mc.MoveSet(ensemble, species_list)
moveset.max_translate = [[14.0*u.angstrom]]
moveset.max_rotate = [[180.0*u.degree]]

eq_length = 1000000
prod_length = eq_length*10

property_list = ["nmols", "density", "pressure", "energy_total"]

def read_prp(filename):
    with open(filename) as fprp:
        fprp.readline()
        colnames = fprp.readline()[1:].strip().split()
        colunits = fprp.readline()[1:].strip().split()
        df = pd.read_csv(fprp, delim_whitespace=True, index_col=False, names=colnames)
    return df

statdf = pd.DataFrame(
        columns=['mu', 'Nmols', 'Nmols_SEM', 'Density', 'Density_SEM', 'Pressure', 'Pressure_SEM', 'Energy_Total', 'Energy_Total_SEM', 'Sim_Time'],
        index=pd.RangeIndex(len(mu_gas)))

for i, mu in enumerate(mu_gas):
    eq_name = "eq"+str(i)
    prod_name = "prod"+str(i)
    boxlength = L[i]
    cutoff = boxlength/2 - 0.1*u.angstrom
    nm_L = boxlength.to_value('nm')
    box = mb.Box([nm_L, nm_L, nm_L])
    box_list = [box]
    system = mc.System(box_list, species_list, mols_to_add=[[N_goal]])
    starttime = time.time()
    mc.run(
            system=system,
            moveset=moveset,
            run_type="equilibration",
            run_length=eq_length,
            temperature=T,
            chemical_potentials=[mu],
            prop_freq=1000,
            coord_freq=10000,
            properties=property_list,
            vdw_cutoff=cutoff,
            charge_cutoff=cutoff,
            run_name=eq_name
    )
    mc.restart(
            restart_from=eq_name,
            run_name=prod_name,
            run_type="production",
            total_run_length=prod_length
    )
    endtime = time.time()
    prpdf = read_prp(prod_name+".out.prp")
    prp_mean = prpdf.mean(axis=0)
    prp_sem = prpdf.sem(axis=0)
    statdf['mu'][i] = mu.to_value('kJ/mol')
    statdf['Pressure'][i] = prp_mean['Pressure']
    statdf['Pressure_SEM'][i] = prp_sem['Pressure']
    statdf['Nmols'][i] = prp_mean['Nmols']
    statdf['Nmols_SEM'][i] = prp_sem['Nmols']
    statdf['Density'][i] = prp_mean['Density']
    statdf['Density_SEM'][i] = prp_sem['Density']
    statdf['Energy_Total'][i] = prp_mean['Energy_Total']
    statdf['Energy_Total_SEM'][i] = prp_sem['Energy_Total']
    statdf['Sim_Time'][i] = endtime-starttime

statdf.to_csv('property_data.csv', index=False)


# write by Mrlv
# coding:utf-8
import time

import tensorflow as tf
import os
from tensorflow.keras import optimizers
from tools import LSTM
from DeepLearn.rnn_classfication import rnnlayer

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
batchsz = 128
total_words = 10000
embedding_len = 100
max_words_len = 80
data_train = rnnlayer.train_data
data_test = rnnlayer.test_data

def main():
    units = 64
    epochs = 10
    model = LSTM(units, embedding_len, max_words_len, total_words)

    time0 = time.time()
    model.compile(optimizer=optimizers.Adam(lr=0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'])

    model.fit(data_train, epochs=epochs, validation_data=data_test)

    model.evaluate(data_test)
    time1 = time.time()
    print()

if __name__ == '__main__':
    main()


from collections import deque

N = int(input())
S = list(input())
A = deque([])

for i in range(N):
    if S[i] == "L":
        A.appendleft(i+1)
    elif S[i] == "R":
        A.append(i+1)
    elif S[i] == "A":
        if len(A) == 0:
            print("ERROR")
        else:
            print(A.popleft())
    elif S[i] == "B":
        if len(A) <= 1:
            print("ERROR")
        else:
            q = A.popleft()
            print(A.popleft())
            A.appendleft(q)
    elif S[i] == "C":
        if len(A) <= 2:
            print("ERROR")
        else:
            q1 = A.popleft()
            q2 = A.popleft()
            print(A.popleft())
            A.appendleft(q2)
            A.appendleft(q1)

    elif S[i] == "D":
        if len(A) == 0:
            print("ERROR")
        else:
            print(A.pop())
    elif S[i] == "E":
        if len(A) <= 1:
            print("ERROR")
        else:
            q = A.pop()
            print(A.pop())
            A.append(q)
    elif S[i] == "F":
        if len(A) <= 2:
            print("ERROR")
        else:
            q1 = A.pop()
            q2 = A.pop()
            print(A.pop())
            A.append(q2)
            A.append(q1)


# coding=UTF-8
# my dataset definition

'''
@File: InputOutputDataset
@Author: WeiWei
@Time: 2023/2/25
@Email: weiwei_519@outlook.com
@Software: PyCharm
'''

import torch
from torch.utils.data import Dataset
from random import sample
import numpy as np
import traceback
import datetime
import logging

today = datetime.datetime.now().strftime("%Y-%m-%d")
logging.basicConfig(filename=f'../logs/utils/utils_{today}.log',
                    level=logging.INFO,
                    format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    filemode='a'
)

def preprocess(text):
    text = text.replace("\n", "\\n").replace("\t", "\\t")
    return text

def postprocess(text):
    return text.replace("\\n", "\n").replace("\\t", "\t").replace(" ", "")

# Input text to Output text dataset
class InputOutputDataset(Dataset):
    """
    创建一个自定义的数据集，用于训练，必须包括两个字段：输入(如source_text)、输出（如target_text）
    Creating a custom dataset for reading the dataset and
    loading it into the dataloader to pass it to the
    neural network for finetuning the model
    """

    def __init__(
            self, tokenizer, source_texts, target_texts, text_length, input_ids_length,
    ):
        """
        Initializes a Dataset class

        Args:
            tokenizer (transformers.tokenizer): Transformers tokenizer
            source_len (int): Max length of source text
            target_len (int): Max length of target text
            source_texts (str): column name of source text
            target_texts (str): column name of target text
        """
        self.tokenizer = tokenizer
        self.text_length = text_length
        self.input_ids_length = input_ids_length
        self.source_texts = source_texts
        self.target_texts = target_texts
        self.source_shape = (len(source_texts), self.text_length)
        self.target_shape = (len(target_texts), self.input_ids_length)

    def __len__(self):
        """returns the length of dataframe"""

        return len(self.target_texts)

    def __getitem__(self, index):
        """return the input ids, attention masks and target ids"""

        source_text = preprocess(str(self.source_texts[index]))
        target_text = preprocess(str(self.target_texts[index]))

        # cleaning data so as to ensure data is in string type
        # source_text = "".join(source_text.split())
        # target_text = "".join(target_text.split())

        source = self.tokenizer.batch_encode_plus(
            [source_text],
            max_length=self.text_length,
            pad_to_max_length=True,
            truncation=True,
            padding="max_length",
            return_tensors="pt",
        )
        target = self.tokenizer.batch_encode_plus(
            [target_text],
            max_length=self.input_ids_length,
            pad_to_max_length=True,
            truncation=True,
            padding="max_length",
            return_tensors="pt",
        )

        return {
            "input_ids": torch.squeeze(source.data['input_ids']),
            "attention_mask": torch.squeeze(source.data['attention_mask']),
            "labels": torch.squeeze(target.data['input_ids'])
        }

    def val_sample(self, sample_rate=0.1):
        indexs = sample(list(range(self.source_shape[0])), int(sample_rate * self.source_shape[0]))
        val_dataset = InputOutputDataset(tokenizer=self.tokenizer,
                                         source_len=self.source_len,
                                         target_len=self.target_len,
                                         source_texts=[self.source_texts[i] for i in indexs],
                                         target_texts=[self.target_texts[i] for i in indexs],
                                         )
        return val_dataset

# Generative Dataset 生成式数据集
class GeneDataset(Dataset):
    """
    创建一个自定义的数据集，用于文本生成
    Creating a custom dataset for reading the dataset and
    loading it into the dataloader to pass it to the
    neural network for finetuning the model
    """

    def __init__(
            self, tokenizer, length, texts
    ):
        """
        Initializes a Dataset class

        Args:
            tokenizer (transformers.tokenizer): Transformers tokenizer
            length (int): Max length of text
            texts：test list
        """
        self.tokenizer = tokenizer
        self.max_len = length
        self.texts = texts
        self.shape = (len(texts), self.max_len)

    def __len__(self):
        """returns the length of dataframe"""

        return len(self.texts)

    def __getitem__(self, index):
        """return the input ids, attention masks and target ids"""

        text = preprocess(str(self.texts[index]))
        # text = self.texts[index]
        # cleaning data so as to ensure data is in string type
        # text = "".join(text.split())

        input_text = text[:-1]
        output_text = text[1:]

        input_encoder = self.tokenizer.batch_encode_plus(
            [input_text],
            max_length=self.max_len,
            # pad_to_max_length=True,
            truncation=True,
            padding="max_length",
            return_tensors="pt",
        )

        label_encoder = self.tokenizer.batch_encode_plus(
            [output_text],
            max_length=self.max_len,
            # pad_to_max_length=True,
            truncation=True,
            padding="max_length",
            return_tensors="pt",
        )

        input_encoder.data['labels'] = label_encoder.data['input_ids']  # 文本生成的自回归任务。

        return input_encoder.data

    def val_sample(self, sample_rate=0.1):
        indexs = sample(list(range(self.shape[0])), int(sample_rate * self.shape[0]))
        val_dataset = GeneDataset(tokenizer=self.tokenizer,
                                  length=self.max_len,
                                  texts=[self.texts[i] for i in indexs]
                                  )
        return val_dataset

class GPT2Dataset(Dataset):
    def __init__(self, data_list, tokenizer):
        self.data_list = data_list
        self.tokenizer = tokenizer

    def __getitem__(self, index):
        input_ids = self.data_list[index]
        return input_ids

    def __len__(self):
        return len(self.data_list)

class TextClassifyDataset(Dataset):
    def __init__(self, tokenizer, texts, labels, max_len):
        self.tokenizer = tokenizer
        self.texts = texts
        self.labels = labels
        self.shape = (len(texts), max_len)

    def __getitem__(self, index):
        text = preprocess(str(self.texts[index]))

        text_encode = self.tokenizer.batch_encode_plus(
            [text],
            max_length=self.shape[1],
            pad_to_max_length=True,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        item = {
            'input_ids': torch.squeeze(text_encode.data['input_ids']),
            # 'token_type_ids': torch.squeeze(text_encode.data['token_type_ids']),
            'attention_mask': torch.squeeze(text_encode.data['attention_mask']),
            'labels': torch.tensor(self.labels[index])
        }
        return item

    def __len__(self):
        return self.shape[0]

class PromptSrcTgtDataset(Dataset):
    """
    创建一个自定义的数据集，用于训练，必须包括两个字段：输入(如source_text)、输出（如target_text）
    Creating a custom dataset for reading the dataset and
    loading it into the dataloader to pass it to the
    neural network for finetuning the model

    """

    def __init__(
            self, dataframe, tokenizer, source_len, target_len, source_text, target_text
    ):
        """
        Initializes a Dataset class

        Args:
            dataframe (pandas.DataFrame): Input dataframe
            tokenizer (transformers.tokenizer): Transformers tokenizer
            source_len (int): Max length of source text
            target_len (int): Max length of target text
            source_text (str): column name of source text
            target_text (str): column name of target text
        """
        self.tokenizer = tokenizer
        self.data = dataframe
        self.source_len = source_len
        self.summ_len = target_len
        self.target_text = self.data[target_text]
        self.source_text = self.data[source_text]

    def __len__(self):
        """returns the length of dataframe"""

        return len(self.target_text)

    def __getitem__(self, index):
        """return the input ids, attention masks and target ids"""

        source_text = str(self.source_text[index])
        target_text = str(self.target_text[index])

        # cleaning data so as to ensure data is in string type
        source_text = " ".join(source_text.split())
        target_text = " ".join(target_text.split())

        source = self.tokenizer.batch_encode_plus(
            [source_text],
            max_length=self.source_len,
            pad_to_max_length=True,
            truncation=True,
            padding="max_length",
            return_tensors="pt",
        )
        target = self.tokenizer.batch_encode_plus(
            [target_text],
            max_length=self.summ_len,
            pad_to_max_length=True,
            truncation=True,
            padding="max_length",
            return_tensors="pt",
        )

        source_ids = source["input_ids"].squeeze()
        source_mask = source["attention_mask"].squeeze()
        target_ids = target["input_ids"].squeeze()
        target_mask = target["attention_mask"].squeeze()

        return {
            "source_ids": source_ids.to(dtype=torch.long),
            "source_mask": source_mask.to(dtype=torch.long),
            "target_ids": target_ids.to(dtype=torch.long),
            "target_ids_y": target_ids.to(dtype=torch.long),
        }

class TextRewardDataset(Dataset):
    """
    创建一个自定义的数据集，用于训练，Text Reward Model

    Args:
        Texts (dict): 训练数据样本，为 \t 分割的句子list
                "texts": [
                            '句子1	句子2	句子3',
                            '句子1	句子2	句子3',
                            ...
                ]

    Returns:
        dict (Tensor) -> tokenized_output = {
                            'input_ids': [
                                            [[101, 3928, ...], [101, 4395, ...], [101, 2135, ...]],
                                            [[101, 3928, ...], [101, 4395, ...], [101, 2135, ...]],
                                            ...
                                        ],
                            'token_type_ids': [
                                                [[0, 0, ...], [0, 0, ...], [0, 0, ...]],
                                                [[0, 0, ...], [0, 0, ...], [0, 0, ...]],
                                                ...
                                            ]
                            'position_ids': [
                                                [[0, 1, 2, ...], [0, 1, 2, ...], [0, 1, 2, ...]],
                                                [[0, 1, 2, ...], [0, 1, 2, ...], [0, 1, 2, ...]],
                                                ...
                                            ]
                            'attention_mask': [
                                                [[1, 1, ...], [1, 1, ...], [1, 1, ...]],
                                                [[1, 1, ...], [1, 1, ...], [1, 1, ...]],
                                                ...
                                            ]
                        }
    """

    def __init__(
            self, texts, tokenizer, max_len
    ):
        """
        Initializes a Dataset class
        """
        self.tokenizer = tokenizer
        self.texts = texts
        self.max_len = max_len

    def __len__(self):
        """returns the length of dataframe"""

        return len(self.texts)

    def __getitem__(self, index):
        """return the input ids, attention masks, position ids and token type ids"""
        tokenized_output = {
            'input_ids': [],
            'token_type_ids': [],
            'position_ids': [],
            'attention_mask': []
        }

        line = self.texts[index]
        rank_texts = line.strip().split('\t')
        rank_texts_prop = {
            'input_ids': [],
            'token_type_ids': [],
            'position_ids': [],
            'attention_mask': []
        }
        for rank_text in rank_texts:
            encoded_inputs = self.tokenizer(
                text=rank_text,
                truncation=True,
                max_length=self.max_len,
                padding='max_length')
            rank_texts_prop['input_ids'].append(encoded_inputs["input_ids"])
            rank_texts_prop['token_type_ids'].append(encoded_inputs["token_type_ids"])
            rank_texts_prop['position_ids'].append([i for i in range(len(encoded_inputs["input_ids"]))])
            rank_texts_prop['attention_mask'].append(encoded_inputs["attention_mask"])

        for k, v in rank_texts_prop.items():
            tokenized_output[k].append(v)

        # for rank_text in rank_texts:
        #     encode = self.tokenizer.batch_encode_plus(
        #         [rank_text],
        #         max_length=self.max_len,
        #         pad_to_max_length=True,
        #         truncation=True,
        #         padding="max_length",
        #         return_tensors="pt",
        #     )
        #     if len(tokenized_output['input_ids']) == 0:
        #         tokenized_output['input_ids'] = encode['input_ids']
        #         tokenized_output['token_type_ids'] = encode['token_type_ids']
        #         tokenized_output['attention_mask'] = encode['attention_mask']
        #         tokenized_output['position_ids'] = torch.tensor([[i for i in range(encode['input_ids'].shape[1])]])
        #     else:
        #         tokenized_output['input_ids'] = torch.cat((tokenized_output['input_ids'], encode['input_ids']), dim=0)
        #         tokenized_output['token_type_ids'] = torch.cat(
        #             (tokenized_output['token_type_ids'], encode['token_type_ids']),
        #             dim=0)
        #         tokenized_output['attention_mask'] = torch.cat(
        #             (tokenized_output['attention_mask'], encode['attention_mask']),
        #             dim=0)
        #         tokenized_output['position_ids'] = torch.cat(
        #             (tokenized_output['position_ids'], torch.tensor([[i for i in range(encode['input_ids'].shape[1])]])),
        #             dim=0
        #         )
        return tokenized_output

def dataset_process(dataset: dict, tokenizer, max_seq_len: int):
    """
    将样本数据转换为模型接收的输入数据。

    Args:
        dataset (dict): 训练数据样本, e.g. -> {
                                                "text": [
                                                            '句子1	句子2	句子3',
                                                            '句子1	句子2	句子3',
                                                            ...
                                                ]
                                            }
        dataset 为 \t 分割的句子list

    Returns:
        dict (str: np.array) -> tokenized_output = {
                            'input_ids': [
                                            [[101, 3928, ...], [101, 4395, ...], [101, 2135, ...]],
                                            [[101, 3928, ...], [101, 4395, ...], [101, 2135, ...]],
                                            ...
                                        ],
                            'token_type_ids': [
                                                [[0, 0, ...], [0, 0, ...], [0, 0, ...]],
                                                [[0, 0, ...], [0, 0, ...], [0, 0, ...]],
                                                ...
                                            ]
                            'position_ids': [
                                                [[0, 1, 2, ...], [0, 1, 2, ...], [0, 1, 2, ...]],
                                                [[0, 1, 2, ...], [0, 1, 2, ...], [0, 1, 2, ...]],
                                                ...
                                            ]
                            'attention_mask': [
                                                [[1, 1, ...], [1, 1, ...], [1, 1, ...]],
                                                [[1, 1, ...], [1, 1, ...], [1, 1, ...]],
                                                ...
                                            ]
                        }
    """
    tokenized_output = {
        'input_ids': [],
        'token_type_ids': [],
        'position_ids': [],
        'attention_mask': []
    }

    for line in dataset['text']:
        try:
            rank_texts = line.strip().split('\t')
        except:
            logging.error(f'"{line}" -> {traceback.format_exc()}')
            exit()

        rank_texts_prop = {
            'input_ids': [],
            'token_type_ids': [],
            'position_ids': [],
            'attention_mask': []
        }
        for rank_text in rank_texts:
            encoded_inputs = tokenizer(
                text=rank_text,
                truncation=True,
                max_length=max_seq_len,
                padding='max_length')
            rank_texts_prop['input_ids'].append(encoded_inputs["input_ids"])
            rank_texts_prop['token_type_ids'].append(encoded_inputs["token_type_ids"])
            rank_texts_prop['position_ids'].append([i for i in range(len(encoded_inputs["input_ids"]))])
            rank_texts_prop['attention_mask'].append(encoded_inputs["attention_mask"])

        for k, v in rank_texts_prop.items():
            tokenized_output[k].append(v)

    for k, v in tokenized_output.items():
        tokenized_output[k] = np.array(v)

    return tokenized_output


# -*- coding: utf-8 -*-
import cv2
import numpy as np
import os
import pickle
import random
import config

from multiprocessing.pool import ThreadPool

import logbook
logger = logbook.Logger(__name__)

SIFT = cv2.xfeatures2d.SIFT_create(contrastThreshold=config.SIFT_CONTRAST_THRESHOLD, edgeThreshold=config.SIFT_EDGE_THRESHOLD, sigma=config.SIFT_SIGMA)
FLANN = cv2.FlannBasedMatcher(config.FLANN_INDEX_PARAMS, config.FLANN_SEARCH_PARAMS)
BF = cv2.BFMatcher(normType=cv2.NORM_HAMMING)

class Match(object):
    def __init__(self, nissl_level, matches, H, mask, result, result2, dist):
        self.nissl_level = nissl_level
        self.matches = matches
        self.H = H
        self.mask = mask
        self.result = result
        self.result2 = result2

        self.dist = dist

        self.matches_count = len(matches)
        self.inlier_count = mask.sum()
        self.inlier_ratio = int(self.inlier_count / self.matches_count * 100)

        self.homography_det = abs(np.linalg.det(H))

        self.svd = np.linalg.svd(self.H, compute_uv=False)
        self.svd_ratio = int(self.svd[0] / self.svd[-1])

    def comparison_key(self):
        return self.inlier_count
        #return -((1/self.inlier_ratio) * (self.svd_ratio) * self.homography_det * self.dist)

    # ['Plate', 'Match Count', 'Inlier Count', 'I/M', 'SVD', 'Det H']
    def to_string_array(self):
        return np.array([
            "Plate #" + str(self.nissl_level),
            str(self.matches_count),
            str(self.inlier_count),
            str(self.inlier_ratio),
            str(self.svd_ratio),
            str(self.homography_det),
            str((1/self.inlier_ratio) * (self.svd_ratio) * self.homography_det * self.dist)
            ])

def warp(im, points, disp_min, disp_max, disp_len = None, disp_angle = None):
    h, w = im.shape[:2]

    # Include the corners
    src_pts = np.array([[0, 0], [w, 0], [0, h], [w, h]])
    dst_pts = np.array([[0, 0], [w, 0], [0, h], [w, h]])

    for i in range(points):
        rand_x = random.uniform(0, w)
        rand_y = random.uniform(0, h)
        p = np.array([rand_x, rand_y])
        if disp_len is None:
            rand_len = random.uniform(disp_min, disp_max)
        else:
            rand_len = disp_len

        if disp_angle is None:
            rand_angle = np.deg2rad(random.uniform(0, 360))
        else:
            rand_angle = disp_angle

        p2_x = rand_x + np.cos(rand_angle) * rand_len
        p2_y = rand_y + np.sin(rand_angle) * rand_len
        p2 = np.array([p2_x, p2_y])

        if src_pts is None:
            src_pts = np.array([p])
        else:
            temp = np.vstack((src_pts, p))
            src_pts = temp

        if dst_pts is None:
            dst_pts = np.array([p2])
        else:
            temp = np.vstack((dst_pts, p2))
            dst_pts = temp

    from skimage.transform import warp, PiecewiseAffineTransform
    tform = PiecewiseAffineTransform()
    tform.estimate(src_pts, dst_pts)
    warped_im = warp(im, tform)

    # Convert to OpenCV
    from skimage import img_as_ubyte
    warped_im = img_as_ubyte(warped_im)
    return warped_im

def im_read(filename, flags=cv2.IMREAD_COLOR):
    im = cv2.imread(filename, flags)

    if flags == cv2.IMREAD_COLOR:
        # Convert from BGR to RGB for some reason
        im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)

    return im

def im_write(filename, im):
    if len(im.shape) == 3:
        # Not grayscale
        # Convert back to the original BGR
        im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)

    cv2.imwrite(filename, im)

def nissl_load(nissl_level, color_flags=cv2.IMREAD_COLOR):
    filename = config.NISSL_PREFIX + str(nissl_level).zfill(config.NISSL_DIGITS) + config.NISSL_EXT
    path = os.path.join(config.NISSL_DIR, filename)

    if not os.path.exists(path):
        logger.error("Tried to load plate {0} but it wasn't found in path {1}", nissl_level, path)
        return None

    return im_read(path, color_flags)

def nissl_load_sift(nissl_level):
    filename = config.NISSL_PREFIX + str(nissl_level).zfill(config.NISSL_DIGITS) + ".sift"
    path = os.path.join(config.NISSL_DIR, filename)

    if not os.path.exists(path):
        logger.info("Creating SIFT for plate {0}", nissl_level)
        nissl = nissl_load(nissl_level, cv2.IMREAD_GRAYSCALE)
        if nissl is None:
            return None

        kp, des = extract_sift(nissl)
        temp = pickle_sift(kp, des)
        pickle.dump(temp, open(path, "wb"))
        return kp, des

    else:
        raw_sift = pickle.load(open(path, "rb"))
        kp, des = unpickle_sift(raw_sift)
        return kp, des

def match(im1, kp1, des1, im2, kp2, des2):
    if config.MATCH_WITH_FLANN:
        matches = FLANN.knnMatch(des1, des2, k=2)
    else:
        matches = BF.knnMatch(des1, des2, k=2)

    # Apply Ratio Test
    good_matches = [m[0] for m in matches if len(m) == 2 and m[0].distance < m[1].distance * config.DISTANCE_RATIO]

    if len(good_matches) < config.MIN_MATCH_COUNT:
        logger.debug("Matches lower than threshold. {0} < {1}", len(good_matches), config.MIN_MATCH_COUNT)
        return None

    # For homography calculation
    src_pts = np.float32([kp1[m.queryIdx].pt for m in good_matches])
    dst_pts = np.float32([kp2[m.trainIdx].pt for m in good_matches])

    # Obtain the homography matrix using RANSAC
    H, mask = cv2.findHomography(src_pts, dst_pts, method=cv2.RANSAC, ransacReprojThreshold=config.RANSAC_REPROJ_TRESHHOLD, maxIters=config.RANSAC_MAX_ITERS, confidence=config.RANSAC_CONFIDENCE)

    # Check homography validity
    if H is None or len(H.shape) != 2:
        logger.debug("Couldn't get homography")
        return None

    # This mask contains the inliers
    matchesMask = mask.ravel().tolist()

    # Calculate the homography determinant
    det = np.linalg.det(H)
    logger.debug("Homography determinant = {0:f}", det)

    # If it's too low for comfort, don't consider the match
    # Or too high
    #if abs(det) < 0.01 or abs(det) > 5:
    #    return None

    # Apply the perspective transformation to the source image corners
    h, w = im1.shape[:2]
    corners = np.float32([[0, 0], [0, h-1], [w-1, h-1], [w-1, 0]]).reshape(-1, 1, 2)

    # Attempt to transform corners based on homography
    try:
        transformedCorners = cv2.perspectiveTransform(corners, H)
    except:
        logger.debug("Couldn't transform corners")
        return None

    # Only accept corners that remain convex after being transformed
    isConvex = cv2.isContourConvex(transformedCorners)
    if not isConvex:
        return None

    # Get the 7 Hu invariant moments
    original_moments = cv2.HuMoments(cv2.moments(corners)).flatten()
    transformed_moments = cv2.HuMoments(cv2.moments(transformedCorners)).flatten()

    # Find the Euclidean distance between the moments
    hu_distance = np.linalg.norm(original_moments - transformed_moments)
    logger.debug("[Hu Moment Distance] = {0}", hu_distance)

    # Ignore moments that are too large
    #if hu_distance > config.HU_DISTANCE_THRESHOLD:
    #    return None

    # Draw a polygon on the second image joining the transformed corners
    im2 = cv2.polylines(im2, [np.int32(transformedCorners)], True, config.MATCH_RECT_COLOR, 2, cv2.LINE_AA)

    # Prepare drawing parameters for drawing lines between matches
    #drawParameters = dict(matchColor=None, singlePointColor=None, matchesMask=None, flags=cv2.DRAW_MATCHES_FLAGS_NOT_DRAW_SINGLE_POINTS)
    drawParameters = dict(matchColor=None, singlePointColor=None, matchesMask=matchesMask, flags=cv2.DRAW_MATCHES_FLAGS_NOT_DRAW_SINGLE_POINTS)

    # Draw the lines between the 2 images connecting matches
    im_matches = cv2.drawMatches(im1, kp1, im2, kp2, good_matches, None, **drawParameters)

    # Warp the first image onto the second image
    im_warp = cv2.warpPerspective(im1, H, (im2.shape[1],im2.shape[0]))

    # Find the pixels which are visible in the warped image
    overlay_mask = (im_warp != 0)

    # Create a copy of the second image
    im_overlay = im2

    # Overlay the warped image on top of the 2nd image
    # If the mask is grayscale the overlay will only be applied to the red channel
    if len(overlay_mask.shape) == 3: # Overlay colors
        im_overlay[overlay_mask] = im_warp[overlay_mask]
    else:
        im_overlay[overlay_mask, 0] = im_warp[overlay_mask]

    return Match(None, good_matches, H, mask, im_matches, im_overlay, hu_distance)

def match_fast(kp1, des1, kp2, des2):
    if config.MATCH_WITH_FLANN:
        matches = FLANN.knnMatch(des1, des2, k=2)
    else:
        matches = BF.knnMatch(des1, des2, k=2)

    # Apply Ratio Test
    good_matches = [m[0] for m in matches if len(m) == 2 and m[0].distance < m[1].distance * config.DISTANCE_RATIO]

    # For homography calculation
    src_pts = np.float32([kp1[m.queryIdx].pt for m in good_matches])
    dst_pts = np.float32([kp2[m.trainIdx].pt for m in good_matches])

    # Obtain the homography matrix using RANSAC
    H, mask = cv2.findHomography(src_pts, dst_pts, method=cv2.RANSAC, ransacReprojThreshold=config.RANSAC_REPROJ_TRESHHOLD, maxIters=config.RANSAC_MAX_ITERS, confidence=config.RANSAC_CONFIDENCE)

    return good_matches

def match_region_nissl(im_region, nissl_level):
    kp1, des1 = extract_sift(im_region)

    kp2, des2 = nissl_load_sift(nissl_level)
    im_nissl = nissl_load(nissl_level)

    m = match(im_region, kp1, des1, im_nissl, kp2, des2)
    if m is None:
        return None
    else:
        m.nissl_level = nissl_level
        logger.debug("Match accepted")
        return m

def match_sift_nissl(im_region, kp1, des1, nissl_level):
    logger.debug("=============== Matching {0}", nissl_level)
    kp2, des2 = nissl_load_sift(nissl_level)
    im_nissl = nissl_load(nissl_level)

    m = match(im_region, kp1, des1, im_nissl, kp2, des2)

    if m is None:
        return None
    else:
        m.nissl_level = nissl_level
        return m

def extract_sift(im):
    if im.dtype != np.uint8:
            im = (im * 255).astype(np.uint8)

    if len(im.shape) == 3:
        im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)

    # Use multithreading to split the affine detection work
    pool = ThreadPool(processes = cv2.getNumberOfCPUs())

    #kp, des = affine_detect(SIFT, im, pool=pool)
    kp, des = SIFT.detectAndCompute(im, None)

    pool.close()
    pool.join()

    return kp, des

def affine_skew(tilt, phi, img, mask=None):
    '''
    affine_skew(tilt, phi, img, mask=None) -> skew_img, skew_mask, Ai
    Ai - is an affine transform matrix from skew_img to img
    '''
    h, w = img.shape[:2]
    if mask is None:
        mask = np.zeros((h, w), np.uint8)
        mask[:] = 255
    A = np.float32([[1, 0, 0], [0, 1, 0]])
    if phi != 0.0:
        phi = np.deg2rad(phi)
        s, c = np.sin(phi), np.cos(phi)
        A = np.float32([[c,-s], [ s, c]])
        corners = [[0, 0], [w, 0], [w, h], [0, h]]
        tcorners = np.int32( np.dot(corners, A.T) )
        x, y, w, h = cv2.boundingRect(tcorners.reshape(1,-1,2))
        A = np.hstack([A, [[-x], [-y]]])
        img = cv2.warpAffine(img, A, (w, h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
    if tilt != 1.0:
        s = 0.8*np.sqrt(tilt*tilt-1)
        img = cv2.GaussianBlur(img, (0, 0), sigmaX=s, sigmaY=0.01)
        img = cv2.resize(img, (0, 0), fx=1.0/tilt, fy=1.0, interpolation=cv2.INTER_NEAREST)
        A[0] /= tilt
    if phi != 0.0 or tilt != 1.0:
        h, w = img.shape[:2]
        mask = cv2.warpAffine(mask, A, (w, h), flags=cv2.INTER_NEAREST)
    Ai = cv2.invertAffineTransform(A)
    return img, mask, Ai

def affine_detect(detector, img, mask=None, pool=None):
    '''
    affine_detect(detector, img, mask=None, pool=None) -> keypoints, descrs
    Apply a set of affine transormations to the image, detect keypoints and
    reproject them into initial image coordinates.
    See http://www.ipol.im/pub/algo/my_affine_sift/ for the details.
    ThreadPool object may be passed to speedup the computation.
    '''
    params = [(1.0, 0.0)]
    for t in 2**(0.5*np.arange(1,6)):
        for phi in np.arange(0, 180, 72.0 / t):
            params.append((t, phi))

    def f(p):
        t, phi = p
        timg, tmask, Ai = affine_skew(t, phi, img)
        keypoints, descrs = detector.detectAndCompute(timg, tmask)
        for kp in keypoints:
            x, y = kp.pt
            kp.pt = tuple( np.dot(Ai, (x, y, 1)) )
        if descrs is None:
            descrs = []
        return keypoints, descrs

    keypoints, descrs = [], []
    if pool is None:
        ires = map(f, params)
    else:
        ires = pool.map(f, params)

    for i, (k, d) in enumerate(ires):
        #logger.debug('affine sampling: %d / %d\r' % (i+1, len(params)))
        keypoints.extend(k)
        descrs.extend(d)

    return keypoints, np.array(descrs)

# https://isotope11.com/blog/storing-surf-sift-orb-keypoints-using-opencv-in-python
def pickle_sift(keypoints, descriptors):
    i = 0
    temp_array = []
    for point in keypoints:
        if i >= len(descriptors):
            break
        temp = (point.pt, point.size, point.angle, point.response, point.octave,
        point.class_id, descriptors[i])
        i += 1
        temp_array.append(temp)
    return temp_array

def unpickle_sift(array):
    keypoints = []
    descriptors = []
    for point in array:
        temp_feature = cv2.KeyPoint(x=point[0][0],y=point[0][1],_size=point[1], _angle=point[2], _response=point[3], _octave=point[4], _class_id=point[5])
        temp_descriptor = point[6]
        keypoints.append(temp_feature)
        descriptors.append(temp_descriptor)
    return keypoints, np.array(descriptors)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('sites', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='AlternativeDomain',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('domain', models.CharField(max_length=500)),
                ('site', models.OneToOneField(related_name='alternative_domain', to='sites.Site')),
            ],
        ),
    ]


import tkinter as tk
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
import matplotlib.pyplot as plt
import matplotlib
from functools import partial
from datetime import datetime

from firmware import Firmware
from GUI.tooltip import create_tool_tip

class App:
    def __init__(self, firmware: Firmware):
        matplotlib.use('TkAgg')
        self.firmware = firmware
        self.root = tk.Tk()
        self.root.geometry("800x600")
        self.root.title('Состояние контроллеров')
        self.show_interface()
        self.root.mainloop()

    def show_interface(self):
        status_frame = tk.Frame(self.root)
        self.show_status_page(status_frame)
        status_frame.pack(expand=1)

    @staticmethod
    def hide_page(page):
        page.pack_forget()

    def show_status_page(self, canvas):
        font = "Arial 12"
        for i, cont in enumerate(self.firmware.controllers[:20]):
            name_lab = tk.Label(canvas, text=f'{cont.name}:', font=font, padx=5)
            btn = tk.Button(canvas, text='Перейти к контроллеру', padx=5,
                            command=partial(self.go_to_cont, cont.name, canvas))
            if cont.is_active:
                status_lab = tk.Label(canvas, text='Активен', fg='green', font=font, padx=5)
                time_from_error = datetime.now() - cont.last_error_time
                time_overall = datetime.now() - cont.start_time
                errors_amount = len(self.firmware.get_controller_errors(cont.name))
                create_tool_tip(status_lab, text=f'Время работы после последнего сбоя: {str(time_from_error)}\n'
                                                 f'Время работы: {str(time_overall)}\n'
                                                 f'Число сбоев за время работы: {str(errors_amount)}')
            else:
                status_lab = tk.Label(canvas, text='Отключен', fg='red', font=font, padx=5)
                btn.configure(state=tk.DISABLED)

            name_lab.grid(row=i, column=0)
            status_lab.grid(row=i, column=1)
            btn.grid(row=i, column=2)

    def go_to_cont(self, cont_name: str, canvas):
        self.hide_page(canvas)
        self.show_controller_page(cont_name)

    def show_controller_page(self, cont_name):
        cont_frame = tk.Frame(self.root)
        self.draw_graph(cont_frame, cont_name)
        cont_frame.pack(expand=1)

    def draw_graph(self, cont_frame, cont_name):
        fig, ax = plt.subplots()
        canvas = FigureCanvasTkAgg(fig, master=cont_frame)
        plot_widget = canvas.get_tk_widget()

        date = []
        activ = []

        log = self.firmware.get_controller_log(cont_name)

        for elem in log['data']:
            if isinstance(elem, dict):
                date.append(elem['timestamp'])
                activ.append(0)
            else:
                date.append(elem)
                activ.append(1)
        ax.set_title('Статус работы')
        ax.step(date, activ, linewidth=1)
        ax.set_xticklabels([])
        ax.set_yticklabels([])

        data = self.firmware.get_controller_data(cont_name)
        btn_frame = tk.Frame(cont_frame)
        btn = tk.Button(btn_frame, text='<-', padx=10,
                        command=partial(self.back_page, cont_frame))
        btn.pack(side=tk.LEFT)
        sensors = list(data[0].keys())[:-1]
        for i, sensor in enumerate(sensors, 1):
            btn = tk.Button(btn_frame, text=sensor, padx=5,
                            command=partial(self.update_graph, sensor, cont_name, fig, ax))
            btn.pack(side=tk.LEFT)

        btn_frame.grid(row=0, column=0)
        plot_widget.grid(row=1, column=0)

    def update_graph(self, sensor, cont, fig, ax):
        plt.cla()
        data = self.firmware.get_controller_data(cont)

        date = []
        value = []

        for elem in data:
            date.append(elem['timestamp'])
            value.append(elem[sensor])
        ax.set_title(sensor)
        ax.plot(date, value, linewidth=1)
        ax.set_xticklabels([])
        fig.canvas.draw()

    def back_page(self, cur_page):
        self.hide_page(cur_page)
        self.show_interface()


import pygame
from debug import debug
class Player(pygame.sprite.Sprite):
    def __init__(self, pos, groups, obstacles):
        super().__init__(groups)
        self.image = pygame.image.load('src/assets/testAssets/player.png');
        self.rect = self.image.get_rect(topleft = pos)
        self.direction = pygame.math.Vector2()
        self.speed = 5
        self.obstacles = obstacles
        self.hitbox = self.rect.inflate(-30, -30)

    def move(self, speed):
        # normalizado de velocidad diagonal
        if self.direction.magnitude() != 0:
            self.direction = self.direction.normalize()

        self.hitbox.x += self.direction.x * speed
        self.collision("horizontal")
        self.hitbox.y += self.direction.y * speed
        self.collision("vertical")
        self.rect.center = self.hitbox.center

    # detectar coliciones con otros objetos de manera vertical y horizontal
    def collision(self, direction):
        if(direction == "horizontal"):
            for obstacle in self.obstacles:
                if obstacle.hitbox.colliderect(self.hitbox):
                    if self.direction.x > 0:
                        self.hitbox.right = self.hitbox.left
                    if self.direction.x < 0:
                        self.hitbox.left = self.hitbox.right
        if(direction == 'vertical'):
            for obstacles in self.obstacles:
                if obstacles.hitbox.colliderect(self.hitbox):
                    if self.direction.y > 0:
                        self.hitbox.bottom = self.hitbox.top
                    if self.direction.y < 0:
                        self.hitbox.top = self.hitbox.bottom

    def input(self):
        # inputs del jugador
        keys = pygame.key.get_pressed()
        if(keys[pygame.K_UP]):
            self.direction.y = -1
        elif(keys[pygame.K_DOWN]):
            self.direction.y = 1
        else:
            self.direction.y = 0

        if(keys[pygame.K_LEFT]):
            self.direction.x = -1
        elif(keys[pygame.K_RIGHT]):
            self.direction.x = +1
        else:
            self.direction.x = 0

    def update(self):
        self.input()
        self.move(self.speed)


from distutils.log import error
import requests
import json
import time

# Global variables with values to be replaced for each customer
# Change values accordingly
auth_token_get_url = "https://api.korewireless.com/Api/api/token"
client_id="<<YOUR CLIENT_ID>>"
client_secret="<<YOUR CLIENT_SECRET>>"
api_gateway_key="<<YOUR API_KEY>>"
api_base_path = "https://api.korewireless.com/connectivity"

#API Parameters
accountid='cmp-pp-org-xxx'
iccid='999XXXX25010000XXXX'
activationprofileid="cmp-prov-ap-xxx"

# =======================================================================================
#                               Get Authentication Token
# ======================================================================================
def getauthtoken():
    try:
        headers = {
            'cache-control': 'no-cache',
            'content-type': 'application/x-www-form-urlencoded',
        }
        # Set the required body values for getting Authentication token
        data = {
            'grant_type': 'client_credentials',
            'client_id': client_id,
            'client_secret': client_secret
        }
        # Raise the request for getting the Authentication token
        response = requests.post(auth_token_get_url, headers=headers, data=data)
        # Convert the response string to Json object so that token value can be extracted
        responseJson = json.loads(response.content)
        return responseJson["access_token"]
    except requests.exceptions.RequestException as err:
        print(err)

# =======================================================================================
#                               Get Scbscription id from iccid
# ======================================================================================
def getSubscriptionId(aceesstoken,accountid,iccid):
    try:
        headers = {
        'Content-Type': 'application/json',
        'Authorization': 'bearer ' + aceesstoken,
        'x-api-key': api_gateway_key
        }
        api_url = "{api_base_path}/v1/accounts/{accountid}/subscriptions?page-index=0&max-page-item=100&iccid={iccid}&show-state-history=false".format(api_base_path=api_base_path,accountid=accountid,iccid=iccid)
        payload={}
        subscriptionresponse = requests.request("GET", api_url, headers=headers)
        subscriptionresponseJson = json.loads(subscriptionresponse.text)
        if(subscriptionresponseJson["page-info-result"]["total-count"]>0):
            return subscriptionresponseJson["subscriptions"][0]["subscription-id"]
        else:
            return ""
    except requests.exceptions.RequestException as err:
        print(err)

def createProvisioningRequest(aceesstoken, accountid, subscriptionid, activationprofileid):
    try:
        headers = {
        'Content-Type': 'application/json',
        'Authorization': 'bearer ' + aceesstoken,
        'x-api-key': api_gateway_key
        }

        api_url = "{api_base_path}/v1/accounts/{accountid}/provisioning-requests".format(api_base_path=api_base_path,accountid=accountid)
        payload = json.dumps({
            "activate": {
                "subscriptions": [
                    {
                        "subscription-id": subscriptionid
                    }
                ],
                "activation-state": "active",
                "activation-profile-id": activationprofileid
            }
        })
        response = requests.request("POST", api_url, headers=headers, data=payload)
        return response.text
    except requests.exceptions.RequestException as err:
        print(err)

def checkProvisioningStatus(aceesstoken, accountid,provisioningrequestid):
    try:
        headers = {
            'Content-Type': 'application/json',
            'Authorization': 'bearer ' + aceesstoken,
            'x-api-key': api_gateway_key
        }
        api_url = "{api_base_path}/v1/accounts/{accountid}/provisioning-requests/{provisioningrequestid}".format(api_base_path=api_base_path,accountid=accountid,provisioningrequestid=provisioningrequestid)
        payload={}
        response = requests.request("GET", api_url, headers=headers, data=payload)
        return response.text
    except requests.exceptions.RequestException as err:
        print(err)

# =======================================================================================
#                               API Access Process
# ======================================================================================
# Get the API Token
accesstoken=getauthtoken()

#Get the Subscription id for the iccid
subscriptionid=getSubscriptionId(accesstoken,accountid,iccid)
#If we got a subscription id to proceed to the next step
if(len(subscriptionid)>0):
    # Provision a SIM
    # A success response will be something like this
    # {"status": "success","data": {"provisioning-request-id": "cmp-cpro-request-1460","message": "Your request has been acknowledged"} }
    provisioningrequestresponse=createProvisioningRequest(accesstoken,accountid,subscriptionid,activationprofileid)
    print(provisioningrequestresponse)
    provisioningrequestJson = json.loads(provisioningrequestresponse)
    if(provisioningrequestJson["status"].casefold() =="success"):
        #Wait for the request to complete
        print('Deliberately waiting 1 minute for request to get completed')
        time.sleep(60)
        #Check Provisioning status
        provisioningrequeststatusresponse=checkProvisioningStatus(accesstoken,accountid,provisioningrequestJson["data"]["provisioning-request-id"])
        print(provisioningrequeststatusresponse)
    else:
        print('Request Failed')
else:
    print('No Subscription id recieved for the iccid')


from django.shortcuts import render, redirect, HttpResponse
from django.views.generic import TemplateView
from .models import Vegetables, Category
from .forms import VegetableCreateForm

class AboutView(TemplateView):
    template_name = "about.html"

def homepage(request):
    products = Vegetables.objects.all()  # return list
    context = {"all_vegetables": products}
    return render(request, "product_list.html", context=context)

def vegetable_detail(request, id):
    try:
        vegetable = Vegetables.objects.get(id=id)
        context = {"vegetable": vegetable}
        return render(request, "vegetable_info.html", context)
    except Vegetables.DoesNotExist:
        return HttpResponse('Такой страницы не существует!', status=404)

def vegetable_add(request):
    context = {}

    if request.method == "GET":
        vegetable_form = VegetableCreateForm()
        context["vegetable_form"] = vegetable_form
        return render(request, 'vegetable_form.html', context)
    elif request.method == "POST":
        vegetable_form = VegetableCreateForm(request.POST)
        if vegetable_form.is_valid():
            new_vegetable = vegetable_form.save()
            return redirect(vegetable_detail, id=new_vegetable.id)
        else:
            return HttpResponse('
Форма не валидна!
', status=400)

def vegetable_update(request, id):
    context = {}
    vegetable_object = Vegetables.objects.get(id=id)

    if request.method == "POST":
        vegetable_form = VegetableCreateForm(request.POST, instance=vegetable_object)
        if vegetable_form.is_valid():
            vegetable_object = vegetable_form.save()
            return HttpResponse(f"Данные обновлены успешно!", status=200)

    vegetable_form = VegetableCreateForm(instance=vegetable_object)
    context["vegetable_form"] = vegetable_form
    return render(request, 'vegetable_form.html', context)

def vegetable_delete(request, id):
    if request.method == "POST":
        vegetable_object = Vegetables.objects.get(id=id)
        vegetable_object.delete()
        return HttpResponse("Информация успешно удалено!", status=200)

def categories_view(request):
    categories = Category.objects.all()
    c = {"categories": categories}
    return render(request, "categories.html", c)

def category_detail(request, id):
    try:
        category_object = Category.objects.get(id=id)
        vegetables_list = Vegetables.objects.filter(category=category_object)
        context = {"all_vegetables": vegetables_list}
        return render(request, 'product_list.html', context)
    except Vegetables.DoesNotExist:
        return HttpResponse('Такой страницы не существует!', status=404)


from bs4 import BeautifulSoup
import requests, os, base64
from selenium import webdriver
from . import singleton

class CrawlTool(metaclass=singleton.Singleton):
    def __init__(self):

        options = webdriver.ChromeOptions()
        options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-dev-shm-usage')

        self.driver = webdriver.Chrome('chromedriver', options=options)
        self.driver.implicitly_wait(3)

    def getSoupContentFromUrl(self, url, **args):

        if type(url) is not str:
            return ""

        url += "?"

        for k, v in args.items():
            url += f'{k}={v}&'

        self.driver.get(url)

        soup = BeautifulSoup(self.driver.page_source, 'html.parser')

        return soup

    def getDaumNews(self, date):

        url = 'https://media.daum.net/ranking/'
        soup = self.getSoupContentFromUrl(url, regDate=date)

        soup = soup.select('a.link_txt')
        news = [ dict(news_link = tag.get('href'), news_text = tag.get_text()) for tag in soup if tag.get('href') is not None ]

        return news

    def getDaumNewsBestReply(self, date):

        url = 'https://media.daum.net/ranking/bestreply'
        soup = self.getSoupContentFromUrl(url, regDate=date)

        soup = soup.select('a.link_txt')
        news = [ dict(news_link = tag.get('href'), news_text = tag.get_text()) for tag in soup if tag.get('href') is not None ]

        return news

    def getReplyOfNews(self, news_link, reply_count):

        news_id = news_link.split("/")[-1]

        print(news_id)

        reply_url = f"https://comment.daum.net/apis/v1/posts/@{news_id}/comments?parentId=0&offset=0&limit={reply_count}&sort=RECOMMEND&isInitial=true"

        soup = self.getSoupContentFromUrl(reply_url)

        print(soup)

        return soup

    def getTextNews(self, url):

        soup = self.getSoupContentFromUrl(url)
        soup = soup.select('div.article_view > section > p')

        text_news = [ article.get_text() for article in soup ]
        text_news = " ".join(text_news)

        return text_news

    def getGoogleImages(self, keyword):
        url = f'https://www.google.com/search?q={keyword}&rlz=1C1CHBD_koKR896KR896&source=lnms&tbm=isch&sa=X&ved=2ahUKEwiAua_v2rTpAhVUVN4KHdZOBIkQ_AUoAXoECBYQAw&biw=1536&bih=674&dpr=1.25'

        self.driver.get(url)
        soup = BeautifulSoup(self.driver.page_source, 'html.parser')
        soup = soup.select('img.rg_i')
        img_binary_type = [ tag.get('src') for tag in soup if tag.get('src') is not None ]
        img_binaries = [ base64.b64decode(img_data.split(',')[1]) for img_data in img_binary_type ]
        img_links = [ tag.get('data-src') for tag in soup if tag.get('data-src') is not None ]

        return img_binaries, img_links


# Priority-Based Keyword Matching (PBKM)
#
# Folder pointed by <name> must contain at least these files:
#
#   <name>_training.json
#   <name>_verify.json
#
# It may also contain a map file for mapping metadata names in
# the above two files to names we expect.
#
#   <name>_map.json

import os
import re
import os.path
import sys
import ijson
import nltk
import mapfile
import priority
from collections import Counter

from nltk.corpus import stopwords
from nltk.corpus import wordnet as wn

# Top 10 keywords

kp = ()

# Corpus

stop_words = set(stopwords.words('english'))

# Tokenise the description and filter out any word in the
# English language corpus.

def text_to_tokens(d):

    # Make all tokens lower-case, limit to 3..15 characters and
    # filter out stop-words
    tk = nltk.word_tokenize(d)
    tk = [x.lower() for x in tk if (len(x) > 2 and len(x) <= 15)]
    tk = [x for x in tk if x not in stop_words]

    # Remove any words which have non-alphabetic characters
    tk = [x for x in tk if re.search('^[a-zA-Z]+$', x)]

    # Filter further to just nouns and specific verbs.
    tagged = nltk.pos_tag(tk)
    tags = ['NN', 'JJ', 'VBG', 'VBD', 'NNP', 'NNS', 'NNPS']
    tk = [x for (x,y) in tagged if y in tags]

    # Restrict results to those in the wordnet corpus
    return [x for x in tk if len(wn.synsets(x)) == 0]

# Build the priority map from the training input file

def build_training(filename):

    global kp

    bn = os.path.basename(filename)
    tf = filename + '/' + bn + '_training.json'

    k_map = Counter()

    with open(tf, "r") as f2:
        bugs = ijson.items(f2, 'item')
        for bug in bugs:

            d = bug[mapfile.map['description']]
            p = bug[mapfile.map['priority']]
            wp = priority.weight_from_priority(p)

            if d and wp > 0:

                tks = text_to_tokens(d)
                for tk in tks:
                    if (wp == 4):   # 4 = Major pri bugs
                        if tk in k_map:
                            k_map[tk] += 1
                        else:
                            k_map[tk] = 1

    # List of 10 most common words in major bugs
    kp = [x for (x,y) in k_map.most_common(10)]

    # Report
    print ('Most common words are ' + ",".join(kp))

# Verify the computed priority against the verification
# input file

def verify(filename):

    global kp

    bn = os.path.basename(filename)
    tf = filename + '/' + bn + '_verify.json'

    with open(tf, "r") as f2:
        bugs = ijson.items(f2, 'item')
        success0 = 0
        success1 = 0
        count = 0
        for bug in bugs:
            d = bug[mapfile.map['description']]
            p = bug[mapfile.map['priority']]
            wp = priority.weight_from_priority(p)
            if d and wp > 0:
                tks = text_to_tokens(d)
                cp = 0
                for tk in tks:
                    if tk in kp:
                        cp = 1

                if cp == 1 and wp == 4:
                    success0 = success0 + 1
                if cp == 1 and (wp >= 3 and wp <= 5):
                    success1 = success1 + 1

                count = count + 1

        if count == 0:
            print (bn + ' : Failed to match any bugs')
        else:
            success0p = int((100/count) * success0)
            success1p = int((100/count) * success1)
            print (bn, ':', str(count), 'records :', success0p, '% success (exact),', success1p, '% success (+/-1)')

# Main program starts here

if len(sys.argv) < 2:
    print ('Syntax: pbkm.py <name>')
    exit()

mapfile.load_map(sys.argv[1])
build_training(sys.argv[1])
verify(sys.argv[1])


import os
import zipfile
import socket

from UNETv3_TrainFunc import TrainingLoop

host_ip = "192.168.86.24"

def communication_send(message_to_send):
    # create a socket object
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        # connect to the server
        s.connect((host_ip, 22))
        print('Connected to', host_ip)

        # send data to the server
        message = message_to_send
        s.sendall(message.encode())
    s.close
    return

def communication_rec():
    # create a socket object
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    # bind the socket to a specific network interface and port number
        s.bind(("0.0.0.0", 1024))
        # listen for incoming connections
        s.listen()
        print('waiting for host message at {}:{}...'.format("0.0.0.0", 1024))

        # accept a nano connection
        conn, addr = s.accept()
        with conn:
            print('Connected by', addr)

            # receive data from the nano
            data = conn.recv(1024)
            print('Received data:', data.decode())
            received_msg = data.decode()
    s.close
    return received_msg

LOAD_DIR = "."
def main():
    print("running nano.py")

    # check and send communication confirmation
    print("sending communication ack.")
    communication_send("connected")
    print("connection established and confirmed")

    for i in range(2):
        print("federated learning loop #", i)
        # wait for files to be sent
        print("waiting for model to be sent")
        while(True):
            if(communication_rec() == "model_sent"):
                break

        # send file size to host for file size confirmation
        file_size = os.path.getsize(r"{}/global_model.zip".format(LOAD_DIR))
        communication_send("{}".format(file_size))

        print("waiting for \"start_train\" function")
        while(True):
            if(communication_rec() == "start_train"):
                break

        # unzip file
        with zipfile.ZipFile("{}/{}".format(LOAD_DIR, "global_model.zip"), 'r') as zip_ref:
            zip_ref.extractall("{}/".format(LOAD_DIR))
            os.remove("{}/{}".format(LOAD_DIR, "global_model.zip"))

        print("starting to train")
        # start training, call train python file
        model = TrainingLoop(2, 1, .0001)

        # zip model file
        with zipfile.ZipFile("trained_model.zip", mode = 'w') as archive:
            archive.write(r"UNET_MBIMAGENET.pth")
        print("training finished")
        communication_send("train_finish")

        print("waiting for model to be transferred back")
        while(True):
            if(communication_rec() == "model_sent_back"):
                break
        # send file size to host for file size confirmation
        file_size = os.path.getsize(r"{}/trained_model.zip".format(LOAD_DIR))
        communication_send("{}".format(file_size))

        os.remove("{}/{}".format(LOAD_DIR, "UNET_MBIMAGENET.pth"))
        os.remove("{}/{}".format(LOAD_DIR, "trained_model.zip"))

    return

if __name__ == "__main__":
    main()


from flask import Flask, render_template  # 플라스크 라이브러리 불러오기
import datetime
import RPi.GPIO as GPIO                     # 시간 라이브러리
app = Flask(__name__)                     # 플라스크 객체 만들기

GPIO.setmode(GIPO.BCM) #BCM 모드

@app.route("/")   # <변수>
def hello():
    now = datetime.datetime.now()
    time = now.strftime("%Y-%m-%d %H:%M")
    templateData = {   #딕션너리 변수
        'title' : "안녕하세요!",
        'time' : time
    }
    return render_template('main.html', **templateData) # 딕셔너리 안에 있는 templateData 변수를 전달
#    return render_template('index.html', name = username)

@app.route("/<pin>") # <변수>
def readPin():
    try:
        GPIC.setup(int(pin), GPIO.IN)
        if GPIO.input(int(pin)) == True:
            response = "핀 넘버"+pin+"은 HIGH"
        else:
            response = "핀 넘버"+pin+"은 LOW"
    except:
        response = "핀을 읽는데 문제가 있음"+pin+"번 핀"
    templateData = {
        'pin' : "핀상태" + pin,
        'response' : response
    }
    return render_template('pin.html', **templataDate)

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=5000, debug=True)


import math
import cv2
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Path and filename settings
def path_set():
	path = "C:/AG/course notes/111_2/IC design/HW5/HW5/file/mosaic/"
	data_name = []
	for i in range(6):
		data_name.append("test"+str(i+1)+".dat")
	return path, data_name

# Hexadecimal to decimal
def H2D(num_str):
	return int(num_str, 16)

# read data(pd) and transfer to np
def read_img(path):
	img_pd = pd.read_csv(path, sep=" ", names=["pixel"])
	img_pd = img_pd.to_numpy()
	img_np = []
	for i in range(img_pd.shape[0]):
		img_np.append(H2D(str(img_pd[i])[2:-2]))
	return np.array(img_np)

# image show
def img_show_gray(img):
	plt.imshow(img,"gray")
	plt.show()

def psnr(img1, img2):
   mse = np.mean( (img1/255. - img2/255.) ** 2 )
   if mse < 1.0e-10:
      return 100
   PIXEL_MAX = 1
   return 20 * math.log10(PIXEL_MAX / math.sqrt(mse))

if __name__=='__main__':
	print(0)


import sys
sys.path.append('..')
import common
import re
import unicodedata

if 1:
	book = common.Bible[26]
	src = 'Aramaic Targum to Psalms - he - Mikraot Gedolot.txt'
	data = open(src, newline='').read()
	chapters = re.split('\n\nChapter [0-9]+\n\n', data)[1:]
	for c in range(len(chapters)):
		dst = '../db/targum/27.%03d.txt'%(c + 1)
		out = chapters[c]
		out = ''.join([unicodedata.normalize('NFD', l) for l in out])
		open(dst, 'w').write(out)
		print (dst)


from collections import namedtuple
from collections.abc import Iterable, Iterator
from typing import NamedTuple
import datetime as dt

def read_line(line:str)->list:
    return line.strip('\n').split(',')

def parse_date(str_date:str)->dt.date:
    return dt.datetime.strptime(str_date, "%m/%d/%Y").date()

# example = (4006478550, 'VAD7274', 'VA', 'PAS', '10/5/2016', 5, '4D', 'BMW', 'BUS LANE VIOLATION',)
dtypes =    (int,       str,        str,  str,  parse_date,  int, str,  str,     str )
def type_cast(data: Iterable)->Iterator:
    return ( dtype(item) for dtype, item in zip(dtypes, data) )

def make_namedtuple_from_header( tuple_name:str, header:str)->NamedTuple:
    field_names = (
        name.replace(' ','_').lower()
        for name in read_line(header)
    )
    return namedtuple(tuple_name, field_names)

def parse_tickets(
        lines: Iterable,
        tuple_class: NamedTuple,
        )->Iterator:
    return (
        tuple_class(
            *type_cast(
                read_line(line)
            )
        ) for line in lines
    )

def count_in_field(tickets: Iterator, field_name: str)->dict:
    count = {}
    for ticket in tickets:
        field_value = getattr(ticket, field_name)
        count[field_value] = count.get(field_value, 0) + 1
    return count

def sort_dict(dct:dict)->dict:
    dict(sorted(
        dct.items(),
        key = lambda item: item[1],
        reverse= True
    ))

def count_tickets_by_make(
        file_path: str,
        *, sort = True
    )->dict:
    with open(file_path) as file:
        Ticket = make_namedtuple_from_header('Ticket', next(file))
        tickets = parse_tickets(file, Ticket)
        count = count_in_field(tickets, 'vehicle_make')
    count = sort and sort_dict(count) or count
    return count

if __name__ == '__main__':
    count_by_maker = count_tickets_by_make('nyc_parking_tickets_extract.csv')
    [print(f'{maker}\t{times}') for maker, times in count_by_maker.items()]