from urllib import request
import sqlite3
import json
import time
from ptuBusCrawling.Crawler.Util.SendSlcakMsg import SendSlackMeg
from ptuBusServer.Models.AppDB.AppDBModel import AppDBModel

now = time.localtime()

class makeSqlite:
    def __init__(self):
        self.msg = SendSlackMeg()
        self.conn = sqlite3.connect("app.db")
        self.cur = self.conn.cursor()
        self.URL = {
            "BusTerminal": "http://127.0.0.1:8000/server/bus/terminal/",
            "TrainStation": "http://127.0.0.1:8000/server/train/station/",
            "SubwayLine": "http://127.0.0.1:8000/server/subway/line/",
            "SubwayStation": "http://127.0.0.1:8000/server/subway/station/",
        }

    def openURL(self, name):
        request_url = request.Request(self.URL[name])
        response = request.urlopen(request_url)
        readData = response.read().decode("utf-8")
        data = json.loads(readData)
        temp = []
        for value in data:
            temp.append(tuple(value.values()))
        return temp

    def deleteTables(self):
        for name in self.URL.keys():
            self.conn.execute("delete from " + name)

    def makeDataBase(self):
        self.cur.execute(
            "CREATE TABLE IF NOT EXISTS BusTerminal (id integer PRIMARY KEY, startStationName text, startStationID text,"
            "endStationName text,endStationID text, isExpress integer)"
        )
        self.cur.execute(
            "CREATE TABLE IF NOT EXISTS TrainStation (id integer PRIMARY KEY, startStationName text, startStationID text,"
            "endStationName text,endStationID text)"
        )
        self.cur.execute(
            "CREATE TABLE IF NOT EXISTS SubwayLine (id integer PRIMARY KEY, lineName text, lineCode text, lineColorCode text,"
            "lineSaidName text)"
        )
        self.cur.execute(
            "CREATE TABLE IF NOT EXISTS SubwayStation (id integer PRIMARY KEY, stationName text, stationCode text,"
            "lineCode text, railLineCode text)"
        )

    def diffDataBase(self):
        for name in self.URL:
            data = self.openURL(name)
            self.cur.execute("SELECT * FROM " + name)
            rows = self.cur.fetchall()
            if data != rows:
                self.msg.sendMsg(name + "에서 새로운 값이 검출되었습니다.")
                self.makeData()
                break

    def makeData(self):
        self.deleteTables()
        self.makeDataBase()
        for name in self.URL:
            data = self.openURL(name)
            if name == "BusTerminal":
                self.cur.executemany(
                    "INSERT INTO BusTerminal VALUES (?, ?, ?, ?, ?, ?)", data
                )
            elif name == "TrainStation":
                self.cur.executemany(
                    "INSERT INTO TrainStation VALUES (?, ?, ?, ?, ?)", data
                )
            elif name == "SubwayLine":
                self.cur.executemany(
                    "INSERT INTO SubwayLine VALUES (?, ?, ?, ?, ?)", data
                )
            elif name == "SubwayStation":
                self.cur.executemany(
                    "INSERT INTO SubwayStation VALUES (?, ?, ?, ?, ?)", data
                )
        self.conn.commit()
        fileKey = "{0}{1}{2}{3}{4}{5}".format(
            now.tm_year, now.tm_mon, now.tm_mday, now.tm_hour, now.tm_min, now.tm_sec
        )
        AppDBModel(fileKey=fileKey).save()
        self.msg.sendMsg("File Key = " + fileKey)

if __name__ == "__main__":
    print(makeSqlite().diffDataBase())


import os
import utils
import numpy as np
try:
    import PIL.Image
except ImportError as e:
    print('Unable to load PIL.Image: {}.\nExporting images won\'t work.'.format(e))
try:
    import librosa as lbr
except ImportError as e:
    print('Unable to load librosa: {}.\nExporting sound won\'t work.'.format(e))
from utils import adjust_dynamic_range, numpy_upsample_nearest

class Postprocessor(object):

    def __init__(self, samples_path='.'):
        self.samples_path = samples_path

class ImageSaver(Postprocessor):

    output_file_format = 'fakes_{}.png'

    def __init__(self, samples_path='.', drange=(-1,1), resolution=512, create_subdirs=True):
        super(ImageSaver, self).__init__(samples_path)
        self.samples_path = samples_path
        if create_subdirs:
            os.makedirs(self.samples_path, exist_ok=True)
        # Setup snapshot image grid.
        self.resolution = resolution
        self.drange = drange
        self.mode = None

    def create_image_grid(self, images):
        (count, channels, img_h, img_w) = images.shape

        grid_w = max(int(np.ceil(np.sqrt(count))), 1)
        grid_h = max((count - 1) // grid_w + 1, 1)

        grid = np.zeros((channels,) + (grid_h * img_h, grid_w * img_w), dtype=images.dtype)
        for i in range(count):
            x = (i % grid_w) * img_w
            y = (i // grid_w) * img_h
            grid[:, y: y + img_h, x: x + img_w] = images[i]
        return grid

    def convert_to_pil_image(self, image):
        format = 'RGB'
        if image.ndim == 3:
            if image.shape[0] == 1:
                image = image[0]
                format = 'L'
            else:
                image = image.transpose(1, 2, 0)
                format = 'RGB'

        image = utils.adjust_dynamic_range(image, self.drange, (0, 255))

        image = image.round().clip(0, 255).astype(np.uint8)
        return PIL.Image.fromarray(image, format)

    def __call__(self, output, description):
        if self.resolution is not None:
            output = numpy_upsample_nearest(output, 2, size=self.resolution)
        im = self.create_image_grid(output)
        im = self.convert_to_pil_image(im)
        fname = self.output_file_format
        if type(description) is int:
            fname = fname.format('{:06}')
        im.save(os.path.join(self.samples_path, fname.format(description)))

class SoundSaver(Postprocessor):

    output_file_format = 'fakes_sound_{}_{}.wav'

    def __init__(self, samples_path='.', drange=(-1, 1), resolution=512, mode='abslog', sample_rate=16000,
                 hop_length=128, create_subdirs=True, verbose=False, griffin_lim_iter=100):
        super(SoundSaver, self).__init__(samples_path)
        self.samples_path = samples_path
        if create_subdirs:
            os.makedirs(self.samples_path, exist_ok=True)
        self.drange = drange
        self.mode = mode
        self.sample_rate = sample_rate
        self.hop_length = hop_length
        self.verbose = verbose
        self.resolution = resolution
        self.griffin_lim_iter = griffin_lim_iter

    def reconstruct_from_magnitude(self, stft_mag):
        n_fft = (stft_mag.shape[0] - 1) * 2
        x = np.random.randn((stft_mag.shape[1] - 1) * self.hop_length)
        for i in range(self.griffin_lim_iter):
            stft_rec = lbr.stft(x, n_fft=n_fft, hop_length=self.hop_length)
            angle = np.angle(stft_rec)
            my_stft = stft_mag * np.exp(1.0j * angle)
            if self.verbose: # and i == it - 1:
                prev_x = x
            x = lbr.istft(my_stft, hop_length=self.hop_length)
            if self.verbose:  # and i == it - 1:
                mse = np.sqrt(np.square(x - prev_x).sum())  # logmse would be more appropriate?
                print('MSE between sub- and ultimate iteration: {}'.format(mse))
        return x

    def image_to_sound(self, image):
        if self.mode == 'reallog' or self.mode == 'abslog':
            x = np.zeros((image.shape[0] + 1, image.shape[1]))  # real spectrograms have 2**i + 1 freq bins
            # x.fill(image.mean())
            x[:image.shape[0], :image.shape[1]] = image
            if self.mode == 'reallog':
                signed = adjust_dynamic_range(x, self.drange, (-1, 1))
                sgn = np.sign(signed)
                real_pt_stft = (np.exp(np.abs(signed)) - 1) * sgn
                signal = lbr.istft(real_pt_stft, self.hop_length)
            else:
                x = adjust_dynamic_range(x, self.drange, (0, 255))
                signal = self.reconstruct_from_magnitude(x)
        elif self.mode == 'raw':
            signal = image.ravel()
        else:
            raise Exception(
                'image_to_sound: unrecognized mode: {}. Available modes are: reallog, abslog, raw.'.format(self.mode)
            )
        signal = signal / np.abs(signal).max()
        return signal

    def output_wav(self, signal, samples_description, ith):
        fname = self.output_file_format
        if type(samples_description) is int:
            fname = fname.format('{:06}', '{:02}')
        else:
            fname = fname.format('{}', '{:02}')
        try:
            lbr.output.write_wav(
                os.path.join(self.samples_path, fname.format(samples_description, ith)),
                signal,
                self.sample_rate,
                norm=True
            )
        except Exception as e:
            with open(os.path.join(self.samples_path, 'error_{}_{}.txt'.format(samples_description, ith)), 'w') as f:
                f.write('Exception trying to save sound: {}'.format(e))

    def __call__(self, output, samples_description):
        times_smaller = self.resolution // output.shape[-1]
        if self.mode == 'raw':
            times_smaller *= times_smaller
        for i, img in enumerate(output):
            signal = self.image_to_sound(img[0])
            signal = numpy_upsample_nearest(signal, 1, scale_factor=times_smaller)
            self.output_wav(signal, samples_description, i)


import random
from django.core.management.base import BaseCommand
from django.contrib.admin.utils import flatten
from django_seed import Seed
from lists import models as list_models
from users import models as user_models
from rooms import models as room_models

class Command(BaseCommand):

    # Metadata about this command.
    help = "this commands creates lists"

    def add_arguments(self, parser):
        parser.add_argument(
            "--number",
            default=2,
            type=int,
            help="How many lists do you want to create?",
        )

    def handle(self, *args, **options):
        number = options.get("number")
        seeder = Seed.seeder()
        users = user_models.User.objects.all()
        rooms_potato = room_models.Room.objects.all()
        # rooms = room_models.Room.objects.all()[4:10] # 모든 방을 주되 4에서 10 사이의 방만 주라는 뜻

        seeder.add_entity(
            list_models.List,
            number,
            {
                # "rooms": lambda x: random.choice(rooms_potato),
                "user": lambda x: random.choice(users),
            },
        )
        created_rooms = seeder.execute()
        created_clean = flatten(list(created_rooms.values()))

        for pk in created_clean:
            list_model = list_models.List.objects.get(pk=pk)
            # 랜덤한 방의 수 얻기
            to_add = rooms_potato[random.randint(0, 5) : random.randint(6, 30)]

            # 왜 to_add 안하고 *to_add? >> 포인터개념으로, 그냥 to_add 하면 쿼리셋,즉 array가 되기 때문에 나는 array 안에 있는 요소 "값" 을 원하므로 *to_add 라고 해주었다.
            list_model.rooms.add(*to_add)
        self.stdout.write(self.style.SUCCESS(f"{number} lists created!!"))


"""
Split posts into separate sentences
"""
import csv
from nltk.tokenize import sent_tokenize, word_tokenize

from common.lib.helpers import UserInput
from backend.abstract.processor import BasicProcessor

__author__ = "Stijn Peeters"
__credits__ = ["Stijn Peeters"]
__maintainer__ = "Stijn Peeters"
__email__ = "4cat@oilab.eu"

class SplitSentences(BasicProcessor):
	"""
	Split sentences
	"""
	type = "sentence-split"  # job type ID
	category = "Text analysis"  # category
	title = "Sentence split"  # title displayed in UI
	description = "Split a body of posts into discrete sentences. Output file has one row per sentence, containing the sentence and post ID."  # description displayed in UI
	extension = "csv"  # extension of result file, used internally and in UI

	options = {
		"language": {
			"type": UserInput.OPTION_CHOICE,
			"default": "english",
			"options": {
				"czech": "Czech",
				"danish": "Danish",
				"dutch": "Dutch",
				"english": "English",
				"estonian": "Estonian",
				"finnish": "Finnish",
				"french": "French",
				"german": "German",
				"greek": "Greek",
				"italian": "Italian",
				"norwegian": "Norwegian",
				"polish": "Polish",
				"portuguese": "Portuguese",
				"russian": "Russian",
				"slovene": "Slovene",
				"spanish": "Spanish",
				"swedish": "Swedish",
				"turkish": "Turkish"
			},
			"help": "Language"
		},
		"min_length": {
			"type": UserInput.OPTION_TEXT,
			"min": 0,
			"max": 100,
			"default": 3,
			"help": "Minimal amount of tokens (words) per sentence"
		}
	}

	def process(self):
		"""
		This takes a 4CAT results file as input, and outputs a number of files containing
		tokenised posts, grouped per time unit as specified in the parameters.
		"""
		self.dataset.update_status("Processing posts")

		num_sentences = 0
		num_posts = 1
		min_length = self.parameters.get("min_length")
		language = self.parameters.get("language")

		with self.dataset.get_results_path().open("w", encoding="utf-8") as output:
			writer = csv.DictWriter(output, fieldnames=("post_id", "sentence",))
			writer.writeheader()

			for post in self.source_dataset.iterate_items(self):
				if num_posts % 100 == 0:
					self.dataset.update_status("Processing post %i" % num_posts)

				num_posts += 1

				if not post["body"]:
					continue

				sentences = sent_tokenize(post.get("body", ""), language=language)

				for sentence in sentences:
					if min_length == 0:
						num_sentences += 1
						writer.writerow({"sentence": sentence})
					else:
						# use NLTK's tokeniser to determine word count
						words = word_tokenize(sentence)
						if len(words) >= min_length:
							num_sentences += 1
							writer.writerow({"post_id": post.get("id", ""), "sentence": sentence})

		# done!
		self.dataset.update_status("Finished")
		self.dataset.finish(num_sentences)


import uuid

from google.protobuf.struct_pb2 import Struct

from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from tests.common import (
    RED_TRUCK_IMAGE_FILE_PATH,
    TRUCK_IMAGE_URL,
    both_channels,
    cleanup_inputs,
    metadata,
    raise_on_failure,
    wait_for_inputs_upload,
)

@both_channels
def test_post_list_patch_get_delete_image(channel):
    stub = service_pb2_grpc.V2Stub(channel)

    post_response = stub.PostInputs(
        service_pb2.PostInputsRequest(
            inputs=[
                resources_pb2.Input(
                    data=resources_pb2.Data(
                        image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True),
                        concepts=[resources_pb2.Concept(id="some-concept")],
                    )
                )
            ]
        ),
        metadata=metadata(),
    )
    raise_on_failure(post_response)
    input_id = post_response.inputs[0].id

    try:
        wait_for_inputs_upload(stub, metadata(), [input_id])

        list_response = stub.ListInputs(
            service_pb2.ListInputsRequest(per_page=1), metadata=metadata()
        )
        raise_on_failure(list_response)
        assert len(list_response.inputs) == 1

        # Most likely we don"t have that many inputs, so this should return 0.
        list_response2 = stub.ListInputs(
            service_pb2.ListInputsRequest(per_page=500, page=1000), metadata=metadata()
        )
        raise_on_failure(list_response2)
        assert len(list_response2.inputs) == 0

        patch_response = stub.PatchInputs(
            service_pb2.PatchInputsRequest(
                action="overwrite",
                inputs=[
                    resources_pb2.Input(
                        id=input_id,
                        data=resources_pb2.Data(
                            concepts=[resources_pb2.Concept(id="some-new-concept")]
                        ),
                    )
                ],
            ),
            metadata=metadata(),
        )
        raise_on_failure(patch_response)

        get_response = stub.GetInput(
            service_pb2.GetInputRequest(input_id=input_id), metadata=metadata()
        )
        raise_on_failure(get_response)
        assert get_response.input.data.concepts[0].name == "some-new-concept"
    finally:
        cleanup_inputs(stub, [input_id], metadata=metadata())

@both_channels
def test_post_delete_batch_images(channel):
    stub = service_pb2_grpc.V2Stub(channel)

    post_response = stub.PostInputs(
        service_pb2.PostInputsRequest(
            inputs=[
                resources_pb2.Input(
                    data=resources_pb2.Data(
                        image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True)
                    )
                ),
                resources_pb2.Input(
                    data=resources_pb2.Data(
                        image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True)
                    )
                ),
            ]
        ),
        metadata=metadata(),
    )
    raise_on_failure(post_response)
    input_id1 = post_response.inputs[0].id
    input_id2 = post_response.inputs[1].id

    wait_for_inputs_upload(stub, metadata(), [input_id1, input_id2])

    cleanup_inputs(stub, [input_id1, input_id2], metadata=metadata())

@both_channels
def test_post_patch_get_image_with_id_concepts_geo_and_metadata(channel):
    stub = service_pb2_grpc.V2Stub(channel)

    input_id = uuid.uuid4().hex[:15]

    input_metadata = Struct()
    input_metadata.update(
        {
            "key1": 123,
            "key2": {
                "inner-key1": "inner-val1",
                "inner-key2": "inner-val2",
            },
        }
    )

    post_response = stub.PostInputs(
        service_pb2.PostInputsRequest(
            inputs=[
                resources_pb2.Input(
                    id=input_id,
                    data=resources_pb2.Data(
                        image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True),
                        concepts=[
                            resources_pb2.Concept(id="some-positive-concept", value=1),
                            resources_pb2.Concept(id="some-negative-concept", value=0),
                        ],
                        geo=resources_pb2.Geo(
                            geo_point=resources_pb2.GeoPoint(longitude=55.0, latitude=66),
                        ),
                        metadata=input_metadata,
                    ),
                )
            ]
        ),
        metadata=metadata(),
    )
    raise_on_failure(post_response)

    wait_for_inputs_upload(stub, metadata(), [input_id])

    get_response = stub.GetInput(
        service_pb2.GetInputRequest(input_id=input_id), metadata=metadata()
    )
    raise_on_failure(get_response)

    inp = get_response.input
    assert inp.id == input_id
    assert inp.data.concepts[0].id == "some-positive-concept"
    assert inp.data.concepts[0].value == 1.0
    assert inp.data.concepts[1].id == "some-negative-concept"
    assert inp.data.concepts[1].value == 0.0
    assert inp.data.metadata["key1"] == 123
    assert inp.data.metadata["key2"]["inner-key1"] == "inner-val1"
    assert inp.data.metadata["key2"]["inner-key2"] == "inner-val2"
    assert inp.data.geo.geo_point.longitude == 55.0
    assert inp.data.geo.geo_point.latitude == 66.0

    new_metadata = Struct()
    new_metadata.update({"new-key": "new-value"})
    patch_response = stub.PatchInputs(
        service_pb2.PatchInputsRequest(
            action="merge",
            inputs=[
                resources_pb2.Input(
                    id=input_id,
                    data=resources_pb2.Data(
                        concepts=[resources_pb2.Concept(id="another-positive-concept", value=1)],
                        geo=resources_pb2.Geo(
                            geo_point=resources_pb2.GeoPoint(longitude=77.0, latitude=88.0)
                        ),
                        metadata=new_metadata,
                    ),
                )
            ],
        ),
        metadata=metadata(),
    )
    raise_on_failure(patch_response)
    inp = patch_response.inputs[0]
    assert inp.data.concepts[2].id == "another-positive-concept"
    assert inp.data.geo.geo_point.longitude == 77.0
    assert inp.data.geo.geo_point.latitude == 88.0
    assert inp.data.metadata["new-key"] == "new-value"
    assert (
        inp.data.metadata["key1"] == 123
    )  # Since we use the merge action, the old values should remain

    cleanup_inputs(stub, [input_id], metadata=metadata())

@both_channels
def test_image_with_bytes(channel):
    stub = service_pb2_grpc.V2Stub(channel)

    with open(RED_TRUCK_IMAGE_FILE_PATH, "rb") as f:
        file_bytes = f.read()

    post_response = stub.PostInputs(
        service_pb2.PostInputsRequest(
            inputs=[
                resources_pb2.Input(
                    data=resources_pb2.Data(image=resources_pb2.Image(base64=file_bytes))
                )
            ]
        ),
        metadata=metadata(),
    )
    raise_on_failure(post_response)
    input_id = post_response.inputs[0].id

    wait_for_inputs_upload(stub, metadata(), [input_id])
    cleanup_inputs(stub, [input_id], metadata=metadata())


import torch
import torch.nn.functional as F
import hyperparameters as HP
import torch.nn as nn

class TargetLoss(torch.nn.Module):
    def __init__(self, T=0.5):
        super().__init__()
        self.T = T
        self.g = nn.Sequential(nn.Linear(2048, 512, bias=False),
                               nn.BatchNorm1d(512),
                               nn.ReLU(inplace=True),
                               nn.Linear(512, 128, bias=True))
    def forward(self, x, y, target):
        '''
        x:tensor, with size (batch_size, emb_len)
        y:tensor, with size (batch_size), is the label of x
        target:tensor, with size (cls_num, emb_len)
        '''
        N = y.size(0)
        y = y.cuda()
        x = x.cuda()
        target = target.cuda()
        index = y.view(-1,1)
        index = index.cuda()
        one_hot = torch.zeros(y.size(0), target.size(0)).cuda().scatter_(1, index, 1) # (bz, cls) one-hot
        one_hot = one_hot.cuda()

        if not HP.attention:
            x = self.g(x)
        similarity_matrix = torch.mm(x, target.T) # (bz, cls)
        similarity_matrix = similarity_matrix.cuda()
        similarity_matrix = torch.div(similarity_matrix, self.T) # (bz, cls)
        similarity_matrix = torch.exp(similarity_matrix) # (bz, cls)

        target_similarity = torch.mul(similarity_matrix, one_hot) # (bz, cls) one hot with sim value
        target_similarity = target_similarity.cuda()
        target_similarity = torch.sum(target_similarity, dim=1) # (bz,1) target similarity of each data in the batch

        total_similarity = torch.sum(similarity_matrix, dim=1) # (bz,1)
        total_similarity = total_similarity.cuda()

        loss = torch.div(target_similarity, total_similarity) # (bz,1)
        loss = torch.log(loss)
        loss = torch.sum(loss,dim=0) # (1,1) the total loss of the batch

        loss = (-loss)/N

        return loss

#loss_func = ContrastiveLoss()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from setuptools import setup,find_packages
def calculate_version():
    initpy = open('DHCdatacleaner/_version.py').read().split('\n')
    version = list(filter(lambda x: '__version__' in x, initpy))[0].split('\'')[1]
    return version

package_version = calculate_version()

setup(
    name='DHCdatacleaner',
    version=package_version,
    author='Eddie Huang',
    author_email='huangyi@dhcc.com.cn',
    packages=find_packages(),
    url='',
    license='',
    entry_points={'console_scripts': ['DHCdatacleaner=DHCdatacleaner:main', ]},
    description=('DHC Python tool that automatically cleans data sets and readies them for analysis.'),
    long_description='''
A Python tool that automatically cleans data sets and readies them for analysis.
''',
    zip_safe=True,
    install_requires=['pandas', 'scikit-learn', 'numpy'],
    classifiers=[
        'Intended Audience :: Developers',
        'Intended Audience :: Information Technology',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Topic :: Utilities'
    ],
    keywords=['data cleaning', 'csv', 'machine learning', 'data analysis', 'data engineering'],
)


from collections import defaultdict
from dataclasses import dataclass
from typing import Tuple, Dict, List

import numpy as np
import pandas as pd

PlayerStats = np.ndarray
GameStats = np.ndarray

@dataclass
class WinLossHistory:
    wins: Dict
    losses: Dict
    N: int

    @staticmethod
    def create_from_dataframe(game_df: pd.DataFrame, N: int):
        K = len(game_df)
        wins = defaultdict(list)
        losses = defaultdict(list)
        for i in range(K):
            winner_idx = int(game_df['winner'].loc[i]) - 1
            losser_idx = int(game_df['losser'].loc[i]) - 1
            wins[winner_idx].append(i)
            losses[losser_idx].append(i)
        return WinLossHistory(wins, losses, N)

    @property
    def player_ids(self):
        return list(range(self.N))

class PlayerStatsFactory:
    """
    Let N be the number of players
    """

    @staticmethod
    def create_from_csv(path, sep=',', name_col='name') -> Tuple[PlayerStats, List, int]:
        player_df = pd.read_csv(path, sep=sep)
        player_names = list(set(player_df[name_col].values))
        N = len(player_names)
        player_stats = np.zeros((N, 3, 2), dtype=np.float32)
        player_stats[:, 1, 0] = np.zeros((N,), dtype=np.float32)
        player_stats[:, 1, 1] = np.ones((N,), dtype=np.float32)
        return player_stats, player_names, N

    @staticmethod
    def create_from_dataframe(player_df: pd.DataFrame, name_col='name'):
        player_names = list(set(player_df[name_col].values))
        N = len(player_names)
        player_stats = np.zeros((N, 3, 2), dtype=np.float32)
        player_stats[:, 1, 0] = np.zeros((N,), dtype=np.float32)
        player_stats[:, 1, 1] = np.ones((N,), dtype=np.float32)
        return player_stats, player_names, N

class GameStatsFactory:
    """
    Let K be the number of Games
    """

    @staticmethod
    def create_from_csv(path, N: int, sep=',') -> Tuple[GameStats, WinLossHistory]:
        game_df = pd.read_csv(path, sep=sep)
        K = len(game_df)
        history = WinLossHistory.create_from_dataframe(game_df, N)
        game_stats = np.zeros((K, 9, 2), dtype=np.float32)
        game_stats[:, -2:, 0] = np.zeros((K, 2,), dtype=np.float32)
        game_stats[:, -2:, 1] = np.zeros((K, 2,), dtype=np.float32)
        return game_stats, history

    @staticmethod
    def create_from_dataframe(game_df: pd.DataFrame, N: int) -> Tuple[GameStats, WinLossHistory]:
        K = len(game_df)
        history = WinLossHistory.create_from_dataframe(game_df, N)
        game_stats = np.zeros((K, 9, 2), dtype=np.float32)
        game_stats[:, -2:, 0] = np.zeros((K, 2,), dtype=np.float32)
        game_stats[:, -2:, 1] = np.zeros((K, 2,), dtype=np.float32)
        return game_stats, history


import argparse
import os
import glob
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import metrics
import ctaplot
import uproot

def main():

    parser = argparse.ArgumentParser(
        description=("plotROC - Python ctaplot-based program to plot ROC curves from MARS melibea (under development) and CTLearn files."))
    parser.add_argument('--input_dir', '-i',
                        help='input directory',
                        default="./")
    parser.add_argument('--pattern', '-p',
                        help='pattern to mask unwanted files',
                        default="*")
    parser.add_argument('--input_offdata', '-j',
                        help='input directory for the off data (melibea)',
                        default=None)
    parser.add_argument('--pattern_offdata', '-q',
                        help='pattern to mask unwanted files for the off data input directory',
                        default="*")
    parser.add_argument('--output_dir', '-o',
                        help='path where to save generated files. By default, the current directory is used.',
                        default="./")

    args = parser.parse_args()

    # Input handling
    abs_file_dir = os.path.abspath(args.input_dir)
    files = glob.glob(os.path.join(abs_file_dir, args.pattern))
    if args.input_offdata:
        abs_file_dir_off = os.path.abspath(args.input_offdata)
        off_files = glob.glob(os.path.join(abs_file_dir_off, args.pattern_offdata))

    fig, ax = plt.subplots(1,figsize=(12,12))

    # color cycle of matplotlib
    color_cycle = plt.rcParams['axes.prop_cycle'].by_key()['color']
    color_cycle = ["orange", "royalblue"]
    naming = ["CTLearn - cleaned images", "CTLearn - raw images"]

    color_counter = 0
    for file in files:
        if file.split('.')[-1] == "h5":
            color_counter += 1
            color_counter %= len(color_cycle)
            data = pd.HDFStore(file, 'r')
            linestyle = {'proton': 'dashed', 'off': 'solid'}
            for bkg in ['proton', 'off']:
                bkg_mc_particle = np.array(data[bkg]['mc_particle']).astype(int)
                bkg_reco_gammaness = np.array(data[bkg]['reco_gammaness']).astype(float)

                gamma_mc_particle = np.array(data['gamma']['mc_particle']).astype(int)
                gamma_reco_gammaness = np.array(data['gamma']['reco_gammaness']).astype(float)

                # Concatenate the background with the gammas (ringwobble)
                mc_particle = np.concatenate((bkg_mc_particle, gamma_mc_particle))
                reco_gammaness = np.concatenate((bkg_reco_gammaness, gamma_reco_gammaness))

                fpr, tpr, thresholds = metrics.roc_curve(mc_particle, reco_gammaness)
                particle_type = "MCs proton" if bkg == 'proton' else "off data"
                ax.plot(fpr, tpr, label="{} (vs {}), AUC={:0.3f}".format(naming[color_counter], particle_type, metrics.auc(fpr, tpr)), color=color_cycle[color_counter], linestyle=linestyle[bkg], linewidth=1.5)
                #ax = ctaplot.plot_roc_curve(mc_particle, reco_gammaness, label="{} (vs {}), AUC={:0.3f}".format(naming[color_counter], particle_type, metrics.auc(fpr, tpr)), color=color_cycle[color_counter], linestyle=linestyle[bkg], linewidth=1.5)

        elif file.split('.')[-1] == "root":
            melibea_file = uproot.open(file)
            evts = melibea_file["Events"]
            if b'MMcEvt_1.' in evts.keys():
                if int(evts["MMcEvt_1.fPartId"].array()[0]) == 1:
                    gamma_mc_particle_root = np.asarray(evts["MMcEvt_1.fPartId"].array())
                    gamma_reco_gammaness_root = 1.0 - np.asarray(evts["MHadronness.fHadronness"].array())

    for file in files:
        if file.split('.')[-1] == "root":
            melibea_file = uproot.open(file)
            evts = melibea_file["Events"]
            if b'MMcEvt_1.' not in evts.keys(): continue
            if int(evts["MMcEvt_1.fPartId"].array()[0]) == 1: continue

            bkg_reco_gammaness = 1.0 - np.asarray(evts["MHadronness.fHadronness"].array())
            bkg_mc_particle = np.zeros(len(bkg_reco_gammaness))

            # Concatenate the background with the gammas (ringwobble) from MARS
            mc_particle = np.concatenate((bkg_mc_particle, gamma_mc_particle_root))
            reco_gammaness = np.concatenate((bkg_reco_gammaness, gamma_reco_gammaness_root))

            fpr, tpr, thresholds = metrics.roc_curve(mc_particle, reco_gammaness)
            ax.plot(fpr, tpr, label="MARS - standard RF (vs MCs proton), AUC={:0.3f}".format(metrics.auc(fpr, tpr)), color='black', linestyle="dashed", linewidth=1.5)
            #ax = ctaplot.plot_roc_curve(mc_particle, reco_gammaness, label="MARS - standard RF (vs MCs proton), AUC={:0.3f}".format(metrics.auc(fpr, tpr)), color='black', linestyle="dashed", linewidth=1.5)

    if args.input_offdata:
        bkg_mc_particle = []
        bkg_reco_gammaness = []
        for file in off_files:
            if file.split('.')[-1] == "root":
                melibea_file = uproot.open(file)
                evts = melibea_file["Events"]
                if b'MMcEvt_1.' in evts.keys(): continue
                file_reco_gammaness = 1.0 - np.asarray(evts["MHadronness.fHadronness"].array())
                file_mc_particle = np.zeros(len(file_reco_gammaness))
                # Concatenate the background with the gammas (ringwobble)
                bkg_mc_particle = np.concatenate((bkg_mc_particle, file_mc_particle))
                bkg_reco_gammaness = np.concatenate((bkg_reco_gammaness, file_reco_gammaness))

        # Concatenate the background with the gammas (ringwobble) from MARS
        mc_particle = np.concatenate((bkg_mc_particle, gamma_mc_particle_root))
        reco_gammaness = np.concatenate((bkg_reco_gammaness, gamma_reco_gammaness_root))

        fpr, tpr, thresholds = metrics.roc_curve(mc_particle, reco_gammaness)
        ax.plot(fpr, tpr, label="MARS - standard RF (vs off data), AUC={:0.3f}".format(metrics.auc(fpr, tpr)), color='black', linestyle="solid", linewidth=1.5)
        #ax = ctaplot.plot_roc_curve(mc_particle, reco_gammaness, label="MARS - standard RF (vs off data), AUC={:0.3f}".format(metrics.auc(fpr, tpr)), color='black', linestyle="solid", linewidth=1.5)

    ax.set_xlabel('False positive rate', fontsize=34)
    ax.set_xbound(0.0,1.0)
    ax.set_xticks([0.0,1.0])

    ax.set_ylabel('True positive rate', fontsize=34)
    ax.set_ybound(0.0,1.0)
    ax.set_yticks([0.0,1.0])

    ax.plot([0, 1], [0, 1], '--', color='grey')
    #major grid lines
    ax.tick_params(labelsize=30)
    #plt.grid(b=True, which='major', color='gray', alpha=0.3, linestyle='dashdot', lw=1.5)

    ax.text(0.5, 0.5, 'Preliminary', fontsize=58, alpha=0.5, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes)

    ax.legend(loc='lower right',fontsize=22)
    ax.set_title("ROC curves",fontsize=40)
    plt.tight_layout()

    plt.savefig(f"{args.output_dir}/roc_curves.pdf", dpi=600)

if __name__ == "__main__":
    main()


####################################################################################################
# __init__.py

'''Tools for analyzing and registering cortical meshes.'''

submodules = ('neuropythy.util.conf',
              'neuropythy.util.core',
              'neuropythy.util.filemap',
              'neuropythy.util.labels',
              'neuropythy.util',
              'neuropythy.math.core',
              'neuropythy.math',
              'neuropythy.java',
              'neuropythy.io.core',
              'neuropythy.io',
              'neuropythy.geometry.util',
              'neuropythy.geometry.mesh',
              'neuropythy.geometry',
              'neuropythy.optimize.core',
              'neuropythy.optimize',
              'neuropythy.mri.core',
              'neuropythy.mri.images',
              'neuropythy.mri',
              'neuropythy.freesurfer.core',
              'neuropythy.freesurfer',
              'neuropythy.hcp.files',
              'neuropythy.hcp.core',
              'neuropythy.hcp',
              'neuropythy.registration.core',
              'neuropythy.registration',
              'neuropythy.vision.models',
              'neuropythy.vision.retinotopy',
              'neuropythy.vision.cmag',
              'neuropythy.vision',
              'neuropythy.graphics.core',
              'neuropythy.graphics',
              'neuropythy.datasets.core',
              'neuropythy.datasets.benson_winawer_2018',
              'neuropythy.datasets.hcp',
              'neuropythy.datasets.visual_performance_fields',
              'neuropythy.datasets.hcp_lines',
              'neuropythy.datasets',
              'neuropythy.plans.core',
              'neuropythy.plans.prfclean',
              'neuropythy.plans',
              'neuropythy.commands.surface_to_ribbon',
              'neuropythy.commands.benson14_retinotopy',
              'neuropythy.commands.register_retinotopy',
              'neuropythy.commands.atlas',
              'neuropythy.commands')
'''neuropythy.submodules is a tuple of all the sub-modules of neuropythy in a loadable order.'''

def reload_neuropythy():
    '''
    reload_neuropythy() reloads all of the modules of neuropythy and returns the reloaded
    neuropythy module. This is similar to reload(neuropythy) except that it reloads all the
    neuropythy submodules prior to reloading neuropythy.

    Example:
      import neuropythy as ny
      # ... some nonsense that breaks the library ...
      ny = ny.reload_neuropythy()
    '''
    import sys, six
    if not six.PY2:
        try:              from importlib import reload
        except Exception: from imp import reload
    for mdl in submodules:
        if mdl in sys.modules:
            sys.modules[mdl] = reload(sys.modules[mdl])
    return reload(sys.modules['neuropythy'])

from   .util       import (config, is_image, library_path, to_affine, is_address, address_data,
                           is_curve_spline, to_curve_spline, curve_spline, flattest,
                           is_list, is_tuple, to_hemi_str, is_dataframe, to_dataframe, auto_dict,
                           label_index, is_label_index, to_label_index)
from   .util       import label_indices as labels
from   .io         import (load, save, to_nifti)
from   .mri        import (is_subject, is_cortex, to_cortex, to_image, to_image_spec,
                           is_image_spec, image_interpolate, image_apply, image_copy, image_clear,
                           is_pimage, to_image_type)
from   .vision     import (retinotopy_data, empirical_retinotopy_data, predicted_retinotopy_data,
                           register_retinotopy, retinotopy_anchors, retinotopy_model,
                           neighborhood_cortical_magnification, as_retinotopy,
                           retinotopy_comparison, to_logeccen, from_logeccen)
from   .geometry   import (mesh, tess, topo, map_projection, path_trace,
                           is_vset, is_mesh, is_tess, is_topo, is_flatmap, paths_to_labels,
                           is_map_projection, is_path, is_path_trace, close_path_traces,
                           to_mesh, to_tess, to_property, to_mask, to_flatmap, to_map_projection,
                           isolines, map_projections)
from   .freesurfer import (subject as freesurfer_subject, to_mgh)
from   .hcp        import (subject as hcp_subject)
from   .datasets   import data
from . import util
from . import math
from . import freesurfer
from . import hcp
from . import plans

# things we might want to load but that might fail
try:
    from .graphics import cortex_plot
    from .         import graphics
except Exception: pass

# Version information...
__version__ = '0.12.13'


import tensorflow as tf
import json
import numpy as np
import cv2
from keras.utils.np_utils import to_categorical
import os, random

class DataHandler:
    def __init__(self, train_json_path, test_json_path, data_path):
        realPath = os.path.realpath(__file__)
        subStr = realPath[:realPath[: realPath[: realPath.rindex('/')].rindex('/')].rindex('/')]
        self.subStr = subStr
        self.train_json_path = train_json_path
        self.test_json_path = test_json_path
        self.data_path = data_path

    def readJson(self, jsonFilePath):
        with open(jsonFilePath, 'r') as rf:
            loadDict = json.load(rf)
            #print(loadDict)
            return loadDict

    def writeJson(self, jsonFilePath, newDict):
        with open(jsonFilePath, 'w') as wf:
            json.dump(newDict, wf)
            print("dump json is over!")

    def prepareImage(self, img, augment=True):
        if augment:
            self.agument(img)
        return img

    def agument(self, img):
        ran = random.random()
        randint = random.randint(0, 2)
        randBri = random.randint(0, 2) / 10
        randCentralCrop = random.randint(6, 8) / 10
        randHue = random.randint(0, 3) / 10  # [0, 0.5]
        # if ran < 0.2:
        #     img = tf.image.flip_left_right(img)
        # ran = random.random()
        # if ran < 0.2:
        #     img = tf.image.flip_up_down(img)
        # ran = random.random()
        if ran < 0.1:
            img = tf.image.adjust_saturation(img, randint)
        ran = random.random()
        if ran < 0.1:
            img = tf.image.adjust_brightness(img, randBri)
        ran = random.random()
        # if ran < 0.2:
        #     img = tf.image.rot90(img)
        #     img = tf.image.rot90(img)
        #     img = tf.image.rot90(img)
        # ran = random.random()
        if ran < 1.0:
            img = tf.image.central_crop(img, central_fraction=randCentralCrop)
        ran = random.random()
        if ran < 0.1:
            img = tf.image.random_hue(img, randHue)
        ran = random.random()
        if ran < 0.1:
            img = tf.image.random_contrast(img, lower=0.5, upper=1.7)
        return img

    def dataGenerator(self, imgLines, batchSize, numClasses):
        n = len(imgLines)
        i = 0
        while True:
            batchImgs = []
            batchLabels = []
            for _ in range(batchSize):
                framesImg5 = []
                if i == 0:
                    np.random.shuffle(imgLines)
                id = imgLines[i]['id']
                keyFrame = imgLines[i]['key_frame']
                keyIndex = 0
                status = imgLines[i]['status']
                # status = tf.one_hot(status, numClasses)
                status = to_categorical(status, num_classes=numClasses)
                # status = tf.expand_dims(status, 0)
                frames = imgLines[i]['frames']
                imgPath = self.data_path + id + '/'
                for index, frame in enumerate(frames):
                    imgName = frame['frame_name']
                    imgData = cv2.imread(imgPath + imgName)
                    # imgData = self.prepareImage(imgData)
                    imgData = np.array(imgData) / 255.
                    imgData = cv2.resize(imgData, (224, 224))
                    if keyFrame == imgName:
                        keyIndex = index
                    framesImg5.append(imgData)
                for _ in range(5 - len(framesImg5)):
                    framesImg5.append(framesImg5[keyIndex])
                i = (i + 1) % n
                batchImgs.append(framesImg5)
                batchLabels.append(status)
            yield (tf.convert_to_tensor(batchImgs), tf.convert_to_tensor(batchLabels))

    def dataPredict(self, imgLines, batchSize):
        n = len(imgLines)
        i = 0
        while True:
            batchImgs = []
            for _ in range(batchSize):
                framesImg5 = []
                id = imgLines[i]['id']
                keyFrame = imgLines[i]['key_frame']
                keyIndex = 0
                frames = imgLines[i]['frames']
                imgPath = self.data_path + id + '/'
                for index, frame in enumerate(frames):
                    imgName = frame['frame_name']
                    imgData = cv2.imread(imgPath + imgName)
                    imgData = tf.image.central_crop(imgData, central_fraction=0.7)
                    imgData = np.array(imgData) / 255.
                    imgData = cv2.resize(imgData, (224, 224))
                    if keyFrame == imgName:
                        keyIndex = index
                    framesImg5.append(imgData)
                for _ in range(5 - len(framesImg5)):
                    framesImg5.append(framesImg5[keyIndex])
                i = (i + 1) % n
                batchImgs.append(framesImg5)
            yield tf.convert_to_tensor(batchImgs)

    def dataAugment(self, path):
        # readJson = self.readJson(self.train_json_path)
        readJson = self.readJson(path)
        items = readJson['annotations']
        label0Items = []
        label1Items = []
        label2Items = []
        for index, item in enumerate(items):
            status = int(item['status'])
            if status == 0:
                label0Items.append(item)
            elif status == 1:
                label1Items.append(item)
            elif status == 2:
                label2Items.append(item)
            else:
                print(item)
        label0 = len(label0Items)
        label1 = len(label1Items)
        label2 = len(label2Items)
        print("label0:", label0, "label1:", label1, "label2:", label2)
        total = label0 + label1 + label2
        print(total)
        print(label0 / total, label1 / total, label2 / total)
        npLabel = np.array([label0, label1, label2])
        maxLabel = npLabel.argmax()
        maxNum = npLabel.max()
        print("maxLabel:", maxLabel, "maxNum:", maxNum)
        for i in range(3):
            if i == 0:
                label = label0
                labelItems = label0Items
            elif i == 1:
                label = label1
                labelItems = label1Items
            else:
                label = label2
                labelItems = label2Items
            num = maxNum - label
            # num = 500
            if num == 0:
                continue
            sliceRet = []
            for ind in range(num):
                randint = random.randint(0, len(labelItems) - 1)
                sliceRet.append(labelItems[randint])
            newSliceList = []
            for imgs in sliceRet:
                id = imgs['id']
                frames = imgs['frames']
                keyFrame = imgs['key_frame']
                status = imgs['status']
                imgPath = self.data_path + id + '/'
                saveId = str((int(id) + 100000))
                imgSavePath = self.data_path + saveId + '/'
                newSlice = {
                    "id": saveId,
                    "key_frame": keyFrame,
                    "status": status,
                    "frames": []
                }
                while True:
                    if os.path.exists(imgSavePath):
                        getId = int(imgSavePath[:-1][imgSavePath[:-1].rfind('/') + 1:])
                        newId = getId + 100000
                        imgSavePath = self.data_path + str(newId) + '/'
                    else:
                        break
                os.mkdir(imgSavePath)
                for img in frames:
                    imgName = img['frame_name']
                    gpsTime = img['gps_time']
                    imgData = cv2.imread(imgPath + imgName)
                    newImgData = self.agument(imgData)
                    cv2.imwrite(imgSavePath + imgName, np.array(newImgData))
                    newFrame = {
                        "frame_name": imgName,
                        "gps_time": gpsTime
                    }
                    newSlice["frames"].append(newFrame)
                newSliceList.append(newSlice)
            items.extend(newSliceList)
        with open(self.subStr + '/data/amap_traffic_augment.json', 'w') as wf:
            json.dump(readJson, wf)

    def framesCount(self, path):
        # readJson = self.readJson(self.train_json_path)
        readJson = self.readJson(path)
        items = readJson['annotations']
        count = {}
        for index, item in enumerate(items):
            framesLen = len(item['frames'])
            if count.get(framesLen) != None:
                count[framesLen] += 1
            else:
                count[framesLen] = 1
        print("count:", count)

    def framesStatus(self, path):
        # readJson = self.readJson(self.train_json_path)
        readJson = self.readJson(path)
        items = readJson['annotations']
        count = {}
        for index, item in enumerate(items):
            framesLen = len(item['frames'])
            status = item['key_frame']
            key = str(status) + '-' + str(framesLen)
            if count.get(key) != None:
                count[key] += 1
            else:
                count[key] = 1
        print("key:", count)

    def dataGeneratorV2(self, imgLines, batchSize, numClasses):
        n = len(imgLines)
        i = 0
        while True:
            batchImgs = []
            batchLabels = []
            for _ in range(batchSize):
                framesImg5 = []
                if i == 0:
                    np.random.shuffle(imgLines)
                id = imgLines[i]['id']
                status = imgLines[i]['status']
                # status = tf.one_hot(status, numClasses)
                status = to_categorical(status, num_classes=numClasses)
                # status = tf.expand_dims(status, 0)
                frames = imgLines[i]['frames']
                imgPath = self.data_path + id + '/'
                for index, frame in enumerate(frames):
                    imgName = frame['frame_name']
                    imgData = cv2.imread(imgPath + imgName)
                    # imgData = self.prepareImage(imgData)
                    imgData = np.array(imgData) / 255.
                    imgData = cv2.resize(imgData, (224, 224))
                    framesImg5.append(imgData)
                for _ in range(5 - len(framesImg5)):
                    framesImg5.append(self.agument(framesImg5[len(frames) - 1]))
                i = (i + 1) % n
                batchImgs.append(framesImg5)
                batchLabels.append(status)
            yield (tf.convert_to_tensor(batchImgs), tf.convert_to_tensor(batchLabels))

if __name__ == '__main__':
    realPath = os.path.realpath(__file__)
    subStr = realPath[:realPath[: realPath[: realPath.rindex('/')].rindex('/')].rindex('/')]
    train_json_path = subStr + '/data/amap_traffic_annotations_train.json'
    # train_json_path = subStr + '/data/amap_traffic_augment.json'
    test_json_path = subStr + '/data/amap_traffic_annotations_test.json'
    test_json_path = subStr + '/data/amap_traffic_annotations_test_answer.json'
    data_path = subStr + '/data/amap_traffic_train_0712/'
    data_test_path = subStr + '/data/amap_traffic_test_0712/'

    # generate train data
    handler = DataHandler(train_json_path, test_json_path, data_path)
    readJson = handler.readJson(handler.train_json_path)
    print(readJson)
    handler.dataAugment()

    handler.framesCount(handler.test_json_path)
    handler.framesStatus(handler.test_json_path)


import os
from datetime import timedelta
import urllib3
from azure.monitor.query import MetricsQueryClient, MetricAggregationType
from azure.identity import DefaultAzureCredential

# provide a cert or disable warnings to run this sample
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# [START metrics_client_auth_with_token_cred]
credential  = DefaultAzureCredential()

client = MetricsQueryClient(credential)
# [END metrics_client_auth_with_token_cred]

# [START send_metrics_query]
metrics_uri = os.environ.get('METRICS_RESOURCE_URI')
response = client.query_resource(
    metrics_uri,
    metric_names=["Ingress"],
    timespan=timedelta(hours=2),
    granularity=timedelta(minutes=5),
    aggregations=[MetricAggregationType.AVERAGE],
    )

for metric in response.metrics:
    print(metric.name + ' -- ' + metric.display_description)
    for time_series_element in metric.timeseries:
        for metric_value in time_series_element.data:
            print('The ingress at {} is {}'.format(
                metric_value.timestamp,
                metric_value.average
            ))
# [END send_metrics_query]


import pandas as pd
import geopandas as gpd
import matplotlib.pyplot as plt

def extract_html_table(row):
    html = row["Description"]

    df = pd.read_html(html, index_col=0)[0]

    columns = pd.Series({
        "Street": df.loc["NAME"][0],
        "Folderpath": df.loc["FOLDERPATH"][0]
    })

    return pd.concat([
        row, columns
    ])

def plot_geojson(df, title, figsize=(5, 5)):
    fig, ax = plt.subplots(figsize=figsize)
    df["geometry"].plot(ax=ax)
    plt.title(title)
    plt.show()

def get_coords_by_code(coordinates, code):
    search = coordinates.copy()
    search = search[search["station_code"].notna()]
    result = search[search["station_code"] == code].iloc[0]
    return [
        result["latitude"],
        result["longitude"],
    ]


from dotenv import load_dotenv
from pydantic_settings import BaseSettings

load_dotenv()

class Settings(BaseSettings):
    USER_SERVICE_BASE_URL: str
    USER_SERVICE_PORT: int

    class Config:
        case_sensitive = False
        env_file = ".env"
        env_file_encoding = "utf-8"

settings: Settings = Settings()


# Estimated win rate class based on the pseudocode provided in the AlphaStar paper [1].
#
# [1] Vinyals, Oriol, et al. "Grandmaster level in StarCraft II using multi-agent reinforcement learning." Nature 575.7782 (2019): 350-354.
import numpy as np
from collections import defaultdict

class PayOff:
    """以下のAlphaStarの論文にて提供されている疑似コードに基づき実装した推定勝率のクラス。
    Vinyals, Oriol, et al. "Grandmaster level in StarCraft II using multi-agent reinforcement learning." Nature 575.7782 (2019): 350-354.
    """
    def __init__(self):
        self._games=defaultdict(lambda:0)
        self._wins=defaultdict(lambda:0)
        self._draws=defaultdict(lambda:0)
        self._losses=defaultdict(lambda:0)
        self._decay=0.99
    def _win_rate(self,home,away):
        if(self._games[home,away]==0):
            return 0.5
        return (self._wins[home,away]+0.5*self._draws[home,away])/self._games[home,away]
    def __getitem__(self,match):
        home,away=match
        if(isinstance(home,str)):
            home=[home]
        if(isinstance(away,str)):
            away=[away]
        win_rates=np.array([[self._win_rate(h,a) for a in away] for h in home])
        if(win_rates.shape[0]==1 or win_rates.shape[1]==1):
            win_rates=win_rates.reshape(-1)
        return win_rates
    def update(self,home,away,result):
        for stats in (self._games,self._wins,self._draws,self._losses):
            stats[home,away]*=self._decay
            stats[away,home]*=self._decay
        self._games[home,away]+=1
        self._games[away,home]+=1
        if(result=="win"):
            self._wins[home,away]+=1
            self._losses[away,home]+=1
        elif(result=="draw"):
            self._draws[home,away]+=1
            self._draws[away,home]+=1
        else:
            self._wins[away,home]+=1
            self._losses[home,away]+=1


#!/usr/bin/env python
# coding: utf-8

# In[1]:

#
#
# =====================
# Training a Classifier
# =====================
#
#

import time, os, copy, math, numpy as np

import torch, torchvision
import torch.multiprocessing
torch.multiprocessing.set_sharing_strategy('file_system')
import torch.nn as nn
from torch.nn import Parameter, init
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.autograd import Variable
import torchvision.models as models
from torch.utils.data import Dataset, DataLoader
import sys
import pickle
from collections import defaultdict
import random

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

print(device)
### Get object features, distances and frame labels
from PIL import Image
object_classes = {'bottle': 0, 'bowl': 1, 'cheese': 2, 'cucumber': 3, 'knife': 4, 'lettuce': 5,
                  'peeler': 6, 'spoon': 7, 'tomato': 8, 'hand': 9}

class FrameFeatures(Dataset):
    features = None
    def __init__(self, object_feature_dir, dt_feature_dir, i3d_feature_dir, sequence_ids, test=False, transform=None):
        self.feature_sequences = []
        self.label_sequences = []
        self.dt_sequences = []
        self.i3d_sequences = []
        self.action_classes = {'SIL':0,'cut_fruit':1,'put_fruit2bowl':2,'peel_fruit':3,'stir_fruit':4,
                               'crack_egg':5,'add_saltnpepper':6,'stir_egg':7,'pour_oil':8,
                               'pour_egg2pan':9,'stirfry_egg':10,'take_plate':11,'put_egg2plate':12,
                               'pour_coffee':13,'pour_milk':14,'spoon_powder':15,'stir_milk':16,
                               'pour_cereals':17,'stir_cereals':18,'pour_flour':19,'stir_dough':20,
                               'pour_dough2pan':21,'fry_pancake':22,'put_pancake2plate':23,
                               'add_teabag':24,'pour_water':25,'cut_orange':26,'squeeze_orange':27,
                               'take_glass':28,'pour_juice':29,'fry_egg':30,'cut_bun':31,
                               'take_butter':32,'smear_butter':33,'put_toppingOnTop':34,
                               'put_bunTogether':35,'spoon_flour':36,'butter_pan':37,'take_eggs':38,
                               'take_cup':39,'pour_sugar':40,'stir_coffee':41,'take_bowl':42,
                               'take_knife':43,'spoon_sugar':44,'take_topping':45,'take_squeezer':46,
                               'stir_tea':47}
        self.transform = transform

        for sequence_id in sequence_ids:
            print(sequence_id)
            feature_sequences_video = []
            label_sequences_video = []
            dt_sequences_video = []
            i3d_sequences_video = []
            sequence_file = os.path.join(object_feature_dir, sequence_id.strip()+'.pkl')
            dt_sequence_file = os.path.join(dt_feature_dir, sequence_id.strip()+'.npy')
            i3d_sequence_file = os.path.join(i3d_feature_dir, sequence_id.strip()+'.npy')
            if os.path.exists(sequence_file) and os.path.exists(dt_sequence_file) \
                and os.path.exists(i3d_sequence_file):

                with open(sequence_file,'rb') as f1:
                    try:
                        sequence_dict = pickle.load(f1)
                        dt_matrix = np.load(dt_sequence_file)
                        i3d_feat = np.load(i3d_sequence_file)
                    except:
                        continue
                    prev_frame = sorted(list(sequence_dict.keys()))[0]
                    starts = []
                    stops = []
                    starts.append(prev_frame)
                    for frame in sorted(list(sequence_dict.keys())):
                        if sequence_dict[frame]['action'] != sequence_dict[prev_frame]['action']:
                            stops.append(prev_frame)
                            starts.append(frame)
                        prev_frame = frame
                    last_frame = min(dt_matrix.shape[0],len(i3d_feat),sorted(list(sequence_dict.keys()))[-1])
                    stops.append(last_frame)
                    feature_sequence = []
                    label_sequence = []
                    dt_sequence = []
                    i3d_sequence = []
                    sequence_count = 0

                    for start, stop in zip(starts,stops):
                        if stop - start < 30:
                            continue
                        for frame_num in range(start,stop-14):
                            frame_feat = sequence_dict[frame_num]
                            features_frame = frame_feat['features']
                            if features_frame:
                                features_frame = features_frame[0]
                            object_labels = frame_feat['labels']
                            if object_labels:
                                object_labels = object_labels[0]
                            #print(object_labels)
                            if len(features_frame) != 0 and 'person' in object_labels:
                                action_label = sequence_dict[frame_num]['action']
                                action_label = self.action_classes[action_label]
                                sequence_count += 1

                                dt_feature_frame = dt_matrix[frame_num-1,:]
                                i3d_feature_frame = i3d_feat[frame_num-1]
                                feature = self.__get_feature_pairs(features_frame, object_labels)
                                if len(feature) != 0 or dt_feature_frame.size != 0:
                                    feature_sequence.append(feature)
                                    label_sequence.append(action_label)
                                    dt_sequence.append(dt_feature_frame)
                                    i3d_sequence.append(i3d_feature_frame)
                        if len(dt_sequence) != 0 or len(feature_sequence) != 0 or len(feature_sequence) != 0:
                            # print(len(i3d_sequence))
                            # print(len(feature_sequence))
                            observation = min(15, len(dt_sequence), len(feature_sequence), len(feature_sequence))
                            feature_sequences_video.append(feature_sequence[-observation:])
                            label_sequences_video.append(label_sequence[-observation:])
                            dt_sequences_video.append(dt_sequence[-observation:])
                            i3d_sequences_video.append(i3d_sequence[-observation:])
                            feature_sequence = []
                            label_sequence = []
                            dt_sequence = []
                            i3d_sequence = []
                    feature_sequences_video = feature_sequences_video[:-1]
                    dt_sequences_video = dt_sequences_video[:-1]
                    i3d_sequences_video = i3d_sequences_video[:-1]
                    label_sequences_video = label_sequences_video[1:]
                    for feature_seq, label_seq, dt_seq, i3d_seq in\
                        zip(feature_sequences_video, label_sequences_video, \
                        dt_sequences_video, i3d_sequences_video):

                        self.feature_sequences.append(feature_seq)
                        self.label_sequences.append(label_seq)
                        self.dt_sequences.append(dt_seq)
                        self.i3d_sequences.append(i3d_seq)

    def __get_feature_pairs(self, features_frame, object_labels):
        feature_pairs_frame = []
        person_feature = features_frame[object_labels.index('person')]
        object_features = [x for i,x in enumerate(features_frame) if i!=object_labels.index('person')]
        #for person_feature in person_features:
            #print(len(person_feature[0]))
        for object_feature in object_features:
            #print(len(object_feature[0]))
            concat_features = np.concatenate((np.array(person_feature[0]), np.array(object_feature[0])))
            #print(concat_features.shape)
            feature_pairs_frame.append(concat_features)
        return feature_pairs_frame

    def __getitem__(self, index):
        feature_seq = self.feature_sequences[index]
        label_seq = self.label_sequences[index]
        dt_seq = self.dt_sequences[index]
        i3d_seq = self.i3d_sequences[index]

        return feature_seq, dt_seq, i3d_seq, label_seq

    def __len__(self):
        return len(self.label_sequences)

object_feature_dir = '/home/roy/breakfast_features/'
dt_feature_dir = '/home/roy/breakfast_dtfv_features/'
i3d_feature_dir = '/home/roy/breakfast_i3dfeatures/'

# train_set_file = 'train_set_seq_1s.pkl'
# if os.path.exists(train_set_file):
    # training_set = pickle.load(open(train_set_file,'rb'))
# else:
    # training_set = FrameFeatures(object_feature_dir, dt_feature_dir, i3d_feature_dir, sequence_ids_train[:], test=False)
    # with open(train_set_file,'wb') as pkl:
        # pickle.dump(training_set, pkl, pickle.HIGHEST_PROTOCOL)
# print('{} training instances.'.format(len(training_set)))

# test_set_file = 'test_set_seq_1s.pkl'
# if os.path.exists(test_set_file):
    # test_set = pickle.load(open(test_set_file,'rb'))
# else:
    # test_set = FrameFeatures(object_feature_dir, dt_feature_dir, i3d_feature_dir, sequence_ids_test[:], test=True)
    # with open(test_set_file,'wb') as pkl:
        # pickle.dump(test_set, pkl, pickle.HIGHEST_PROTOCOL)
# print('{} test instances.'.format(len(test_set)))

# In[11]:

# ### Network definition

class SummationObj(nn.Module):
    def __init__(self, feature_length, embedding_length):
        super(SummationObj, self).__init__()

        """
        Arguments
        ---------
        output_size : 18
        feature_length : 512

        --------

        """

        self.feature_length = feature_length
        self.embedding_length = embedding_length
        self.dropout = nn.Dropout(0.5)
        self.W = nn.Linear(feature_length, self.embedding_length)

    def forward(self, input1, batch_size=None):
        weighted_out = self.W(input1)
        weighted_out = F.relu(weighted_out)
        #print(weighted_out.shape)
        summed_out = torch.sum(weighted_out, dim=1)
        summed_out = F.relu(summed_out)
        summed_out = self.dropout(summed_out)

        return summed_out

class SummationDTResnet(nn.Module):
    def __init__(self, feature_length, embedding_length):
        super(SummationDTResnet, self).__init__()

        """
        Arguments
        ---------
        output_size : 18
        feature_length : 512

        --------

        """

        self.feature_length = feature_length
        self.embedding_length = embedding_length
        self.dropout = nn.Dropout(0.5)
        self.W = nn.Linear(feature_length, self.embedding_length)

    def forward(self, input, batch_size=None):
        weighted_out = self.W(input)
        weighted_out = F.relu(weighted_out)
        summed_out = self.dropout(weighted_out)
        return summed_out

# In[12]:

class EncoderTransformer(nn.Module):
    def __init__(self, embedding_length, hidden_size, output_size, attention_heads):
        super(EncoderTransformer, self).__init__()
        self.input_size = embedding_length
        self.hidden_size = hidden_size
        self.output_size = output_size
        obj_feature_length = 512
        dt_feature_length = 64
        i3d_feature_length = 2048
        self.obj_embedding = SummationObj(obj_feature_length, 256)
        self.dt_embedding = SummationDTResnet(dt_feature_length, 64)
        self.i3d_embedding = SummationDTResnet(i3d_feature_length, 512)
        obj_encoder_layer = nn.TransformerEncoderLayer(d_model=256, nhead=attention_heads,
            dim_feedforward=32)
        self.obj_encoder = nn.TransformerEncoder(obj_encoder_layer, 2)
        obj_decoder_layer = nn.TransformerDecoderLayer(d_model=256, nhead=attention_heads)
        self.obj_decoder = nn.TransformerDecoder(obj_decoder_layer, 2)
        self.obj_fc = nn.Linear(256, self.output_size)

        dt_encoder_layer = nn.TransformerEncoderLayer(d_model=64, nhead=attention_heads,
            dim_feedforward=64)
        self.dt_encoder = nn.TransformerEncoder(dt_encoder_layer, 2)
        dt_decoder_layer = nn.TransformerDecoderLayer(d_model=64, nhead=attention_heads,
            dim_feedforward=64)
        self.dt_decoder = nn.TransformerDecoder(dt_decoder_layer, 2)
        self.dt_fc = nn.Linear(64, self.output_size)

        i3d_encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=attention_heads,
            dim_feedforward=self.hidden_size)
        self.i3d_encoder = nn.TransformerEncoder(i3d_encoder_layer, 2)
        i3d_decoder_layer = nn.TransformerDecoderLayer(d_model=512, nhead=attention_heads)
        self.i3d_decoder = nn.TransformerDecoder(i3d_decoder_layer, 2)
        self.i3d_fc = nn.Linear(512, self.output_size)

    def forward(self, obj_sequence, dt_sequence, i3d_sequence):
        obj_embedded = []
        for input1 in obj_sequence:
            obj_embedded.append(self.obj_embedding(input1))
        obj_embedded = torch.stack(obj_embedded).permute(1,0,2)*math.sqrt(256)
        dt_embedded = self.dt_embedding(dt_sequence).permute(1,0,2)*math.sqrt(64)
        i3d_embedded = self.i3d_embedding(i3d_sequence).permute(1,0,2)*math.sqrt(512)

        obj_out = self.obj_encoder(obj_embedded)
        dt_out = self.dt_encoder(dt_embedded)
        i3d_out = self.i3d_encoder(i3d_embedded)

        #print(obj_out.shape)
        min_seq_len = min(obj_out.shape[1], dt_out.shape[1], i3d_out.shape[1])

        #total_out = torch.cat([obj_out[:,:min_seq_len,:], dt_out[:,:min_seq_len,:], i3d_out[:,:min_seq_len,:]], dim=2)
        obj_target = torch.rand(1,min_seq_len,256).to(device)
        obj_out = self.obj_decoder(obj_target, obj_out[:,:min_seq_len,:])
        obj_out = self.obj_fc(obj_out).squeeze()

        dt_target = torch.rand(1,min_seq_len,64).to(device)
        dt_out = self.dt_decoder(dt_target, dt_out[:,:min_seq_len,:])
        dt_out = self.dt_fc(dt_out).squeeze()

        i3d_target = torch.rand(1,min_seq_len,512).to(device)
        i3d_out = self.i3d_decoder(i3d_target, i3d_out[:,:min_seq_len,:])
        i3d_out = self.i3d_fc(i3d_out).squeeze()

        out = obj_out + dt_out + i3d_out
        return out

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)

class TrainTest():

    def __init__(self, model, trainset, testset, criterion, batch_size, nepoch, ckpt_path):
        self.model = model
        self.optimizer = optim.Adam(model.parameters(), lr=0.00001)
        #self.optimizer = optim.SGD(model.parameters(), lr=0.0005, weight_decay=0.9 )
        self.criterion = criterion
        self.trainloader = torch.utils.data.DataLoader(trainset, batch_size=1,
                                          shuffle=True, num_workers=0)
        self.testloader = torch.utils.data.DataLoader(testset, batch_size=1,
                                          shuffle=False, num_workers=0)

        self.model.to(device)
        self.chkpath = ckpt_path
        self.batch_size = batch_size
        self.nepoch = nepoch
        self.trainset_size = len(trainset)
        if not os.path.exists('ckpt/'):
            os.mkdir('ckpt/')
        print(self.chkpath)
        if os.path.exists(self.chkpath) == True:
            print('load from ckpt', end=' ')
            self.state = torch.load(self.chkpath)
            self.model.load_state_dict(self.state['model'])
            best_acc = self.state['acc']
            start_epoch = self.state['epoch']
            print('Epoch {}'.format(start_epoch))
            if start_epoch == self.nepoch:
                print('existing as epoch is max.')
            self.details = self.state['details']
            self.best_acc = best_acc
            self.start_epoch = start_epoch + 1
            self.model.to(device)
        else:
            self.best_acc = -1.
            self.details = []
            self.start_epoch = 0

    def test(self):
        correct = 0
        count = 0
        sequence = []
        with torch.no_grad():
            for i, data in enumerate(self.testloader, 0):
                #print(len(data))
                obj_feat_seq = []
                dt_feat_seq = []
                i3d_feat_seq = []
                target_seq = []
                obj_sequence, dt_sequence, i3d_sequence, target_sequence = data
                for obj_feats in obj_sequence:
                    if len(obj_feats) == 0:
                        continue
                    obj_feats = torch.stack(obj_feats)
                    obj_feats = obj_feats.permute(1, 0, 2)

                    obj_feat_seq.append(obj_feats.float().to(device))
                dt_feat_seq = torch.stack(dt_sequence).float().to(device)
                i3d_feat_seq = torch.stack(i3d_sequence).float().to(device)
                target_seq = torch.stack(target_sequence).squeeze().to(device)

                output = self.model(obj_feat_seq, dt_feat_seq, i3d_feat_seq)
                if output.shape[0] == 48:
                    output = output.unsqueeze(0)
                seq_length = min(output.shape[0],target_seq.shape[0])
                pred_seq = torch.argmax(output, dim=1)[:seq_length]
                target_seq = target_seq[:seq_length]
                correct = correct + torch.sum(pred_seq == target_seq).item()

                loss = self.criterion(output[:seq_length], target_seq)
                #print(len(self.testloader))
                print("\rIteration: {}/{}, Loss: {}.".format(i+1, len(self.testloader), loss), end="")
                sys.stdout.flush()
                count += output.shape[0]
        return correct/count*100., correct

    def train(self):
        for epoch in range(self.start_epoch,self.nepoch):
            start_time = time.time()
            running_loss = 0.0
            correct = 0.
            count = 0.
            total_loss = 0
            self.optimizer.zero_grad()

            iterations = 0
            for i, data in enumerate(self.trainloader, 0):
                obj_feat_seq = []
                dt_feat_seq = []
                i3d_feat_seq = []
                target_seq = []
                obj_sequence, dt_sequence, i3d_sequence, target_sequence = data
                for obj_feats in obj_sequence:
                    if len(obj_feats) == 0:
                        continue
                    obj_feats = torch.stack(obj_feats)
                    obj_feats = obj_feats.permute(1, 0, 2)

                    obj_feat_seq.append(obj_feats.float().to(device))
                dt_feat_seq = torch.stack(dt_sequence).float().to(device)
                i3d_feat_seq = torch.stack(i3d_sequence).float().to(device)
                target_seq = torch.stack(target_sequence).squeeze().to(device)

                output = self.model(obj_feat_seq, dt_feat_seq, i3d_feat_seq)
                #print(output.shape)
                if output.shape[0] == 48:
                    output = output.unsqueeze(0)
                with torch.no_grad():
                    seq_length = min(output.shape[0],target_seq.shape[0])
                    pred_seq = torch.argmax(output, dim=1)[:seq_length]
                    target_seq = target_seq[:seq_length]
                    correct = correct + torch.sum(pred_seq == target_seq).item()

                loss = self.criterion(output[:seq_length], target_seq)
                loss.backward()
                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5)
                count += output.shape[0]
                if i % batch_size == 0 and i>1:
                    print("\rIteration: {}/{}, Loss: {}.".format(i+1, len(self.trainloader), loss), end="")
                    self.optimizer.step()
                    self.optimizer.zero_grad()
                    running_loss = running_loss + loss
                    iterations += 1
            TRAIN_LOSS =  running_loss/iterations
            TRAIN_ACC = correct/count*100
            TEST_ACC, TEST_COUNT = self.test()
            self.details.append((TRAIN_LOSS,TRAIN_ACC,0.,TEST_ACC))
            #utils.draw_Fig(self.details)
            #plt.savefig('results/sum_classifier.png')
            #plt.close()
            if TEST_ACC > self.best_acc:
                self.state = {
                    'model': self.model.state_dict(),
                    'acc': TEST_ACC,
                    'epoch': epoch,
                    'details':self.details,
                }
                torch.save(self.state, self.chkpath)
                self.best_acc = TEST_ACC
            else:
                self.state['epoch'] = epoch
                torch.save(self.state, self.chkpath)
            elapsed_time = time.time() - start_time
            print('[{}] [{:.1f}] [Loss {:.3f}] [Correct : {}] [Trn. Acc {:.1f}] '.format(epoch, elapsed_time,
                    TRAIN_LOSS, correct,TRAIN_ACC),end=" ")
            print('[Test Cor {}] [Acc {:.1f}]'.format(TEST_COUNT,TEST_ACC))

# In[17]:

# ### define hyperparameters

embedding_length = 1024
dropout = 0.5
output_size = 48 # number of classes
hidden_size = 64
attention_heads = 1

#instantiate the model
model_ft = EncoderTransformer(embedding_length, hidden_size, output_size, attention_heads)
model_dict = model_ft.state_dict()

ckpt_sum = 'ckpt/sum_transformer_1s.pt'
pretrained_dict = torch.load(ckpt_sum)['model']
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
model_dict.update(pretrained_dict)
model_ft.load_state_dict(model_dict)

ckpt_i3d = 'ckpt/i3d_transformer_1s.pt'
pretrained_dict = torch.load(ckpt_i3d)['model']
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
model_dict.update(pretrained_dict)
model_ft.load_state_dict(model_dict)

ckpt_dt = 'ckpt/dt_transformer_1s.pt'
pretrained_dict = torch.load(ckpt_dt)['model']
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
model_dict.update(pretrained_dict)
model_ft.load_state_dict(model_dict)

ckpt_path = 'ckpt/sum_i3d_dt_transformer_1s.pt'

criterion = nn.CrossEntropyLoss()

batch_size = 8

sequence_ids_test = open('test.s1').readlines()
test_set = FrameFeatures(object_feature_dir, dt_feature_dir, i3d_feature_dir, sequence_ids_test[:10], test=True)
print('{} test instances.'.format(len(test_set)))

# sequence_ids_train = sorted(open('train.s1').readlines())

# nepochs = 10
# training_set = FrameFeatures(object_feature_dir, dt_feature_dir, i3d_feature_dir, sequence_ids_train[:], test=False)
# print('{} training instances.'.format(len(training_set)))
# EXEC = TrainTest(model_ft, training_set, test_set, criterion, batch_size, nepochs, ckpt_path)
# EXEC.train()

from sklearn.metrics import classification_report
pred_labels = []
target_labels = []
testloader = torch.utils.data.DataLoader(test_set, batch_size=1,
                                          shuffle=False, num_workers=0)
pred_labels = []
target_labels = []
model_ft.to(device)
with torch.no_grad():
    for i, data in enumerate(testloader, 0):
        #print(len(data))
        obj_feat_seq = []
        dt_feat_seq = []
        i3d_feat_seq = []
        target_seq = []
        obj_sequence, dt_sequence, i3d_sequence, target_sequence = data
        for obj_feats in obj_sequence:
            if len(obj_feats) == 0:
                continue
            obj_feats = torch.stack(obj_feats)
            obj_feats = obj_feats.permute(1, 0, 2)

            obj_feat_seq.append(obj_feats.float().to(device))
        dt_feat_seq = torch.stack(dt_sequence).float().to(device)
        i3d_feat_seq = torch.stack(i3d_sequence).float().to(device)

        target_seq = torch.stack(target_sequence).squeeze().to(device)

        output = model_ft(obj_feat_seq, dt_feat_seq, i3d_feat_seq)
        if output.shape[0] == 48:
            output = output.unsqueeze(0)
        seq_length = min(output.shape[0],target_seq.shape[0])
        pred_seq = torch.argmax(output, dim=1)[:seq_length]
        target_seq = target_seq[:seq_length]
        #print(target_seq)
        #print(pred_seq)
        pred_labels.append(pred_seq)
        target_labels.append(target_seq)

pred_mod = []
target_mod = []
for preds, targets in zip(pred_labels, target_labels):
    pred_mod.append(preds[0].item())
    if targets.dim() == 0:
        targets = torch.unsqueeze(targets,0)
    target_mod.append(targets[0].item())
print(classification_report(target_mod, pred_mod,digits=4))


#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import sys
from os import listdir
from os.path import isfile, join
import csv
import pymorphy2

morph = pymorphy2.MorphAnalyzer()

tags_dict = {}

tag_files = [ f for f in listdir('./tags/') if isfile(join('./tags/',f)) ]
for tag_file in tag_files:
    if tag_file[-4:] == '.txt':
    	print(tag_file)
    	with open(('./tags/' + tag_file)) as tag_file_file:
    		for line in tag_file_file:
    			a = []
    			for token in line.strip().split():
    				a.append(morph.parse(token.lower())[0].normal_form)
	    		tags_dict[' '.join(a)] = tag_file[:-4]
print(len(tags_dict))
facets = ['acm', 'action', 'agregator', 'amn', 'city', 'contacts', 'date', 'geo', 'poi', 'price', 'stars', 'time', 'wh']

with open('data.csv/data_lem.csv') as data_file:
	reader = csv.DictReader(data_file, doublequote=False, escapechar='\\')
	with open('data.csv/data_lem_tags.csv', 'w') as out_file:
		fieldnames = ['lemmatized', 'id', 'conversion', 'conversion1', 'conversion2', 'conversion3', 'tags', 'acm', 'action', 'agregator', 'amn', 'city', 'contacts', 'date', 'geo', 'poi', 'price', 'stars', 'time', 'wh', 'other']
		writer = csv.DictWriter(out_file, fieldnames=fieldnames, doublequote=False, escapechar='\\')
		writer.writeheader()

		for row in reader:
			#print(row)
			try:
				try:
					if int(row['id']) % 500 == 0:
						print(row['id'])
				except TypeError:
					pass
				tokens = row['lemmatized'].split()
				row_out = {}
				row_out['lemmatized'] = row['lemmatized']
				row_out['id'] = row['id']
				row_out['conversion'] = row['conversion']
				row_out['conversion1'] = row['conversion1']
				row_out['conversion2'] = row['conversion2']
				row_out['conversion3'] = row['conversion3']
				row_tags = []
				for entry in tags_dict:
					if entry in row['lemmatized']:
						row_tags.append(tags_dict[entry])
					if entry in row['keyword']:
						row_tags.append(tags_dict[entry])
				row_tags = list(set(row_tags))
				row_out['tags'] = ' '.join(row_tags)
				row_out['other'] = ''
				for i in row_tags:
					in_facets = False
					for facet in facets:
						if i.strip().startswith(facet) and not (i.strip() == facet):
							in_facets = True
							if facet in row_out:
								row_out[facet] += ' ' + re.sub(facet, '', i)
							else:
								row_out[facet] = re.sub(facet, '', i)
					if not in_facets:
						row_out['other'] += i + ' '
				writer.writerow(row_out)
			except AttributeError:
				pass
			#print(tokens)


import numpy as np
from numpy import ma, pi
from maud import window_1Dmean
import pylab
from window_func import _weight_blackman as blackman
from window_func import _weight_triangular as triangular

x =  ma.array(np.sort(100*np.random.random(100)))
y = 5*np.sin(2*pi*x/43.) + 3*np.sin(2*pi*(x-7.)/12.) + 2*np.random.rand(len(x))-1

import pdb; pdb.set_trace()

l1=10
l2 = 30
yf = ma.masked_all(y.shape)
y2f = ma.masked_all(y.shape)

for i in np.arange(len(x)):
    w1 = blackman(x-x[i],l1)
    w1 = w1/sum(w1)
    yf[i] = sum(w1*y)
    w2 = blackman(x-x[i],l2)
    w2 = w2/sum(w2)
    y2f[i] = sum(w2*y)

zf = window_1Dmean(y,l1,method='blackman')
zf2 = window_1Dmean(y,l1,method='hann')
zf3 = window_1Dmean(y,l1,method='triangular')

pylab.figure()
pylab.plot(x,y)
#pylab.plot(x,yf,'r')
#pylab.plot(x,y2f,'og')
#pylab.plot(x,y-y2f,'k')
#pylab.plot(x,zf,'r')
#pylab.plot(x,zf2,'b')
pylab.plot(x,zf3,'g')
pylab.show()


import itertools as it

def solution(number):
    answer = 0

    for a,b,c in it.combinations(number, 3):
        if a+b+c == 0:
            answer += 1

    return answer

print(f'test1 = {solution([-2,3,0,2,-5])}')
print(f'test2 = {solution([-3,-2,-1,0,1,2,3])}')
print(f'test3 = {solution([-1,1,-1,1])}')


from collections import defaultdict
from math import sqrt

def d(p1, p2):
    dist = 0
    for i in range(len(p1)):
        dist += (p1[i] - p2[i]) ** 2
    return sqrt(dist)

# новый центр соответствует центру тяжести
def NewCenter(cluster):
    center = [0] * len(cluster[0])
    for point in cluster:
        for i in range(len(cluster[0])):
            center[i] += point[i] / len(cluster)
    return center

def NearestCenter(point, centers):
    dist = float(1e9)
    center = centers[0]
    for i in centers:
        current = d(i, point)
        if current < dist:
            dist = current
            center = i
    return center

# назначаем каждую точку данных кластеру
def GetClustersWithPoints(dataPoints, centers):
    clusters = defaultdict(list)
    for point in dataPoints:
            center = NearestCenter(point, centers)
            clusters[tuple(center)].append(point)
    return clusters

# назначаем центр тяжести каждому кластеру
def NewCentersForClusters(clusters, centers):
    currCenters = [[]] * k
    for i in range(k):
        currCenters[i] = NewCenter(clusters[tuple(centers[i])])
    return currCenters

def LloydAlgorithm(dataPoints, k, m):
    centers = dataPoints[0:k]
    while True:
        clusters = GetClustersWithPoints(dataPoints, centers)
        currCenters = NewCentersForClusters(clusters, centers)
        # если центры перестали меняться, алгоритм сошелся и можно заканчивать
        if currCenters == centers:
            break
        else:
            centers = currCenters[:]
    return centers

if __name__ == "__main__":
    f = open('input.txt', 'r')
    input_lines = f.read().strip().split("\n")

    k, m = [int(x) for x in input_lines[0].split()]
    dataPoints = [[float(x) for x in line.split()] for line in input_lines[1:]]
    centers = LloydAlgorithm(dataPoints, k, m)
    for center in centers:
        print(" ".join(map(str, center)))


from collections import defaultdict
import math

from alphaconnect4.engines.mcts import MCTS
from alphaconnect4.interfaces.neural_interface import NeuralInterface

class NeuralMCTS(MCTS):
    "Neural Monte Carlo tree searcher. First rollout the tree then choose a move."

    def __init__(self, exploration_weight=1, model_path=None):
        super().__init__(exploration_weight)
        self.p_value = defaultdict(int)  # total visit count for each node
        self.neural_interface = NeuralInterface(model_path=model_path)

    def do_rollout(self, node):
        """Make the tree one layer better. (Train for one iteration.)
        In the neural version, score leafs with NN and store policy vectors """
        path = self._select(node)
        leaf = path[-1]
        self._expand(leaf)
        if leaf.is_terminal():
            reward = 1 - leaf.reward()
        else:
            score, policy = self.neural_interface.score(leaf)
            reward = 1 - score

            for child in self.children[leaf]:
                self.p_value[child] = policy[child.last_move]

        self._backpropagate(path, reward)

    def _uct_select(self, node):
        "Select a child of node, balancing exploration & exploitation"

        # All children of node should already be expanded:
        # assert all(n in self.children for n in self.children[node])

        vertex_count = math.sqrt(self.visit_count[node])

        def uct(n):
            "Upper confidence bound for trees"
            return self.q_value[n] / self.visit_count[n] + self.exploration_weight * self.p_value[n] * vertex_count / (
                        1 + self.visit_count[n])

        return max(self.children[node], key=uct)


"""empty message

Revision ID: c13ced84655c
Revises: a76b4120a840
Create Date: 2022-07-21 19:21:07.363561

"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "c13ced84655c"
down_revision = "a76b4120a840"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column("acronym", sa.Column("created_at", sa.DateTime(), nullable=True))
    op.add_column(
        "acronym", sa.Column("last_modified_at", sa.DateTime(), nullable=True)
    )
    op.drop_column("acronym", "created")
    op.drop_column("acronym", "last_modified")
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column(
        "acronym",
        sa.Column(
            "last_modified", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
        ),
    )
    op.add_column(
        "acronym",
        sa.Column(
            "created", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
        ),
    )
    op.drop_column("acronym", "last_modified_at")
    op.drop_column("acronym", "created_at")
    # ### end Alembic commands ###


#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import pytest
from pages.image_quality_page import ImageQualityPage
from common.logs import Log
from config.test_flag import skip_flag

class TestImageTaskList(object):
    """图像质检任务列表接口测试：/api/uniqc/image-qc/task-list"""
    @classmethod
    def setup_class(cls):
        cls.log = Log()
        cls.page = ImageQualityPage()
        cls.skip_flag = True

    # 参数正常场景测试
    case1 = [
        (1, 10, "2021-11")
    ]

    @pytest.mark.parametrize('pageNumber,pageSize,months', case1)
    def test_image_task_list_01(self, pageNumber, pageSize, months):
        """
        传参正确，列表数据获取成功
        :return:
        """
        result = self.page.image_task_list(pageNumber=pageNumber, pageSize=pageSize, months=months)
        # print(reslut)
        assert result["status"] == 200
        self.log.info(result)

    # 参数不全场景测试
    # pageSize为空，默认10
    # months为空，默认查全部
    case2 = [
        (None, 10, "2021-11"),
        (1, None, "2021-11"),
        (1, 12, None)
    ]

    @pytest.mark.parametrize('pageNumber,pageSize,months', case2)
    def test_image_task_list_02(self, pageNumber, pageSize, months):
        """
        参数不全
        :return:
        """
        result = self.page.image_task_list(pageNumber=pageNumber, pageSize=pageSize, months=months)
        # print(reslut)
        assert result["status"] == 200
        self.log.info(result)

    # pageNumber参数值异常场景测试
    case3 = [
        (-1, 10, "2021-11"),
        (0, 10, "2021-11")
    ]

    @pytest.mark.skipif(skip_flag=skip_flag, reason="跳过该测试用例")
    @pytest.mark.parametrize('pageNumber,pageSize,months', case3)
    def test_image_task_list_03(self, pageNumber, pageSize, months):
        """
        pageNumber参数值异常
        :return:
        """
        result = self.page.image_task_list(pageNumber=pageNumber, pageSize=pageSize, months=months)
        assert result["status"] == 5003
        self.log.info(result)

    # pageNumber参数异常场景测试
    # pageNumber赋值0.5
    case4 = [(0.5, 10, "2021-11")]

    @pytest.mark.parametrize('pageNumber,pageSize,months', case4)
    def test_image_task_list_04(self, pageNumber, pageSize, months):
        """
        pageNumber参数值异常
        :return:
        """
        result = self.page.image_task_list(pageNumber=pageNumber, pageSize=pageSize, months=months)
        assert result["status"] == 1004
        self.log.info(result)

    # pageSize参数值异常场景测试
    case5 = [
        (-1, 10, "2021-11"),
        (0, 10, "2021-11")
    ]

    @pytest.mark.parametrize('pageNumber,pageSize,months', case5)
    def test_image_task_list_05(self, pageNumber, pageSize, months):
        """
        pageSize参数值异常
        :return:
        """
        result = self.page.image_task_list(pageNumber=pageNumber, pageSize=pageSize, months=months)
        assert result["status"] == 5003
        self.log.info(result)

    # months参数异常场景
    case6 = [
        (1, 10, "测试"),
        (1, 10, "202111"),
        (1, 10, 202111)
    ]

    @pytest.mark.parametrize('pageNumber,pageSize,months', case6)
    def test_image_task_list_06(self, pageNumber, pageSize, months):
        """
        months参数值异常
        :return:
        """
        result = self.page.image_task_list(pageNumber=pageNumber, pageSize=pageSize, months=months)
        assert result["status"] == 3002
        self.log.info(result)

if __name__ == '__main__':
    pytest.main(["-v","-s", "test_image_task_list.py"])


from collections import deque
import math
import sys

input = sys.stdin.readline

def TopologicalSort():
    basic = []
    Q = deque()
    for i in range(1, N+1):
        if indegree[i] == 0:
            Q.append(i)
            basic.append(i)  # 기본 부품
    DP = [[0]*(N+1) for _ in range(N+1)]
    while Q:
        node = Q.popleft()
        for edge, parts in graph[node]:
            indegree[edge] -= 1

            if node in basic:
                DP[edge][node] = parts
            else:
                for i in range(len(DP[edge])):
                    DP[edge][i] += DP[node][i] * parts
            if indegree[edge] == 0:
                Q.append(edge)

    result = []
    for i in range(1, len(DP[N])):
        if i in basic:
            result.append([i, DP[N][i]])
    return result

N = int(input())
M = int(input())

graph = [[]*(N+1) for _ in range(N+1)]
indegree = [0]*(N+1)

for i in range(M):
    X, Y, K = map(int, input().split())
    indegree[X] += 1
    graph[Y].append([X, K])

dp = TopologicalSort()

for i in dp:
    print(i[0], i[1])

백준/Gold/2637. 장난감 조립/장난감 조립.py


import gradio as gr
from modules.whisper_inference import WhisperInference
import os
from ui.htmls import *
from modules.youtube_manager import get_ytmetas

def open_folder(folder_path):
    if os.path.exists(folder_path):
        os.system(f"start {folder_path}")
    else:
        print(f"The folder {folder_path} does not exist.")

def on_change_models(model_size):
    translatable_model = ["large", "large-v1", "large-v2"]
    if model_size not in translatable_model:
        return gr.Checkbox.update(visible=False, value=False, interactive=False)
    else:
        return gr.Checkbox.update(visible=True, value=False, label="Translate to English?", interactive=True)

whisper_inf = WhisperInference()
block = gr.Blocks(css=CSS).queue(api_open=False)

with block:

    with gr.Tabs():
        with gr.TabItem("File"):
            with gr.Row():
                input_file = gr.Files(type="file", label="Upload File here")
            with gr.Row():
                dd_model = gr.Dropdown(choices=whisper_inf.available_models, value="large-v2", label="Model")
                dd_lang = gr.Dropdown(choices=["Automatic Detection"] + whisper_inf.available_langs,
                                      value="Automatic Detection", label="Language")
                dd_subformat = gr.Dropdown(["SRT", "WebVTT"], value="SRT", label="Subtitle Format")
            with gr.Row():
                cb_translate = gr.Checkbox(value=False, label="Translate to English?", interactive=True)
            with gr.Row():
                btn_run = gr.Button("GENERATE SUBTITLE FILE", variant="primary")
            with gr.Row():
                tb_indicator = gr.Textbox(label="Output")
                btn_openfolder = gr.Button('ðŸ“‚').style(full_width=False)

            btn_run.click(fn=whisper_inf.transcribe_file,
                          inputs=[input_file, dd_model, dd_lang, dd_subformat, cb_translate], outputs=[tb_indicator])
            btn_openfolder.click(fn=lambda: open_folder("outputs"), inputs=None, outputs=None)
            dd_model.change(fn=on_change_models, inputs=[dd_model], outputs=[cb_translate])

        with gr.TabItem("Youtube"):
            with gr.Row():
                tb_youtubelink = gr.Textbox(label="Youtube Link")
            with gr.Row().style(equal_height=True):
                with gr.Column():
                    img_thumbnail = gr.Image(label="Youtube Thumbnail")
                with gr.Column():
                    tb_title = gr.Label(label="Youtube Title")
                    tb_description = gr.Textbox(label="Youtube Description", max_lines=15)
            with gr.Row():
                dd_model = gr.Dropdown(choices=whisper_inf.available_models, value="large-v2", label="Model")
                dd_lang = gr.Dropdown(choices=["Automatic Detection"] + whisper_inf.available_langs,
                                      value="Automatic Detection", label="Language")
                dd_subformat = gr.Dropdown(choices=["SRT", "WebVTT"], value="SRT", label="Subtitle Format")
            with gr.Row():
                cb_translate = gr.Checkbox(value=False, label="Translate to English?", interactive=True)
            with gr.Row():
                btn_run = gr.Button("GENERATE SUBTITLE FILE", variant="primary")
            with gr.Row():
                tb_indicator = gr.Textbox(label="Output")
                btn_openfolder = gr.Button('ðŸ“‚').style(full_width=False)

            btn_run.click(fn=whisper_inf.transcribe_youtube,
                          inputs=[tb_youtubelink, dd_model, dd_lang, dd_subformat, cb_translate],
                          outputs=[tb_indicator])
            tb_youtubelink.change(get_ytmetas, inputs=[tb_youtubelink],
                                  outputs=[img_thumbnail, tb_title, tb_description])
            btn_openfolder.click(fn=lambda: open_folder("outputs"), inputs=None, outputs=None)
            dd_model.change(fn=on_change_models, inputs=[dd_model], outputs=[cb_translate])

block.launch()