#!/usr/bin/env python3

import copy
import bisect

from defect.util import zip_variadic, zip_matching_length

import numpy as np

# FIXME this file currently simultaneously tries to support multiple paradigms
# (a bunch of free functions operating on dicts, versus gratituous use of structured numpy arrays)
#
# As a result, some properties of the data can be computed via two completely independent code paths.
# This is the *bad* kind of redundancy (the kind which can easily become inconsistent!)

def read_info(path):
	import json
	with open(path) as f:
		s = f.read()
	return json.loads(s)

def trial_defects_stepwise(trial_info):
	return map(len, trial_info['steps']['deleted'])

def trial_defects_cumulative(trial_info):
	return prefix_sums(trial_defects_stepwise(trial_info), zero=0)

def trial_max_defects_possible(trial_info):
	# FIXME this assumes there's exactly 2 non-deletable vertices
	return trial_info['graph']['num_vertices'] - 2

def trial_defects_ratio(trial_info):
	n = trial_max_defects_possible(trial_info)
	return [float(x) / n for x in trial_defects_stepwise(trial_info)]

def trial_current(trial_info):
	return trial_info['steps']['current']

def trial_resistance(trial_info):
	return map(lambda x: 1./x, trial_current(trial_info))

def trial_array(trial_info):
	steps = trial_info['steps']
	data_iter = zip_matching_length(*steps.values())

	# numpy is REALLY picky about container types
	data_tuples = list(map(tuple, data_iter))

	# autogenerate record type based on python value types
	# not sure how reliable this will be versus hardcoded names/types
	spec = []
	for name, value in zip(steps.keys(), data_tuples[0]):
		spec.append((name, type(value)))

	assert all(type(v) == typ for v,(name,typ) in zip(data_tuples[0], spec))
	return np.array(data_tuples, dtype=spec)

def trialset_array(trial_infos):
	return np.vstack(list(trial_array(info) for info in trial_infos))

# FIXME there's some redundant functionality between this and previously existing "trialset"
#  functions currently as the two different interfaces kind of work at odds to eachother
def trialset_augmented_array(trialset_info):
	arr = trialset_array(trialset_info)

	fields = listify_dtype(arr.dtype)
	oldnames = [x[0] for x in fields]
	fields.append(('step', 'i8'))
	fields.append(('deleted_cum', 'O'))
	fields.append(('deleted_count', 'i8'))
	fields.append(('deleted_count_cum', 'i'))
	fields.append(('deleted_ratio', 'f8'))

	out = np.zeros(arr.shape, dtype=fields)
	for name in oldnames:
		out[name] = arr[name]

	for t_out,t_info in zip(out, trialset_info):
		t_out['step'] = np.arange(len(t_out))
		t_out['deleted_cum'] = trial_deleted_cum_array(t_info)

	out['deleted_count'] = np.vectorize(len)(out['deleted'])
	out['deleted_count_cum'] = np.vectorize(len)(out['deleted_cum'])

	out['deleted_ratio'] = np.float64(out['deleted_count_cum']) / trial_max_defects_possible(trialset_info[0])
	assert (out['deleted_count_cum'] == np.vectorize(len)(out['deleted_cum'])).all()
	return out

# converts a np.dtype into a python list in the format accepted by np.array
def listify_dtype(dtype):
	# np.dtype is not iterable o_O
	types = [dtype[i] for i in range(len(dtype))]
	return list(zip(dtype.names, types))

# Provides a complete list of all vertices deleted up to a point for any given step.
# To avoid O(N^2) memory requirements, each element is actually a slice of a shared master array.
def trial_deleted_cum_array(trial_info):
	arr = trial_array(trial_info)

	assert isinstance(arr['deleted'][0], list)
	pieces = [np.array(x, dtype='O') for x in arr['deleted']]
	master = np.hstack(pieces)

	lengths = np.array([len(x) for x in pieces])
	stops   = np.cumsum(lengths)
	views   = [master[:x] for x in stops]

	return object_array(views, ndim=1)

# Force creation of an object-type numpy array (``dtype='O'``) with the specified number of
#  dimensions, rather than letting numpy wing it with its crazy value-based magicks:
#
# >>> np.array([[1,2],[3,4],[5,6],[7]], dtype='O').shape    # 1d array of lists
# (4,)
# >>> np.array([[1,2],[3,4],[5,6],[7,8]], dtype='O').shape  # 2d array of integers  (!!!)
# (4, 2)
def object_array(objects, ndim=1):
	# listify objects in case it is iterable.
	# keep in mind ``np.array`` cannot be trusted with this as it auto-detects ndim
	#   and may coerce the actual objects themselves into array if they happen to be
	#   list-like and of equal length
	objects = ndim_list_create(objects, ndim)
	shape = ndim_list_shape(objects, ndim)

	out = np.zeros(shape, dtype='O')
	out[...] = objects
	return out

# create an "ndim_list" (a multidimensional list of known depth) from any iterable.
# lengths are assumed to be equal along each axis.
def ndim_list_create(src, ndim):
	assert ndim > 0
	if ndim == 1:
		return list(src)
	else:
		inner = lambda x: ndim_list_create(x, ndim-1)
		return list(map(inner, src))

def ndim_list_shape(lst, ndim):
	assert ndim > 0
	shape = [None]*ndim
	for i in range(ndim):
		shape[i] = len(lst)
		lst = lst[0]
	assert None not in shape
	return tuple(shape)

# Note: because omitting steps from the beginning/middle of the trial may mess
#  with "cumulative" properties, it's not a very good idea to use this for any
#  purpose other than cutting stuff off from the end.
def slice_steps(step_info, *args):
	sl = slice(*args)
	return {k:v[sl] for k,v in step_info.items()}

# ignores shorter trials once they are zero
def trialset_average_current(trial_infos):
	assert are_lists_consistent(trial_defects_cumulative(x) for x in trial_infos)

	currents = [trial_current(x) for x in trial_infos]

	return map(average, zip_variadic(*currents))

def trialset_defects_cumulative(trial_infos):
	return reduce_consistent_lists(trial_defects_cumulative(x) for x in trial_infos)

def trim_trial_by_current(trial_info, threshold=0.0):
	current = trial_current(trial_info)

	# current monotonically decreases --> this monotonically increases
	arr = [-x for x in current]
	val = -threshold

	zero_idx = bisect.bisect_left(arr, val)

	assert all(abs(x) > threshold*(1 - 1e-14) for x in arr[:zero_idx])
	assert all(abs(x) <= threshold*(1 + 1e-14) for x in arr[zero_idx:])

	result = copy.deepcopy(trial_info)
	result['steps'] = slice_steps(result['steps'], 0, zero_idx)
	return result

# FIXME annoying signature; unlike other "trial" functions, this takes the
#  complete info and a trial index.  This is because the deletion mode is
#  not present in the individual trial infos (can we change that?)
# FIXME also don't like the idea/placement of this in general; it basically
#  tries to simulate a trial, which means any changes to the trial runner may
#  need to be reflected here
def trial_edge_currents_at_step(g, cycles, info, trialid, step):
	from defect.trial import node_deletion
	from defect.trial.cyclebasis_provider import builder_cbupdater
	from defect.circuit import MeshCurrentSolver

	# get the deletion func
	deletion_mode = node_deletion.from_info(info['defect_mode'])
	deletion_func = deletion_mode.deletion_func

	# gather all vertices deleted at the specified step
	arr = trialset_augmented_array(info['trials'])
	deleted = arr['deleted_cum'][trialid][step]

	solver = MeshCurrentSolver(g, cycles, builder_cbupdater())
	for v in deleted:
		deletion_func(solver, v)

	currents = solver.get_all_currents()
	return currents

# True if all provided lists contain the same values up to where each is defined
# (the lists may be of different length)
def are_lists_consistent(its):
	return all(map(all_equal, zip_variadic(*its)))

# are all elements equal?
def all_equal(vals):
	vals = list(vals)
	if len(vals) == 0:
		return True

	return all(x == vals[0] for x in vals)

# takes consistent lists (lists which have the same values but possibly different
# lengths) and returns a copy of the longest list.
def reduce_consistent_lists(its):
	its = list(its)
	if len(its) == 0:
		return []

	its = [list(x) for x in its]
	if are_lists_consistent(its):
		lengths = list(map(len, its))
		return its[lengths.index(max(lengths))]
	else:
		raise ValueError('Lists are not consistent!')

def average(xs, zero=0.0):
	xs = list(xs)
	return sum(xs, 0.0)/len(xs)

def prefix_sums(xs, zero=0.0):
	sums = []
	s = zero
	for x in xs:
		s += x
		sums.append(s)
	return sums

# tests I'm too lazy to give a proper place >_>
assert all_equal([])
assert all_equal([2.])
assert all_equal([2.,2.,2.,2.])
assert not all_equal([2.,2.,1.,2.])

assert reduce_consistent_lists([]) == []
assert reduce_consistent_lists([[5.,6.],[5.,6.,7.],[5.]]) == [5.,6.,7.]

try: reduce_consistent_lists([[5.,3.],[5.,6.,7.],[5.]])
except ValueError: pass
else: assert False


#!/usr/bin/env python
# coding: utf-8

# In[1]:

#!/usr/bin/python

"""
    This is the code to accompany the Lesson 3 (decision tree) mini-project.

    Use a Decision Tree to identify emails from the Enron corpus by author:
    Sara has label 0
    Chris has label 1
"""

import sys
from time import time
sys.path.append("../tools/")
from email_preprocess import preprocess
from datetime import datetime

# In[2]:

### features_train and features_test are the features for the training
### and testing datasets, respectively
### labels_train and labels_test are the corresponding item labels
t1 = datetime.now()
features_train, features_test, labels_train, labels_test = preprocess()
t2 = datetime.now()
delta = t2 - t1
print(delta.total_seconds())
features_train.shape

# In[ ]:

t1 = datetime.now()
from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(min_samples_split=40)
clf.fit(features_train,labels_train)
t2 = datetime.now()
delta = t2 - t1
print(delta.total_seconds())

# In[4]:

t1 = datetime.now()
y_pred = clf.predict(features_test)
t2 = datetime.now()
delta = t2 - t1
print(delta.total_seconds())

# In[5]:

print("Number of mislabeled points out of a total %d points : %d"
      % (features_test.shape[0], (labels_test != y_pred).sum()))

# In[6]:

from sklearn.metrics import accuracy_score
scor = accuracy_score(y_pred,labels_test)
scor

# In[7]:

print(len(features_train[0]))

# In[8]:

features_train

# In[ ]:


# test that lammpslib handle nonperiodic cases where the cell size
# in some directions is small (for example for a dimer).
from __future__ import print_function
import os
import numpy as np
from ase.calculators.lammpslib import LAMMPSlib
from ase import Atoms

potential_path = os.environ.get('LAMMPS_POTENTIALS_PATH', '.')
cmds = ["pair_style eam/alloy",
        "pair_coeff * * {path}/NiAlH_jea.eam.alloy Ni H"
        "".format(path=potential_path)]
lammps = LAMMPSlib(lmpcmds=cmds,
                   atom_types={'Ni': 1, 'H': 2},
                   log_file='test.log', keep_alive=True)
a = 2.0
dimer = Atoms("NiNi", positions=[(0,0,0),(a,0,0)],
              cell=(1000*a, 1000*a, 1000*a),pbc=(0,0,0))
dimer.set_calculator(lammps)

energy_ref = -1.10756669119
energy = dimer.get_potential_energy()
print("Computed energy: {}".format(energy))
diff = abs((energy - energy_ref) / energy_ref)
np.testing.assert_allclose(energy, energy_ref, atol=1e-10)

np.set_printoptions(precision=16)
forces_ref = np.array([[-0.9420162329811532, 0., 0.],[ 0.9420162329811532, 0., 0. ]])
forces = dimer.get_forces()
print(np.array2string(forces))
np.testing.assert_allclose(forces, forces_ref, atol=1e-10)


import pygame
from board import Board
SCREEN_WIDTH = 600
SCREEN_HEIGHT = 600

class Text:
    def __init__(self,text,pos):

        font = pygame.font.Font('freesansbold.ttf', 72)
        self.text = font.render(text, True, (255,0,0), (255,255,255))
        self.textRect = self.text.get_rect()
        self.textRect.center = self._convert_position(pos)

    def blit(self,screen):
        screen.blit(self.text, self.textRect)

    def _convert_position(self,pos):
        w = SCREEN_WIDTH//3
        h = SCREEN_HEIGHT//3
        p1 = pos[0]
        p2 = pos[1]
        return (w*p1 + w//2,h*p2+ h//2)

class pygameBoard(Board):
    def __init__(self):
        super().__init__()

    def pygame_display(self,screen):
        '''
        display the current board on the screen
        '''
        #There are 3 rows and 3 columns
        w1 = SCREEN_WIDTH//3
        h1 = SCREEN_HEIGHT//3

        for row in range(3):
            for col in range(3):
                if self.board[row][col] != ' ':
                    text = Text(self.board[row][col],(row ,col ))

                    text.blit(screen)

def reset_screen(screen):
    '''
    reset the screen and draw
    the tic tac toe lines
    '''
    w1 = SCREEN_WIDTH//3
    w2 = (2*SCREEN_WIDTH)//3
    h1 = SCREEN_HEIGHT//3
    h2 = (2*SCREEN_HEIGHT)//3
    screen.fill("White")
    pygame.draw.line(screen, (0,0,0), (w1, 0), (w1, SCREEN_HEIGHT))
    pygame.draw.line(screen, (0,0,0), (w2, 0), (w2, SCREEN_HEIGHT))
    pygame.draw.line(screen, (0,0,0), (0,h1), ( SCREEN_HEIGHT,h1))
    pygame.draw.line(screen, (0,0,0), (0,h2), (SCREEN_HEIGHT,h2))

    return screen


import numpy as np
import scipy.io as sio

"""
Converts weights provided by authors in `B1.npy`, `B2.npy`, `W1.npy`
and `W2.npy` into a single `two-layer.mat` file.
"""

d = {}

b1 = np.load("B1.npy")
b2 = np.load("B2.npy")
w1 = np.load("W1.npy")
w2 = np.load("W2.npy")

d["fc1/weight"] = w1
d["fc1/bias"] = b1
d["logits/weight"] = w2
d["logits/bias"] = b2

sio.savemat("two-layer.mat", d)


import enemies
import pygame
import math
class Projectile:
    def __init__(self, grid_pos, enemy, path:list, game):
        if not isinstance(enemy, enemies.Enemy):
            raise TypeError("Input must be an enemy_class")

        self.done = False
        self.grid_pos = grid_pos
        self.enemy = enemy
        self.path = path
        self.screen = game.screen
        self.speed = 10
        self.pos = [self.grid_pos[0]*80 + 40, self.grid_pos[1]*80 + 40]
        self.original_pos = self.pos
        self.image = pygame.image.load("sprites/projectiles/box_cutter_projectile.png")
        self.image = pygame.transform.rotate(self.image, 90)
        self.rect = self.image.get_rect()
        self.rotated_image = self.image.copy()
        self.rotation = 0
        self.coords = [0, 0]

        self._update_rect()

    def update_self(self):

        self._move_projectile()
        self._check_for_enemy()
        self._update_rect()
        self._rotate()
        self._blit_self()

    def _move_projectile(self):
        self.coords = [(self.enemy.pos[0]*80+40)-self.pos[0],(self.enemy.pos[1]*80+40)-self.pos[1]] #Get's the difference between the current position in pixels, and the enemy's grid position in pixels
        hyp = math.hypot(self.coords[0], self.coords[1]) #Hypotenuse between arrow and target calculated

        # Unless the distance is 0 it updates the position of the arrow relative to the angle
        if hyp != 0:
            cos = self.coords[0]/hyp
            sin = self.coords[1]/hyp

            self.pos[0] += cos * self.speed
            self.pos[1] += sin * self.speed

    def _update_rect(self):
        self.rect.x = self.pos[0]
        self.rect.y = self.pos[1]

    def _check_for_enemy(self):
        is_at_enemy_x = self.enemy.pos[0] * 80 <= self.pos[0] <= self.enemy.pos[0]*80 + 80
        is_at_enemy_y = self.enemy.pos[1] * 80 <= self.pos[1] <= self.enemy.pos[1]*80 + 80
        if is_at_enemy_x and is_at_enemy_y:
            #Do something
            self.enemy.take_damage(100)
            self.done = True

    def _rotate(self):
        original_rect = self.image.get_rect()

        # To be honest I'm not quite sure what i was thinking at the time i wrote this
        #       Negative atan towards the enemy. It just works  ¯\_(ツ)_/¯
        rot = -math.degrees(math.atan2(self.coords[1], self.coords[0]))

        self.rotated_image = pygame.transform.rotate(self.image, rot+180)   #Rotates the image the rotation plus 180 degrees to compensate for image orientation
        rotated = self.rotated_image.get_rect()
        self.rotation = rot
        dif = [original_rect[2]-rotated[2], original_rect[3]-rotated[3]]    #Difference in widt, height, in the rect

        #Compensates for the picture shifting
        self.rect.x += dif[0]
        self.rect.y += dif[1]

    def _blit_self(self):
        self.screen.blit(self.rotated_image, self.rect)

class ScissorArrow(Projectile):
    def __init__(self, grid_pos, enemy, path: list, game):
        super().__init__(grid_pos, enemy, path, game)
        self.speed = 15
        self.image = pygame.image.load("sprites/projectiles/Tower_2_projectile.png")
        self.image = pygame.transform.rotate(self.image, 315)
        self.rect = self.image.get_rect()
        self.rotated_image = self.image.copy() # Deep copy not needed

class KnifeArrow(Projectile):
    def __init__(self, grid_pos, enemy, path: list, game):
        super().__init__(grid_pos, enemy, path, game)
        self.speed = 15
        self.image = pygame.image.load("sprites/projectiles/box_cutter_projectile.png")
        self.image = pygame.transform.rotate(self.image, 0)
        self.rect = self.image.get_rect()
        self.rotated_image = self.image.copy() # Deep copy not needed


from __future__ import absolute_import

import six

from collections import OrderedDict
from rest_framework.response import Response

from sentry import tsdb
from sentry.api.base import StatsMixin
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import ProjectKey

class ProjectKeyStatsEndpoint(ProjectEndpoint, StatsMixin):
    def get(self, request, project, key_id):
        try:
            key = ProjectKey.objects.get(
                project=project,
                public_key=key_id,
                roles=ProjectKey.roles.store,
            )
        except ProjectKey.DoesNotExist:
            raise ResourceDoesNotExist

        stat_args = self._parse_args(request)

        stats = OrderedDict()
        for model, name in (
            (tsdb.models.key_total_received,
             'total'), (tsdb.models.key_total_blacklisted, 'filtered'),
            (tsdb.models.key_total_rejected, 'dropped'),
        ):
            result = tsdb.get_range(model=model, keys=[key.id], **stat_args)[key.id]
            for ts, count in result:
                stats.setdefault(int(ts), {})[name] = count

        return Response(
            [
                {
                    'ts': ts,
                    'total': data['total'],
                    'dropped': data['dropped'],
                    'filtered': data['filtered'],
                    'accepted': data['total'] - data['dropped'] - data['filtered'],
                } for ts, data in six.iteritems(stats)
            ]
        )


from collections import namedtuple
import functools

from cbtk.core import groupby
from cbtk.speedup import make_speedup_matrices, SpeedupMatrix

def print_speedups(matrix):
    for suite in matrix:
        runners = matrix[suite].runners()
        for r0 in runners:
            for r1 in runners:
                record = matrix[suite].get(r0, r1)
                speedups = record.get_values_by_metric("_speedup")
                average = speedups["_average"]
                print(
                    f"{str(suite):20} {r0.longname:40} -> {r1.longname:40}: "
                    f"{average:.3}")

def get_oldest(records):

    def cmp_run_at(x, y):
        return x if x.run_at < y.run_at else y

    return functools.reduce(cmp_run_at, records)

def get_latest(records):

    def cmp_run_at(x, y):
        return x if x.run_at > y.run_at else y

    return functools.reduce(cmp_run_at, records)

def convert_to_table(suite, matrix):
    Table = namedtuple("Table", ["caption", "header", "rows"])

    def fmt_runner(runner):
        name = [f"{runner.name}-{runner.version}"]
        if runner.tags:
            name += [f"({runner.tags})"]
        return name

    header = [fmt_runner(r) for r in matrix.runners()]

    rows = []
    for r0 in matrix.runners():
        row = [fmt_runner(r0)]
        row += [
            f"{matrix.get(r0, r1).value('_speedup', '_average'):.2f}"
            for r1 in matrix.runners()
        ]
        rows += [row]

    return Table(caption=str(suite), header=header, rows=rows)

def make_host_section(config, hostname, records, matrix):
    Section = namedtuple(
        "Data",
        ["hostname", "latest_run_at", "oldest_run_at", "num_runs", "tables"])

    oldest = get_oldest(records)
    latest = get_latest(records)

    tables = [convert_to_table(k, v) for k, v in matrix.items()]

    return Section(hostname=hostname,
                   latest_run_at=latest.run_at.strftime("%c"),
                   oldest_run_at=oldest.run_at.strftime("%c"),
                   num_runs=len(records),
                   tables=tables)

def make_html(maker, config, sections):
    home_template = maker.get_template("home.html")
    contents = home_template.render(sections=sections)

    data = {
        "title": "Home",
        "contents": contents,
    }

    return maker.render_page(config, **data)

def drop_older_patch(runners):
    dropped = []
    for r0 in runners:
        is_older = any([r0.is_older_patch(r1) for r1 in runners])
        if not is_older:
            dropped += [r0]

    return dropped

def drop_patch(matrix):
    runners = matrix.runners()
    runners = drop_older_patch(runners)

    dropped = SpeedupMatrix()
    for r0 in runners:
        for r1 in runners:
            dropped.set(r0, r1, matrix.get(r0, r1))

    return dropped

def make_page(maker, config, records):
    groups = groupby(records, key=lambda r: r.hostname)

    sections = []
    for hostname in groups:
        matrices = make_speedup_matrices(groups[hostname], config)
        matrices = {k: drop_patch(v) for k, v in matrices.items()}
        # print_speedups(matrices)
        section = make_host_section(config, hostname, groups[hostname],
                                    matrices)
        sections += [section]

    maker.write("index.html", make_html(maker, config, sections))


import os
import sys
import pickle
import codecs
root_path = os.path.abspath('.')

from collections import defaultdict
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
import pandas as pd
import numpy as np

# global variable
PAD_ID = 0
_GO="_GO"
_END="_END"
_PAD="_PAD"

def __csv2label__():
    train_label  = pd.read_csv('../input/train_first.csv')
    train_hankcs = pd.read_csv('../input/train_word.csv')
    train = pd.merge(train_label, train_hankcs, on = 'Id', how = 'left')
    words = train['words'].values
    label = train['Score'].values
    with open('../input/tourist.zh.label.txt', 'a+', encoding='utf-8') as f:
        for i, word in enumerate(words):
            word = word[1:-1]
            word = word.replace(';', ' ')
            word += '\t' + '__label__' + str(label[i]) + '\n'
            f.write(word)

def create_voabulary(word2vec_vocabulary_path, name_scope=''):
    cache_path ='cache_vocabulary_label_pik/'+ name_scope + "_word_voabulary.pik"
    cache_path = os.path.join(root_path, cache_path)
    print("cache_path:",cache_path,"file_exists:",os.path.exists(cache_path))
    if os.path.exists(cache_path):
        with open(cache_path, 'rb') as data_f:
            vocabulary_word2index, vocabulary_index2word = pickle.load(data_f)
            return vocabulary_word2index, vocabulary_index2word
    else:
        vocabulary_word2index={}
        vocabulary_index2word={}

        vocabulary_word2index['PAD_ID']=0
        vocabulary_index2word[0]='PAD_ID'

        special_index = 0
        with open(word2vec_vocabulary_path, 'r', encoding='utf-8') as f:
            for _, line in enumerate(f.readlines()):
                vocab = line.split(' ')[0]
                vocabulary_word2index[vocab] = _ + 1 + special_index
                vocabulary_index2word[_ + 1 + special_index] = vocab

        #save to file system if vocabulary of words is not exists.
        if not os.path.exists(cache_path):
            with open(cache_path, 'ab') as data_f:
                pickle.dump((vocabulary_word2index, vocabulary_index2word), data_f)
    return vocabulary_word2index, vocabulary_index2word

# create vocabulary of lables. label is sorted. 1 is high frequency, 2 is low frequency.
def create_voabulary_label(vocabulary_label, regression_flag, name_scope='', use_seq2seq=False):
    print("create_voabulary_label_sorted.started.traning_data_path:",vocabulary_label)
    cache_path ='cache_vocabulary_label_pik/'+ name_scope + "_label_voabulary.pik"
    cache_path = os.path.join(root_path, cache_path)
    if os.path.exists(cache_path):
        with open(cache_path, 'rb') as data_f:
            vocabulary_word2index_label, vocabulary_index2word_label=pickle.load(data_f)
            return vocabulary_word2index_label, vocabulary_index2word_label
    else:
        train = codecs.open(vocabulary_label, 'r', 'utf-8')
        lines = train.readlines()

        vocabulary_word2index_label = {}
        vocabulary_index2word_label = {}
        vocabulary_label_count_dict = defaultdict(int)

        for i, line in enumerate(lines):
            if '__label__' in line:  #'__label__-2051131023989903826
                label = line[line.index('__label__') + len('__label__'):].strip().replace("\n","")
                vocabulary_label_count_dict[label] += 1

        list_label = sort_by_value(vocabulary_label_count_dict)
        print("length of list_label:",len(list_label))
        count=0

        ##########################################################################################
        if use_seq2seq: #if used for seq2seq model,insert two special label(token):_GO AND _END
            i_list = [0, 1, 2]
            label_special_list = [_GO, _END, _PAD]
            for _, label in enumerate(label_special_list):
                vocabulary_word2index_label[label] = i_list[_]
                vocabulary_index2word_label[i_list[_]] = label
        #########################################################################################
        if regression_flag:
            print('vocabulary_index2word_label is used as a classification regression')
            for label in list_label:
                label = int(label)
                vocabulary_word2index_label[label] = label
                vocabulary_index2word_label[label] = label
        else:
            for i, label in enumerate(list_label):
                if i < 10:
                    count_value = vocabulary_label_count_dict[label]
                    print("label:", label, "count_value:", count_value)
                    count = count + count_value
                index = i + 3 if use_seq2seq else i
                if use_seq2seq: print('use_seq2seq, please check the code...')
                vocabulary_word2index_label[label] = index
                vocabulary_index2word_label[index] = label
        print("count top10:", count)

        if not os.path.exists(cache_path):
            with open(cache_path, 'ab') as data_f:
                pickle.dump((vocabulary_word2index_label,vocabulary_index2word_label), data_f)

    print("create_voabulary_label_sorted.ended.len of vocabulary_label:", len(vocabulary_index2word_label))
    return vocabulary_word2index_label, vocabulary_index2word_label

def sort_by_value(d):
    items=d.items()
    backitems=[[v[1],v[0]] for v in items]
    backitems.sort(reverse=True)
    return [backitems[i][1] for i in range(0,len(backitems))]

def load_data_multilabel_new(vocabulary_word2index,
                             vocabulary_word2index_label,
                             using_kfold=False,
                             valid_portion=0.3,
                             max_training_data=1000000,
                             training_data_path='../input/tourist.zh.train.txt',
                             multi_label_flag=False,
                             use_seq2seq=False,
                             seq2seq_label_length=6):  # n_words=100000,
    """
    input: a file path
    :return: train, test, valid. where train=(trainX, trainY). where
                trainX: is a list of list.each list representation a sentence.trainY: is a list of label. each label is a number
    """
    # 1. load a tourist data from file
    # example: "userid 好 大 的 一个 游乐 公园 已经 去 了 2 次 但 感觉 还 没有 玩 够 似的 会 有 第 三 第 四 次 的	__label__5"
    print("load_data.started...")
    print("load_data_multilabel_new.training_data_path:", training_data_path)
    data_f = codecs.open(training_data_path, 'r', 'utf8')
    lines = data_f.readlines()

    # 2.transform X as indices
    # 3.transform  y as scalar
    X = []
    Y = []
    Y_decoder_input=[] #ADD 2017-06-15
    ID = []
    for i, line in enumerate(lines):
        data, y = line.split('__label__')
        y = y.strip().replace('\n', '')

        data = data.strip()

        if i < 1:
            print(i, "x0:", data)

        data = data.split(" ")
        id = data[0:1]
        x = data[1:]
        # if can't find the word, set the index as '0'.(equal to PAD_ID = 0)
        x = [vocabulary_word2index.get(e, 0) for e in x]
        if i < 2:
            print(i, "x1:", x)

        # 1) prepare label for seq2seq format(ADD _GO,_END,_PAD for seq2seq)
        if use_seq2seq:
            ys = y.replace('\n', '').split(" ")  # ys is a list
            _PAD_INDEX = vocabulary_word2index_label[_PAD]
            ys_mulithot_list = [_PAD_INDEX] * seq2seq_label_length
            ys_decoder_input = [_PAD_INDEX] * seq2seq_label_length
            # below is label.
            for j, y in enumerate(ys):
                if j < -1:
                    ys_mulithot_list[j]=vocabulary_word2index_label[y]
            if len(ys)>seq2seq_label_length-1:
                ys_mulithot_list[seq2seq_label_length-1]=vocabulary_word2index_label[_END]# ADD END TOKEN
            else:
                ys_mulithot_list[len(ys)] = vocabulary_word2index_label[_END]

            # below is input for decoder.
            ys_decoder_input[0]=vocabulary_word2index_label[_GO]
            for j, y in enumerate(ys):
                if j < seq2seq_label_length - 1:
                    ys_decoder_input[j+1]=vocabulary_word2index_label[y]
            if i < 10:
                print(i,"ys:==========>0", ys)
                print(i,"ys_mulithot_list:==============>1", ys_mulithot_list)
                print(i,"ys_decoder_input:==============>2", ys_decoder_input)
        else:
            # 2) prepare multi-label format for classification
            if multi_label_flag:
                ys = y.replace('\n', '').split(" ")  # ys is a list
                ys_index = []
                for y in ys:
                    y_index = vocabulary_word2index_label[y]
                    ys_index.append(y_index)
                ys_mulithot_list = transform_multilabel_as_multihot(ys_index)
            else:
                # 3) prepare single label format for classification
                ys_mulithot_list = vocabulary_word2index_label[y]
        if i <= 3:
            print("ys_index:")
            print(i, "y:", y, " ;ys_mulithot_list:", ys_mulithot_list)

        X.append(x)
        Y.append(ys_mulithot_list)
        ID.append(id)

        if use_seq2seq:
            Y_decoder_input.append(ys_decoder_input) #decoder input

    # 4.split to train,test and valid data
    number_examples = len(X)
    print("number_examples:", number_examples)

    if using_kfold:
        X = np.array(X)
        Y = np.array(Y)

        kf = list(StratifiedKFold(n_splits=5, random_state=2018, shuffle=False).split(X, Y))
        for train_index, test_index in kf:
            ID_ = []
            for index in test_index:
                ID_.append(ID[index][0])

            yield X[train_index], Y[train_index], X[test_index], Y[test_index], ID_
    else:
        X, X_, y, y_ = train_test_split(X, Y, test_size=valid_portion, random_state=0)
        X, y = data_augmentation(X, y, method=None)
        number_examples = len(X)
        print("number_examples:", number_examples)

        train = (X, y)
        valid = (X_, y_)

        if use_seq2seq:
            train = train + (Y_decoder_input[0:int((1 - valid_portion) * number_examples)],)
            test = valid + (Y_decoder_input[int((1 - valid_portion) * number_examples) + 1:],)

        # 5.return
        print("load_data.ended...")
        yield train, valid, valid

def data_augmentation(X, y, method = None):
    if method is not None:
        return method(X, y)
    else:
        return X, y

def shuffle(X, y):
    X = X.copy()
    y = y.copy()

    X_ = []
    y_ = []
    for x, label in zip(X, y):
        X_.append(np.random.permutation(x[::]))
        y_.append(label)

    X = np.concatenate((X, X_), axis = 0)
    y = np.concatenate((y, y_), axis = 0)
    return X, y

def transform_multilabel_as_multihot(label_list):
    """
    :param label_list: e.g.[0,1,4]
    :param label_size: e.g.199
    :return:e.g.[1,1,0,1,0,0,........]
    """
    result = np.zeros(len(label_list))
    result[label_list] = 1
    return result

def load_final_test_data(filename):
    with open(filename, 'r', encoding='utf-8') as outf:
        question_lists_result = []
        for line in outf.readlines():
            line = line.replace('\n', '')
            id, val = line.split(' ')[0], line.split(' ')[1:]
            question_lists_result.append((id, val))
        print("length of total question lists:", len(question_lists_result))
        return question_lists_result

def process_one_sentence_to_get_ui_bi_tri_gram(sentence, n_gram = 3):
    """
    :param sentence: string. example:'w17314 w5521 w7729 w767 w10147 w111'
    :param n_gram:
    :return:string. example:'w17314 w17314w5521 w17314w5521w7729 w5521 w5521w7729 w5521w7729w767 w7729 w7729w767 w7729w767w10147 w767 w767w10147 w767w10147w111 w10147 w10147w111 w111'
    """
    result=[]
    word_list=sentence.split(" ") #[sentence[i] for i in range(len(sentence))]
    unigram='';bigram='';trigram='';fourgram=''
    length_sentence=len(word_list)
    for i,word in enumerate(word_list):
        unigram=word                           #ui-gram
        word_i=unigram
        if n_gram>=2 and i+2<=length_sentence: #bi-gram
            bigram="".join(word_list[i:i+2])
            word_i=word_i+' '+bigram
        if n_gram>=3 and i+3<=length_sentence: #tri-gram
            trigram="".join(word_list[i:i+3])
            word_i = word_i + ' ' + trigram
        if n_gram>=4 and i+4<=length_sentence: #four-gram
            fourgram="".join(word_list[i:i+4])
            word_i = word_i + ' ' + fourgram
        if n_gram>=5 and i+5<=length_sentence: #five-gram
            fivegram="".join(word_list[i:i+5])
            word_i = word_i + ' ' + fivegram
        result.append(word_i)
    result=" ".join(result)
    return result

def load_data_predict(vocabulary_word2index, vocabulary_word2index_label, questionid_question_lists, uni_to_tri_gram=False):
    final_list = []
    for i, tuple in enumerate(questionid_question_lists):
        id, question_string_list = tuple
        # question_string_list = question_string_list[1:-1]
        # question_string_list = question_string_list.replace(";"," ")
        if uni_to_tri_gram:
            x_ = process_one_sentence_to_get_ui_bi_tri_gram(question_string_list)
            x = x_.split(" ")
        else:
            # x = question_string_list.split(" ")
            x = question_string_list
        x = [vocabulary_word2index.get(e, 0) for e in x]
        if i <= 2:
            print("question_id:", id)
            print("question_string_list:", question_string_list)
            print("x_indexed:", x)

        final_list.append((id, x))

    number_examples = len(final_list)
    print("number_examples:",number_examples) #
    return final_list

def split_data(filename = '../input/tourist.zh.label.txt'):
    with open(filename, 'r', encoding='utf-8') as f:
        train_outf = open(filename.replace('label', 'train'), 'w', encoding='utf-8')
        test_outf = open(filename.replace('label', 'test'), 'w', encoding='utf-8')
        for line in f.readlines():
            label = line[line.index('__label__') + len('__label__'):].strip().replace("\n", "")
            if label != '-1':
                train_outf.write(line)
            else:
                test_outf.write(line[:line.index('__label__')].strip() + '\n')
    pass

# vocabulary_word2index, vocabulary_index2word = create_voabulary(word2vec_vocabulary_path='../utils/dump/vocabulary', name_scope="TextCNN")
# vocab_size = len(vocabulary_word2index)
# vocabulary_word2index_label, vocabulary_index2word_label = create_voabulary_label(vocabulary_label='../input/tourist.zh.label.txt', name_scope="TextCNN")
# final_list = load_data_predict(vocabulary_word2index, vocabulary_word2index_label, load_final_test_data('../input/predict_word.csv'))
# print(final_list)

if __name__ == '__main__':
    # split_data()

    stacking_0 = pd.read_csv('../models/TextCNN/__models__/TextCNN_TFIDF_tra_0.csv')
    stacking_1 = pd.read_csv('../models/TextCNN/__models__/TextCNN_TFIDF_tra_1.csv')
    stacking_2 = pd.read_csv('../models/TextCNN/__models__/TextCNN_TFIDF_tra_2.csv')
    stacking_3 = pd.read_csv('../models/TextCNN/__models__/TextCNN_TFIDF_tra_3.csv')
    stacking_4 = pd.read_csv('../models/TextCNN/__models__/TextCNN_TFIDF_tra_4.csv')
    stacking = pd.concat([stacking_0, stacking_1, stacking_2, stacking_3, stacking_4])

    stacking.to_csv('TextCNN_train.csv', index = False)

    test_0 = pd.read_csv('../models/TextCNN/__models__/tourist_result_cnn_multilabel_v6_e14_kf_0.csv')
    test_1 = pd.read_csv('../models/TextCNN/__models__/tourist_result_cnn_multilabel_v6_e14_kf_1.csv')
    test_2 = pd.read_csv('../models/TextCNN/__models__/tourist_result_cnn_multilabel_v6_e14_kf_2.csv')
    test_3 = pd.read_csv('../models/TextCNN/__models__/tourist_result_cnn_multilabel_v6_e14_kf_3.csv')
    test_4 = pd.read_csv('../models/TextCNN/__models__/tourist_result_cnn_multilabel_v6_e14_kf_4.csv')

    test_0 = pd.merge(test_0, test_1, on = 'Id', how = 'left')
    test_0 = pd.merge(test_0, test_2, on = 'Id', how = 'left')
    test_0 = pd.merge(test_0, test_3, on = 'Id', how='left')
    test_0 = pd.merge(test_0, test_4, on = 'Id', how='left')

    Id = test_0[['Id']]
    Id['textCNN_label'] = test_0.drop(['Id'], axis=1).mean(axis=1)

    Id.to_csv('TextCNN_test.csv', index = False)


import os
import sys
import random
from flask import Flask, request, jsonify

import config
from proxy_pool import ProxyPool

app = Flask(__name__)

@app.route('/')
def index():
    return 'Proxy Pool'

@app.route('/get')
def get():
    p = ProxyPool.get()
    return p

@app.route('/count')
def count():
    n = ProxyPool.count()
    return str(n)

@app.route('/delete')
def delete():
    pass
    return 'not complete'

application = app

def run():
    app.run(host=config.web_host, port=config.web_port)

if __name__ == '__main__':
    run()


import io
import numpy as np
from torch import nn
import torch.utils.model_zoo as model_zoo
import torch.onnx

import torch.nn as nn
import torch.nn.init as init
from gen_data import gen_np_args, args_adaptor

import onnx

def bbox_overlaps(bboxes1, bboxes2, mode="iou", is_aligned=False, eps=1e-6):
    """Calculate overlap between two set of bboxes.

    If ``is_aligned`` is ``False``, then calculate the ious between each bbox
    of bboxes1 and bboxes2, otherwise the ious between each aligned pair of
    bboxes1 and bboxes2.

    Args:
        bboxes1 (Tensor): shape (m, 4) in <x1, y1, x2, y2> format or empty.
        bboxes2 (Tensor): shape (n, 4) in <x1, y1, x2, y2> format or empty.
            If is_aligned is ``True``, then m and n must be equal.
        mode (str): "iou" (intersection over union) or iof (intersection over
            foreground).

    Returns:
        ious(Tensor): shape (m, n) if is_aligned == False else shape (m, 1)

    Example:
        >>> bboxes1 = torch.FloatTensor([
        >>>     [0, 0, 10, 10],
        >>>     [10, 10, 20, 20],
        >>>     [32, 32, 38, 42],
        >>> ])
        >>> bboxes2 = torch.FloatTensor([
        >>>     [0, 0, 10, 20],
        >>>     [0, 10, 10, 19],
        >>>     [10, 10, 20, 20],
        >>> ])
        >>> bbox_overlaps(bboxes1, bboxes2)
        tensor([[0.5000, 0.0000, 0.0000],
                [0.0000, 0.0000, 1.0000],
                [0.0000, 0.0000, 0.0000]])

    Example:
        >>> empty = torch.FloatTensor([])
        >>> nonempty = torch.FloatTensor([
        >>>     [0, 0, 10, 9],
        >>> ])
        >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1)
        >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0)
        >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0)
    """

    assert mode in ["iou", "iof"]
    # Either the boxes are empty or the length of boxes's last dimenstion is 4
    assert bboxes1.size(-1) == 4 or bboxes1.size(0) == 0
    assert bboxes2.size(-1) == 4 or bboxes2.size(0) == 0

    rows = bboxes1.size(0)
    cols = bboxes2.size(0)
    if is_aligned:
        assert rows == cols

    if rows * cols == 0:
        return bboxes1.new(rows, 1) if is_aligned else bboxes1.new(rows, cols)

    if is_aligned:
        lt = torch.max(bboxes1[:, :2], bboxes2[:, :2])  # [rows, 2]
        rb = torch.min(bboxes1[:, 2:], bboxes2[:, 2:])  # [rows, 2]

        wh = (rb - lt).clamp(min=0)  # [rows, 2]
        overlap = wh[:, 0] * wh[:, 1]
        area1 = (bboxes1[:, 2] - bboxes1[:, 0]) * (bboxes1[:, 3] -
                                                   bboxes1[:, 1])

        if mode == "iou":
            area2 = (bboxes2[:, 2] - bboxes2[:, 0]) * (bboxes2[:, 3] -
                                                       bboxes2[:, 1])
            union = area1 + area2 - overlap
        else:
            union = area1
    else:
        lt = torch.max(bboxes1[:, None, :2], bboxes2[:, :2])  # [rows, cols, 2]
        rb = torch.min(bboxes1[:, None, 2:], bboxes2[:, 2:])  # [rows, cols, 2]

        wh = (rb - lt).clamp(min=0)  # [rows, cols, 2]
        overlap = wh[:, :, 0] * wh[:, :, 1]
        area1 = (bboxes1[:, 2] - bboxes1[:, 0]) * (bboxes1[:, 3] -
                                                   bboxes1[:, 1])

        if mode == "iou":
            area2 = (bboxes2[:, 2] - bboxes2[:, 0]) * (bboxes2[:, 3] -
                                                       bboxes2[:, 1])
            union = area1[:, None] + area2 - overlap
        else:
            union = area1[:, None]

    eps = union.new_tensor([eps])
    union = torch.max(union, eps)
    ious = overlap / union

    return ious

def iou_loss(pred, target, eps=1e-6):
    """IoU loss.

    Computing the IoU loss between a set of predicted bboxes and target bboxes.
    The loss is calculated as negative log of IoU.

    Args:
        pred (torch.Tensor): Predicted bboxes of format (x1, y1, x2, y2),
            shape (n, 4).
        target (torch.Tensor): Corresponding gt bboxes, shape (n, 4).
        eps (float): Eps to avoid log(0).

    Return:
        torch.Tensor: Loss tensor.
    """
    ious = bbox_overlaps(pred, target, is_aligned=True).clamp(min=eps)
    loss = -ious.log()
    return loss

class Bbox(nn.Module):
    def __init__(self):
        super(Bbox, self).__init__()
        self.bbox_overlaps = bbox_overlaps
        self.iou_loss = iou_loss

    def forward(self, pred, target):
        loss = self.iou_loss(pred, target)

        return loss

torch_model = Bbox()

torch_model.eval()

pred, target = args_adaptor(gen_np_args(128, ))
torch_out = torch_model(pred, target)

torch.onnx.export(torch_model,
        (pred, target),
        "iou_loss.onnx",
        verbose=True,
        export_params=True,
        opset_version=12,
        input_names=['pred', 'target'],
        output_names = ['output'])


from django.contrib.auth import get_user_model
from django.db.utils import ProgrammingError

def disable_admin_login():
    """
    Disable admin login, but allow editing.

    amended from: https://stackoverflow.com/a/40008282/517560
    """
    User = get_user_model()

    try:
        user, created = User.objects.update_or_create(
            id=1,
            defaults=dict(
                first_name="Default Admin",
                last_name="User",
                is_superuser=True,
                is_active=True,
                is_staff=True,
            ),
        )
    except ProgrammingError:
        # auth_user doesn't exist, this allows the migrations to run properly.
        user = None

    def no_login_has_permission(request):
        setattr(request, "user", user)

        return True

    return no_login_has_permission


import argparse

def _setup_parser():
    parser = argparse.ArgumentParser(description="Aprendendo a fazer um autoencoder simples")

    # Show linear transformation using nn.Linear
    parser.add_argument('--lintrans', action='store_true', help='Runs the linear transformation snippet')

    # Import MNIST dataset
    parser.add_argument('--mnist', action='store_true', help='Imports the MNIST dataset')

    # Create construction-test split
    parser.add_argument('--split', action='store_true', help='Splits dataset into train-validation-test subsets')

    # Normalize construction and test sets
    parser.add_argument('--prep', action='store_true', help='Preprocess dataset')

    # Create a simple autoencoder and run it without training
    parser.add_argument('--simpleae', action='store_true', help='Running simple autoencoder (without training)')

    return parser.parse_args()


#!/usr/bin/env python3
import datetime
import os

import numpy as np
import tensorflow as tf
from keras import backend as K
from sklearn.model_selection import KFold
from tensorflow import keras
from tensorflow.keras.layers import (
    Activation,
    BatchNormalization,
    Concatenate,
    Conv2D,
    Conv2DTranspose,
    Input,
    MaxPool2D,
)
from tensorflow.keras.losses import CategoricalCrossentropy
from tensorflow.keras.metrics import BinaryIoU, MeanIoU
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tqdm import tqdm

from utils import prep_data

# Config
BASE_LOGS_DIR = "logs"
DATA_DIR = "data"
SET_NAMES = ["classes_2"]
SELECT_LABELS = [2]
N_CLASSES = 1
MULTICLASS = False
MASK_VALUE = -1

# Hyperparams
N_FOLDS = 5
EPOCHS = 50
ETA = 1e-2
BATCH_SIZE = 4

tf.get_logger().setLevel("ERROR")

# Construct the U-Net
def conv_block(input, num_filters):
    x = Conv2D(num_filters, 3, padding="same")(input)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = Conv2D(num_filters, 3, padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    return x

def encoder_block(input, num_filters):
    x = conv_block(input, num_filters)
    p = MaxPool2D((2, 2))(x)
    return x, p

def decoder_block(input, skip_features, num_filters):
    x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding="same")(input)
    x = Concatenate()([x, skip_features])
    x = conv_block(x, num_filters)
    return x

def data_augmentation():
    augs = keras.Sequential(
        [
            keras.layers.RandomFlip("horizontal_and_vertical"),
            keras.layers.RandomRotation(0.5),
            # keras.layers.RandomContrast(0.1),
            # keras.layers.RandomBrightness(0.1),
        ]
    )
    return augs

def build_unet(input_shape, aug=False):
    inputs = Input(input_shape)
    if aug:
        inputs = data_augmentation()(inputs)
    s1, p1 = encoder_block(inputs, 64)
    s2, p2 = encoder_block(p1, 128)
    s3, p3 = encoder_block(p2, 256)
    s4, p4 = encoder_block(p3, 512)
    b1 = conv_block(p4, 1024)
    d1 = decoder_block(b1, s4, 512)
    d2 = decoder_block(d1, s3, 256)
    d3 = decoder_block(d2, s2, 128)
    d4 = decoder_block(d3, s1, 64)

    if MULTICLASS:
        outputs = Conv2D(filters=N_CLASSES, kernel_size=(1,1), padding="same", activation="softmax")(d4)
    else:
        outputs = Conv2D(1, 1, padding="same", activation="sigmoid")(d4)

    model = Model(inputs, outputs, name="U-Net")
    return model

def callbacks(name):
    cb = []
    now = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    LOGS_DIR = os.path.join(BASE_LOGS_DIR, f"{now}_{name}")
    if not os.path.exists(LOGS_DIR):
        os.makedirs(LOGS_DIR)
    tensorboard = keras.callbacks.TensorBoard(
        log_dir=LOGS_DIR,
        histogram_freq=1,
    )
    checkpoint = keras.callbacks.ModelCheckpoint(
        os.path.join(LOGS_DIR, f"{name}_best_wg.h5"),
        save_best_only=True,
        mode="min",
        monitor="val_loss",
        save_weights_only=True,
        verbose=1,
    )
    lr_decay = keras.callbacks.ReduceLROnPlateau(
        monitor="val_loss",
        factor=0.1,
        patience=5,
        verbose=1,
        mode="auto",
        min_delta=0.0001,
        cooldown=1,
        min_lr=0.00001,
    )
    log = keras.callbacks.CSVLogger(os.path.join(LOGS_DIR, f"{now}_{name}_history.csv"))
    cb.append(tensorboard)
    cb.append(checkpoint)
    cb.append(lr_decay)
    cb.append(log)
    return cb

def train_unet(
    X_train,
    y_train,
    X_test,
    y_test,
    batch_size=16,
    epochs=100,
    eta=1e-2,
    cb=None,
    masked_loss=False,
):
    input_shape = X_train.shape[1:]
    model = build_unet(input_shape)
    batch_size = batch_size
    epochs = epochs

    if MULTICLASS:
        loss = "categorical_crossentropy"
        metrics = MeanIoU(num_classes=N_CLASSES)
    else:
        loss = "binary_crossentropy"
        metrics = BinaryIoU(target_class_ids=[1], threshold=0.5)

    model.compile(
        optimizer=Adam(learning_rate=eta),
        loss=loss,
        metrics=[metrics],
    )
    history = model.fit(
        X_train,
        y_train,
        batch_size=batch_size,
        epochs=epochs,
        validation_data=(X_test, y_test),
        verbose=1,
        callbacks=cb,
    )
    return model, history

def concat_train_test(X_train, y_train, X_test, y_test, X_train_add, y_train_add, X_test_add, y_test_add):
    X_train = np.concatenate((X_train, X_train_add), axis=0)
    y_train = np.concatenate((y_train, y_train_add), axis=0)
    X_test = np.concatenate((X_test, X_test_add), axis=0)
    y_test = np.concatenate((y_test, y_test_add), axis=0)
    return X_train, y_train, X_test, y_test

def load_datasets(include_nir=False, add_ndvi=False, select_labels=False, squash=True):
    # Load Kivalina
    X_train, y_train, X_test, y_test = prep_data(
        "data/WA_Kivalina_01_20219703/20cm/Ortho",
        include_nir=include_nir,
        add_ndvi=add_ndvi,
        select_labels=select_labels,
        squash=squash,
        img_size_override=2500,
    )

    # Load Kotzebue
    X_train_add, y_train_add, X_test_add, y_test_add = prep_data(
        "data/WA_Kotzebue_01_20210625/20cm/Ortho",
        include_nir=include_nir,
        add_ndvi=add_ndvi,
        select_labels=select_labels,
        squash=squash,
        img_size_override=2500,
    )

    # Concatenate Kivalina and Kotzebue
    X_train, y_train, X_test, y_test = concat_train_test(
        X_train, y_train, X_test, y_test, X_train_add, y_train_add, X_test_add, y_test_add
    )

    # Load Shishmaref
    X_train_add, y_train_add, X_test_add, y_test_add = prep_data(
        "data/WA_Shishmaref_01_20210628/20cm/Ortho",
        include_nir=include_nir,
        add_ndvi=add_ndvi,
        select_labels=select_labels,
        squash=squash,
        img_size_override=2500,
    )

    # Concatenate Shishmaref to train test sets
    X_train, y_train, X_test, y_test = concat_train_test(
        X_train, y_train, X_test, y_test, X_train_add, y_train_add, X_test_add, y_test_add
    )

    print("\nFinal concatenated sizes:\n")
    print("-------------------------\n")
    print("X_train:", X_train.shape)
    print("y_train:", y_train.shape)
    print("X_test:", X_test.shape)
    print("y_test:", y_test.shape)

    return X_train, y_train, X_test, y_test

def train_set(include_nir=False, add_ndvi=False, squash=True):

    X_train, y_train, X_test, y_test = load_datasets(include_nir=include_nir, add_ndvi=add_ndvi, select_labels=SELECT_LABELS, squash=True)

    if MULTICLASS:
        y_train = tf.keras.utils.to_categorical(y_train, N_CLASSES)
        y_test = tf.keras.utils.to_categorical(y_test, N_CLASSES)

    # MODEL
    # Data structure
    stats = ["loss", "val_loss", "iou", "val_iou", "mean_biou", "human_biou", "bg_biou"]

    data = np.zeros((N_FOLDS, len(stats)), dtype=object)

    # Initialize the KFold
    kf = KFold(n_splits=N_FOLDS, shuffle=True)

    for i, (itrain, itest) in enumerate(
        tqdm(
            kf.split(
                X_train,
                y_train,
            ),
            desc="K-Folds",
            position=0,
            leave=True,
            total=N_FOLDS,
        )
    ):
        X_train_fold = X_train[itrain]
        y_train_fold = y_train[itrain]
        X_test_fold = X_train[itest]
        y_test_fold = y_train[itest]

        # Set callbacks each iteration so that logs are stored in new
        # directory
        cb = callbacks(f"KF{i+1}of{N_FOLDS}_{set_name}")
        model, history = train_unet(
            X_train_fold,
            y_train_fold,
            X_test_fold,
            y_test_fold,
            batch_size=BATCH_SIZE,
            epochs=EPOCHS,
            eta=ETA,
            cb=cb,
        )

        # Loss and accuracies from each epoch
        loss = history.history["loss"]
        val_loss = history.history["val_loss"]
        iou = history.history[list(history.history.keys())[1]]
        val_iou = history.history[list(history.history.keys())[3]]

        # Test the model on the preserved test data
        y_pred = model.predict(X_test, batch_size=2)

        # Get the IoU for the test data
        biou = BinaryIoU(target_class_ids=[0, 1], threshold=0.5)
        biou.update_state(y_pred=y_pred, y_true=y_test)
        pred_biou = biou.result().numpy()

        # only for human-built
        human_biou = BinaryIoU(target_class_ids=[1], threshold=0.5)
        human_biou.update_state(y_pred=y_pred, y_true=y_test)
        pred_human_biou = human_biou.result().numpy()

        # only for non-tree pixel (background)
        bg_biou = BinaryIoU(target_class_ids=[0], threshold=0.5)
        bg_biou.update_state(y_pred=y_pred, y_true=y_test)
        pred_bg_biou = bg_biou.result().numpy()

        # Log the five stats according to their K-Fold and parameter iteration
        stats = [
            loss,
            val_loss,
            iou,
            val_iou,
            pred_biou,
            pred_human_biou,
            pred_bg_biou,
        ]

        for s, stat in enumerate(stats):
            data[i, s] = stat

        # Save the updated array each iteration
        np.save(f"stats/{N_FOLDS}folds_CV_{set_name}.npy", data)

if __name__ == "__main__":
    for set_name in tqdm(SET_NAMES, desc="Set", total=len(SET_NAMES)):
        train_set(include_nir=True, add_ndvi=True, squash=True)


from MainWindow import Ui_MainWindow
from pictures_screen import Ui_Picture_Window
from PyQt5 import QtWidgets, QtGui, QtCore
import sys
import zipfile
import datetime
import os
import numpy as np
import cv2

class MyWindow(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self):
        super(MyWindow, self).__init__()
        self.setupUi(self)
        self.setStyleSheet('.QWidget {background-image: url(background.png);}')
        self.pushButton_start.clicked.connect(self.start)

    def start(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self, "Выбрать архив", '',
                                                      'Архив (*.zip);;Картинка (*.jpg);;Картинка (*.png)')[0]
        self.screen = Picture_Show(fname)
        self.screen.show()

class Picture_Show(QtWidgets.QMainWindow, Ui_Picture_Window):
    def __init__(self, name):
        super(Picture_Show, self).__init__ ()
        self.setupUi(self)
        self.setStyleSheet('.QWidget {background-image: url(background.png);}')
        if name[-3:] == 'zip':
            files = zipfile.ZipFile(name)
            self.name_of_directory = str(datetime.date.today())
            os.mkdir(self.name_of_directory)
            files.extractall(self.name_of_directory)
            self.many_files()
        else:
            self.one_file(name.split('/')[-1])

    def change_photo(self, fname, directory_name):
        img = cv2.imread(fname)
        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        h = hsv[:, :, 0]

        img[h >= 45] = (0, 0, 0)
        img[h <= 19] = (0, 0, 0)

        gray = np.copy(img[:, :, 0])

        thr, wb = cv2.threshold(gray, 3, 255, cv2.THRESH_BINARY)

        num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(wb, 4, cv2.CV_32S)

        MAX_W = 70
        MAX_H = 70
        MIN_W = 20
        MIN_H = 20

        labels_to_add = []

        for i, stat in enumerate(stats):
            if stat[cv2.CC_STAT_WIDTH] > MAX_W or stat[cv2.CC_STAT_HEIGHT] > MAX_H:
                continue

            if stat[cv2.CC_STAT_WIDTH] < MIN_W or stat[cv2.CC_STAT_HEIGHT] < MIN_H:
                continue

            labels_to_add.append (i)
            print(f'Added {i}')

        wb[:, :] = 0

        for lbl in labels_to_add:
            print(f'Added {lbl}')
            wb[labels == lbl] = 255

        kernel = np.ones((9, 9), np.uint8)
        eroded = cv2.erode(wb, kernel, iterations = 1)
        dilated = cv2.dilate(eroded, kernel, iterations = 1)
        img[dilated == 255] = [0, 0, 255]

        cv2.imwrite(directory_name, img)
        return directory_name

    def many_files(self):
        directory = os.fsdecode(self.name_of_directory)
        self.files_path = []
        self.name_of_directory_new = self.name_of_directory + '_new'
        os.mkdir(self.name_of_directory_new)
        for file in os.listdir(directory):
            fname = self.name_of_directory + '/' + str(file)
            directory_name = self.name_of_directory_new + '/' + str(file)
            self.files_path.append(self.change_photo(fname, directory_name))
        self.last_ind_of_photo = 0
        self.set_pixmap(self.files_path[self.last_ind_of_photo])
        print(self.files_path)
        self.pushButton_left.setEnabled(False)
        self.pushButton_right.clicked.connect(self.right)
        self.pushButton_left.clicked.connect(self.left)

    def right(self):
        self.last_ind_of_photo += 1
        self.set_pixmap(self.files_path[self.last_ind_of_photo])
        if self.last_ind_of_photo == 1:
            self.pushButton_left.setEnabled(True)
        if self.last_ind_of_photo == len(self.files_path) - 1:
            self.pushButton_right.setEnabled(False)

    def left(self):
        self.last_ind_of_photo -= 1
        self.set_pixmap(self.files_path[self.last_ind_of_photo])
        if self.last_ind_of_photo == 0:
            self.pushButton_left.setEnabled(False)
        if self.last_ind_of_photo == len(self.files_path) - 2:
            self.pushButton_right.setEnabled(True)

    def one_file(self, fname):
        self.pushButton_left.setEnabled(False)
        self.pushButton_right.setEnabled(False)
        print('processing...')
        self.change_photo(fname, fname[:-4] + '_new' + fname[-4:])
        self.set_pixmap(fname[:-4] + '_new' + fname[-4:])

    def set_pixmap(self, fname):
        print('setting photo...')
        pixmap = QtGui.QPixmap(fname)
        pixmap2 = pixmap.scaledToHeight(1024)
        self.label.setPixmap(pixmap2)

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    app.setWindowIcon(QtGui.QIcon('icon.png'))
    window = MyWindow()
    window.setWindowIcon(QtGui.QIcon('icon.png'))
    window.show()
    sys.exit(app.exec())


from docx import Document

class Gather:
    code = {"CS8451":"DAA", "MA8402":"PQT"}
    subjDepartment = {"DAA":"CSE/IT", "PQT":"CSE"}
    def __init__(self, subject, exam, timetable):
        self.subject = subject
        self.exam = exam
        self.timetable = timetable
        self.tt = Document(self.timetable)

    def getDate(self):
        table = self.tt.tables[0]
        for j in range(2,12):
            for i in range(1,7):
                if table.cell(i,j).text == Gather.code[self.subject]:
                    self.date = table.cell(i,1).text
        return self.date

    def getTime(self):
        if int(self.exam[-1]) in [1,2]:
            return "1.30hours"
        else:
            return "3 hours"

    def getTotal(self):
        if int(self.exam[-1]) in [1,2]:
            return "50 marks"
        else:
            return "100 marks"

if __name__ == "__main__":
    obj = Gather("MA8402", "IA2", "TimeTable.docx")
    print(obj.getDate())
    print(obj.getTime())
    print(obj.getTotal())


from __future__ import annotations

from collections import Counter
from typing import Dict, List, Optional, Tuple
from fastq_demux.parser import FastqFileParser
from fastq_demux.writer import FastqFileWriterHandler

class DemultiplexResults:
    """
    Class to handle the statistics from the demultiplexing
    """
    def __init__(self, barcode_to_sample_mapping: Dict[str, str]):
        """
        Create a new DemultiplexResults instance

        :param barcode_to_sample_mapping: a dict having barcodes as keys and the corresponding
        sample-id as value, representing the known barcodes used for demultiplexing
        """
        self.barcode_to_sample_mapping: Dict[str, str] = barcode_to_sample_mapping
        self.barcode_counts: Counter = Counter()
        self.barcode_mismatches_count: Dict[str, Counter] = {
            barcode: Counter() for barcode in barcode_to_sample_mapping.keys()}

    def add(self, barcode: str, n: int = 1, distance: int = 0) -> None:
        """
        Increase the count of the supplied barcode by n (default = 1)

        :param barcode: the barcode to increase the count for
        :param n: the value to increase the count with (default = 1)
        :param distance: the number of mismatches encountered in the barcode (default = 0)
        """
        self.barcode_counts[barcode] += n
        try:
            self.barcode_mismatches_count[barcode][str(distance)] += n
        except KeyError:
            pass

    def known_barcode_counts(self) -> Counter:
        """
        Get a Counter representing only the known barcodes
        :return: a Counter representing only the known barcodes
        """
        return Counter({barcode: count for barcode, count in self.barcode_counts.items()
                        if barcode in self.barcode_to_sample_mapping})

    def unknown_barcode_counts(self) -> Counter:
        """
        Get a Counter representing only the unknown barcodes
        :return: a Counter representing only the unknown barcodes
        """
        return Counter({barcode: count for barcode, count in self.barcode_counts.items()
                        if barcode not in self.barcode_to_sample_mapping})

    def _sample_to_barcode(self) -> Dict[str, List[str]]:
        sample_to_barcode = {
            sample: []
            for sample in set(list(self.barcode_to_sample_mapping.values()))}
        for barcode, sample in self.barcode_to_sample_mapping.items():
            sample_to_barcode[sample].append(barcode)
        return sample_to_barcode

    def barcode_mismatch_counts(self, barcode: str) -> List[int]:
        max_distance = max([0] + [
            int(distance)
            for distance in self.barcode_mismatches_count[barcode].keys()])
        return [
            self.barcode_mismatches_count[barcode][str(distance)]
            for distance in range(max_distance + 1)]

    def summarize_counts(
            self,
            barcode_set: str = "all",
            n_values: Optional[int] = None) -> List[Tuple[str, int, float]]:
        """
        Summarize the counts for a set of barcodes, presenting counts and percent of total for each
        barcode in the set

        :param barcode_set: the set of barcodes to present a summary for, this must be one of
        "all", "known" or "unknown"
        :param n_values: if specified, present only the n_values top values
        :return: a list of Tuples, each element being a barcode, count and percent of total
        """
        total: int = sum(self.barcode_counts.values())
        counter: Counter = self.barcode_counts
        if barcode_set == "known":
            counter: Counter = self.known_barcode_counts()
        elif barcode_set == "unknown":
            counter: Counter = self.unknown_barcode_counts()

        return [(
            barcode,
            count,
            round(100. * count / total, 1))
            for barcode, count in counter.most_common(n_values)]

    def stats_json(self) -> Dict[str, dict]:
        """
        Summarize the results in a data structure mimicking the output in bcl2fastq's Stats.json

        :return: a dict mimicking the output in bcl2fastq's Stats.json
        """
        demux_results: List[dict] = []

        for sample, barcodes in self._sample_to_barcode().items():
            demux_results.append({
                "SampleId": sample,
                "NumberReads": sum([self.barcode_counts[barcode] for barcode in barcodes]),
                "IndexMetrics": [{
                    "IndexSequence": barcode,
                    "MismatchCounts": {
                        str(distance): self.barcode_mismatches_count[barcode][str(distance)]
                        for distance, count in enumerate(self.barcode_mismatch_counts(barcode))
                    }
                } for barcode in barcodes]
            })

        return {
            "ConversionResults": {
                "DemuxResults": demux_results,
                "Undetermined": {
                    "NumberReads": sum(self.unknown_barcode_counts().values())}},
            "UnknownBarcodes": {
                "Barcodes": self.unknown_barcode_counts()}}

class FastqDemultiplexer:
    """
    Class handling the demultiplexing of a FASTQ read according to the barcode in the header
    """

    def __init__(
            self,
            fastq_parser: FastqFileParser,
            fastq_writer: FastqFileWriterHandler,
            demultiplex_results: DemultiplexResults,
            unknown_barcode: str,
            mismatches: int = 0):
        """
        Create a new FastqDemultiplexer instance

        :param fastq_parser: a FastqFileParser instance used to parse the input FASTQ file(s)
        :param fastq_writer: a FastqFileWriterHandler instance used to write the demultiplexed
        output to corresponding FASTQ file(s)
        :param demultiplex_results: a Demultiplexresults instance used for collecting statistics
        :param unknown_barcode: a string to collect the unknown barcodes under
        :param mismatches: the number of mismatches allowed when matching barcodes (default 0)
        """
        self.fastq_parser: FastqFileParser = fastq_parser
        self.fastq_writer: FastqFileWriterHandler = fastq_writer
        self.demultiplex_results: DemultiplexResults = demultiplex_results
        self.unknown_barcode: str = unknown_barcode
        self.mismatches = mismatches
        self.mismatched_barcodes: Dict[str, Tuple[str, int, str]] = {}

    @staticmethod
    def create_fastq_demultiplexer(
        fastq_parser: FastqFileParser,
        fastq_writer: FastqFileWriterHandler,
        demultiplex_results: DemultiplexResults,
        unknown_barcode: str,
        mismatches: int = 0) -> FastqDemultiplexer:
        if mismatches == 0:
            return FastqDemultiplexer(
                fastq_parser,
                fastq_writer,
                demultiplex_results,
                unknown_barcode,
                mismatches)
        return FastqMismatchDemultiplexer(
                fastq_parser,
                fastq_writer,
                demultiplex_results,
                unknown_barcode,
                mismatches)

    def demultiplex(self) -> DemultiplexResults:
        """
        Iterate over the fastq_parser and write output to the fastq_writer

        :return: the DemultiplexResults instance containing the statistics from the demultiplexing
        """
        for record, barcode in self.fastq_parser.fastq_records():
            self.demultiplex_record(record, barcode)
        return self.demultiplex_results

    def _write_matching_barcode(self, fastq_record: List[List[str]], barcode: str) -> None:
        self.fastq_writer.write_fastq_record(barcode, fastq_record)

    def demultiplex_record(self, fastq_record: List[List[str]], barcode: str) -> None:
        """
        Demultiplex a FASTQ read (or read pair) using the barcode in the FASTQ header and write it
        to the corresponding FastqFileWriter

        :param fastq_record: a 1- or 2-element list of FASTQ reads for single-end or paired-end,
        respectively. Each read is a list of 4 strings
        :param barcode: a string containing the barcode belonging to the record
        """
        try:
            self._write_matching_barcode(fastq_record, barcode)
        except KeyError:
            self._write_matching_barcode(fastq_record, self.unknown_barcode)
        self.demultiplex_results.add(barcode)

class FastqMismatchDemultiplexer(FastqDemultiplexer):

    @staticmethod
    def hamming_distance(str1: str, str2: str) -> int:
        return sum([int(s1 != s2) for (s1, s2) in zip(str1, str2)])

    def match_mismatched_single_barcode(
            self, barcode: str, known_barcodes: List[str]) -> Tuple[str, int]:
        distances = list(map(
            lambda x: self.hamming_distance(x, barcode),
            known_barcodes))
        no_mm = -1
        while no_mm < self.mismatches:
            no_mm += 1
            try:
                i = distances.index(no_mm)
                if distances.count(no_mm) > 1 and \
                        len(set([
                            known_barcodes[x] for x, d in enumerate(distances) if d == no_mm])) > 1:
                    raise Exception(
                        f"the barcode {barcode} is ambiguous when allowing {self.mismatches} "
                        f"mismatches")
                return known_barcodes[i], no_mm
            except ValueError:
                pass
        return "", -1

    def match_mismatched_barcode(self, barcode: str) -> None:
        keys = list(
            self.demultiplex_results.barcode_to_sample_mapping.keys())
        matched_key = []
        distance = 0
        for ix, sindex in enumerate(barcode.split("+")):
            known_barcodes = [k.split("+")[ix] for k in keys if k != self.unknown_barcode]
            mk, d = self.match_mismatched_single_barcode(sindex, known_barcodes)
            matched_key.append(mk)
            distance = max([d, distance])
        matched_barcode = "+".join(matched_key)
        counted_barcode = matched_barcode
        if matched_barcode not in keys:
            matched_barcode = self.unknown_barcode
            distance = 0
            counted_barcode = barcode
        self.mismatched_barcodes[barcode] = (matched_barcode, distance, counted_barcode)

    def demultiplex_record(self, fastq_record: List[List[str]], barcode: str) -> None:
        """
        Demultiplex a FASTQ read (or read pair) using the barcode in the FASTQ header and write it
        to the corresponding FastqFileWriter

        :param fastq_record: a 1- or 2-element list of FASTQ reads for single-end or paired-end,
        respectively. Each read is a list of 4 strings
        :param barcode: a string containing the barcode belonging to the record
        """
        try:
            self._write_matching_barcode(fastq_record, self.mismatched_barcodes[barcode][0])
        except KeyError:
            self.match_mismatched_barcode(barcode)
            return self.demultiplex_record(fastq_record, barcode)
        self.demultiplex_results.add(
            self.mismatched_barcodes[barcode][2],
            distance=self.mismatched_barcodes[barcode][1])


#!/usr/bin/env python3
# coding: utf-8
"""
@file: test_qc.py
@description:
@author: Ping Qiu
@email: qiuping1@genomics.cn
@last modified by: Ping Qiu

change log:
    2021/07/05  create file.
"""
from stereo.core.stereo_exp_data import StereoExpData
from stereo.preprocess.qc import cal_qc
from stereo.io.reader import read_h5ad

def make_data():
    import numpy as np
    from scipy import sparse
    genes = ['g1', 'mt-111', 'g3']
    rows = [0, 1, 0, 1, 2, 0, 1, 2]
    cols = [0, 0, 1, 1, 1, 2, 2, 2]
    cells = ['c1', 'c2', 'c3']
    v = [2, 3, 4, 5, 6, 3, 4, 5]
    exp_matrix = sparse.csr_matrix((v, (rows, cols)))
    # exp_matrix = sparse.csr_matrix((v, (rows, cols))).toarray()
    position = np.random.randint(0, 10, (len(cells), 2))
    out_path = '/home/qiuping//workspace/st/stereopy_data/test.h5ad'
    data = StereoExpData(bin_type='cell_bins', exp_matrix=exp_matrix, genes=np.array(genes),
                         cells=np.array(cells), position=position, output=out_path)
    return data

def quick_test():
    data = make_data()
    data = cal_qc(data)
    print(data.gene_names.dtype)
    print(data.cells.total_counts)
    print(data.cells.n_genes_by_counts)
    print(data.cells.pct_counts_mt)
    return data

def test_file():
    path = '/home/qiuping//workspace/st/stereopy_data/mource_bin100.h5ad'
    data = read_h5ad(path)
    print(data.gene_names.dtype)

    data = cal_qc(data)
    print(data.cells.total_counts)
    print(data.cells.n_genes_by_counts)
    print(data.cells.pct_counts_mt)

if __name__ == '__main__':
    quick_test()
    test_file()


import pygame
import random
import colorama
from colorama import *

colorama.init()

screen_size = [360, 600]
score = 0
screen = pygame.display.set_mode(screen_size)

backgound = pygame.image.load("forest.png")
player = pygame.image.load("bowl.svg")
chicken = pygame.image.load("apple.svg")

# define color
green = Fore.GREEN
red = Fore.RED

def random_offset():
    return -1 * random.randint(100, 2000)

chicken_y = [random_offset(), random_offset(), random_offset()]
user_x = 150

def chicken_position(idx):
    if chicken_y[idx] > 600:
        chicken_y[idx] = random_offset()
    else:
        chicken_y[idx] = chicken_y[idx] + 5

keep_alive = True
clock = pygame.time.Clock()
while keep_alive:
    # Event controler
    pygame.event.get()
    keys = pygame.key.get_pressed()
    if keys[pygame.K_RIGHT] and user_x < 303:
        user_x += 10
    elif keys[pygame.K_LEFT] and user_x > 0:
        user_x -= 10

    chicken_position(0)
    chicken_position(1)
    chicken_position(2)

    screen.blit(backgound, [0, 0])
    screen.blit(player, [user_x, 520])
    screen.blit(chicken, [0, chicken_y[0]])
    screen.blit(chicken, [150, chicken_y[1]])
    screen.blit(chicken, [280, chicken_y[2]])

    if chicken_y[0] > 500 and user_x < 70  or chicken_y[2] > 500 and user_x < 220 or chicken_y[1] > 500 and user_x > 100 and user_x < 200:
        score += 1
        print("Score:"+green+str(score))
        chicken_y = [random_offset(), random_offset(), random_offset()]

    else:
        if chicken_y[0] > screen_size[1] or chicken_y[1] > screen_size[1] or chicken_y[2] > screen_size[1]:
            print(red+"Game Over")
            keep_alive = False

    pygame.display.update()
    clock.tick(60)
pygame.quit()

Game Project/Apple Catching Game/text.py


from django.shortcuts import render
from .models import Education, Experience
# Create your views here.

def home(request):
	"""
	renders the resume home template
	"""
	name = "Pritha Dutta"
	address = "172 Forest Park, Durham, NH-03824"
	email = "pd1057@wildcats.unh.edu"
	skills = ["Java","C++", "Python", "Django", "Software Testing"]
	context = {}
	educations = Education.objects.order_by('degree')
	experiences = Experience.objects.order_by('title')
	context = {'my_education':educations,'my_experiences':experiences,
	'my_name':name,'my_addr':address,"my_email":email,
	"my_skills":skills}
	return render(request,'resume/home.html',context)


from flask import Flask, render_template, flash, request, Markup
from wtforms import Form, StringField, validators
from src.opt import get_sos
from sympy import poly, latex, sympify, nan
from src.poly import get_latex_from_poly, is_polynomial

DEBUG = True
app = Flask(__name__)
app.config.from_object(__name__)
app.config['SECRET_KEY'] = 'SjdnUends821Jsdlkvxh391ksdODnejdDw'

class ReusableForm(Form):
    polynomial = StringField('Polynomial', validators=[validators.DataRequired()])

@app.route('/', methods=['GET', 'POST'])
def hello():
    form = ReusableForm(request.form)
    if request.method == 'POST':
        _input = request.form['polynomial']
        if form.validate() and is_polynomial(_input):
            _polynomial = poly(_input)
            msg, sos = get_sos(_polynomial)
            if sos == nan:
                flash(Markup(f'Input: \({latex(sympify(_input))}\)'))
                flash(Markup(f'Result: {msg}'))
            else:
                flash(Markup(f'Result: \({latex(sympify(_input))} = {get_latex_from_poly(sos)}\)'))
        else:
            flash('Error: Non-constant polynomial required')
    return render_template('index.html', form=form)

if __name__ == '__main__':
    app.run()


import os
import sys
import numpy as np
import pandas as pd
import torch
import glob
import torch
import matplotlib.pyplot as plt

niter=2000
seed=1
outdir='/projects/b1042/ClareLab/saya/train_record_mlp'

for fn in glob.glob('bbcar/model_performance/results_mlp/results_*.txt'):
    try:
        # find best performing model's parameters
        results = pd.read_csv(fn, skiprows=1)
        bestpf = results.sort_values('val bal acc', ascending=False).iloc[0,:]
        C, lr = bestpf['C'], bestpf['lr']
        if C >=1:
            C = int(C)
        # get feature names
        gen_feature = fn.split('.')[0].split('/')[-1].split('_')[1]
        # using the best parameters, find the training record for best performing model
        train_record_fn = '%s/bbcarmlp_%s_C%s_lr%s_niter%s_seed%s.p' % (outdir, gen_feature, C, lr, niter, seed)
        try:
            record = torch.load(train_record_fn, map_location=torch.device('cpu'))
            plt.plot(record['report']['loss'])
            plt.xlabel('Epochs')
            plt.ylabel('Loss')
            plt.title('%s, C=%s, lr=%s' % (gen_feature, C, lr))
            plt.savefig('bbcar/model_performance/results_mlp/learning_curve_mlp_%s_C%s_lr%s.png' % (gen_feature, C, lr))
            plt.close()
        except FileNotFoundError:
            print('Could not load file: %s' % train_record_fn)
            continue
    except:
        print('Could not read file %s' % fn)
        continue


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 31 09:47:52 2023

@author: tono
"""
import pandas as pd
import numpy as np
import os
from datetime import date, datetime, timedelta
from iconsdk.icon_service import IconService
from iconsdk.providers.http_provider import HTTPProvider
from iconsdk.builder.call_builder import CallBuilder
from iconsdk.wallet.wallet import KeyWallet
import matplotlib.ticker as ticker
import six
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.patheffects as pe
from typing import Union
import requests
import random
from time import time, sleep

# loop to icx converter
def loop_to_icx(loop):
    icx = loop / 1000000000000000000
    return(icx)

def hex_to_int(val: str) -> Union[int, float]:
    """
    Attempts to convert a string based hex into int
        Parameters:
            val (str): Value in hex
        Returns:
            (Union[int, float]): Returns the value as an int if successful, otherwise a float NAN if not
    """
    try:
        return int(val, 0)
    except ValueError:
        print(f"failed to convert {val} to int")
        return float("NAN")

def parse_icx(val: str) -> Union[float, int]:
    """
    Attempts to convert a string loop value into icx
        Parameters:
            val (str): Loop value
        Returns:
            (Union[float, int]): Will return the converted value as an int if successful, otherwise it will return NAN
    """
    try:
        return loop_to_icx(int(val, 0))
    except ZeroDivisionError:
        return float("NAN")
    except ValueError:
        return float("NAN")

def request_sleep_repeat(url, repeat=3, verify=True):
    for i in range(0,repeat):
        print(f"Trying {str(i)}...")
        try:
            # this is from Blockmove's iconwatch -- get the destination address (known ones, like binance etc)
            url_info = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, verify=verify)
            random_sleep_except = random.uniform(3,6)
            print("Just pausing for " + str(random_sleep_except) + " seconds and try again \n")
            sleep(random_sleep_except)

        except:
            random_sleep_except = random.uniform(30,60)
            print("I've encountered an error! I'll pause for " + str(random_sleep_except) + " seconds and try again \n")
            sleep(random_sleep_except)
    return url_info

walletPath = '/home/tono/ICONProject/data_analysis/wallet'

## Creating Wallet if does not exist (only done for the first time)
tester_wallet = os.path.join(walletPath, "test_keystore_1")

if os.path.exists(tester_wallet):
    wallet = KeyWallet.load(tester_wallet, "abcd1234*")
else:
    wallet = KeyWallet.create()
    wallet.get_address()
    wallet.get_private_key()
    wallet.store(tester_wallet, "abcd1234*")

tester_address = wallet.get_address()
SYSTEM_ADDRESS = "cx0000000000000000000000000000000000000000"
icon_service = IconService(HTTPProvider("https://ctz.solidwallet.io/api/v3"))

def get_iiss_info():
    call = CallBuilder().from_(tester_address) \
        .to(SYSTEM_ADDRESS) \
        .method("getIISSInfo") \
        .build()
    result = icon_service.call(call)['variable']

    df = {'Icps': hex_to_int(result['Icps']), 'Iglobal': parse_icx(result['Iglobal']),
          'Iprep': hex_to_int(result['Iprep']),
          'Irelay': hex_to_int(result['Irelay']), 'Ivoter': hex_to_int(result['Ivoter'])}

    return df

def icx_text_format(icx_float: float) -> str:
    return '{:,}'.format(round(icx_float)) + ' ICX'

iglobal = get_iiss_info()['Iglobal']
daily_issuance = iglobal*12/365
daily_issuance_text = icx_text_format(daily_issuance)
# daily_issuance_voter =

daily_icps = icx_text_format(daily_issuance*get_iiss_info()['Icps']/100)
daily_iprep = icx_text_format(daily_issuance*get_iiss_info()['Iprep']/100)
daily_ivoter = icx_text_format(daily_issuance*get_iiss_info()['Ivoter']/100)
daily_irelay = icx_text_format(daily_issuance*get_iiss_info()['Irelay']/100)

total_supply = round(loop_to_icx(icon_service.get_total_supply()))
total_supply_text = icx_text_format(total_supply)
yearly_inflation = '{:.2%}'.format(iglobal*12/total_supply)

today = datetime.utcnow() - timedelta(days=1)
day_today = today.strftime("%Y-%m-%d")
day_today_fn = today.strftime("%Y_%m_%d")
# day_today_text = today.strftime("%Y/%m/%d")
this_year = day_today_fn[0:4]

# making path for saving
currPath = '/home/tono/ICONProject/data_analysis/'
inPath = '/home/tono/ICONProject/data_analysis/output/'
prepvotePath = os.path.join(inPath, "prep_votes")
savePath = os.path.join(prepvotePath, this_year)

fn = f'prep_top_100_votes_and_bond_{day_today_fn}.csv'
df = pd.read_csv(os.path.join(savePath, fn), low_memory=False)

df['Bond Status'] = np.where(df['bond'] != 0, 'Bonded', 'Not bonded')
df = df.rename(columns = {'prep_type': 'P-Rep Type'})

df_plot_count = df.groupby(['Bond Status','P-Rep Type'])\
    .size()\
    .reset_index()\
    .pivot(columns='P-Rep Type', index='Bond Status', values=0)
df_plot_count = df_plot_count[['main','sub','candidate']]

df_plot_votes = df.groupby(['Bond Status','P-Rep Type'])['delegation']\
    .sum()\
    .reset_index()\
    .pivot(columns='P-Rep Type', index='Bond Status', values='delegation')
# df_plot = df_plot[['candidate','sub','main']]
df_plot_votes = df_plot_votes[['main','sub','candidate']]

def plot_bonded_status(df, my_title, ylab):

    def shorten_number(x):
        if x>= 1000000:
            x = '{:,.1f}'.format(x / 1e6) + ' M'
        elif x>= 100000:
            x = '{:,.0f}'.format(x / 1e3) + ' K'
        elif x>= 1000:
            x = '{:,.0f}'.format(x)
        elif x< 1000:
            x = int(x)
        return x

    sns.set(style="ticks", rc={"lines.linewidth": 1})
    plt.style.use(['dark_background'])

    ax = df.plot(kind='bar', stacked=True)
    plt.setp(ax.get_xticklabels(), rotation=0)
    handles, labels = ax.get_legend_handles_labels()
    # ax.legend(reversed(handles), reversed(labels))

    for c in ax.containers:

        labels = ['{:,.1f}'.format(x / 1e6) + ' M' if x>= 1000000\
                  else '{:,.0f}'.format(x / 1e3) + ' K' if x>= 100000\
                  else '{:,.0f}'.format(x) if x>= 1000\
                  else int(x) if x< 1000\
                  else x for x in c.datavalues]

        labels = [a if a else "" for a in labels]

        ax.bar_label(c, labels=labels, label_type='center', color='black',
                     path_effects=[pe.withStroke(linewidth=2, foreground='white')])

    my_labels = [shorten_number(i) for i in df.sum(axis=1).to_list()]

    ax.bar_label(ax.containers[2], labels=my_labels, padding=10, color='cyan', weight='bold')

    ax.margins(y=0.2)

    ax.legend(title='P-Rep Type', bbox_to_anchor=(1.05, 1), loc='upper left')
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.set_xlabel('Bond status', fontsize=14, weight='bold', labelpad=10)
    ax.set_ylabel(ylab, fontsize=14, weight='bold', labelpad=10)
    ax.set_title(my_title, fontsize=14, weight='bold', loc='left', pad=10)

    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()

    if ymax >= 1000:
        ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, pos: '{:,.0f}'.format(x)))
    if ymax >= 100000:
        ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, pos: '{:,.0f}'.format(x / 1e3) + ' K'))
    if ymax >= 1000000:
        ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, pos: '{:,.1f}'.format(x / 1e6) + ' M'))
    plt.tight_layout()

    plt.savefig(os.path.join(savePath, my_title\
                             .replace(' ', '_')\
                             .replace('(','')\
                             .replace(')','')\
                             .replace('$','')\
                             .replace('-','_')\
                             .replace('\n','_')\
                                 + '.png'))

plot_bonded_status(df_plot_count, f'Number of P-Rep types by\nbond status ({day_today})', 'Number of P-Reps')
plot_bonded_status(df_plot_votes, f'$ICX delegated to P-Rep types by\nbond status ({day_today})', '$ICX')

## wallet counts
wallet_data_url = request_sleep_repeat(url = f'https://api.iconwat.ch/daily/?from=1970-01-18&to={day_today}', repeat=1, verify=True)
watllet_data_list = wallet_data_url.json()['data']

wallet_date = [i['date'] for i in watllet_data_list]
wallet_transacted_count = [i['walletsTransacted'] for i in watllet_data_list]
wallet_count = [i['totalWallets'] for i in watllet_data_list]

wallet_df = pd.DataFrame(list(zip(wallet_date, wallet_transacted_count, wallet_count)),
             columns = ['Date', 'Transacted wallets', 'Total wallets'])


import itchat
import itchat.content
import requests
import json

with open('data.json','r')as f:
    dt=f.read()
data=json.loads(dt)
token=data['token']
my_id=data['my_id']
qr=data['qr']

def replay_tg(my_id,token,msg):
    try:
        text='来自用户: '+msg['FromUserName']+' - 备注: '+msg['User']['RemarkName']+'\n发送给用户: '+msg['ToUserName']+'\n的内容: '+msg['Text']
    except KeyError:
        text='来自: 你自己\n发送给用户: '+msg['ToUserName']+'\n的内容: '+msg['Text']
    params={'chat_id':my_id,'text':text}
    requests.post("https://api.telegram.org/bot"+token+"/sendMessage",params=params)

@itchat.msg_register(itchat.content.TEXT)
def get_msg(msg):
    global my_id
    global token
    replay_tg(my_id,token,msg)

if qr=='y':
    itchat.auto_login(hotReload=True,enableCmdQR=2)
else:
    itchat.auto_login(hotReload=True)
itchat.run()


import pygame
import random
import sys

pygame.init()

# Configurações da tela
SCREEN_WIDTH = 400
SCREEN_HEIGHT = 600
screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
pygame.display.set_caption("Jogo Simples com Pygame")

WHITE = (255, 255, 255)
RED = (255, 0, 0)

# Configurações do jogador
player_width = 50
player_height = 50
player_x = (SCREEN_WIDTH - player_width) // 2
player_y = SCREEN_HEIGHT - player_height
player_speed = 5

# Configurações dos obstáculos
obstacle_width = 50
obstacle_height = 50
obstacle_x = random.randint(0, SCREEN_WIDTH - obstacle_width)
obstacle_y = 0
obstacle_speed = 3

# Variável para controlar o estado do jogo
game_over = False

# Variável para controlar o aumento de velocidade dos obstáculos
obstacle_speed_increase = 0.1

# Loop principal do jogo
while not game_over:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()

    keys = pygame.key.get_pressed()
    if keys[pygame.K_LEFT]: player_x -= player_speed
    if keys[pygame.K_RIGHT]: player_x += player_speed

    # Atualize a posição do obstáculo
    obstacle_y += obstacle_speed

    # Verifique se o obstáculo saiu da tela
    if obstacle_y > SCREEN_HEIGHT:
        obstacle_x = random.randint(0, SCREEN_WIDTH - obstacle_width)
        obstacle_y = 0
        obstacle_speed += obstacle_speed_increase

    # Verifique a colisão entre o jogador e o obstáculo
    if (
        player_x < obstacle_x + obstacle_width and player_x + player_width > obstacle_x
        and player_y < obstacle_y + obstacle_height and player_y + player_height > obstacle_y
    ):
        game_over = True

    screen.fill(WHITE)

    pygame.draw.rect(screen, RED, (player_x, player_y, player_width, player_height))
    pygame.draw.rect(screen, RED, (obstacle_x, obstacle_y, obstacle_width, obstacle_height))

    pygame.display.update()

    pygame.time.Clock().tick(60)

pygame.quit()
sys.exit()


import pandas as pd

from IPython.core.interactiveshell import InteractiveShell

InteractiveShell.ast_node_interactivity = "all"

pd.set_option('display.max_columns', 99)

pd.set_option('display.max_rows', 99)

import os

import numpy as np

from matplotlib import pyplot as plt

from tqdm import tqdm

import datetime as dt
import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = [16, 10]

plt.rcParams['font.size'] = 14

import seaborn as sns

sns.set_palette(sns.color_palette('tab20', 20))

import plotly.express as px

import plotly.graph_objects as go
DATEFORMAT = '%Y-%m-%d'

def get_comp_data(COMP):

    train = pd.read_csv(f'{COMP}/train.csv')

    test = pd.read_csv(f'{COMP}/test.csv')

    print(train.shape, test.shape)

    train['Country_Region'] = train['Country_Region'].str.replace(',', '')

    test['Country_Region'] = test['Country_Region'].str.replace(',', '')

    train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')

    test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')

    train['LogConfirmed'] = to_log(train.ConfirmedCases)

    train['LogFatalities'] = to_log(train.Fatalities)

    train = train.drop(columns=['Province_State'])

    test = test.drop(columns=['Province_State'])

    country_codes = pd.read_csv('../input/covid19-metadata/country_codes.csv', keep_default_na=False)

    train = train.merge(country_codes, on='Country_Region', how='left')

    test = test.merge(country_codes, on='Country_Region', how='left')

    train['DateTime'] = pd.to_datetime(train['Date'])

    test['DateTime'] = pd.to_datetime(test['Date'])

    return train, test

def process_each_location(df):

    dfs = []

    for loc, df in tqdm(df.groupby('Location')):

        df = df.sort_values(by='Date')

        df['Fatalities'] = df['Fatalities'].cummax()

        df['ConfirmedCases'] = df['ConfirmedCases'].cummax()

        df['LogFatalities'] = df['LogFatalities'].cummax()

        df['LogConfirmed'] = df['LogConfirmed'].cummax()

        df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)

        df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)

        df['DateNextDay'] = df['Date'].shift(-1)

        df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)

        df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)

        df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']

        df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']

        df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']

        df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']

        dfs.append(df)

    return pd.concat(dfs)

def add_days(d, k):

    return dt.datetime.strptime(d, DATEFORMAT) + dt.timedelta(days=k)

def to_log(x):

    return np.log(x + 1)

def to_exp(x):

    return np.exp(x) - 1
COMP = '../input/covid19-global-forecasting-week-4/'

start = dt.datetime.now()

train, test = get_comp_data(COMP)

train.shape, test.shape

train.head(2)

test.head(2)
train[train.geo_region.isna()].Country_Region.unique()

train = train.fillna('#N/A')

test = test.fillna('#N/A')

train[train.duplicated(['Date', 'Location'])]

train.count()

TRAIN_START = train.Date.min()

TEST_START = test.Date.min()

TRAIN_END = train.Date.max()

TEST_END = test.Date.max()

TRAIN_START, TRAIN_END, TEST_START, TEST_END

train_w2, test_w2 = get_comp_data('../input/covid19-global-forecasting-week-2')

train_w2.shape, test_w2.shape

train_w3, test_w3 = get_comp_data('../input/covid19-global-forecasting-week-3')

train_w3.shape, test_w3.shape
top_submissions = dict(

    beluga = '../input/covid-19-w3-a-few-charts-and-submission/submission.csv',

    Kaz = '../input/gbr-169v3-v2/submission.csv',

    Lockdown = '../input/gbt3n/submission.csv',

    PDD = '../input/cv19w3-v3fix-cpmp-oscii-belug-full-8118/submission.csv',

    KGMON = '../input/covid19-w3-submission-blend-4-models/submission.csv',

    Vopani = '../input/covid-19-w3-lgb-mad/submission.csv',

    OsciiArt = '../input/covid-19-lightgbm-with-weather-2/submission.csv',

    Northquay = '../input/c19-wk3-uploader/submission.csv',

)

top_submissions

predictions = train.copy()

predictions.shape

SUBM_DIR = './data/subms/'

for team, f in top_submissions.items():

    s = pd.read_csv(f)

    s = s.merge(test_w3, on='ForecastId')

    s = s[['Location', 'Date', 'ConfirmedCases', 'Fatalities']]

    predictions = predictions.merge(s, on=['Location', 'Date'], suffixes = ['', f' ({team})'], how='outer')

    s.shape

    s.tail(2)

daily_total = predictions.groupby('Date').sum().reset_index()

daily_total = daily_total[daily_total.Date > train_w3.Date.max()]

cols = ['Date'] + [c for c in predictions.columns if c.startswith('Confirmed')]

melted = pd.melt(daily_total[cols], id_vars='Date')

fig2 = px.line(melted, x='Date', y='value', color='variable')

_ = fig2.update_layout(

    yaxis_type="log",

    title_text=f'W3 Submissions - Confirmed Cases [Updated: {TRAIN_END}]'

)

fig2.show()
daily_total = predictions.groupby('Date').sum().reset_index()

daily_total = daily_total[daily_total.Date > train_w3.Date.max()]

cols = ['Date'] + [c for c in predictions.columns if c.startswith('Fatalities')]

melted = pd.melt(daily_total[cols], id_vars='Date')

fig2 = px.line(melted, x='Date', y='value', color='variable')

_ = fig2.update_layout(

    yaxis_type="log",

    title_text=f'W3 Submissions - Fatalities [Updated: {TRAIN_END}]'

)

fig2.show()
top_submissions = dict(

    beluga = '../input/covid-19-a-few-charts-and-a-simple-baseline/submission.csv',

    Kaz = '../input/gr1621-v2/submission.csv',

    KGMON = '../input/covid19-w2-final-v2/submission.csv',

    Vopani = '../input/covid19-metadata/w2_submission_Vopani.csv',

    OsciiArt = '../input/covid19-metadata/w2_submission_OsciiArt.csv',

    DAV = '../input/kernel303a0b031a/submission.csv'

)

top_submissions

predictions = train.copy()

predictions.shape

SUBM_DIR = './data/subms/'

for team, f in top_submissions.items():

    s = pd.read_csv(f)

    s = s.merge(test_w2, on='ForecastId')

    s = s[['Location', 'Date', 'ConfirmedCases', 'Fatalities']]

    predictions = predictions.merge(s, on=['Location', 'Date'], suffixes = ['', f' ({team})'], how='outer')

    s.shape

    s.tail(2)
melted
daily_total = predictions.groupby('Date').sum().reset_index()

daily_total = daily_total[daily_total.Date > train_w2.Date.max()]

cols = ['Date'] + [c for c in predictions.columns if c.startswith('Confirmed')]

melted = pd.melt(daily_total[cols], id_vars='Date')

fig2 = px.line(melted[melted.Date <= '2020-04-30'], x='Date', y='value', color='variable')

_ = fig2.update_layout(

    yaxis_type="log",

    title_text=f'W2 Submissions - Confirmed Cases [Updated: {TRAIN_END}]',

    width = 1600,

    height = 800,

)

fig2.show()
daily_total = predictions.groupby('Date').sum().reset_index()

daily_total = daily_total[daily_total.Date > train_w2.Date.max()]

cols = ['Date'] + [c for c in predictions.columns if c.startswith('Fatalities')]

melted = pd.melt(daily_total[cols], id_vars='Date')

fig2 = px.line(melted[melted.Date <= '2020-04-30'], x='Date', y='value', color='variable')

_ = fig2.update_layout(

    yaxis_type="log",

    title_text=f'W2 Submissions - Fatalities [Updated: {TRAIN_END}]',

    width = 1600,

    height = 800,

)

fig2.show()
train_w1, test_w1 = get_comp_data('../input/covid19-metadata')

train_w1.shape, test_w1.shape

top_submissions = dict(

    RalphNeumann='../input/covid19-metadata/w1_RalphNeumann_submission.csv',

    StephenKeller='../input/covid19-metadata/w1_StephenKeller_submission.csv',

    beluga='../input/covid19-metadata/w1_beluga_submission.csv',

    OsciiArt='../input/covid19-metadata/w1_OsciiArt_submission.csv'

)

top_submissions

predictions = train.copy()

predictions.shape

SUBM_DIR = './data/subms/'

for team, f in top_submissions.items():

    s = pd.read_csv(f)

    s = s.merge(test_w1, on='ForecastId')

    s = s[['Location', 'Date', 'ConfirmedCases', 'Fatalities']]

    predictions = predictions.merge(s, on=['Location', 'Date'], suffixes = ['', f' ({team})'], how='outer')

    s.shape

    s.tail(2)
daily_total = predictions.groupby('Date').sum().reset_index()

daily_total = daily_total[daily_total.Date > train_w1.Date.max()]

cols = ['Date'] + [c for c in predictions.columns if c.startswith('Confirmed')]

melted = pd.melt(daily_total[cols], id_vars='Date')

fig2 = px.line(melted[melted.Date <= '2020-04-23'], x='Date', y='value', color='variable')

_ = fig2.update_layout(

    yaxis_type="log",

    title_text=f'W1 Submissions - Confirmed Cases [Updated: {TRAIN_END}]'

)

fig2.show()
daily_total = predictions.groupby('Date').sum().reset_index()

daily_total = daily_total[daily_total.Date > train_w2.Date.max()]

cols = ['Date'] + [c for c in predictions.columns if c.startswith('Fatalities')]

melted = pd.melt(daily_total[cols], id_vars='Date')

fig2 = px.line(melted[melted.Date <= '2020-04-23'], x='Date', y='value', color='variable')

_ = fig2.update_layout(

    yaxis_type="log",

    title_text=f'W1 Submissions - Fatalities [Updated: {TRAIN_END}]'

)

fig2.show()