#!/usr/bin/env python3

import pandas as pd
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from mlxtend.preprocessing import TransactionEncoder
from mlxtend.frequent_patterns import apriori

engine = create_engine("sqlite:///mag.db")
con = engine.connect()
Session = sessionmaker(bind=engine)

clusters_df = pd.read_sql_table("clusters", con)
nc_df = clusters_df[clusters_df["novel"]==1]
catalog = set(nc_df[nc_df["size"] > 10]["cluster_id"])

bag_fp = "bag.csv"
bag_df = pd.read_csv(bag_fp)
transactions = []
for bag in bag_df["bag"].dropna():
    transaction = [int(e) for e in bag.split(';') if int(e) in catalog]
    if len(transaction) > 0:
        transactions.append(transaction)

te = TransactionEncoder()
te_ary = te.fit(transactions).transform(transactions)
df = pd.DataFrame(te_ary, columns=te.columns_)
apriori_df = apriori(df, min_support=20/len(df))
apriori_df.to_csv("apriori.csv", index=False)


import logging
import string
from requests import get
from requests.exceptions import RequestException
from contextlib import closing
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from scraper.models import Base, Game
from bs4 import BeautifulSoup
from scraper.models import Game
from scraper.loggers import error_logger, info_logger

domain = 'https://gamesystemrequirements.com/'

engine = create_engine('sqlite:////home/anon/PycharmProjects/canirunthis-scraper/games.db', echo=True)
Session = sessionmaker(bind=engine)
Base.metadata.create_all(engine)
session = Session()

def fetch_page(url):

    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None
    except RequestException as e:
        error_logger.exception(e)

def is_good_response(resp):
    """
    Returns True if the response seems to be HTML, False otherwise.
    """
    content_type = resp.headers['Content-Type'].lower()
    return (resp.status_code == 200
            and content_type is not None
            and content_type.find('html') > -1)

def test():
    letters = list(string.ascii_lowercase)
    for l in ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']:
        letters.remove(l)
    for char in letters:
        root_page = "https://gamesystemrequirements.com/database/{}".format(char)
        root_page_html = fetch_page(root_page)
        bs4_root_page = BeautifulSoup(root_page_html, 'html.parser')
        nav = bs4_root_page.find_all('div', class_='pagenav_c')
        pages = len(nav[0].find_all('a')) + 1 if len(nav) > 0 else 1
        for page in range(pages):
            print("STARTING PAGE {} !!!!!!!!!!!!!!!!!!!!!".format(page))
            page_html = root_page_html if page == 0 else fetch_page("{}/page/{}".format(root_page, page+1))
            get_games_on_page(page_html)
            print('FINISHED PAGE!!!!!!!!!!!!!!!!!!!!!')

def get_games_on_page(html):
    bs4_root_page = BeautifulSoup(html, 'html.parser')
    table = bs4_root_page.find_all('table', class_='database_t')[0]
    rows = table.find_all('tr')
    for row in rows:
        game_link = domain + row.find_all('a', href=True)[0]['href']
        get_game_specs(game_link)

def get_game_specs(url):
    gaml_page_html = fetch_page(url)
    bs4_game_page = BeautifulSoup(gaml_page_html, 'html.parser')
    title = bs4_game_page.find_all('div', class_='game_head_title')[0].text
    main_panel = bs4_game_page.find_all('div', class_='main-panel')[0].find_all('div')[0]
    rows = main_panel.find_all('div', class_='srb_row')
    game = {}
    game['title'] = title
    for row in rows:
        subrows = row.find_all('div', class_='tbl')
        len_subrows = len(subrows)
        if len_subrows == 2 or len_subrows == 4:
            col = subrows[0].find_all('b')[0].text.replace(':', '').strip().lower()
            val = subrows[1].text.strip()
            if val.find(domain) > -1:
                val = val.replace(domain, '')
            game[col] = val
    game = Game(**game)
    session.add(game)
    session.commit()


import random

from mesa import Agent
import math

#from random_walk import RandomWalker

#define default probability values
flamability_probs = {0:1,1:0.7,2:0.49,3:0.343,4:0.2401}
#min and max time stored by vegetation type
succession_times = {0:(5,15),1:(10,20),2:(10,20),3:(20,40),4:(20,40)}

class Landscape(Agent):
    '''
    A patch of grass that grows at a fixed rate and it is eaten by sheep
    '''

    def __init__(self, pos, model, elevation, burn_s_m_p, burn_s_t_p, vegetation_type, is_patch_burned,
                 time_colonised, is_patch_colonised, succ_s_m_p, succ_s_t_p):

        super().__init__(pos, model)
        self.elevation = elevation
        self.burn_s_m_p=burn_s_m_p
        self.burn_s_t_p=burn_s_t_p
        self.succ_s_m_p = succ_s_m_p
        self.succ_s_t_p = succ_s_t_p
        self.vegetation_type = vegetation_type
        self.is_patch_burned = is_patch_burned
        self.breed = "Landscape"
        self.time_since_last_burnt = 0
        self.time_colonised = time_colonised
        self.is_patch_colonised = is_patch_colonised
        # get all agents in dispersal range
        radius = self.model.grid.get_neighbors(pos, 50, False)
        # filter agent list so that only those in landscape are considered
        self.dispersal_radius = list(filter(lambda x: x.breed == "Landscape", radius))
        self.dispesal_patch_count = len(self.dispersal_radius)

        # flamability probs stored in dictionary by veg type

        self.succ_prob = {0: succession_prob_calc(succession_times[0][0],succession_times[0][1]),
                     1: succession_prob_calc(succession_times[1][0],succession_times[1][1]),
                     2: succession_prob_calc(succession_times[2][0],succession_times[2][1]),
                     3: succession_prob_calc(succession_times[3][0],succession_times[3][1]),
                     4: succession_prob_calc(succession_times[4][0],succession_times[4][1])}

    #automatically update the flamability value when vegetation type is changed
    @property
    def flamability(self):
        return flamability_probs[self.vegetation_type]

    #automatically update succession times based on veg type
    @property
    def min_succession_time(self):
        return succession_times[self.vegetation_type][0]

    @property
    def max_succession_time(self):
        return succession_times[self.vegetation_type][1]

    @property
    def succession_probability(self):
        return self.succ_prob[self.vegetation_type]

    @staticmethod
    def fire_front(patches_to_burn):
        while len(patches_to_burn) != 0:
            patches_to_burn_new = []
            for this_patch in patches_to_burn:
                neighbours = this_patch.model.grid.get_neighbors(this_patch.pos, 1, False)
                #filter list of neighbours to only those that are landcape based and not burned
                land_scape_neighs = list(filter(lambda x: x.breed == 'Landscape' and x.is_patch_burned == False, neighbours))
                for neigh_patch in land_scape_neighs:
                    slope = math.atan(abs(neigh_patch.elevation - this_patch.elevation) / 1000)
                    if this_patch.elevation >= neigh_patch.elevation:
                        slope_burn_prob = (math.exp(3.533 * (math.tan(slope) ** 1.2)))
                    else:
                        slope_burn_prob = (1 - 0.033 * slope + 0.000749 * slope * 2) * math.cos(slope)
                    if random.random() <= 0.57 * slope_burn_prob and does_this_patch_burn(neigh_patch):
                        neigh_patch.is_patch_burned = True
                        patches_to_burn_new.append(neigh_patch)

            patches_to_burn = patches_to_burn_new

    def step(self):
        if self.is_patch_burned == True:
            # this will automatically update flamability
            self.vegetation_type = 0
            self.is_patch_burned = False
            self.is_patch_colonised = 0
            self.time_colonised = 0
        if self.is_patch_colonised == 1:
            self.succession()
        elif self.is_patch_colonised == 0:
            self.dispersal()

    def succession(self):
        self.time_colonised += 1
        #Should maybe include rule to succeed if max colonised time is reached?
        if self.vegetation_type <= 3 and self.time_colonised >= self.min_succession_time:
            if random.random() <= self.succession_probability * self.succ_s_m_p * self.succ_s_t_p * self.model.rainfall_this_year * self.model.temp_this_year:
                self.vegetation_type += 1
                #will further colonisation be required for succession?
                if self.vegetation_type == 1 or self.vegetation_type == 2:
                    self.is_patch_colonised = 0
            self.time_colonised = 0

    def dispersal(self):
        if self.vegetation_type == 0:
            self.is_patch_colonised = 1
        else:
            potential_dis_patches = list(filter(lambda x: x.vegetation_type > self.vegetation_type, self.dispersal_radius))
            try:
                if random.random() <= (len(potential_dis_patches) / self.dispesal_patch_count):
                    self.is_patch_colonised = 1
            except ZeroDivisionError:
                return
        return

class Lightning(Agent):

    def __init__(self, pos, model):
        super().__init__(pos, model)
        self.breed = "Lightning"

    def step(self):
        # Create lightning strikes

        x = random.randrange(self.model.width)
        y = random.randrange(self.model.height)
        strike = Lightning((x, y), self.model)
        self.model.grid.place_agent(strike, (x, y))
        self.model.schedule.add(strike)
        potential_to_burn = []
        struck_land = False

        this_cell = self.model.grid.get_cell_list_contents((x, y))

        #Check to see if lighning hits land
        for obj in this_cell:
            if obj.breed == "Landscape":
                patch_veg = obj
                struck_land = True

        if struck_land == True:
            if patch_veg.is_patch_burned == False:
                if does_this_patch_burn(patch_veg):
                    patch_veg.is_patch_burned = True
                    potential_to_burn.append(patch_veg)
                    Landscape.fire_front(potential_to_burn)

        self.model.grid._remove_agent(self.pos, self)
        self.model.schedule.remove(self)

'''
Early ideas:
This agent should be created as a result of fire - several factors to consider:
1. Size of fire - Macroscopic/Microscopic - will determine how far it travels
2. Wind direction
3. Distance to deposition site - this will mean accurately identifying swamps/lakes on grid
4. Type of vegetation burning
'''
class charcoal(Agent):
    def __init__(self, pos, model, size_of_fire, vegetation_type_burning):
        print("charoal born!")

'''
Early ideas:
This agent should be created each tick??:
1. Amount of pollen produced - could be tricky to portray accurately as varied at species level
2. Age of species/functional type to be considered
2. Distance from deposition site - Can travel along way, maybe not so important
3. New class may not be needed and could just simply be a function of vegetation class?

First Steps!
1. Get center of deposition site...
'''
class pollen(Agent):
    def __init__(self, pos, model, pollen_type):
        print("pollen born!")

'''
Early ideas:
Should be born/die given set birth and death rates:
1. How many fires how often?
2. How many people of GBI before European arrival
3. Where did they go, why? Look at settlement records
'''
class tribe(Agent):
    def __init__(self, pos, model):
        print("Human born, we're all fucked!")

class Deposition_site(Agent):
    def __init__(self, pos, model, deposition_area):
        self.deposition_area = deposition_area
        self.breed = "deposition"

def does_this_patch_burn(patch_veg):
    if random.random() <= patch_veg.flamability * patch_veg.burn_s_m_p * patch_veg.burn_s_t_p\
            * patch_veg.model.rainfall_this_year * patch_veg.model.temp_this_year:
        return True
    else:
        return False

def succession_prob_calc(min, max):
    prob = 1 - (1 - 0.95) ** (1 / (min - max))
    return abs(prob)


from datetime import datetime
import json
import os
from django.test import TestCase
from corehq.apps.commtrack.models import Product as Prod
from corehq.apps.commtrack.tests.util import bootstrap_domain as initial_bootstrap
from custom.ilsgateway.api import Product, ILSGatewayAPI
from custom.logistics.commtrack import synchronization
from custom.logistics.models import MigrationCheckpoint
from custom.ilsgateway.tests.mock_endpoint import MockEndpoint

TEST_DOMAIN = 'ilsgateway-commtrack-product-test'

class ProductSyncTest(TestCase):

    def setUp(self):
        self.endpoint = MockEndpoint('http://test-api.com/', 'dummy', 'dummy')
        self.api_object = ILSGatewayAPI(TEST_DOMAIN, self.endpoint)
        self.datapath = os.path.join(os.path.dirname(__file__), 'data')
        initial_bootstrap(TEST_DOMAIN)
        for product in Prod.by_domain(TEST_DOMAIN):
            product.delete()

    def test_create_product(self):
        with open(os.path.join(self.datapath, 'sample_products.json')) as f:
            product = Product(json.loads(f.read())[0])
        self.assertEqual(0, len(Prod.by_domain(TEST_DOMAIN)))
        ilsgateway_product = self.api_object.product_sync(product)
        self.assertEqual(product.sms_code, ilsgateway_product.code.lower())
        self.assertEqual(product.name, ilsgateway_product.name)
        self.assertEqual(product.description, ilsgateway_product.description)
        self.assertEqual(product.units, str(ilsgateway_product.unit))

    def test_locations_migration(self):
        checkpoint = MigrationCheckpoint(
            domain=TEST_DOMAIN,
            start_date=datetime.now(),
            date=datetime.now(),
            api='product',
            limit=100,
            offset=0
        )
        synchronization('product',
                        self.endpoint.get_products,
                        self.api_object.product_sync, checkpoint, None, 100, 0)
        self.assertEqual('product', checkpoint.api)
        self.assertEqual(100, checkpoint.limit)
        self.assertEqual(0, checkpoint.offset)
        self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))


import socket
import array as arr
import statistics as st
import os, time, sys
import errno
import math
# import pyttsx3
import Heat_Handler as hh
import TCP_Server_m as Server
import utility
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui
import numpy as np
import sys
import time
import pyqtgraph.console
import matplotlib.pyplot as plt

# ------------- Begin: variables declaration -------------
FACTOR_MOT = 50  # use this factor to standardize motion standard deviation to 50
STEP_FOOT = 0.2  # 'step' second to check footstep samples, e.g. 0.2 means check samples of every 0.2 second
THRES_FOOT = 10  # footstep threshold

server = None
# fall_words = ['help', 'ouch', 'fell', 'fall']
# r = sr.Recognizer()
help_list = []

AUD_FILTER_THRES = 3 # for audio sound median filter threshold

# buf_aud_val = []
buf_aud_val = np.empty((0,2), float)
buf_aud_val_last = np.empty((0,2), float)
# buf_vib_val = []
buf_vib_val = np.empty((0,2), float)
buf_mot_val = []
buf_heat_val = []
buf_humn_val = []
buf_heat_mtr = []

buf_mot_val_tag0 = []
buf_mot_val_tag2 = []

# buf_aud_val_real = []
buf_aud_val_real = np.empty((0,2), float)
# buf_vib_val_real = []
buf_vib_val_real = np.empty((0,2), float)
buf_mot_val_real = []
buf_heat_val_real = []
buf_humn_val_real = []

# buf_mot_std = []  # [ts, mot_std]
buf_mot_std_real = []  # [ts, mot_std]
# ts_mot_std_last = -1

buf_power_S_norm_real = np.empty((0,6), float) #  [] # sound normalized power
buf_power_V_norm_real = np.empty((0,6), float) # vibration normalized power
NUM_BUF_POWER = 1200       # number of normalized sound and vibration power buffer

# v_max_powers = np.empty((0,5), float)

noise_power_vib = -1
noise_power_aud = -1

checking_buffer_aud = []
checking_buffer_vib = []
checking_buffer_mot = []
checking_buffer_heat = []

checking_buffer_mot_tag0 = []
checking_buffer_mot_tag2 = []

pre_min_buff_time = 0

MAX_BUF = 1000
MAX_BUF_HEAT = 1500  # for heat samples for 1 hour
TIME_CHK = 5  # 5 seconds
TIME_PLOT = 0.5  # plot time

# Create figure for plotting
fig = plt.figure()
sub_plot_aud = fig.add_subplot(4, 1, 1)
sub_plot_vib = fig.add_subplot(4, 1, 2)
sub_plot_mot = fig.add_subplot(4, 1, 3)
sub_plot_heat = fig.add_subplot(4, 1, 4)

time_next = -1
matr_thresh = None

MAX_FLOAT = 10 ** 20  # simulate a largest float number
FIFO_NAME = '/tmp/pipefifo'
BUFFER_SIZE = 40
STATE = 0  # state 1: training session. State 2: analyzing session
DURATION_CHECK = 1  # 2 seconds check
time_check = -1

min_buff_time = -1
new_min_buff_time = -1
pre_floor_time = -1
pre_floor_time_next = -1
pre_floor_time_y = -1  # the previous buffer cutting floor time

calib_std_tag0 = -1
calib_std = -1
calib_std_tag2 = -1

rf_threshold_tag0 = -1
rf_threshold = -1
rf_threshold_tag2 = -1

time_start = -1

motion_threshold = 2 # 1 for footstep detection # 3 for fall detection
motion_threshold_tag0 = 2 # 1 for footstep detection # 3 for fall detection
motion_threshold_tag2 = 2 # 1 for footstep detection # 3 for fall detection

# tidx = 0
# time_arr = []
mov_arr = []
mov_val = []
fcandidates = []

mov_arr_tag0 = []
mov_val_tag0 = []

mov_arr_tag2 = []
mov_val_tag2 = []

# vec_power = [] # signal power vector using maximal ratio combining to combine sound signal and vibration signal

aud_thres = -1
prev_ewma = 0
aud_std = 0
prev_var = 0

# aud_thres_filter = -1 # an audio threshold used to filter spike noise from sound

# vib_threshold = [31000, 15000, 10000, 10000]
vib_threshold = [6500, 6500, 6500, 6500]
vibration_threshold = 7000 # 6500

time_transition = -1
vib_check1 = -1

TIME_LIMITED = 20  # 20 seconds
time_chk_heat = -1
time_heat_start = -1
time_heat_thrsh = -1
is_candidate = False
time_curr = -1
SECOND_CALIB = 5 # 10 # 5   # number of seconds used for motion sensor calibration
ts_1sec_begin = -1
ts_1sec_end = -1

mflag = 0
vflag = 0
aflag = 0
hflag = 0
flag_stand = -1

vflag_last = -1
aflag_last = -1

is_start = False
is_plot_on = False
candidates = []
NUM_CANDIDATE = 3
speech_flag = -1
speech_timer = -1

asave = []
vsave = []
hsave = []

is_lower_human = 0
time_1sec = -1
pipe_out = []
samples_remain = {}
offset_time = {}

mot_ts = -1
vib_ts = -1

time_update = -1

# ------------- End: variables declaration -------------


import pytest  # flake8: noqa

from os.path import dirname
from pathlib import Path

from seminario.config import config

tests_path = Path(dirname(__file__))

default_seminar_name = 'Seminar'
default_path = {
    'database': 'data/database.csv',
    'abstract': 'data/abstract/',
    'css': 'data/poster/css/poster.css',
}
default_tba = {
    'date': 'TBA',
    'begin_time': '',
    'end_time': '',
    'place': 'TBA',
    'speaker': 'TBA',
    'affiliation': '',
    'title': 'TBA',
    'abstract': 'TBA',
}

new_seminar_name = 'New Seminar'
new_path = {
    'database': 'new.csv',
    'abstract': 'new/',
    'css': 'new.css',
}
new_tba = {
    'date': 'New TBA date',
    'begin_time': 'New TBA begin_time',
    'end_time': 'New TBA end_time',
    'place': 'New TBA place',
    'speaker': 'New TBA speaker',
    'affiliation': 'New TBA affiliation',
    'title': 'New TBA title',
    'abstract': 'New TBA abstract',
}

params_default = [
    ('path', default_path),
    ('seminar_name', default_seminar_name),
    ('tba', default_tba),
]

params_new = [
    ('path', new_path),
    ('seminar_name', new_seminar_name),
    ('tba', new_tba),
]

# --------------------------------------------------------------------------------

@pytest.fixture(scope='module', autouse=True)
def reset_setup():
    yield
    config.setup(tests_path / 'data/config/default.yml')

@pytest.mark.parametrize('key, value', params_default)
def test_default(key, value):
    assert getattr(config, key) == value

@pytest.mark.parametrize('key, value', params_new)
def test_setup(key, value):
    config.setup(tests_path / 'cases/config/01.yml')
    assert getattr(config, key) == value


from django.shortcuts import HttpResponseRedirect
from django.conf import settings

class AuthRequiredMiddleware(object):
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        # Code to be executed for each request before
        # the view (and later middleware) are called.

        response = self.get_response(request)
        if not request.user.is_authenticated():
            path = request.path_info.lstrip('/')
            if path not in ["account/login/", "admin/"]:
                return HttpResponseRedirect(settings.LOGIN_URL)
        else:
            path = request.path_info.lstrip('/')
            if path in "account/login/":
                return HttpResponseRedirect(settings.LOGIN_REDIRECT_URL)
        # Code to be executed for each request/response after
        # the view is called.

        return response


import optuna
import subprocess

def objective(trial):

    TEMP_RADIX = trial.suggest_uniform('temp radix', 50, 3000)
    PROBABILITY_COEF = trial.suggest_uniform('probability coefficient', 0.000001, 1)

    total_dist = float(subprocess.check_output(['./route_builder', str(TEMP_RADIX), str(PROBABILITY_COEF)]))

    print('temp radix: %1.3f, probability coefficient: %1.3f, total_dist: %1.3f' % (TEMP_RADIX, PROBABILITY_COEF, total_dist))

    return total_dist

study = optuna.create_study()
study.optimize(objective, n_trials=10000)

print()
print('best parameters: ', study.best_params)
print('minimum total dist: ', study.best_value)
print()
print(study.best_trial)


# vim:ts=4:sts=4:sw=4:expandtab

import copy
import datetime
import dateutil.parser
import glob
import json
import logging
import math
import os
import shutil
import subprocess
import sys
import tempfile
from threading import Thread
import time
import uuid

from kolejka.common import kolejka_config, foreman_config
from kolejka.common import KolejkaTask, KolejkaResult, KolejkaLimits
from kolejka.common import MemoryAction, TimeAction
from kolejka.client import KolejkaClient
from kolejka.worker.stage0 import stage0

def foreman_single(temp_path, client, task):
    config = foreman_config()
    with tempfile.TemporaryDirectory(temp_path) as jailed_path:
        if task.limits.storage is not None:
            subprocess.run(['mount', '-t', 'tmpfs', '-o', 'size='+str(task.limits.storage), 'none', jailed_path], check=True)
        try:
            task_path = os.path.join(jailed_path, 'task')
            result_path = os.path.join(jailed_path, 'result')
            temp_path = os.path.join(jailed_path, 'temp')
            os.makedirs(task_path, exist_ok=True)
            os.makedirs(result_path, exist_ok=True)
            os.makedirs(temp_path, exist_ok=True)
            task.path = task_path
            client.task_get(task.id, task_path)
            for k,f in task.files.items():
                f.path = k
            task.commit()
            stage0(task.path, result_path, temp_path=temp_path, consume_task_folder=True)
            result = KolejkaResult(result_path)
            result.tags = config.tags
            client.result_put(result)
        finally:
            if task.limits.storage is not None:
                subprocess.run(['umount', '-l', jailed_path])

def foreman():
    config = foreman_config()
    limits = KolejkaLimits()
    limits.cpus = config.cpus
    limits.memory = config.memory
    limits.pids = config.pids
    limits.storage = config.storage
    limits.time = config.time
    limits.network = config.network
    client = KolejkaClient()
    while True:
        try:
            tasks = client.dequeue(config.concurency, limits, config.tags)
            if len(tasks) == 0:
                time.sleep(config.interval)
            else:
                while len(tasks) > 0:
                    resources = KolejkaLimits()
                    resources.update(limits)
                    processes = list()
                    cpus_offset = 0
                    for task in tasks:
                        if len(processes) >= config.concurency:
                            break
                        if task.exclusive and len(processes) > 0:
                            break
                        task.limits.update(limits)
                        task.limits.cpus_offset = cpus_offset
                        ok = True
                        if resources.cpus is not None and task.limits.cpus > resources.cpus:
                            ok = False
                        if resources.memory is not None and task.limits.memory > resources.memory:
                            ok = False
                        if resources.pids is not None and task.limits.pids > resources.pids:
                            ok = False
                        if resources.storage is not None and task.limits.storage > resources.storage:
                            ok = False
                        if ok:
                            proc = Thread(target=foreman_single, args=(config.temp_path, client, task))
                            proc.start()
                            processes.append(proc)
                            cpus_offset += task.limits.cpus
                            if resources.cpus is not None:
                                resources.cpus -= task.limits.cpus
                            if resources.memory is not None:
                                resources.memory -= task.limits.memory
                            if resources.pids is not None:
                                resources.pids -= task.limits.pids
                            if resources.storage is not None:
                                resources.storage -= task.limits.storage
                            tasks = tasks[1:]
                            if task.exclusive:
                                break
                        else:
                            break
                    for proc in processes:
                        proc.join()
        except:
            time.sleep(config.interval)

def config_parser(parser):
    parser.add_argument('--auto-tags', type=bool, help='add automatically generated machine tags', default=True)
    parser.add_argument('--tags', type=str, help='comma separated list of machine tags')
    parser.add_argument('--temp', type=str, help='temp folder')
    parser.add_argument('--interval', type=float, help='dequeue interval (in seconds)')
    parser.add_argument('--concurency', type=int, help='number of simultaneous tasks')
    parser.add_argument('--cpus', type=int, help='cpus limit')
    parser.add_argument('--memory', action=MemoryAction, help='memory limit')
    parser.add_argument('--pids', type=int, help='pids limit')
    parser.add_argument('--storage', action=MemoryAction, help='storage limit')
    parser.add_argument('--time', action=TimeAction, help='time limit')
    parser.add_argument('--network',type=bool, help='allow netowrking')
    def execute(args):
        kolejka_config(args=args)
        foreman()
    parser.set_defaults(execute=execute)


import pytest
from pymilvus import DataType

import utils.utils as ut
from common.common_type import CaseLabel

default_entities = ut.gen_entities(ut.default_nb, is_normal=True)
raw_vectors, default_binary_entities = ut.gen_binary_entities(ut.default_nb)
default_int_field_name = "int64"
default_float_field_name = "float"
default_pos = 5
default_term_expr = f'{default_int_field_name} in {[i for i in range(default_pos)]}'

def init_data(connect, collection, nb=ut.default_nb, partition_names=None, auto_id=True):
    """
    Generate entities and add it in collection
    """
    if nb == 3000:
        insert_entities = default_entities
    else:
        insert_entities = ut.gen_entities(nb, is_normal=True)
    if partition_names is None:
        if auto_id:
            res = connect.insert(collection, insert_entities)
        else:
            res = connect.insert(collection, insert_entities, ids=[i for i in range(nb)])
    else:
        if auto_id:
            res = connect.insert(collection, insert_entities, partition_name=partition_names)
        else:
            res = connect.insert(collection, insert_entities, ids=[i for i in range(nb)],
                                 partition_name=partition_names)
    connect.flush([collection])
    ids = res.primary_keys
    return insert_entities, ids

def init_binary_data(connect, collection, nb=3000, insert=True, partition_names=None):
    """
    Generate entities and add it in collection
    """
    ids = []
    # global binary_entities
    global raw_vectors
    if nb == 3000:
        insert_entities = default_binary_entities
        insert_raw_vectors = raw_vectors
    else:
        insert_raw_vectors, insert_entities = ut.gen_binary_entities(nb)
    if insert is True:
        if partition_names is None:
            res = connect.insert(collection, insert_entities)
        else:
            res = connect.insert(collection, insert_entities, partition_name=partition_names)
        connect.flush([collection])
        ids = res.primary_keys
    return insert_raw_vectors, insert_entities, ids

class TestQueryPartition:
    """
    test Query interface
    query(collection_name, expr, output_fields=None, partition_names=None, timeout=None)
    """

    @pytest.mark.tags(CaseLabel.L0)
    def test_query_partition(self, connect, collection):
        """
        target: test query on partition
        method: create a partition and query
        expected: verify query result
        """
        connect.create_partition(collection, ut.default_tag)
        entities, ids = init_data(connect, collection, partition_names=ut.default_tag)
        assert len(ids) == ut.default_nb
        connect.load_partitions(collection, [ut.default_tag])
        res = connect.query(collection, default_term_expr, partition_names=[ut.default_tag], output_fields=["*", "%"])
        for _id, index in enumerate(ids[:default_pos]):
            if res[index][default_int_field_name] == entities[0]["values"][index]:
                assert res[index][default_float_field_name] == entities[1]["values"][index]
                ut.assert_equal_vector(res[index][ut.default_float_vec_field_name], entities[2]["values"][index])

def insert_entities_into_two_partitions_in_half(connect, collection):
    """
    insert default entities into two partitions(default_tag and _default) in half(int64 and float fields values)
    :param connect: milvus connect
    :param collection: milvus created collection
    :return: entities of default_tag and entities_2 of _default
    """
    connect.create_partition(collection, ut.default_tag)
    half = ut.default_nb // 2
    entities, _ = init_data(connect, collection, nb=half, partition_names=ut.default_tag)
    vectors = ut.gen_vectors(half, ut.default_dim)
    entities_2 = [
        {"name": "int64", "type": DataType.INT64, "values": [i for i in range(half, ut.default_nb)]},
        {"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(half, ut.default_nb)]},
        {"name": ut.default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": vectors}
    ]
    connect.insert(collection, entities_2)
    connect.flush([collection])
    connect.load_collection(collection)
    return entities, entities_2


import os
import sys
import glob
import argparse
import numpy as np

parser = argparse.ArgumentParser()
parser.add_argument('--path', type=str,
    help='path to training samples')
parser.add_argument('--train-ratio', '-tr', dest='tr', default=1, type=float,
    help='the ratio of training samples')
parser.add_argument('--save-path', type=str,
    help='path to the generated dataset')
args = parser.parse_args()

image_filenames = []
for p in args.path.split(','):
    image_filenames += list(sorted(glob.glob(os.path.join(p, '*.jpg'))))
    image_filenames += list(sorted(glob.glob(os.path.join(p, '*.JPG'))))
    image_filenames += list(sorted(glob.glob(os.path.join(p, '*.jpeg'))))
    image_filenames += list(sorted(glob.glob(os.path.join(p, '*.png'))))

num_samples = len(image_filenames)
rand_index = np.random.permutation(num_samples)
num_train = int(args.tr * num_samples)
num_test = num_samples - num_train

with open(os.path.join(args.save_path, 'train.txt'), 'w') as file:
    for i in range(num_train):
        root, ext = os.path.splitext(image_filenames[rand_index[i]])
        root = root.replace('img1', 'labels_with_ids')
        file.write(f"{image_filenames[rand_index[i]]} {root}.txt\n")
    file.close()

if num_test < 1:
    sys.exit()

with open(os.path.join(args.save_path, 'test.txt'), 'w') as file:
    for i in range(num_train, num_samples):
        root, ext = os.path.splitext(image_filenames[rand_index[i]])
        root = root.replace('img1', 'labels_with_ids')
        file.write(f"{image_filenames[rand_index[i]]} {root}.txt\n")
    file.close()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

import matplotlib.pyplot as plt
lista = [11, 18, 3, 1, 16, 12, 6, 19, 5, 0, 14, 4, 17, 9, 13, 7, 10, 15, 2, 8] #Valores da Lista
plt.figure()
#Criamos uma figura vazia
plt.plot(range(0,20), lista, 'ok')
#Definimos x como 'range(0,20)' e y como a lista original, antes de ser modificada pelo algoritmo
plt.title("Lista Original")
#Título do Gráfico
plt.xlabel("Posição na lista")
#Nomeamos X
plt.ylabel("Valores da lista")
#Nomeamos Y
plt.savefig("bubble-inicio.png")
#Salvamos a figura
plt.close()
#Fechamos a figura
print("lista original", lista) #para imprimir a lista original sem modificações
a = 0
N = 20 #Numero de Elementos
for i in range(0, N-1, 1): #Andando do primeiro elemento até o penultimo de 1 em 1
    for j in range(i+1, N, 1): #Andando do elemento seguinte a 'i' até o ultimo de 1 em 1
        plt.figure()
        plt.plot(range(0,20), lista, 'ok')
        plt.plot(i, lista[i], 'or') #Marcar em vermelho a carta i
        plt.plot(j, lista[j], 'ob') #Marvar em azul a carta j
        plt.title("Lista Em Todas as Trocas")
        plt.xlabel("Posição na lista")
        plt.ylabel("Valores da lista")
        a = a + 1
        plt.savefig("bubble-it{}.png".format(a))
        plt.close()
        if lista[i] < lista[j]:
            continue
        else:
            temp = lista[i] #Para inverter os valores
            lista[i] = lista[j]
            lista[j] = temp
            plt.figure()
            plt.plot(range(0,20), lista, 'ok')
            plt.title("Lista Em Cada Troca")
            plt.xlabel("Posição na lista")
            plt.ylabel("Valores da lista")
            a = a + 1
            plt.savefig("bubble-troca{}.png".format(a))
            plt.close()
print("lista em ordem crescente", lista) #para imprimir a lista em ordem crescente com modificações
plt.figure()
plt.plot(range(0,20), lista, 'ok')
plt.title("Lista Em Ordem Crescente")
plt.xlabel("Posição na lista")
plt.ylabel("Valores da lista")
plt.savefig("bubble-fim.png")
plt.close()

print("cinco menores valores", lista[0:5]) #usamos esse comando para imprimir apenas os cinco menores valores da lista em ordem crescente, e o nomeamos como 'cinco menores valores'
print("cinco maiores valores", lista[15:20]) #usamos esse comando para imprimir apenas os cinco maiores valores da lista em ordem crescente, e o nomeamos como 'cinco maiores valores'

#Para selecionar o primeiro elemento da lista 'lista[0]'
#Para selecionar os cincos primeiros elementos da lista 'lista[0:4]'


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun 29 09:31:48 2018

@author: eduardo-ssr
"""

import sys
import serial
import glob
import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from drawnow import drawnow
from time import sleep
from datetime import datetime

read1 = []
read2 = []
read3 = []
read4 = []
read5 = []
read6 = []
read7 = []

acelX = []
acelY = []
acelZ = []
gyroX = []
gyroY = []
gyroZ = []

constant_Calib_Acel = (16384/9.81)
constant_Calib_Gyro = 131

os.system('clear')
print ("======= List of USB devices =======")
serial_ports = glob.glob('/dev/ttyUSB*')
while(len(serial_ports)==0):
    print ("Connect the Arduino...\n")
    sleep(5);
    serial_ports = glob.glob('/dev/ttyUSB*')

for i in range(len(serial_ports)):
    print (i, " - ", serial_ports[i])
    port = input("Choose the arduino port (e.g. 0): ")

ser = serial.Serial(serial_ports[int(port)], 115200)

def main():
    global contador
    #ser.open()
    if ser.is_open:
        print("Established serial communication\n")
    else :
        print("Serial communication error\n")

   # ax.clear()
    input("Pressione 'Enter' para iniciar a leitura e \"Ctrl+C\" para pausar: \n")

    while(True):
        try:
            line = ser.readline().decode("utf-8")
            sleep(0.0001)
            print(line)
            try:
                entry = line.split("\t")
                AcX = np.float(entry[0])
                AcY = np.float(entry[1])
                AcZ = np.float(entry[2])
                temp = np.float(entry[3])
                Gx = np.float(entry[4])
                Gy = np.float(entry[5])
                Gz = np.float(entry[6])

                #print(AcX, AcY, AcZ, temp, Gx, Gy, Gz)

                ACX = AcX/constant_Calib_Acel -9.81 # entre--20 m/s² e + 20 m/s²
                ACY = AcY/constant_Calib_Acel -9.81 # entre--20 m/s² e + 20 m/s²
                ACZ = AcZ/constant_Calib_Acel -9.81 # entre--20 m/s² e + 20 m/s²

                GX = Gx/constant_Calib_Gyro # entre +250º/s e -250º/s
                GY = Gy/constant_Calib_Gyro # entre +250º/s e -250º/s
                GZ = Gz/constant_Calib_Gyro # entre +250º/s e -250º/s

                read1.append(AcX)
                read2.append(AcY)
                read3.append(AcZ)
                read4.append(Gx)
                read5.append(Gy)
                read6.append(Gz)
                read7.append(temp)

                acelX.append(ACX)
                acelY.append(ACY)
                acelZ.append(ACZ)
                gyroX.append(GX)
                gyroY.append(GY)
                gyroZ.append(GZ)

               # print(ACX,ACY,ACZ,GX,GY,GZ)

            except (ValueError):
                print("Erro de valor.")
                pass

        except (KeyboardInterrupt):
            now = datetime.now()
            print ("Voce pressionou Ctrl+C para interromper este programa! Seus dados foram salvos em 'Dados_%s.csv'"%(str(now)[:-7]))
            ser.close()
            #plt.plot(acelX, '-r')
            plt.plot(gyroX)
            plt.show()
           # x = np.vstack((read1,read2,read3,read4,read5,read6,read7))
           # np.savetxt('Dados_%s.csv'%(str(now)[:-7]), np.transpose(x), delimiter=';')
            break
if __name__ == "__main__":
    main()


#导入程序运行必须模块
from win32com.client import Dispatch
import sys
import os
import time
import shutil
import openpyxl as xl
import xlrd
from xlutils.copy import copy

from PyQt5 import QtCore, QtGui
from PyQt5.QtCore import QEventLoop, QTimer
from PyQt5.QtGui import QPainter, QColor
#PyQt5中使用的基本控件都在PyQt5.QtWidgets模块中
from PyQt5.QtWidgets import QWidget, QApplication, QMainWindow,QGroupBox, QPushButton, QLabel, QHBoxLayout, QVBoxLayout, QGridLayout, QFormLayout, QLineEdit, QTextEdit, QInputDialog, QFileDialog, QMessageBox, QDesktopWidget

#导入UI文件类
from pyqt5_office_keywords_replace_ui import Ui_MainWindow

class MyMainForm(QMainWindow, Ui_MainWindow):

    def __init__(self, parent=None):
        super(MyMainForm, self).__init__(parent)
        self.setupUi(self)

        # 添加选择替换目录 按钮信号和槽
        self.ori_dir_btn.clicked.connect(self.replacePath)

        # 添加添加选择存储目录 按钮信号和槽
        self.replace_dir_btn.clicked.connect(self.savePath)

        #添加开始转换按钮信号和槽
        self.start_transfer.clicked.connect(self.transfer)

        self.word = Dispatch('kwps.Application')

    #需要替换的目录
    def replacePath(self):
        path = QFileDialog.getExistingDirectory(self, "请选择您要替换的目录")
        # 判断选择的文件是否存在
        if os.path.exists(path):
            # 将保存url放入路径文本框中
            self.ori_dir_input.setText(path)
        else:
            self.showMsg('错误', '您选择的目录不存在，请重新选择！')
            return False

    def savePath(self):
        path = QFileDialog.getExistingDirectory(self, "请选择您要保存的位置")
        if os.path.exists(path):
            # 将保存url放入路径文本框中
            self.replace_dir_input.setText(path)
        else:
            self.showMsg('错误', '您选择的目录不存在，请重新选择！')
            return False

    # 显示消息
    def showMsg(self, title, content, icon=3):
        box = QMessageBox(QMessageBox.Warning, title, content)
        # 设置左上角消息框图标
        # box.setWindowIcon(QIcon(r'E:\site\python\cutimg\favicon.ico'))
        # 添加按钮，可用中文
        yes = box.addButton('确定', QMessageBox.YesRole)
        # 设置消息框中内容前面的图标
        box.setIcon(icon)
        # 显示该问答框
        box.exec()
        return False

    #开始转换
    def transfer(self):

        #需要替换的目录
        self.replace_dir = self.ori_dir_input.text()
        #保存的目录
        self.save_dir = self.replace_dir_input.text()
        #关键词，并且去除收尾空格
        self.keywords = self.replace_words.toPlainText().strip()

        if not os.path.isdir(self.replace_dir) or not os.path.isdir(self.save_dir):
            self.showMsg('错误', '您选择的目录不存在，请重新选择！')
            return False

        #存储目录必须为空目录
        if os.listdir(self.save_dir):
            self.showMsg('错误', '存储目录必须为空目录！')
            return False

        #文件目录和子目录不能相同
        if self.replace_dir == self.save_dir:
            self.showMsg('错误', '替换目录和存储目录不能相同！')
            return False

        # 替换目录和文件目录不能互为子目录
        if self.replace_dir.startswith(self.save_dir) or self.save_dir.startswith(self.replace_dir):
            self.showMsg('错误', '替换目录和存储目录不能相互包含！')
            return False

        if self.keywords == "":
            self.showMsg('错误', '关键词不能为空')
            return False

        #存储目录不存在就直接创建
        if not os.path.exists(self.save_dir):
            os.makedirs(self.save_dir)

        #开始替换
        self.replace_tool()
        #将转换结果输出到界面上
        # self.result_display.setPlainText(self.keywords)

    #删除目录及目录下的所有文件
    def del_dir(self, filepath):
        del_list = os.listdir(filepath)
        for f in del_list:
            file_path = os.path.join(filepath, f)
            if os.path.isfile(file_path):
                os.remove(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)

    #将doc替换成docx
    def doc2docx(self, file):
        #获取文件目录，文件名带后缀，文件名，后缀
        file_dir, tmpfilename = os.path.split(file)
        file_name, extension = os.path.splitext(tmpfilename)
        doc = self.word.Documents.Open(file)

        new_file = file_dir + "/" +file_name + '.docx'
        doc.SaveAs(new_file, 12)  #12表示docx格式
        doc.Close()
        #删除doc文件
        os.remove(file)

        #docx文件转换
        self.docx_keywords_replace(new_file)

    #将目录及子目录下的文件copy到另一个目录
    def deep_copy_dir(self, origin_dir, target_dir):
        for files in os.listdir(origin_dir):
            name = os.path.join(origin_dir, files)
            back_name = os.path.join(target_dir, files)
            if os.path.isfile(name):
                if os.path.isfile(back_name):
                    shutil.copy(name, back_name)
                else:
                    shutil.copy(name, back_name)
            else:
                if not os.path.isdir(back_name):
                    os.makedirs(back_name)
                self.deep_copy_dir(name, back_name)

    #获取目录及子目录下的文件，并转换成可以处理的格式
    def recurse_transfer_file(self, path):
        for root, dirs, files in os.walk(path):
            for file in files:
                file = file.replace('\\', '/')
                src_file = os.path.join(root, file)

                #doc文件，先进行转换
                if src_file.endswith(".doc") and not src_file.startswith('~$'):
                    self.display_result("{src_file}格式转换".format(src_file=src_file))
                    self.doc2docx(src_file)
                elif src_file.endswith(".docx") and not src_file.startswith('~$'):
                    self.docx_keywords_replace(src_file)
                elif src_file.endswith(".xls") and not src_file.startswith('~$'):
                    # self.xls_keywords_replace(src_file)
                    pass
                elif src_file.endswith(".xlsx") and not src_file.startswith('~$'):
                    # self.xlsx_keywords_replace(src_file)
                    pass
                elif src_file.endswith(".pdf") and not src_file.startswith('~$'):
                    pass

    #word文件关键字替换
    def docx_keywords_replace(self, file):

        # 获取文件目录，文件名带后缀，文件名，后缀
        file_dir, tmpfilename = os.path.split(file)
        file_name, extension = os.path.splitext(tmpfilename)
        origin_file_name = file_name
        doc = self.word.Documents.Open(file)
        a = self.word.ActiveDocument.Sections

        # 每行
        every_line = self.keywords.split('\n')

        for i in range(len(a)):

            for line in every_line:
                split_list = line.split('|')
                old_word = split_list[0].strip().replace('\r', '').replace('\n', '').replace('\t', '')
                new_word = split_list[1].strip().replace('\r', '').replace('\n', '').replace('\t', '')

                #对页眉进行替换
                self.word.ActiveDocument.Sections[i].Headers[0].Range.Find.Execute(old_word, False, False, False, False, False, True, 1, True, new_word, 2)

                #对页脚进行替换
                self.word.ActiveDocument.Sections[i].Footers[0].Range.Find.Execute(old_word, False, False, False, False, False, True, 1, True, new_word, 2)

                #替换正文
                self.word.Selection.Find.Execute(old_word, False, False, False, False, False, True, 1, True, new_word, 2)

                file_name = file_name.replace(old_word, new_word)
                display_content = "{file}进行{old_word}->{new_word}替换".format(file=file, old_word = old_word, new_word = new_word)
                self.display_result(display_content)

        #存储替换文件
        doc.SaveAs(r"{0}/{1}.docx".format(file_dir, file_name))
        if origin_file_name != file_name:
            os.remove(file)
        doc.Close()
        # self.word.Quit()

    #替换xlsx关键词
    def xlsx_keywords_replace(self, file):
        wb = xl.load_workbook(file)
        ws = wb.worksheets[0]

        every_line = self.keywords.split('\n')
        file_dir, tmpfilename = os.path.split(file)
        file_name, extension = os.path.splitext(tmpfilename)

        # 每个关键词替换
        for line in every_line:

            split_list = line.split('|')
            old_word = split_list[0].strip().replace('\r', '').replace('\n', '').replace('\t', '')
            new_word = split_list[1].strip().replace('\r', '').replace('\n', '').replace('\t', '')

            for row in range(1, ws.max_row + 1):
                for col in range(1, ws.max_column + 1):
                    content = ws.cell(row=row, column=col).value
                    ws.cell(row=row, column=col).value = content.replace(old_word, new_word, 1)

            file_name = file_name.replace(old_word, new_word)

        #存储
        wb.save(r"{0}/{1}{2}".format(file_dir, file_name,extension))
        #删除源文件
        os.remove(file)
        wb.close()
        display_content = file + "关键词转换成功"
        self.display_result(display_content)

    #替换xls文件关键词
    def xls_keywords_replace(self, file):

        wb = xlrd.open_workbook(file, formatting_info=True)  # 获取xls，保留原格式
        ws = wb.sheet_by_index(0)  # 根据index获取sheet
        rows = ws.nrows
        cols = ws.ncols
        newbook = copy(wb)  # 复制xls
        newsheet = newbook.get_sheet(0)

        every_line = self.keywords.split('\n')
        file_dir, tmpfilename = os.path.split(file)
        file_name, extension = os.path.splitext(tmpfilename)
        # 每个关键词替换
        for line in every_line:
            split_list = line.split('|')
            old_word = split_list[0].strip().replace('\r', '').replace('\n', '').replace('\t', '')
            new_word = split_list[1].strip().replace('\r', '').replace('\n', '').replace('\t', '')

            # 遍历每个单元格，进行替换操作
            for row in range(1, rows):
                for col in range(1, cols):
                    content = ws.cell(row, col).value
                    if (content != None and isinstance(content, str)):  # 判断不为空且为字符
                        if (content.find(old_word) != -1):  # 找到需要替换的字符
                            newsheet.write(row, col, content.replace(old_word, new_word))

            file_name = file_name.replace(old_word, new_word)

        # 保存新的xls以替换原有的xls
        newbook.save(r"{0}/{1}{2}".format(file_dir, file_name, extension))
        # 删除源文件
        os.remove(file)
        display_content = file + "关键词转换成功"
        self.display_result(display_content)

    #将处理结果实时显示到控制台
    def display_result(self,msg):
        self.result_display.append("{msg}\n".format(msg=msg))
        # self.cursor = self.result_display.textCursor()
        # self.result_display.moveCursor(self.cursor.End)
        QApplication.processEvents()

    #核心替换功能
    def replace_tool(self):
        try:
            self.result_display.clear()
            self.display_result("正进行目录清理")
            # 转换前先清空目录下文件
            self.del_dir(self.save_dir)
            self.display_result("正进行目录文件拷贝")
            #将原始目录及子目录文件全部拷贝至新目录
            self.deep_copy_dir(self.replace_dir, self.save_dir)

            self.display_result("正进行文件关键词替换")
            #递归处理存储目录下的文件
            self.recurse_transfer_file(self.save_dir)
            self.display_result("全部操作完毕")
        except Exception as e:
            self.display_result("==================出现异常，请钉钉联系管理员==================")
            self.display_result(str(e))
            self.display_result("=============================================================")

if __name__ == "__main__":
    app = QApplication(sys.argv)
    # 初始化
    myWin = MyMainForm()
    # 将窗口控件显示在屏幕上
    myWin.show()
    # 程序运行，sys.exit方法确保程序完整退出。
    sys.exit(app.exec_())


from models import model
from college.api.college_schema import *
from config.dataBase.data_base_client import engine, DbLife
model.Base.metadata.create_all(bind=engine)
from models.model import paginate

def get_college(college_id:int):
    with DbLife() as db:
        college = db.query(model.College).filter(model.College.id == college_id,model.College.is_delete != 1).first()
    return college
def get_all_college():
    with DbLife() as db:
        colleges = db.query(model.College).filter(model.College.is_delete != 1).all()
    return colleges
def add_college(college_name,college_eng_name,address,telephone,filename):
    with DbLife() as db:
        college = model.College(college_name=college_name,college_eng_name=college_eng_name,
                                    address=address,telephone=telephone,icon=filename,is_delete=0)
        db.add(college)
        db.commit()
        db.refresh(college)
        return college
def update_college(id,college_name,college_eng_name,address,telephone,filename):
    with DbLife() as db:
        college = db.query(model.College).filter(model.College.id == id).first()
        setattr(college,"college_name",college_name)
        setattr(college, "college_eng_name", college_eng_name)
        setattr(college, "address", address)
        setattr(college, "telephone", telephone)
        setattr(college, "icon", filename)
        db.commit()
        db.refresh(college)
        return college

def delete_college(id):
    with DbLife() as db:
        college = db.query(model.College).filter(model.College.id == id).first()
        setattr(college,"is_delete",1)
        db.commit()
        db.refresh(college)


import tkinter
from tkinter.constants import END
import tkinter.ttk
from typing import Text
import serial
import threading
import time

'''
    調試好了：
        tcp調試好了用tk.place.froget()
        串口連接調試好了，引入一個全局變量控制綫程的開關
    未能調試好的：
        調整tcpsocket 連接

'''

# 窗口初始化，大小
tk=tkinter.Tk()
tk.title('網絡串口連接工具')
tk.geometry('1000x700')
# 創建框架
comout=tkinter.Frame(tk,bd=2,relief='ridge',width=680,height=360).place(x=300,y=30)     #輸出窗口
showconnect=tkinter.Frame(tk,bd=2,relief='ridge',width=250,height=360).place(x=30,y=30) #連接窗口
connect=tkinter.Frame(tk,bd=2,relief='ridge',width=250,height=360)                      #連接窗口
scdconnect=tkinter.Frame(tk,bd=2,relief='ridge',width=250,height=360)                   #連接窗口
osframe=tkinter.Frame(tk,bd=2,relief='ridge',width=950,height=270).place(x=30,y=400)    #輸入窗口

# 創建顯示窗口

text1= tkinter.Text(comout,width=61,height=21)
text1.place(x=305,y=35)
text1.tag_config("tag_1", foreground="red")# 創建窗口時間顯示的字體大小和顔色
text1.tag_config("tag_2", foreground="blue")# 創建窗口數據顯示的字體大小和顔色
text1.insert(END,'歡迎光臨'+'\n')

# 創建輸入窗口

text2= tkinter.Text(comout,width=90,height=16)
text2.place(x=35,y=405)
text2.tag_config("tag_2", foreground="blue")# 創建窗口數據顯示的字體大小和顔色
text2.insert(END,'在這裏輸入數據'+'\n')

# 獲取連接方式,ip地址，端口號，串口號，波特率A
whatcnt=tkinter.IntVar()
ip=tkinter.StringVar()
ip.set('192.168.0.1')
port=tkinter.IntVar()
com=tkinter.StringVar()
bote=tkinter.IntVar()
bote.set('115200')
closecon=tkinter.StringVar()

# 顯示網絡連接tcp ip 端口號
def showcnt():
    if whatcnt.get() == 1:
        scdconnect.place_forget()
        connect.place(x=30,y=30)
        # 顯示tcp窗口
        tkinter.Label(connect,text='IP地址:').place(x=20,y=120)
        tkinter.Label(connect,text='端口號:').place(x=20,y=220)
        tkinter.Entry(connect,textvariable=ip,width=12).place(x=120,y=120)
        tkinter.Entry(connect,textvariable=port,width=12).place(x=120,y=220)
    elif whatcnt.get() == 2:
        connect.pack_forget()
        scdconnect.place(x=30,y=30)
        tkinter.Label(scdconnect,text='串口號:').place(x=20,y=120)
        bote=tkinter.Label(scdconnect,text='波特率:').place(x=20,y=220)
        combobox=tkinter.ttk.Combobox(scdconnect,textvariable=com,values=('com1','com2','com3','com4','com5'),width=10)
        combobox.current(0)
        combobox.place(x=120,y=120)
        botebobox=tkinter.ttk.Combobox(scdconnect,textvariable=bote,values=(115200,57600,38400,19200,9600),width=10)
        botebobox.current(0)
        botebobox.place(x=120,y=220)
    else :
        print('打印錯誤')

#打開串口，調用串口

ser = serial.Serial()#初始化串口數據
ser.baudrate = bote.get()   #設置波特率
ser.bytesize = 8
ser.parity = serial.PARITY_NONE
ser.stopbits = 1
over_time = 30
bool=True

# 讀取串口數據
def serialcon():
    global bool
    def test():
        while bool:
            end_time=time.time()
            if end_time - starttime < over_time:
                ch = ser.readline() # 讀取一行數據   # 如果需要處理發來的數據請在這裏添加函數處理ch
                text1.insert(END,time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+'\n',"tag_1")
                text1.insert(END,ch,"tag_2")
                text1.see(tkinter.END)
                text1.update()
    if ser.is_open== False:
        try:
            ser.port = com.get() #獲取串口號
            ser.open() # 开启串口
            starttime=time.time()
            thread=threading.Thread(target=test)
            thread.start()
        except:
            ser.close()

#他開TCP，連接TCP

# 關閉串口連接

def serialclose():
    global bool
    try:
        bool=False
        ser.close()
    except:
        print('錯誤')

def open():
    global bool
    if whatcnt.get() ==1:
        pass
    elif whatcnt.get() ==2:
        bool=True
        serialcon()
    else:
        pass

def close():
    if whatcnt.get() ==1:
            pass
    elif whatcnt.get() ==2:
        serialclose() # 關閉串口
    else:
        pass

# 創建tcp串口選擇按鈕
tcp=tkinter.Radiobutton(tk,text='網絡連接',variable=whatcnt,value=1,command=showcnt,width=10,height=1)
tcp.place(x=38,y=40)
serial2=tkinter.Radiobutton(tk,text='串口連接',variable=whatcnt,value=2,command=showcnt,width=10,height=1)
serial2.place(x=140,y=40)
# 創建連接，斷開按鈕
tkinter.Button(tk,text='連接設備',command=open,width=10,height=1).place(x=50,y=320)
tkinter.Button(tk,text='斷開設備',command=close,width=10,height=1).place(x=160,y=320)

def send():
    if whatcnt.get() ==1:
        pass
    elif whatcnt.get() ==2:
        serialsenddata=text2.get(1.0,END)
        # 如果需要處理發出去的數據請在這裏添加一個函數處理
        ser.write(serialsenddata.encode('GBK'))
    else:
        pass
# 創建發送數據按鈕
senddatabut=tkinter.Button(osframe,text='發送數據',command=send,width=10,height=1)
senddatabut.place(x=758,y=405)

tk.mainloop()


#!usr/bin/python
# -*- coding: utf-8 -*-

#        +=========================================+
#        |........♚ łαbørαŧøriø Ŧαηŧαรмα...........|
#        +-----------------------------------------+
#        |♚Coded: @DreadPirateRobertt (Telegram)   |
#        |♚Contact:telegram.me/FullPythonAlchemist |
#        |♚Date: 15/02/2017                        |
#        |♚Chanell:telegram.me/Phantasm_Lab        |
#        |♚Changing the Description of this tool   |
#        |Won't made you the coder ^_^ !!!         |
#        |♚Respect Coderz ^_^ (Open_Source_Project)|
#        |♚I take no responsibilities for the      |
#        |  use of this program !                  |
#        +=========================================+
#        |........♚ łαbørαŧøriø Ŧαηŧαรмα...........|
#        +-----------------------------------------+

import os #----------------#
import sys #---------------#
import logging #-----------#

verde = '\033[32;1m'
amarelo = '\033[1;33m'
vermelho = '\033[31;1m'
cyanClaro = '\033[1;36m'

os.system('clear')
Banner = vermelho + """
─────────────────────────────────────────────────────────────────────────────
─████████████───██████████████─██████──────────██████─██████──────────██████─
─██░░░░░░░░████─██░░░░░░░░░░██─██░░██──────────██░░██─██░░██████████──██░░██─
─██░░████░░░░██─██░░██████░░██─██░░██──────────██░░██─██░░░░░░░░░░██──██░░██─
─██░░██──██░░██─██░░██──██░░██─██░░██──────────██░░██─██░░██████░░██──██░░██─
─██░░██──██░░██─██░░██──██░░██─██░░██──██████──██░░██─██░░██──██░░██──██░░██─
─██░░██──██░░██─██░░██──██░░██─██░░██──██░░██──██░░██─██░░██──██░░██──██░░██─
─██░░██──██░░██─██░░██──██░░██─██░░██──██░░██──██░░██─██░░██──██░░██──██░░██─
─██░░██──██░░██─██░░██──██░░██─██░░██████░░██████░░██─██░░██──██░░██████░░██─
─██░░████░░░░██─██░░██████░░██─██░░░░░░░░░░░░░░░░░░██─██░░██──██░░░░░░░░░░██─
─██░░░░░░░░████─██░░░░░░░░░░██─██░░██████░░██████░░██─██░░██──██████████░░██─
─████████████───██████████████─██████──██████──██████─██████──────────██████─
─────────────────────────────────────────────────────────────────────────────
───────────────────────────────────────────────────────────────
─██████████████─██████──██████─██████████████───██████████████─
─██░░░░░░░░░░██─██░░██──██░░██─██░░░░░░░░░░██───██░░░░░░░░░░██─
─██████░░██████─██░░██──██░░██─██░░██████░░██───██░░██████████─
─────██░░██─────██░░██──██░░██─██░░██──██░░██───██░░██─────────
─────██░░██─────██░░██──██░░██─██░░██████░░████─██░░██████████─
─────██░░██─────██░░██──██░░██─██░░░░░░░░░░░░██─██░░░░░░░░░░██─
─────██░░██─────██░░██──██░░██─██░░████████░░██─██░░██████████─
─────██░░██─────██░░██──██░░██─██░░██────██░░██─██░░██─────────
─────██░░██─────██░░██████░░██─██░░████████░░██─██░░██████████─
─────██░░██─────██░░░░░░░░░░██─██░░░░░░░░░░░░██─██░░░░░░░░░░██─
─────██████─────██████████████─████████████████─██████████████─
───────────────────────────────────────────────────────────────"""

Banner2 = verde + """
▒▓▒▓▒▓▒▓▒▓▒▓─▄▀▀▀▄       +=================================================+
─██▀████▀██──▀▄▀──█      |     Download: Videos, Musicas e Playlists       |
O▀████████▀O─────█       +=================================================+
───▀█▄▄█▀────────█       | Coded: @DreadPirateRobertt                      |
──▓▒▓▒▓▒▓▒───────█       | ☎ Contact: t.me/Phantasm_Lab (@Phantasm_Lab)    |
                         | ☮ date : 15.02.2017                             |
                         +=================================================+"""

BARER = '#' * 88
DOLLA = '$' * 78
space = " " * 34

def _header(x, y):
    return "\n{}\n{}  {} | {}  {}\n".format(BARER, BARER[:17], x, y, BARER[:18])

def _formatos(x, y):
    return "{}\n{}  {} : {} {}{}".format(
        BARER, BARER[:3], x, ", ".join(y), space[:22], BARER[:3], BARER
        )

def _comands(x, y):
    return "{}\n{} {}{}|{}{}{}{}\n{}".format(
        BARER, BARER[:3], x, space[:11], space[:11], y, space, BARER[:3], BARER
        )

def _generate_options(flag, llike):
    result = []
    if flag:
        msg = ['Download Faixa de Audio (Youtube)', 'Download de uma Playlist de Audio (Youtube)']
        for item in llike:
            result.append([item, msg[0]])
            result.append(['--playlist{}'.format(item), msg[1]])
        return result
    else:
        msg = ['Download de video em formato ', 'Download de uma Playlist']
        for item in llike:
            result.append([item, msg[0] + item.split('-')[1]])
            result.append(['--playlist{}'.format(item), msg[1]])
        return result

def _generate_help_option(llike):
    result = ""
    for option, msg in llike:
        result += "\n###  {}{} >>> {}{} ###".format(
            option, " "*(24 - len(option)), msg, " "*(50 - len(msg))
            )
        if "playlist" in option:
            result += "\n{}".format(BARER)
    return result

def _usage():
    AUNIQUE = _generate_options(True, ['-mp3', '-best', '-m4a'])
    VUNIQUE = _generate_options(False, ['-mp4', '-fly', '-ogg', '-mkv', '-avi', '-webm'])

    AUDIO = "$$$ AUDIO {}\n{}".format(DOLLA, BARER) + _generate_help_option(AUNIQUE)
    VIDEOS = "$$$ VIDEOS {}\n{}".format(DOLLA[1:], BARER) + _generate_help_option(VUNIQUE)

    print(
        """{}{}\n{}""".format(
        _header('Download Music', 'Videos and Playlist Music Videos'),
        _formatos('Formatos disponiveis', ['mp3', 'mp4', 'flv', 'ogg', 'mkv', 'avi', 'webm']),
        _comands('Comands Options', 'FUNCTIONS')
        )
    )
    print(cyanClaro + AUDIO)
    print(amarelo + VIDEOS)

EXAMPLE = verde + 'Example > python2 D0nwTube.py '
MODO_DE_USO = 'Modo de uso:\n$ python2 download.py [opção] [url]'

################ HELPS ###############
MP3 = EXAMPLE + "-mp3 https://www.youtube.com/watch?v=txmvb7tbAxs&t=57s"
MP4 = EXAMPLE + "-mp4 https://www.youtube.com/watch?v=txmvb7tbAxs&t=57s"
PMP3 = EXAMPLE + "--playlist-mp3 https://www.youtube.com/playlist?list=PLQimmOQ9bzurCStHdAoBnZeRyzESSsc7q\n"
PMP4 = EXAMPLE + "--playlist-mp4 https://www.youtube.com/playlist?list=PLQimmOQ9bzurCStHdAoBnZeRyzESSsc7q\n"
OPENED_PLAYLIST = EXAMPLE + "--playlist-mp4 \"https://.../watch?v=...&index=2&list=PLQimmOQ9bzurCStHdAoBnZeRyzESSsc7q\""
LINE = (MP3, PMP3, MP4, PMP4, OPENED_PLAYLIST, " ")

def help():
    _usage()
    print(verde +  MODO_DE_USO)
    print(vermelho + '$ sudo apt-get install youtube-dl | if not installed\n')
    for LIN in LINE:
        print(LIN)

print(Banner)
print(Banner2)

ARGS_ON = False
GO_NEXT = True
EXISTS_OPTIONS = False

def check_exist_option(opt, options):
    global EXISTS_OPTIONS
    for option in options:
        if opt in option:
            EXISTS_OPTIONS = True

def program_option(nop, url):
    yep = nop.replace('--no', '--yes')
    url = "".join(url.split("\\"))  # zsh: watch\?v\=, playlist\?list\=
    NOP = 'youtube-dl {} {}'.format(nop, url)
    YEP = 'youtube-dl {} {}'.format(yep, url)
    return NOP, YEP

def _unique_url(wf, msg):
    _unique = False
    if os.path.exists(wf):
        with open(wf, 'r') as log_list:
            if msg not in log_list.readlines():
                _unique = True
    return _unique

def parse_args(opt, option, command, url):
    """
    LINUX COMMAND LINE CAN'T IGNORE [SPECIAL CHARACTERS]
    https://unix.stackexchange.com/questions/296141/how-to-use-a-special-character-as-a-normal-one

    INPUT LINK: https:// ... /watch?v=wUDkg8wyROU&list=PLlhaET2L7ba0Vp3otZf6ipXPDhFceWWCz
    SCRIPT GOT: https:// ... /watch?v=wUDkg8wyROU

    """
    lfile = "../urls.log"   # DIRECTORY WAS CHANGED IN LINE: 266
    logging.basicConfig(
        filename=lfile, level=logging.INFO, format="%(message)s"
    )
    HLINK = 'https://www.youtube.com/playlist?'
    PLIST = 'playlist?list='
    LLIST = PLIST.split('?')[1]
    HTTPS = HLINK.split(':')[0]
    global GO_NEXT
    NOP, YEP = program_option(command, url)
    single, with_video, playlist = option
    if opt == single or opt == with_video:
        if ARGS_ON:
            if _unique_url(lfile, NOP):
                logging.info(NOP)
            os.system(NOP)
            GO_NEXT = False
    elif opt == playlist:
        if ARGS_ON:
            if PLIST in YEP:
                if _unique_url(lfile, YEP):
                    logging.info(YEP)
                os.system(YEP)
                GO_NEXT = False
            elif LLIST in YEP:
                get_option = YEP.split(HTTPS)
                LIST = None
                for list in get_option[1].split('&'):
                    if list.startswith(LLIST):
                        LIST = list
                YEP = "{}{}{}".format(get_option[0], HLINK, LIST)
                if _unique_url(lfile, YEP):
                    logging.info(YEP)
                os.system(YEP)
                GO_NEXT = False

def main():
    global ARGS_ON
    global GO_NEXT
    global EXISTS_OPTIONS
    DOWNLOADED_FILES = 'DOWNLOADED_FILES'
    OPTIONS = [
        ['-mp3', '--audio', '--playlist-mp3'], ['-mp4', '--video-mp4', '--playlist-mp4'],
        ['-flv', '--video-flv', '--playlist-flv'], ['-ogg', '--video-ogg', '--playlist-ogg'],
        ['-webm', '--video-webm', '--playlist-webm'], ['-mkv', '--video-mkv', '--playlist-mkv'],
        ['-avi', '--video-avi', '--playlist-avi']
    ]
    YOUTUBE_DL = [
        '-x --no-playlist --audio-format mp3', '--no-playlist --recode-video mp4',
        '--no-playlist --recode-video flv', '--no-playlist --recode-video ogg',
        '--no-playlist --recode-video webm', '--no-playlist --recode-video mkv',
        '--no-playlist --recode-video avi'
    ]
    SINGLE_DL = ['-x ' + option for option in YOUTUBE_DL[1:-1]]
    SINGLE_OPTIONS = [['NOP', 'YEP', option] for _, _, option in OPTIONS[1:-1]]
    try:
        if len(sys.argv) == 2:
            opt = sys.argv[1]
            if opt == '-h' or opt == '--help':
                help()
        elif len(sys.argv) <= 3 and len(sys.argv) >= 2:
            opt = sys.argv[1]
            url = sys.argv[2]
            ARGS_ON = True
            check_exist_option(opt, OPTIONS)
            if EXISTS_OPTIONS:
                ########## CHANGE DIRECTORY ##########
                if not os.path.exists(DOWNLOADED_FILES):
                    os.mkdir(DOWNLOADED_FILES)
                os.chdir(DOWNLOADED_FILES)
                ########## MUSIC, VIDEOS #############
                for idx, itm in enumerate(YOUTUBE_DL):
                    if GO_NEXT:
                        parse_args(opt, OPTIONS[idx], itm, url)
                    else: break
                ########## PLAYLISTS #################
                for idx, itm in enumerate(SINGLE_DL):
                    if GO_NEXT:
                        parse_args(opt, SINGLE_OPTIONS[idx], itm, url)
                    else: break
            else:
                print('Opção invalida, para ver o menu de ajuda use: \n')
                print('$ python2 download.py -h ou | python download.py --help')
        else:
            print('Você deve digitar alguma opção!\n')
            print(MODO_DE_USO)
            print('$ python2 download.py -h, --help')
            quit()
    except KeyboardInterrupt:
        print('[!] 404 - Error Not Found;....[./]')

if __name__ == '__main__':
    main()


import mxnet as mx
import time
import os
import logging
import numpy as np
import json
import datetime
import glob
from east_symbol import east_symbol
from data_iter.dataset import TrainDataListDataset as dataset
from data_iter.dataloader import DataLoader as dataloader
from icdar_mx import save_train_data

def main():
    batch_per_gpu = 10
    num_gpus = 1
    num_workers = 5
    batch_size = int(batch_per_gpu * num_gpus)
    im_size = 512
    save_model_steps = 2
    update_data_steps = 1
    optimizer = 'adadelta'
    optimizer_params = {'rho': 0.99, 'wd': 0.00001}
    load_epoch = 0
    begin_epoch, num_epoch = load_epoch+1, 300
    train_eval_ratio = 10
    train_data_path = "/home/wangpan/Dataset/OCR/icdar2015/all_train_data/"
    model_prefix = "/home/wangpan/Workspace/OCR/EAST/model/text_det"
    result_path = "/home/wangpan/Workspace/OCR/EAST/output/"

    logger=set_logging(result_path)

    train_data, eval_data = train_eval_dataloader(train_data_path, batch_size, im_size, num_workers, train_eval_ratio)

    _, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, load_epoch)
    train_net = east_symbol()
    sym = train_net.get_symbol(is_training=True)
    mod = mx.mod.Module(symbol=sym, context=[mx.gpu()], logger=logger,
                        data_names=('data',), label_names=('gt_cls','gt_geo','training_mask'))

    mod.bind(data_shapes=[('data',(batch_size, 3, im_size, im_size))],
             label_shapes=[('gt_cls',(batch_size, 1, im_size/4, im_size/4)),
                           ('gt_geo', (batch_size, 5, im_size/4, im_size/4)),
                           ('training_mask', (batch_size, 1, im_size/4, im_size/4))],
             for_training=True, force_rebind=False)
    mod.init_params(arg_params=arg_params, aux_params=aux_params,
                     allow_missing=True, force_init=False)
    mod.init_optimizer(kvstore='device', optimizer=optimizer,
                       optimizer_params=optimizer_params)

    for epoch in range(begin_epoch, begin_epoch+num_epoch):
        tic = time.time()
        nbatch = 0
        train_data_iter = iter(train_data)
        end_of_batch = False
        next_data_batch = next(train_data_iter)
        losses=[]
        while not end_of_batch:
            data_batch = next_data_batch
            mod.forward_backward(data_batch)
            mod.update()
            try:
                next_data_batch = next(train_data_iter)
                mod.prepare(next_data_batch)
            except StopIteration:
                end_of_batch = True
            get_loss= [loss.asnumpy()[0]*batch_size for loss in mod.get_outputs()]
            losses.append(get_loss)
            nbatch+=1
            print('Batch[{}] cls loss={:.4f} loc loss={:.4f}'.format(nbatch, get_loss[0], get_loss[1]))
        toc = time.time()
        mean_loss = np.array(losses).mean(0)
        info = 'Epoch[{}] Time cost={:.3f} Cls loss={:.4f} Loc loss={:.4f}'.format(epoch, (toc - tic), mean_loss[0], mean_loss[1])
        logger.info(info)

        eval_data_iter=iter(eval_data)
        next_data_batch = next(eval_data_iter)
        end_of_batch = False
        losses = []
        while not end_of_batch:
            data_batch = next_data_batch
            mod.forward(data_batch)
            try:
                next_data_batch = next(train_data_iter)
                mod.prepare(next_data_batch)
            except StopIteration:
                end_of_batch = True
            get_loss = [loss.asnumpy()[0] * batch_size for loss in mod.get_outputs()]
            losses.append(get_loss)
        mean_loss = np.array(losses).mean(0)
        info = 'Epoch[{}] evalidation Cls loss={:.4f} Loc loss={:.4f}'.format(epoch,mean_loss[0], mean_loss[1])
        logger.info(info)

        # sync aux params across devices
        arg_params, aux_params = mod.get_params()
        mod.set_params(arg_params, aux_params)

        if epoch%save_model_steps==0:
            mod.save_checkpoint(model_prefix, epoch)
        if (epoch+1-begin_epoch) % update_data_steps == 0:
            tic = time.time()
            save_train_data(input_data_path='/home/wangpan/Dataset/OCR/icdar2015/train/',
                            input_text_path='/home/wangpan/Dataset/OCR/icdar2015/traingt/',
                            ouput_data_path='/home/wangpan/Dataset/OCR/icdar2015/all_train_data/')
            train_data, eval_data = train_eval_dataloader(train_data_path, batch_size, im_size, num_workers, train_eval_ratio)
            print("gen data cost time={:.2f}s".format(time.time() - tic))
        train_data.reset()

def train_eval_dataloader(train_data_path, batch_size, im_size, num_workers, train_eval_ratio):
    samples_list = glob.glob(os.path.join(train_data_path, '*.npy'))
    ids = range(len(samples_list))
    np.random.shuffle(ids)
    train_list = [samples_list[i] for i in ids[len(ids) / train_eval_ratio:]]
    eval_list = [samples_list[i] for i in ids[:len(ids) / train_eval_ratio]]
    train_dataset = dataset(train_data_list=train_list)
    train_data = dataloader(train_dataset,
                            batch_size=batch_size,
                            num_workers=num_workers,
                            provide_data=[('data', (batch_size, 3, im_size, im_size))],
                            provide_label=[('gt_cls', (batch_size, 1, im_size / 4, im_size / 4)),
                                           ('gt_geo', (batch_size, 5, im_size / 4, im_size / 4)),
                                           ('training_mask', (batch_size, 1, im_size / 4, im_size / 4))])
    eval_dataset = dataset(train_data_list=eval_list)
    eval_data = dataloader(eval_dataset,
                           batch_size=batch_size,
                           num_workers=num_workers,
                           provide_data=[('data', (batch_size, 3, im_size, im_size))],
                           provide_label=[('gt_cls', (batch_size, 1, im_size / 4, im_size / 4)),
                                          ('gt_geo', (batch_size, 5, im_size / 4, im_size / 4)),
                                          ('training_mask', (batch_size, 1, im_size / 4, im_size / 4))])
    return train_data, eval_data

def set_logging(output_path):
    now = datetime.datetime.now()
    nowtime = now.strftime("%m-%d-%H:%M:%S")
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                        datefmt='%a, %d %b %Y %H:%M:%S',
                        filename=os.path.join(output_path,nowtime+'.log'),
                        filemode='w')
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
    return logging

if __name__ == '__main__':
    main()


import requests
from bs4 import BeautifulSoup
import re
from os import getcwd, path, rename

import datetime
from helpers import ensure_dirs, ensure_consistency

URL = 'https://es.wikipedia.org/wiki/Pandemia_de_enfermedad_por_coronavirus_de_2020_en_Chile'

def scrape_chile():
    cwd = getcwd()
    chile_dir = path.join(cwd, 'data', 'chile')
    tmp_dir = path.join(cwd, 'tmp')
    ensure_dirs(chile_dir, tmp_dir)

    today = str(datetime.date.today())
    page = requests.get(URL)
    soup = BeautifulSoup(page.content, 'html.parser')
    not_number_regexp = re.compile(r'\D')

    per_region_table = None
    tables = soup.find_all('table')

    for table in tables:
        headers = table.find_all('th')
        if len(headers) > 0 and 'Regiones' in headers[0].get_text():
            per_region_table = table
            break

    updated_files = []
    header = 'date,region,region_iso,province,city,place_type,cases,deaths\n'
    for tr in per_region_table.find_all('tr')[2:-1]:
        cols = [td.get_text() for td in tr.find_all('td')]
        if len(cols) != 6:
            continue

        iso = None
        for region in REGION_ISO:
            if region in cols[0]:
                iso = REGION_ISO[region]
                break

        if iso is None:
            continue

        region = ISO_REGION[iso]

        line = ','.join([
            today,
            region,
            iso,
            '',
            '',
            'region',
            not_number_regexp.sub('', cols[2]),
            not_number_regexp.sub('', cols[4]),
        ])

        region_file = path.join(chile_dir, f'{iso.lower()}.csv')
        is_empty = not path.exists(region_file)

        with open(region_file, 'a+') as f:
            if is_empty:
                f.write(header)
            f.write(f'{line}\n')

        if not is_empty:
            updated_files.append(region_file)

    ensure_consistency(updated_files, lambda row: row[:5])

    with open(path.join(getcwd(), 'data', 'chile', 'README.md'), 'w') as readme_f:
        readme_f.write(get_readme_contents())

def get_readme_contents():
    toc = [f'| {name} | [`{iso.lower()}.csv`]({iso.lower()}.csv) |' for name,
           iso in REGION_ISO.items()]
    toc_contents = '\n'.join(toc)

    return f"""## Chile

> Last updated at {datetime.datetime.now(datetime.timezone.utc).strftime('%b %d %Y %H:%M:%S UTC')}.

| Region | Dataset |
| ------ | ------- |
{toc_contents}

"""

REGION_ISO = {
    "Tarapacá": "CL-TA",
    "Antofagasta": "CL-AN",
    "Atacama": "CL-AT",
    "Coquimbo": "CL-CO",
    "Araucanía": "CL-AR",
    "Valparaíso": "CL-VS",
    "O'Higgins": "CL-LI",
    "Maule": "CL-ML",
    "Biobío": "CL-BI",
    "Los Lagos": "CL-LL",
    "Aysén": "CL-AI",
    "Magallanes": "CL-MA",
    "Metropolitana": "CL-RM",
    "Los Ríos": "CL-LR",
    "Arica y Parinacota": "CL-AP",
    "Ñuble": "CL-NB",
}

ISO_REGION = {v: k for k, v in REGION_ISO.items()}


from model.config import Config
from model.ner_model import NERModel
from model.ner_learner import NERLearner
from utils import parse_dataset, parse_dataset_laser
import time
from torch.cuda import empty_cache

#from model.ent_model import EntModel
#from model.ent_learner import EntLearner
from models import *
from subprocess import run

def main(config = None, embedders_to_train=None):
    # create instance of config
    if config is None:
        config = Config()
    if embedders_to_train is None:
        embedders_to_train = [
            # 'LASEREmbedderBase',
            #             # 'LASEREmbedderBaseGRU',
            'LASEREmbedderI',
            # 'LASEREmbedderIII',
         # 'LASEREmbedderIIIELMo',
        ]

    encoding = 'utf-8'
    static_lstm = False

    # parse datasets
    train_laser, tr_pad_len = parse_dataset_laser(config.filename_train, config.label_to_idx,  config.word_to_idx, pos_target = config.pos_target, encoding=encoding)
    dev_laser, dev_pad_len = parse_dataset_laser(config.filename_dev, config.label_to_idx, config.word_to_idx, pos_target = config.pos_target, encoding=encoding)
    # else:
    train_base, tr_pad_len = parse_dataset(config.filename_train, config.label_to_idx, config.word_to_idx, pos_target = config.pos_target, encoding=encoding)
    dev_base, dev_pad_len = parse_dataset(config.filename_dev, config.label_to_idx, config.word_to_idx, pos_target = config.pos_target, encoding=encoding)
    # # build model
    embedder_base = LASEREmbedderBase #(config.model_path, tr_pad_len)
    embedder_base_gru = LASEREmbedderBaseGRU#(config.model_path, tr_pad_len)
    embedderI = LASEREmbedderI#(config.model_path, static_lstm = False)
    embedderIII = LASEREmbedderIII#(config.model_path, static_lstm = False)
    embedderIIIElmo = LASEREmbedderIIIELMo

    embedders = {
        'LASEREmbedderBase':embedder_base,
        'LASEREmbedderBaseGRU':embedder_base_gru,
        'LASEREmbedderI':embedderI,
        'LASEREmbedderIII':embedderIII,
        'LASEREmbedderIIIELMo':embedderIIIElmo
    }
    # model_name = {
    #     embedder_base:'LASEREmbedderBase',
    #     embedder_base_gru:'LASEREmbedderBaseGRU',
    #     embedderI:'LASEREmbedderI',
    #     embedderIII:'LASEREmbedderIII',
    #     embedderIIIElmo:'LASEREmbedderIIIELMo',
    # }

    use_laser = {
        'LASEREmbedderBase': False,
        'LASEREmbedderBaseGRU': False,
        'LASEREmbedderI': True,
        'LASEREmbedderIII': True,
        'LASEREmbedderIIIELMo': True
    }

    for embedder in embedders_to_train:

        # set output filename
        laser = use_laser[embedder]
        config.set_model_name(embedder)
        config.use_laser = laser
        # config.set_params(laser)
        train = train_laser if laser else train_base
        dev = dev_laser if laser else dev_base
        model = embedders[embedder](config.model_path, bpe_pad_len=tr_pad_len, static_lstm = static_lstm,
                         drop_before = config.drop_before_laser, drop_after = config.drop_after_laser, drop_within=config.drop_in_laser)

        # try:
        fit(config, model, tr_pad_len, dev_pad_len, train, dev)
        del model
        empty_cache()
        time.sleep(60) # free up CUDA memory
        # except:
        #     time.sleep(60)
        #     with open('log.txt', 'a') as f:
        #         f.write(str(embedder)+config.filename_train)

def fit(config, embedder, tr_pad_len, dev_pad_len, train, dev):

    # Initiate model
    model = NERModel(config, embedder,
                     tr_pad_len, dropout = config.transformer_drop,
                     num_heads=config.num_heads, num_layers = config.num_layers,
                     filter_size = config.filter_size)
    # train
    learn = NERLearner(config, model, tr_pad_len, dev_pad_len)
    learn.fit(train, dev)

if __name__ == "__main__":
    main()


# Copyright 2020 MONAI Consortium
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import torch
from parameterized import parameterized

from monai.losses import DiceLoss

TEST_CASE_1 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
    {"include_background": True, "do_sigmoid": True},
    {
        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
        "smooth": 1e-6,
    },
    0.307576,
]

TEST_CASE_2 = [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
    {"include_background": True, "do_sigmoid": True},
    {
        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
        "ground": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
        "smooth": 1e-4,
    },
    0.416657,
]

TEST_CASE_3 = [  # shape: (2, 2, 3), (2, 1, 3)
    {"include_background": False, "to_onehot_y": True},
    {
        "pred": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
        "ground": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
        "smooth": 0.0,
    },
    0.0,
]

TEST_CASE_4 = [  # shape: (2, 2, 3), (2, 1, 3)
    {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
    {
        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
        "smooth": 1e-4,
    },
    0.435050,
]

TEST_CASE_5 = [  # shape: (2, 2, 3), (2, 1, 3)
    {"include_background": True, "to_onehot_y": True, "do_softmax": True},
    {
        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
        "smooth": 1e-4,
    },
    0.383713,
]

TEST_CASE_6 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
    {"include_background": True, "do_sigmoid": True},
    {
        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
        "smooth": 1e-6,
    },
    0.307576,
]

TEST_CASE_7 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
    {"include_background": True, "do_sigmoid": True, "squared_pred": True},
    {
        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
        "smooth": 1e-5,
    },
    0.178337,
]

TEST_CASE_8 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
    {"include_background": True, "do_sigmoid": True, "jaccard": True},
    {
        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
        "smooth": 1e-5,
    },
    -0.059094,
]

class TestDiceLoss(unittest.TestCase):
    @parameterized.expand(
        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
    )
    def test_shape(self, input_param, input_data, expected_val):
        result = DiceLoss(**input_param).forward(**input_data)
        self.assertAlmostEqual(result.item(), expected_val, places=5)

if __name__ == "__main__":
    unittest.main()


import boto3
from datetime import datetime
import os
import pandas as pd
from io import StringIO

# Initialize AWS resources
s3 = boto3.client('s3')
s3r = boto3.resource('s3')  # Needed to write to S3
rek = boto3.client('rekognition')

# Today's date for filename
today = datetime.today()
day = f"{today.year}-{today.month}-{today.day}"

def login(key_id: str, secret_key: str, region: str):
    # Set environment variables for boto3
    os.environ['AWS_ACCESS_KEY_ID'] = key_id
    os.environ['AWS_SECRET_ACCESS_KEY'] = secret_key
    os.environ['REGION'] = region

def download(bucket: str, key: str, file: str):
    # Download file from bucket
    # Key and file are the same if file is not in a
    # folder within the bucket
    try:
        s3.download_file(bucket, key, file)
    except Exception as e:
        print(e)
        print(f"Error downloading {file} from {bucket}")
        raise e

def upload(bucket: str, key: str, file: str):
    # Upload file back to S3
    # File and key should be the same,
    # unless you want the file under a different folder
    try:
        s3.upload_file(file, bucket, key)
    except Exception as e:
        print(e)
        print(f"Error uploading {file} to {bucket}")
        raise e

class S3_Rekognition:
    """Run Rekognition on S3 Objects

    """

    def __init__(self):
        self._buckets = s3.list_buckets()
        self.buckets = [bucket['Name'] for bucket in self._buckets['Buckets']]
        self._file = None

    @property
    def file(self):
        return self._file

    @file.setter
    def file(self, filename: str):
        # Set filename for appending to final CSV
        if isinstance(filename, str):
            self._file = filename
        else:
            raise ValueError("File needs to be a String of a filename.")

    @staticmethod
    def bucket_content(bucket: str) -> list:
        # List files within an s3 bucket
        content = s3.list_objects(Bucket=bucket)['Contents']
        return [key['Key'] for key in content]

    @staticmethod
    def get_labels(bucket: str, file: str):
        # Generate labels for S3 image
        rek_response = rek.detect_labels(Image={"S3Object":{"Bucket": bucket,
                                                            "Name": file}})
        rek_labels = rek_response['Labels']
        rek_data = pd.DataFrame(rek_labels)
        rek_data['Image'] = file
        rek_data['Date'] = day
        return rek_data

    def to_s3(self, bucket: str, dataframe):
        # Write datafram back to S3 bucket
        # Name will be:
        # year-month-day-image.csv
        csv_buffer = StringIO()
        dataframe.to_csv(csv_buffer)
        filename = f"{day}-{self._file}.csv"
        s3r.Object(bucket, filename).put(Body=csv_buffer.getvalue())