"""A simple example of how to access the Google Analytics API."""

import argparse

from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials

import httplib2
from oauth2client import client
from oauth2client import file
from oauth2client import tools
import os

from . import googleaccount

def get_service(api_name, api_version, scope, key_file_location,
                service_account_email):
  """Get a service that communicates to a Google API.

  Args:
    api_name: The name of the api to connect to.
    api_version: The api version to connect to.
    scope: A list auth scopes to authorize for the application.
    key_file_location: The path to a valid service account p12 key file.
    service_account_email: The service account email address.

  Returns:
    A service that is connected to the specified API.
  """

  credentials = ServiceAccountCredentials.from_p12_keyfile(
    service_account_email, key_file_location, scopes=scope)

  http = credentials.authorize(httplib2.Http())

  # Build the service object.
  service = build(api_name, api_version, http=http)

  return service

def get_first_profile_id(service):
  # Use the Analytics service object to get the first profile id.

  # Get a list of all Google Analytics accounts for this user
  accounts = service.management().accounts().list().execute()

  if accounts.get('items'):
    # Get the first Google Analytics account.
    account = accounts.get('items')[0].get('id')

    # Get a list of all the properties for the first account.
    properties = service.management().webproperties().list(
        accountId=account).execute()

    if properties.get('items'):
      # Get the first property id.
      property = properties.get('items')[0].get('id')

      # Get a list of all views (profiles) for the first property.
      profiles = service.management().profiles().list(
          accountId=account,
          webPropertyId=property).execute()

      if profiles.get('items'):
        # return the first view (profile) id.
        return profiles.get('items')[0].get('id')

  return None

#This is a sample from the original with some modifications which has been left here as an example
def get_results(service, profile_id):
  # Use the Analytics Service Object to query the Core Reporting API
  # for the number of sessions within the past seven days.
  return service.data().ga().get(
    ids='ga:' + profile_id,
    start_date='30daysAgo',
    end_date='today',
    metrics='ga:pageviews',
    dimensions='ga:PagePathLevel1,ga:PagePathLevel2'
  ).execute()

#Accepts a function to call against the api after setting up the required variables. This is essentially just a module to run any function with the analytics api.
def run(request_func):
  # Define the auth scopes to request.
  scope = ['https://www.googleapis.com/auth/analytics.readonly']

  # Use the developer console and replace the values with your
  # service account email and relative location of your key file.
  # service account in git ignored file
  service_account_email = googleaccount.ANALYTICS_ACCOUNT
  file_path=os.path.dirname(os.path.abspath(__file__))
  key_file_location = file_path+'/googlekey.p12'

  # Authenticate and construct service.
  service = get_service('analytics', 'v3', scope, key_file_location,
    service_account_email)
  profile = get_first_profile_id(service)
  return request_func(service, profile)


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 25 01:30:31 2018

@author: jaide
"""
import tkinter as tk
import pickle
from tkinter.filedialog import askopenfilename
import sqlite3
import pandas as pd
import numpy as np
import json
import os
import matplotlib.pyplot as plt
import datetime
from flask import Flask,request,render_template,send_file,make_response,Response
from io import StringIO
app = Flask(__name__)

@app.route('/Test', methods=['GET', 'POST'])
def function():
    if request.method == 'POST':
        abspath = os.path.abspath(__file__)
        dname = os.path.dirname(abspath)
        print(dname)
        os.chdir(dname)

        root = tk.Tk()

        root.update()

        name = askopenfilename(title='Select data file',filetypes=[("CSV Files","*.csv")])
        root.withdraw()
        print(name)

    #filename = 'finalized_model.sav'
        loaded_model = pickle.load(open('finalized_model_clustering.sav', 'rb'))
        loaded_model1 = pickle.load(open('rf.sav', 'rb'))

#y_test=pd.read_csv('C:\\Users\\jaide\\pred.csv',header=None)
#X_test=pd.read_csv('C:\\Users\\jaide\\new_file.csv')
#X_test=X_test.drop('Unnamed: 0',axis=1)

#print(loaded_model.score(X_test,y_test))
        X_test=pd.read_csv(name)
        predictions=pd.DataFrame(columns=['Name','Predicted_position'])
        predictions['Name']=X_test.player_name
        predictions['Current_rating']=X_test.overall_rating
        cols=['gk_diving', 'finishing','standing_tackle','sliding_tackle','interceptions','marking','volleys','weight','strength','jumping']
        predictions['Predicted_position']=loaded_model.predict(X_test.filter(items=cols))
        predictions['Predicted_potential']=loaded_model1.predict(X_test.iloc[:,3:44])
        #pos_dict = { 3: 'Forward',  2: 'Midfielder',1: 'Defender', 0: 'Goalkeeper'}
        pos_dict = { 3: 'Midfielder',  2: 'Goalkeeper',1: 'Defender', 0: 'Forward',4: 'Midfielder'}
        predictions.loc[:, 'Predicted_position'] = predictions.loc[:, 'Predicted_position'].map(pos_dict)

        pot_dict = { 3: 'Reserve (70-80)',  2: 'Professional (80-90)',1: 'World Class (90+)' ,4: 'Amateur (60-70)',5: 'Novice (50-60)'}
        predictions.loc[:, 'Predicted_potential'] = predictions.loc[:, 'Predicted_potential'].map(pot_dict)
        X_test['Predicted_position']=predictions['Predicted_position']
        print(predictions)

        jsonfiles = json.loads(predictions.to_json(orient='records'))

        return render_template('index.html', ctrsuccess=jsonfiles)

        #JSONP_data
        #return Response(predictions.to_csv())

    return '''<form method="POST">

                    <title>Player Analytics</title>

Player Analytics

Select a CSV file containing Player Attributes for Analysis

                   <button class="button" color="blue">Select Input Data File</button>

              </form>'''

if __name__ == "__main__":
    app.run()
#<input type="submit"  value="Select data file">


#!/bin/python

import torch
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import Dataset, random_split
import matplotlib.pyplot as plt
import numpy as np

def LoadCifar10DatasetTrain(save, transform=None):
    trainset = torchvision.datasets.CIFAR10(root=save, train=True,
                                        download=True, transform=transform)
    return trainset

def LoadCifar10DatasetTest(save, transform):
    return torchvision.datasets.CIFAR10(root=save, train=False,
                                       download=False, transform=transform)

def GetCustTransform():
    transform_train = transforms.Compose([
        transforms.RandomRotation(20),
        transforms.RandomCrop(32, (2, 2), pad_if_needed=False, padding_mode='constant'),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    return transform_train

def Dataloader_train_valid(save, batch_size):

    # See utils/dataloader.py for data augmentations
    transform_train_valid = GetCustTransform()

    # Get Cifar 10 Datasets
    trainset = LoadCifar10DatasetTrain(save, transform_train_valid)
    train_val_abs = int(len(trainset) * 0.8)
    train_subset, val_subset = random_split(trainset, [train_val_abs, len(trainset) - train_val_abs])

    # Get Cifar 10 Dataloaders
    trainloader = torch.utils.data.DataLoader(train_subset, batch_size=batch_size,
                                              shuffle=True, num_workers=4)

    valloader = torch.utils.data.DataLoader(val_subset, batch_size=batch_size,
                                            shuffle=True, num_workers=4)
    return trainloader, valloader

def Dataloader_train(save, batch_size):

    # See utils/dataloader.py for data augmentations
    transform_train = GetCustTransform()

    # Get Cifar 10 Datasets
    trainset = LoadCifar10DatasetTrain(save, transform_train)
    # Get Cifar 10 Dataloaders
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
                                              shuffle=True, num_workers=4)

    return trainloader

def Dataloader_test(save, batch_size):

    # transformation test set
    transform_test = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    # initialize test dataset and dataloader
    testset = LoadCifar10DatasetTest(save, transform_test)
    testloader = torch.utils.data.DataLoader(testset, batch_size=64,
                                             shuffle=False, num_workers=4)

    return testloader

def imshow(im):
    image = im.cpu().clone().detach().numpy()
    image = image.transpose(1, 2, 0)
    image = image * np.array((0.5, 0.5, 0.5)) + np.array((0.5, 0.5, 0.5)) # unnormalize
    plt.imshow(image)
    plt.show()

def imretrun(im):
    image = im.cpu().clone().detach().numpy()
    image = image.transpose(1, 2, 0)
    image = image * np.array((0.5, 0.5, 0.5)) + np.array((0.5, 0.5, 0.5)) # unnormalize
    return image


import cPickle
import os.path
import hashlib
import uuid

# TODO: load/save should not include the player_dir
# TODO: Create Saveable class, for player, pass, and map?  include version?

class Password(object):
    def __init__(self, name, player_dir):
        self.name = name
        self.hash = ""
        self.salt = ""
        self.player_dir = player_dir

    def do_hash(self, password, salt):
        return hashlib.sha512(password + salt).hexdigest()

    def load(self):
        with open(self.player_dir + self.name + '.pwd', 'rb') as in_file:
            self.name, self.hash, self.salt = cPickle.load(in_file)

    def save(self, password):
        # TODO: Atomic save, by rotating old files?
        self.salt = uuid.uuid4().hex;
        self.hash = self.do_hash(password, self.salt)
        with open(self.player_dir + self.name + '.pwd', 'wb') as out_file:
            cPickle.dump((self.name, self.hash, self.salt), out_file, cPickle.HIGHEST_PROTOCOL)

    def check(self, password):
        return self.hash == self.do_hash(password, self.salt)

class Item(object):
    def __init__(self, name, description):
        self.name = name
        self.description = description

class Equipment(Item):

    # Wear slots
    HEAD = 0
    TORSO = 1
    LEFT_HAND = 2
    RIGHT_HAND = 3
    LEGS = 4
    BACK = 5
    MAX_WEAR = 6

    # Name of wear slots
    wear_names = {
        HEAD: "Head",
        TORSO: "Torso",
        LEFT_HAND: "Left Hand",
        RIGHT_HAND: "Right Hand",
        LEGS: "Legs",
        BACK: "Back"
    }

    def __init__(self, name, description, wear=[], mods=[]):
        Item.__init__(self, name, description)
        self.wear = wear
        self.mods = mods

    def to_string(self):
        mod_strs = []
        for mod in self.mods:
            if mod.value < 0:
                if mod.is_percent:
                    mod_strs.append('{} {}%'.format(Stat.names[mod.stat], int(mod.value*100)))
                else:
                    mod_strs.append('{} {}'.format(Stat.names[mod.stat], mod.value))
            else:
                if mod.is_percent:
                    mod_strs.append('{} +{}%'.format(Stat.names[mod.stat], int(mod.value*100)))
                else:
                    mod_strs.append('{} +{}'.format(Stat.names[mod.stat], mod.value))
        return '{} ({})'.format(self.name, ', '.join(mod_strs))

class Character(object):
    def __init__(self, name, hp, mp, location):
        self.name = name
        self.hp = hp
        self.mp = mp
        self.stats = StatTable(stats={Stat.STRENGTH: 11, Stat.SPEED: 8, Stat.INTELLIGENCE: 5, Stat.MELEE: 4})
        self.location = location
        self.wear = []

    def mod_hp(self, hp):
        self.hp += hp
        if self.hp < 0:
            self.hp = 0

class NPC(Character):

    AGGRO_FRIENDLY = 0
    AGGRO_NEUTRAL = 1
    AGGRO_HOSTILE = 2

    def __init__(self, name, hp, mp, location, aggro, game_server):
        Character.__init__(self, name, hp, mp, location)
        self.aggro = aggro
        self.engaged = None  # Character this NPC is fighting
        self.game_server = game_server

    def act(self):
        # 1) If engaged, fight that character
        if self.engaged is not None:
            self.game_server.add_action(AttackAction(actor=self, target_character=self.engaged))

        # 2) If not engaged, find someone to fight if hostile
        #elif self.AGGRO_HOSTILE:

    def engage(self, character):
        """
        Engage a character as they enter the room. This could be a shop owner saying hi to a shopper, or a hostile
        monster attacking a PC
        """
        if self.aggro == NPC.AGGRO_HOSTILE:
            self.engaged = character
            self.act()
            return '{} prepares to fight {}'.format(self.name, character.name)

        if self.aggro == NPC.AGGRO_FRIENDLY:
            return '{} waves to {}'.format(self.name, character.name)

        return None

    def disengage(self):
        self.engaged = None

class Player(Character):
    def __init__(self, name, player_dir, net_handler):
        # TODO: This is all test stuff
        Character.__init__(self, name, 10, 10, (0,0,0))
        self.player_dir = player_dir
        self.net_handler = net_handler
        self.inventory = [
            Equipment(
                'Iron Longsword',
                'Sturdy sword made of iron.  Might be useful in a fight',
                wear=[Equipment.RIGHT_HAND, Equipment.LEFT_HAND],
                mods=[StatMod(Stat.MELEE, 2, False)]
            ),
            Equipment(
                'Staff of Healing',
                'Smooth maple staff that emits a dim yellow aura',
                wear=[Equipment.RIGHT_HAND, Equipment.LEFT_HAND],
                mods=[StatMod(Stat.INTELLIGENCE, 2, False), StatMod(Stat.CAST_HEAL, .25, True)]
            ),
            Equipment(
                'Berserker Helmet',
                'Helmet that enrages the wearer',
                wear=[Equipment.HEAD],
                mods=[StatMod(Stat.ARMOR, 3, False), StatMod(Stat.MELEE, 5, False), StatMod(Stat.INTELLIGENCE, -5, False)]
            ),
            Equipment(
                'Heavy Chainmail',
                'Thick coat of chainmail.  Offers excellent protection from physical damage.',
                wear=[Equipment.TORSO],
                mods=[StatMod(Stat.ARMOR, 7, False), StatMod(Stat.SPEED, -.5, True)]
            )
        ]

    def serialize(self):
        return self.name, self.hp, self.mp, self.location

    def deserialize(self, data):
        self.name, self.hp, self.mp, self.location = data

    def save(self):
        # TODO: Atomic pickle, by rotating old pickle files?
        with open(self.player_dir + self.name + '.save', 'wb') as out_file:
            cPickle.dump(self.serialize(), out_file, cPickle.HIGHEST_PROTOCOL)
        self.net_handler.update_prompt()

    def load(self):
        with open(self.player_dir + self.name + '.save', 'rb') as in_file:
            self.deserialize(cPickle.load(in_file))

def player_exists(name, player_dir):
    return os.path.isfile(player_dir + name + '.save')

class Action(object):
    def __init__(self, actor, target=None, target_character=None, kwargs={}):
        self.actor = actor
        self.target = target
        self.target_character = target_character
        self.location = actor.location
        self.kwargs = kwargs
        self.pre_msg_str = None
        self.zero_target_error_msg_str = None

    def pre_msg(self):
        if self.time > 0 and self.pre_msg_str is not None:
            return self.pre_msg_str.format(self.actor.name)
        return None

    def zero_target_error_msg(self):
        if self.zero_target_error_msg_str is not None:
            return self.zero_target_error_msg_str.format(self.target)
        return None

class SayAction(Action):
    def __init__(self, actor, target=None, target_character=None, kwargs={}):
        super(SayAction, self).__init__(actor, target, target_character, kwargs)
        self.time = 0
        self.pre_msg_str = '{} prepares to speak...'

    def execute(self):
        return '{} says \"{}\"'.format(self.actor.name, self.kwargs["msg"])

class AttackAction(Action):
    def __init__(self, actor, target=None, target_character=None, kwargs={}):
        super(AttackAction, self).__init__(actor, target, target_character, kwargs)
        self.time = 1.0  # TODO: Make the time actually dependent on the actor (e.g. actor.get_attack_speed())
        # NPCs will hit over and over.  Don't actually say this every single time
        if isinstance(actor, Player):
            self.pre_msg_str = '{} makes ready their weapon...'

    def execute(self):
        damage = 1
        self.target_character.mod_hp(-damage)

        if isinstance(self.target_character, Player):
            self.target_character.save()

        # If an NPC is attacked, set aggro to hostile and engage
        # If an NPC did the attacking, see if they need to attack again
        if isinstance(self.target_character, NPC) and self.target_character.hp > 0:
            self.target_character.aggro = NPC.AGGRO_HOSTILE
            if self.target_character.engaged is None:
                self.target_character.engage(self.actor)
        elif isinstance(self.actor, NPC) and self.target_character.hp > 0:
            self.actor.act()

        msg = '{} strikes {} for {} damage'.format(self.actor.name, self.target_character.name, damage)
        if self.target_character.hp <= 0:
            msg += '...and {} is slain.'.format(self.target_character.name)
        return msg

    def target_gone(self, target):
        if isinstance(self.actor, NPC) and self.actor.engaged == target:
            self.actor.disengage()

class AwarenessAction(Action):
    """
    NPCs use this action to detect new PCs in the room
    """
    def __init__(self, actor, target=None, target_character=None, kwargs={}):
        super(AwarenessAction, self).__init__(actor, target, target_character, kwargs)
        self.time = 1

    def execute(self):
        # TODO: possibly some check to see if the NPC can detect the PC
        if self.actor.engaged is None:
            return self.actor.engage(self.target_character)

class Stat(object):

    # TODO: Are these class variables??

    # Base stats
    STRENGTH = 0
    SPEED = 1
    INTELLIGENCE = 2

    # Secondary stats
    MELEE = 10
    ARMOR = 11
    RANGED = 20
    EVASION = 21

    WILLPOWER = 30
    CAST_FLAME = 31
    CAST_HEAL = 32
    CAST_CHILL = 33
    CAST_FLAMING_SWORD = 34

    # Name of the stat
    names = {
        STRENGTH: "Strength",
        SPEED: "Speed",
        INTELLIGENCE: "Intelligence",
        MELEE: "Melee Attack",
        ARMOR: "Armor",
        RANGED: "Ranged Attack",
        EVASION: "Evasion",
        WILLPOWER: "Willpower",
        CAST_FLAME: "Cast Flame",
        CAST_HEAL: "Cast Heal",
        CAST_CHILL: "Cast Chill",
        CAST_FLAMING_SWORD: "Cast Flaming Sword"
    }

    # Description of the stat
    # TODO: Descriptions like this should be in some sort of documentation resource file
    descriptions = {
        STRENGTH: "Physical strength and stamina.  Used to determine effectiveness of melee attacks and amount of HP.",
        SPEED: "Speed and dexterity.  Used to determine effectiveness of ranged attacks and evasion.",
        INTELLIGENCE: "Intellect and willpower.  Used to determine effectiveness of magical attacks and magical defense.",
        MELEE: "Strike an opponent with a weapon at close range.  Usage: equip a melee weapon and type 'attack <target>'",
        ARMOR: "Physical defense due to armor, physical constitution, or both.  Damage from melee and ranged attacks is reduced",
        RANGED: "Fire a ranged weapon at an opponent.  Usage: equip a ranged weapon and type 'attack <target>'",
        EVASION: "Ability to dodge melee and ranged attacks",
        WILLPOWER: "Damaged is reduced from magic attacks",
        CAST_FLAME: "Launch a fireball about as big as a fist towards an opponent.  Usage: 'cast flame <target>'",
        CAST_HEAL: "TBD",
        CAST_CHILL: "TBD",
        CAST_FLAMING_SWORD: "TBD"
    }

    def __init__(self, value):
        self._value = value
        self._mod_value = value

        self._mod_number = 0
        self._mod_percent = 1.0
        self._modifiers = []

    @property
    def base(self):
        return int(self._value)

    @property
    def modified(self):
        mod_val = int(round((self._value + self._mod_number) * self._mod_percent))
        if mod_val < 0:
            return 0
        return mod_val

    def add_modifier(self, value, is_percent):
        self._modifiers.append([value, is_percent])
        if is_percent:
            self._mod_percent += value
        else:
            self._mod_number += value

    def remove_modifier(self, value, is_percent):
        if [value, is_percent] in self._modifiers:
            self._modifiers.remove([value, is_percent])
            if is_percent:
                self._mod_percent -= value
            else:
                self._mod_number -= value

class StatTable(object):

    # Base modifiers (which base stats enhance a secondary stat)
    base_mod = {
        Stat.MELEE:                Stat.STRENGTH,
        Stat.ARMOR:                Stat.STRENGTH,
        Stat.RANGED:               Stat.SPEED,
        Stat.EVASION:              Stat.SPEED,
        Stat.WILLPOWER:            Stat.INTELLIGENCE,
        Stat.CAST_FLAME:           Stat.INTELLIGENCE,
        Stat.CAST_HEAL:            Stat.INTELLIGENCE,
        Stat.CAST_CHILL:           Stat.INTELLIGENCE,
        Stat.CAST_FLAMING_SWORD:   Stat.INTELLIGENCE
    }

    # Dependencies (which stats do you need to be able to access new stats?)
    deps = {
        Stat.STRENGTH: [],
        Stat.SPEED: [],
        Stat.INTELLIGENCE: [],
        Stat.MELEE: [],
        Stat.ARMOR: [],
        Stat.RANGED: [],
        Stat.EVASION: [],
        Stat.WILLPOWER: [],
        Stat.CAST_FLAME: [],
        Stat.CAST_HEAL: [
            {'stat': Stat.CAST_FLAME, 'level': 5}
        ],
        Stat.CAST_CHILL: [
            {'stat': Stat.CAST_HEAL, 'level': 5}
        ],
        Stat.CAST_FLAMING_SWORD: [
            {'stat': Stat.CAST_FLAME, 'level': 5},
            {'stat': Stat.MELEE, 'level': 10}
        ]
    }

    # Initialize with {stat_enum: int, stat_enum: int}
    def __init__(self, stats={}):
        self._stats = {}
        for stat in StatTable.deps:
            if stat in stats:
                self._stats[stat] = Stat(stats[stat])
            else:
                self._stats[stat] = Stat(0)

    def get_combined_modified(self, stat):
        modified = self._stats[stat].modified
        if modified <= 0:
            return 0  # Base mods should not buff stats that are 0 to begin with
        if stat in StatTable.base_mod:
            return self._stats[stat].modified + self._stats[StatTable.base_mod[stat]].modified
        else:
            return self._stats[stat].modified

    # Get a list of all stats where deps have been met
    def get_available_stats(self):
        ret = []
        for stat in StatTable.deps:
            should_add = True
            for dep in StatTable.deps[stat]:
                if self._stats[dep['stat']].base < dep['level']:
                    should_add = False
                    break
            if should_add:
                ret.append(stat)
        return ret

    def to_string(self):
        display_stats = []
        for s in self._stats:
            combined_modified = self.get_combined_modified(s)
            if combined_modified > 0:
                display_stats.append('{}:\t{} ({})'.format(Stat.names[s], combined_modified, self._stats[s].base))
        return '\n'.join(display_stats)

class StatMod(object):
    def __init__(self, stat, value, is_percent, time=0):
        self.stat = stat
        self.value = value
        self.is_percent = is_percent
        self.time = time


import pandas as pd
import numpy as np
import os
import sys
import os.path
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.utils import np_utils, plot_model
#from keras import backend as K
# from keras.regularizers import L1L2, l1
# activity_l1 = L1L2(l1=1)
from keras import regularizers
from keras.callbacks import ModelCheckpoint

from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.cross_validation import train_test_split
import itertools

import matplotlib.pyplot as plt
import matplotlib.cm as cm

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, int(cm[i, j]*100)/100.0,
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

#convert flatten data to 48*48 matrix
def reshapeTo60and160(dataset):
    #extract pixels value from original pandas dataframe
    pixels_values = dataset.pixels.str.split(" ").tolist()
    #convert pixels of each image to 48*48 formats
    images = []
    for image in np.array(pixels_values, dtype=float):
        images.append(image.reshape(60, 160))
    return np.array(images, dtype=float)

def sample_model():
    #initial model
    model = Sequential()
    #add dropout to reduce overfitting
    #model.add(Dropout(0.2, input_shape=(60, 160, 1)))
    #with 64 filters, 5*5 for convolutional kernel and activation 'relu'
    model.add(Conv2D(64, (5, 5), input_shape=(60, 160, 1), activation='relu'))
    #pooling layer
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.1))
    model.add(Conv2D(128, (5, 5), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))
    model.add(Flatten())
    #fully connected layer
    model.add(Dense(600, activation='relu'))
    model.add(Dropout(0.1))
    model.add(Dense(200, activation='relu'))
    model.add(Dense(num_classes, kernel_regularizer=regularizers.l2(0.01),
                    activity_regularizer=regularizers.l1(0.01),
                    activation='softmax'))
    return model
#concatenate data
path = os.getcwd()
data = []

for i in range(12):
    pData = []
    for j in range(10):
        if os.path.isfile(path+"/data/generalData/part%d/%ddata.csv" %(i+1,j)):
            tmp = pd.read_csv(path+"/data/generalData/part%d/%ddata.csv" %(i+1,j))
            if len(tmp) > 100:
                pData.append(tmp[:100])
            else:
                pData.append(tmp)
        else:
            pData.append(None)
    data.append(pData)

# for i in range(len(data)):
#     ratio = float(size)/len(data[i])
#     if ratio != 1:
#         exData, sData = train_test_split(data[i], test_size = ratio)
#         train, tmp_test = train_test_split(sData, test_size = 0.2)
#         #train, tmp_test = train_test_split(data[i], test_size = 0.2)
#     else:
#         train, tmp_test = train_test_split(data[i], test_size = 0.2)
#     #split each expression into k-fold data
#     trainSplit = []
#     k = 10
#     for j in range(k):
#         if j == k-1:
#             trainSplit.append(train)
#             break
#         rTrain, sTrain = train_test_split(train, test_size = 1./(k-j))
#         trainSplit.append(sTrain)
#         train = rTrain
#     trainList.append(trainSplit)
#     frames = [test, tmp_test]
#     test = pd.concat(frames)
# test_images = reshapeTo60and160(test)

# #reshape to [# of samples][width][height][pixels] for tensorflow-keras input format
# #train_images = train_images.reshape(train_images.shape[0], 60, 160, 1).astype('float32')
# test_images = test_images.reshape(test_images.shape[0], 60, 160, 1).astype('float32')

# #check input format
# # train_images.shape

# #normilize the data
# #train_images = train_images/255
# test_images = test_images/255.

# #One hot encode outputs: change target(emotion) values to input format with one-hot encode
# #train_targets = np_utils.to_categorical(train.expression.values)
# test_targets = np_utils.to_categorical(test.expression.values)

# set number of prediction classes
# num_classes = test_targets.shape[1]
#10-fold cross validation
cvscores = []
test = None
train = None
model = None
TrainSize = 0
TestSize = 0
for i in range(1):
    # get the size for each expression
    tmpTrainSize, tmpTestSize = 0, 0
    for exp in range(10):
        tmpTrainSize = 0
        tmpTestSize = 0
        for j in range(12):
            if i==j and data[j][exp] is not None:
                tmpTestSize = len(data[j][exp])
            elif data[j][exp] is not None :
                tmpTrainSize += len(data[j][exp])
        if (TrainSize,TestSize) == (0,0):
            TrainSize = tmpTrainSize
            TestSize = tmpTestSize
        if TrainSize > tmpTrainSize and tmpTrainSize != 0:
            TrainSize = tmpTrainSize
        if TestSize > tmpTestSize and tmpTestSize != 0:
            TestSize = tmpTestSize
    #     print (TestSize, tmpTestSize)
    # sys.exit(-1)
    for exp in range(10):
        expTrain = None
        for j in range(12):
            if i == j and data[j][exp] is not None:
                # concatenate testData
                ratio = float(TestSize)/len(data[j][exp])
                if ratio < 1:
                    discard, sTest = train_test_split(data[j][exp], test_size = ratio)
                    frames = [test,sTest]
                else:
                    frames = [test,data[j][exp]]
                test = pd.concat(frames)
            elif data[j][exp] is not None:
                # concatenate expTrain
                frames = [expTrain, data[j][exp]]
                expTrain = pd.concat(frames)
        #concatenate train
        ratio = float(TrainSize)/len(expTrain)
        if ratio < 1:
            discard, sTrain  = train_test_split(expTrain, test_size = ratio)
            frames = [train,sTrain]
        else:
            frames = [train,expTrain]
        train = pd.concat(frames)

    # #concatenate 9 train and 1 validation
    # train = None
    # val = None
    # for exp in range(11):
    #     strain = None
    #     sval = None
    #     for j in range(k):
    #         if j == i:
    #             sval = trainList[exp][j]
    #             continue
    #         frames = [strain, trainList[exp][j]]
    #         strain = pd.concat(frames)
    #     frames = [train, strain]
    #     train = pd.concat(frames)
    #     frames = [val, sval]
    #     val = pd.concat(frames)

    test_images = reshapeTo60and160(test)
    train_images = reshapeTo60and160(train)
    #reshape to [# of samples][width][height][pixels] for tensorflow-keras input format
    test_images = test_images.reshape(test_images.shape[0], 60, 160, 1).astype('float32')
    train_images = train_images.reshape(train_images.shape[0], 60, 160, 1).astype('float32')
    #normilize the data
    test_images = test_images/255
    train_images = train_images/255
    #One hot encode outputs: change target(emotion) values to input format with one-hot encode
    test_targets = np_utils.to_categorical(test.expression.values)
    train_targets = np_utils.to_categorical(train.expression.values)
    #set number of prediction classes
    num_classes = test_targets.shape[1]

    model = sample_model()
    # model.summary()
    # sys.exit(-1)
    filename = path+"/data/generalData/general_Model.hdf5"
    check_point = ModelCheckpoint(filename, monitor='val_acc', verbose=2, save_best_only=True, mode='max')
    callbacks_list = [check_point]
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    history = model.fit(train_images, train_targets, validation_data=(test_images, test_targets),
          epochs=10, batch_size=80, callbacks=callbacks_list, verbose=2)
    scores = model.evaluate(test_images, test_targets, verbose=0)
    print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))
    cvscores.append(scores[1] * 100)
    #plot CM
    #print(test.expression.values)
    Saved_prediction = model.predict_classes(test_images, verbose=0)
    True_prediction = test.expression.values
    cm = confusion_matrix(True_prediction, Saved_prediction)

    class_names = ['blink', 'blink_left', 'blink_right', 'squint', 'frown', 'raise_eyebrow', 'enlarge',  'smile', 'smile_left', 'others']
    plot_confusion_matrix(cm, classes=class_names, normalize=True,
                          title='Confusion Matrix for Test Dataset')
    plt.show()

print("%.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores)))

# #plot CM
# print(test.expression.values)
# Saved_prediction = model.predict_classes(test_images, verbose=0)
# True_prediction = test.expression.values
# cm = confusion_matrix(True_prediction, Saved_prediction)

# class_names = ['blink_both', 'blink_left', 'blink_right', 'squint', 'squint_left', 'squint_right', 'frown', 'raise_eyebrow', 'enlarge',  'smile_both', 'others']
# plot_confusion_matrix(cm, classes=class_names, normalize=True,
#                       title='Confusion Matrix for Test Dataset')
# plt.show()


import math
import time
from decimal import Decimal
import random
import signal
import chess
import h5py
import numpy as np
import chess.syzygy
import sys
import traceback

def db(t,v):
    if debug == True:
        print(t,v)

def Db(t,v):
    if Debug == True:
        print(t,v)

def restart_line():
    sys.stdout.write('\r')
    sys.stdout.flush()

def printProgress(j, l, saving):
    prog = round(100 * j / l, 7)
    t2 = time.time()
    timeElapsed = t2 - t1
    timeLeft = timeElapsed * (l - j) / (j + 1)
    if timeElapsed > 3600*24:
        timeElapsed = str("%.3f" %round(timeElapsed/3600./24, 3) + " days.")
    elif timeElapsed > 3600:
        timeElapsed = str("%.3f" %round(timeElapsed/3600., 3) + " hours.")
    else:
        timeElapsed = str("%.3f" %round(timeElapsed, 3) + " seconds.")
    if timeLeft > 3600*24:
        timeLeftFormated = str("%.3f" % float(round(timeLeft/3600/24,3))) + ' days.'
    elif timeLeft > 3600:
        timeLeftFormated = str("%.3f" % float(round(timeLeft/3600,3))) + ' hours.'
    else:
        timeLeftFormated = str("%.3f" % float(round(timeLeft,3))) + ' seconds.'
        # timeLeftFormated = str("%.3f" % round(prog,3))
    restart_line()
    if saving:
        sys.stdout.write("Progress: " + str("%.3f" % round(prog,3)) + '%. Runtime: ' + timeElapsed + " Time left: " + timeLeftFormated + " At state: %.3e" %Decimal(j) + "/ %.3e" %Decimal(l) + ". ==>> Saving to disk")
    else:
        sys.stdout.write("                                                                                ")
        restart_line()
        sys.stdout.write("Progress: " + str("%.3f" % round(prog,3)) + '%. Runtime: ' + timeElapsed + " Time left: " + timeLeftFormated + " At state: %.3e" %Decimal(j) + "/ %.3e" %Decimal(l))
    sys.stdout.flush()

def run_program():
    ####################
    #
    #   Signal Handler
    #
    ####################
    def exit_gracefully(signum, frame):
        # restore the original signal handler as otherwise evil things will happen
        # in raw_input when CTRL+C is pressed, and our signal handler is not re-entrant
        signal.signal(signal.SIGINT, original_sigint)

        try:
            if confirmQuit:
                if input("\nReally quit? (y/n)> ").lower().startswith('y'):
                    print("Flushing data to disk")
                    f.close()
                    sys.exit(1)
            else:
                print("\nFlushing data to disk")
                f.close()
                sys.exit(1)

        except KeyboardInterrupt:
            print("\nOk ok, quitting")
            print("Flushing data to disk")
            f.close()
            sys.exit(1)

        # restore the exit gracefully handler here
        signal.signal(signal.SIGINT, exit_gracefully)

    # store the original SIGINT handler
    original_sigint = signal.getsignal(signal.SIGINT)
    signal.signal(signal.SIGINT, exit_gracefully)

    ##############################
    #
    #      Load datasets
    #
    ##############################

    f = h5py.File(fileName, 'a')
    states = f[dataSetNameSource]
    wdl = f[dataSetNameWdlSource]
    wdl = wdl[:]

    try:

        ##############################
        #
        #   Create Legal dataset
        #
        ##############################

        def createLegalDataset():
            dsName = dataSetNameOnlyLegal
            def legalCreator():
                # return f.create_dataset(dataSetWdlName, (len(states),1), dtype='b', chunks=True, maxshape=(None), data = np.full((len(states),1), 10, dtype = int))#, compression="gzip", compression_opts=9)
                return f.create_dataset(dsName, states.shape, dtype = 'b', chunks=True, maxshape=states.shape)
                #, compression="gzip", compression_opts=9)
            if dsName in f:
               if confirmDSOverwrite:
                    action = input("Dataset " + str(dsName) + " already exists. Overwrite? [y/n]")
                    if action == 'y' or action == 'Y':
                        del f[dsName]
                        Db("Overwriting dataset: ", dsName)
                        ds = legalCreator()
                    else:
                        Db("Aborting.",'')
                        sys.exit()
               else:
                    del f[dsName]
                    Db("Overwriting dataset: ", dsName)
                    ds = legalCreator()
            else:
                Db("Creating dataset: " + dsName, '')
                ds = legalCreator()
                Db("...done",'')
            return ds
        x = 1
        statesLegal = createLegalDataset()
        # wdl = f[dataSetWdlName]

        ##############################
        #
        #   Create WDL dataset
        #
        ##############################

        def createWdlDataset():
            dsName = dataSetNameWdlOnlyLegal
            def wdlCreator():
                # return f.create_dataset(dataSetWdlName, (len(states),1), dtype='b', chunks=True, maxshape=(None), data = np.full((len(states),1), 10, dtype = int))#, compression="gzip", compression_opts=9)
                return f.create_dataset(dsName, (len(states), 1), dtype='b', chunks=True, maxshape=(len(states),1), data = np.full((len(states), 1), 10))
                #, compression="gzip", compression_opts=9)
            if dsName in f:
                if confirmDSOverwrite:
                    action = input("Dataset " + str(dsName) + " already exists. Overwrite? [y/n]")
                    if action == 'y' or action == 'Y':
                        del f[dsName]
                        Db("Overwriting dataset: ", dsName)
                        ds = wdlCreator()
                    else:
                        Db("Aborting.",'')
                        sys.exit()
                else:
                    del f[dsName]
                    Db("Overwriting dataset: ", dsName)
                    ds = wdlCreator()
            else:
                Db("Creating dataset: " + dsName, '')
                ds = wdlCreator()
                Db("...done",'')
            return ds
        x = 1
        wdlLegal = createWdlDataset()

        ##############################
        #
        #   Saving only legal states
        #
        ##############################
        # iterate through all
        #     add stuff to legalStatesTemp if legalStatesTemp empty
        #     if not error then add both to temp
        #     write temp to file if temp full
        # write temp to file

        l = len(states)
        percentageUpdateInterval = l // progressDivisions
        legalTemp = []
        legalLocation = 0
        wdlTemp = []
        wdlLocation = 0
        statesBuffer = states[:readStatesBufferSize]
        iStatesBuffer = 0
        nrOfLegalSates = 0

        # ------------------------------ ADD TO TEMP IF BOARD LEGAL  ------------------------------
        def flushDisk(wdlTemp, legalTemp, wdlLocation, legalLocation):
            if saveToDisk:
                # print("\nsaving to disk\n")
                printProgress(j, l, True)
                wdlLegal[wdlLocation:wdlLocation + len(wdlTemp)] = np.array(wdlTemp, dtype = np.int8)
                wdlLegal.flush()
                wdlLocation += len(wdlTemp)
                wdlTemp = []

                statesLegal[legalLocation:legalLocation + len(legalTemp)] = np.array(legalTemp, dtype = np.int8)
                statesLegal.flush()
                legalLocation += len(legalTemp)
                legalTemp = []
            return wdlTemp, legalTemp, wdlLocation, legalLocation

        for j in range(len(states)):
            if j%percentageUpdateInterval == 0:
                printProgress(j, l, False)

            if iStatesBuffer == readStatesBufferSize:
                statesBuffer = states[j:j + readStatesBufferSize]
                iStatesBuffer = 0

            if len(wdlTemp) >= writeBufferSize:
                wdlTemp, legalTemp, wdlLocation, legalLocation = flushDisk(wdlTemp, legalTemp, wdlLocation, legalLocation)

            if wdl[j][0] < 3:
                nrOfLegalSates += 1
                legalTemp.append(statesBuffer[iStatesBuffer])
                wdlTemp.append(wdl[j])
            iStatesBuffer += 1

        # print('wdllocation', wdlLocation)
        # print('legalLocation', legalLocation)

        flushDisk(wdlTemp, legalTemp, wdlLocation, legalLocation)
        nrOfIllegalStates = l - nrOfLegalSates

        print()
        Db('Done checking all states','')
        Db('Resising arrays','')
        oldShape = wdlLegal.shape
        newShape = (oldShape[0] - nrOfIllegalStates, oldShape[1])
        wdlLegal.resize(newShape)
        oldShape = statesLegal.shape
        newShape = (oldShape[0] - nrOfIllegalStates, oldShape[1])
        statesLegal.resize(newShape)

        # Db("orginal shape:", oldShape)
        # Db("new     shape:", newShape)
        Db('Number of Legal States',nrOfLegalSates)
        Db('Number of Illegal States', nrOfIllegalStates)

    except Exception as e:
        f.close()
        print("Exception!!!!!!!!!!!!!!!")
        print(e)
        print(traceback.format_exc())

if __name__ == '__main__':

    #######################
    #
    #       Description
    #
    #######################

    # Wdl values:
    #   Syzygy values: -2..2
    #   Initial value: 10
    #   Invalid boardstate value: 11

    #######################
    #
    #       Settings
    #
    #######################

    baseName = '5PPpKk'
    fileName = baseName + '.hdf5'
    dataSetNameSource = baseName #source dataset
    dataSetNameOnlyLegal = baseName + '_onlyLegal'
    dataSetNameWdlSource = baseName + '_Wdl'
    dataSetNameWdlOnlyLegal = baseName + '_Wdl_onlyLegal'

    nPi =  int(baseName[0])
    nPa = nPi - 2
    nWPa = math.ceil(nPa/2)

    # fileName = '4PpKk.hdf5'
    # dataSetNameSource = '4PpKk' #source dataset
    # dataSetNameOnlyLegal = '4PpKk_onlyLegal'
    # dataSetNameWdlSource = '4PpKk_Wdl'
    # dataSetNameWdlOnlyLegal = '4PpKk_Wdl_onlyLegal'
    # nPi = 3
    # nPa = 1
    # nWPa = 1
    confirmQuit = True
    confirmDSOverwrite = False
    saveToDisk = True
    progressDivisions = 100
    debug = False
    Debug = True
    writeBufferSize = 10000 #buffer for writing wdl and states
    readStatesBufferSize = 10000 #buffer for reading states

    t0 = time.time()
    t1 = time.time()
    run_program()


from django import forms
from orderapp.models import Order, OrderItem
from decimal import Decimal
from django.core.validators import MaxValueValidator

from bookshopapp.models import Book

class OrderForm(forms.ModelForm):

    class Meta:
        model = Order
        exclude = (
            'user',
        )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs.update({'class': 'form-control'})

class OrderItemForm(forms.ModelForm):

    book_shower = forms.CharField(label='книга', required=False)
    price = forms.CharField(label='цена', required=False)
    cost = forms.CharField(label='стоимость', required=False)

    field_order = [
        'book_shower',
        'quantity',
        'price',
    ]

    class Meta:
        model = OrderItem
        exclude = ()

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['book'].queryset = Book.get_items()
        for field in self.fields.values():
            field.widget.attrs.update({'class': 'form-control'})
            if field.label == 'Книга':
                field.widget = forms.widgets.HiddenInput()
            if field.label == 'Количество':
                try:
                    field.widget.attrs.update({'max': self.instance.book.quantity + self.instance.quantity, 'class': 'quantity form-control order'})
                    # field.widget.attrs.update({'class': 'quantity form-control'})
                except OrderItem.book.RelatedObjectDoesNotExist:
                    pass


import functools
from collections import OrderedDict

from django.db import models
from django.db.models import Count, BooleanField
from django.db.models.base import ModelBase
from django.db.models.expressions import Col, BaseExpression
from django.db.models.sql import Query
from django.db.models.sql.where import WhereNode, ExtraWhere
from django.utils.datastructures import OrderedSet

class SphinxTableName(str):
    is_table_name = True

# noinspection PyAbstractClass
class SphinxCount(Count):
    """ Replaces Mysql-like COUNT('*') with COUNT(*) token."""
    template = '%(function)s(*)'

    def as_sql(self, compiler, connection, function=None, template=None):
        sql, params = super().as_sql(
            compiler, connection, function=function, template=template)
        try:
            params.remove('*')
        except ValueError:
            pass
        return sql, params

class SphinxWhereExpression(BaseExpression):
    def __init__(self, where, where_params):
        self.where = where
        self.where_params = where_params
        super().__init__(output_field=BooleanField())

    def as_sql(self, compiler, connection):
        return self.where, self.where_params

class SphinxExtraWhere(ExtraWhere):

    def as_sql(self, qn=None, connection=None):
        sqls = ["%s" % sql for sql in self.sqls]
        return " AND ".join(sqls), tuple(self.params or ())

class SphinxWhereNode(WhereNode):
    pass

class SphinxQuery(Query):
    _clonable = ('options', 'match', 'group_limit', 'group_order_by',
                 'with_meta')

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('where', SphinxWhereNode)
        super().__init__(*args, **kwargs)

    def clone(self):
        query = super().clone()
        for attr_name in self._clonable:
            value = getattr(self, attr_name, None)
            if value:
                setattr(query, attr_name, value)
        return query

    def add_match(self, *args, **kwargs):
        if not hasattr(self, 'match'):
            # noinspection PyAttributeOutsideInit
            self.match = OrderedDict()
        for expression in args:
            self.match.setdefault('*', OrderedSet())
            if isinstance(expression, (list, tuple)):
                self.match['*'].update(expression)
            else:
                self.match['*'].add(expression)
        for field, expression in kwargs.items():
            self.match.setdefault(field, OrderedSet())
            if isinstance(expression, (list, tuple, set)):
                self.match[field].update(expression)
            else:
                self.match[field].add(expression)

    def get_count(self, using):
        """
        Performs a COUNT() query using the current filter constraints.
        """
        obj = self.clone()
        obj.add_annotation(SphinxCount('*'), alias='__count', is_summary=True)
        number = obj.get_aggregation(using, ['__count'])['__count']
        if number is None:
            number = 0
        return number

# noinspection PyAbstractClass
class SphinxCol(Col):
    def as_sql(self, compiler, connection):
        # As column names in SphinxQL couldn't be escaped with `backticks`,
        # simply return column name
        return self.target.column, []

class SphinxModelBase(ModelBase):

    # noinspection PyProtectedMember
    def __new__(mcs, name, bases, attrs, **kwargs):
        manual_pk = None
        # Each field must be monkey-patched with SphinxCol class to prevent
        # `tablename`.`attr` appearing in SQL
        for attr in attrs.values():
            if isinstance(attr, models.Field):
                col_patched = getattr(attr, '_col_patched', False)
                if not col_patched:
                    mcs.patch_col_class(attr)
                if attr.primary_key:
                    manual_pk = attr
        # Force non-autoincrement primary key
        if manual_pk is None:
            attrs['id'] = models.IntegerField(primary_key=True,
                                              verbose_name='ID')

        new_class = super().__new__(mcs, name, bases, attrs, **kwargs)

        # mark db_table to be processed with quote_name
        new_class._meta.db_table = SphinxTableName(new_class._meta.db_table)
        return new_class

    def add_to_class(cls, name, value):
        col_patched = getattr(value, '_col_patched', False)
        if not col_patched and isinstance(value, models.Field):
            cls.patch_col_class(value)
        super().add_to_class(name, value)

    @classmethod
    def patch_col_class(mcs, field):
        @functools.wraps(field.get_col)
        def wrapper(alias, output_field=None):
            col = models.Field.get_col(field, alias, output_field=output_field)
            col.__class__ = SphinxCol
            return col
        field.get_col = wrapper


# %% import library
import tushare as ts
import datetime
import pandas as pd
import os
import time
start = time.time()

# %%  获取数据参数
pro = ts.pro_api()
# idx = 4  # 第几个
# market = ['MSCI指数', '中证指数', '上交所指数', '深交所指数', '中金指数', '申万指数',
#           '其他指数'][idx - 1]  # 交易所或服务商
# index_name = '深证成指'  # 具体的指数名字
nom_index = {
    '上证指数': '上交所指数',
    '沪深300': '上交所指数',
    '上证50': '上交所指数',
    '深证综指': '深交所指数',
    '中小板指': '深交所指数',
    '创业板指': '深交所指数',
    '深证成指': '深交所指数'
}
rehabilitation = 'qfq'  # 是否复权
asset = 'I'  # 金融工具种类
freq = 'D'  # 数据周期
factors = ['tor', 'vr']  # 增加换手率和量比

# %% set mdir
mdir = "E:/data/stock/csv"
if not os.path.exists("{}/stock_index".format(mdir)):
    os.makedirs("{}/stock_index".format(mdir))

# %% 设定获取日线行情的初始日期和终止日期，其中终止日期设定为今天。
start_dt = '20100101'
time_temp = datetime.datetime.now()  # - datetime.timedelta(days=1)
end_dt = time_temp.strftime('%Y%m%d')
idx = 0
total = len(nom_index)

def smf(s):
    m, s = divmod(s, 60)
    h, m = divmod(m, 60)
    return "%d:%02d:%02d" % (h, m, s)

# %% run
for k in nom_index.keys():
    idx += 1
    index_name = k
    market = nom_index[k]
    df_index = pd.read_csv("{}/index_list/{}.csv".format(mdir, market))
    index_code = df_index[df_index["name"] == index_name]['ts_code'].to_numpy()[0]
    if not os.path.exists("{}/stock_index/{}".format(mdir, market)):
        os.makedirs("{}/stock_index/{}".format(mdir, market))

    print('Seq: ' + str(idx) + ' of ' + str(total) + '\tCode: ' +
          str(index_name))
    # 处理表格
    try:
        df = ts.pro_bar(ts_code=index_code,
                        start_date=start_dt,
                        end_date=end_dt,
                        adj=rehabilitation,
                        asset=asset,
                        freq=freq,
                        factors=factors,
                        adjfactor=True,
                        ma=[5, 20, 60, 120, 250])
        df = df[::-1]
        df.drop_duplicates(subset=['trade_date'], inplace=True)
        df.to_csv("{}/stock_index/{}/{}.csv".format(mdir, market, index_name), index=False)
    except Exception as err:
        with open('domestic_index_err.log', "a", encoding='utf-8') as f:
            f.write("{}\t({}.{})\n".format(
                err, index_name,
                datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) +
                    "-" * 30 + "\n")
        with open('domestic_index_name.log', "a", encoding='utf-8') as f:
            f.write("'{}', ".format(index_name))
        continue
    # 计时
    end = time.time()
    total_time = end - start
    avg_time = total_time / (idx + 1)
    remain_time = (total - idx - 1) * avg_time
    print("Total: " + smf(total_time) + "\tRemain: " + smf(remain_time))
    print("-" * 32)

print('domestic_index Finished!')

# %%


# -*- coding: utf-8 -*-

import unittest2 as unittest

from StringIO import StringIO

from zope.app.file.tests.test_image import zptlogo
from zope.interface import directlyProvides

from plone.app.customerize import registration

from plone.app.testing import TEST_USER_ID
from plone.app.testing import setRoles

from collective.nitf.interfaces import INITFBrowserLayer
from collective.nitf.testing import INTEGRATION_TESTING

class BrowserLayerTest(unittest.TestCase):

    layer = INTEGRATION_TESTING

    def setUp(self):
        self.portal = self.layer['portal']
        self.request = self.layer['request']
        directlyProvides(self.request, INITFBrowserLayer)

        setRoles(self.portal, TEST_USER_ID, ['Manager'])
        self.portal.invokeFactory('Folder', 'test-folder')
        setRoles(self.portal, TEST_USER_ID, ['Member'])
        self.folder = self.portal['test-folder']

        self.folder.invokeFactory('collective.nitf.content', 'n1')
        self.n1 = self.folder['n1']

    def test_views_registered(self):
        views = ['view', 'scrollable', 'folder_summary_view']
        registered = [v.name for v in registration.getViews(INITFBrowserLayer)]
        # empty set only if all 'views' are 'registered'
        self.assertEquals(set(views) - set(registered), set([]))

    def test_default_view(self):
        pt = getattr(self.portal, 'portal_types')
        self.assertEquals(pt['collective.nitf.content'].default_view, 'view')

    def test_view(self):
        name = '@@view'
        try:
            self.n1.unrestrictedTraverse(name)
        except AttributeError:
            self.fail('%s has no view %s' % (self.n1, name))

        view = self.n1.restrictedTraverse(name)
        self.assertEquals(len(view.get_images()), 0)
        self.assertEquals(len(view.get_files()), 0)
        self.assertEquals(len(view.get_links()), 0)
        self.assertEquals(len(view.get_media()), 0)
        self.assertEquals(view.getImage(), None)

    def test_get_images(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@view')
        images = view.get_images()
        self.assertEquals(len(images), 1)
        self.assertEquals(images[0].getObject().id, 'foo')
        self.assertEquals(images[0].getObject().Title(), 'Foo')
        self.assertEquals(images[0].getObject().Description(), 'FOO')

    def test_has_images(self):
        view = self.n1.restrictedTraverse('@@view')
        self.assertEquals(view.has_images(), 0)
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        self.assertEquals(view.has_images(), 1)

    def test_get_files(self):
        self.n1.invokeFactory('File', 'bar', title='Bar', description='BAR',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@view')
        files = view.get_files()
        self.assertEquals(len(files), 1)
        self.assertEquals(files[0].getObject().id, 'bar')
        self.assertEquals(files[0].getObject().Title(), 'Bar')
        self.assertEquals(files[0].getObject().Description(), 'BAR')

    def test_has_files(self):
        view = self.n1.restrictedTraverse('@@view')
        self.assertEquals(view.has_files(), 0)
        self.n1.invokeFactory('File', 'bar', title='Bar', description='BAR',
                              image=StringIO(zptlogo), filename='zpt.gif')
        self.assertEquals(view.has_files(), 1)

    def test_get_links(self):
        self.n1.invokeFactory('Link', 'baz', title='Baz', url='http://baz/')
        view = self.n1.restrictedTraverse('@@view')
        links = view.get_links()
        self.assertEquals(len(links), 1)
        self.assertEquals(links[0].getObject().id, 'baz')
        self.assertEquals(links[0].getObject().title, 'Baz')
        self.assertEquals(links[0].getObject().description, '')

    def test_has_links(self):
        view = self.n1.restrictedTraverse('@@view')
        self.assertEquals(view.has_links(), 0)
        self.n1.invokeFactory('Link', 'baz', title='Baz', url='http://baz/')
        self.assertEquals(view.has_links(), 1)

    def test_get_media(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', image=StringIO(zptlogo))
        self.n1.invokeFactory('File', 'bar', title='Bar', file=StringIO(zptlogo))
        self.n1.invokeFactory('Link', 'baz', title='Baz', url='http://baz/')
        view = self.n1.restrictedTraverse('@@view')
        media = view.get_media()
        self.assertEquals(len(media), 3)
        self.assertEquals(media[0].getObject().id, 'foo')
        self.assertEquals(media[1].getObject().id, 'bar')
        self.assertEquals(media[2].getObject().id, 'baz')

    def test_getImage(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@view')
        image = view.getImage()
        self.assertEquals(image.id, 'foo')
        self.assertEquals(image.Title(), 'Foo')
        self.assertEquals(image.Description(), 'FOO')

    def test_tag(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@view')
        tag = view.tag
        self.assertEquals(tag(),
            '')
        self.assertEquals(tag(scale='preview'),
            '')
        self.assertEquals(tag(css_class='myClass'),
            '')

    def test_imageCaption(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@view')
        self.assertEquals(view.imageCaption(), 'FOO')

    def test_chunks(self):
        """ Test is chunks are created from a list.
        """
        view = self.n1.restrictedTraverse('@@view')
        chunks = view._chunks
        # create chunks of 3 elements
        data = chunks([1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4], 3)
        # generator elements are accessed calling next()
        self.assertEquals(data.next(), [1, 1, 1])
        self.assertEquals(data.next(), [2, 2, 2])
        self.assertEquals(data.next(), [3, 3, 3])
        self.assertEquals(data.next(), [4, 4])

    def test_get_images_in_groups(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@view')
        groups = len([i for i in view.get_images_in_groups()])
        # we only have one image, so we only have one group
        self.assertEquals(groups, 1)

    def test_scrollable(self):
        name = '@@scrollable'
        try:
            self.n1.unrestrictedTraverse(name)
        except AttributeError:
            self.fail('%s has no view %s' % (self.n1, name))

    def test_folder_summary_view(self):
        name = '@@folder_summary_view'
        try:
            self.n1.unrestrictedTraverse(name)
        except AttributeError:
            self.fail('%s has no view %s' % (self.n1, name))

    def test_viewlets_registered(self):
        NotImplemented

class NITFViewTest(unittest.TestCase):

    layer = INTEGRATION_TESTING

    def setUp(self):
        self.portal = self.layer['portal']
        self.request = self.layer['request']
        directlyProvides(self.request, INITFBrowserLayer)

        setRoles(self.portal, TEST_USER_ID, ['Manager'])
        self.portal.invokeFactory('Folder', 'test-folder')
        setRoles(self.portal, TEST_USER_ID, ['Member'])
        self.folder = self.portal['test-folder']

        self.folder.invokeFactory('collective.nitf.content', 'n1')
        self.n1 = self.folder['n1']

    def test_nitf(self):
        # this view is available but not registered
        name = '@@nitf'
        try:
            self.n1.unrestrictedTraverse(name)
        except AttributeError:
            self.fail('%s has no view %s' % (self.n1, name))

    def test_get_mediatype(self):
        view = self.n1.restrictedTraverse('@@nitf')
        _get_mediatype = view._get_mediatype
        self.assertEquals(_get_mediatype('application/pdf'), 'application')
        self.assertEquals(_get_mediatype('audio/mpeg3'), 'audio')
        self.assertEquals(_get_mediatype('image/jpeg'), 'image')
        self.assertEquals(_get_mediatype('image/png'), 'image')
        self.assertEquals(_get_mediatype('multipart/signed'), 'other')
        self.assertEquals(_get_mediatype('text/plain'), 'text')
        self.assertEquals(_get_mediatype('video/avi'), 'video')

    def test_get_media(self):
        self.n1.invokeFactory('Image', 'foo', title='Foo', description='FOO',
                              image=StringIO(zptlogo), filename='zpt.gif')
        view = self.n1.restrictedTraverse('@@nitf')
        media = view.get_media()
        self.assertEquals(len(media), 1)

def test_suite():
    return unittest.defaultTestLoader.loadTestsFromName(__name__)


from tokenizer import tokenizer
from mini_parser import Parser,SymbolTable
from icg import CodeGenerator as cg

while(True):
    try:
        f_p=input(">>Enter file name: ")
        inp=open(f_p)
        file=inp.read()
        # print(file)

        tokens=tokenizer(file)
        print("Tokens: ", tokens)
        P=Parser(tokens)
        P.parse()
        print("Mini Tree\n",repr(P.tree))
        print("SymbolTable\n",SymbolTable.symbols)
        print("code generator")
        cg(SymbolTable.symbols)

    except KeyboardInterrupt:
        exit()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright (c) 2015 Jérémie DECOCK (http://www.jdhp.org)

"""
This is the simplest Python GTK+3 snippet.

See: http://python-gtk-3-tutorial.readthedocs.org/en/latest/label.html
"""

from gi.repository import Gtk as gtk

def main():
    window = gtk.Window()

    label = gtk.Label(label="Hello! ¡Buenos días! Bonjour! 你好！")
    window.add(label)

    window.connect("delete-event", gtk.main_quit) # ask to quit the application when the close button is clicked
    window.show_all()                             # display the window
    gtk.main()                                    # GTK+ main loop

if __name__ == '__main__':
    main()


from collections import Counter
N, P = map(int, input().split())
S = input()

def largeSol():
    T = [0] * (N + 1)
    for i, s in enumerate(map(int, S[:: -1])):
        T[i + 1] = (s * pow(10, i, P) + T[i]) % P

    ans = 0
    cnt = Counter(T[1:])
    for p in range(P):
        x = cnt[p]
        ans += x * (x - 1) // 2
    ans += cnt[0]
    return ans

def smallSol():
    dp = [0] * P
    ans = 0
    for s in map(int, S):
        newDp = [0] * P
        newDp[s % P] += 1
        for p in range(P):
            newDp[(p * 10 + s) % P] += dp[p]
        dp = newDp
        ans += dp[0]
    return ans

print(smallSol() if N * P <= 10**7 else largeSol())


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Question',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('question_types', models.IntegerField(default=1, choices=[(1, b'Multiple Choice')])),
                ('taxonomy_types', models.IntegerField(default=1, choices=[(1, b'Remembering'), (2, b'Understanding'), (3, b'Applying')])),
                ('difficulty_level', models.IntegerField(default=1, choices=[(1, b'EASY'), (2, b'MILD'), (3, b'HARD')])),
                ('question', models.TextField(max_length=200)),
                ('created_date', models.DateTimeField(verbose_name=b'date')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]


from .Function_RGB import Class_EmailKonto
import time
from PIL import Image
from PIL import ImageFont
from PIL import ImageDraw
import random
import socket
import poplib
from rgbmatrix import Adafruit_RGBmatrix

def draw_Message(message, Color1, Color2):
    text_width, text_height = fnt.getsize(message)
    img = Image.new('RGBA', (text_width + 10, 16), Color2)
    draw = ImageDraw.Draw(img)
    draw.text((5, (16 - text_height) / 2), message, Color1, font=fnt)
    for n in range(32, -img.size[0], -1):
        matrix.Clear()
        matrix.SetImage(img.im.id, n, 0)
        time.sleep(0.05)

# festlegen der Gesamten Grund

Konto = Class_EmailKonto.EmailKontoBox('pythontest1@outlook.de', 'simpsons1', 'pop-mail.outlook.com')
Konto.load_colors()
Konto.load_messages()
Konto.error_log("Hi")

matrix = Adafruit_RGBmatrix(16, 1)
fnt = ImageFont.truetype("Ticketing.ttf", 14)
random.seed()
Colors = ((255, 0, 0, 0),
          (0, 255, 0, 0),
          (0, 0, 255, 0),
          (255, 255, 0, 0),
          (0, 255, 255, 0),
          (255, 0, 255, 0),
          (255, 255, 255, 0))

draw_Message("Willkommen", (255, 255, 255), (0, 0, 0))

Konto.error_log("draw Welcome error")
t = time.time()
Konto.load_messages()
Konto.update_message_file()
Konto.delete_old_messages()
Konto.load_messages()

while True:

    try:

        if time.time() - t > 120.0:
            Konto.update_message_file()
            Konto.delete_old_messages()
            Konto.load_messages()

        Emails = Konto.get_messages()
        spec_color = Konto.get_colors()
        final_Message = []
        if (Emails == []):
            draw_Message("Keine Nachrichten", (255, 0, 0), (0, 0, 0))
        else:
            for x in Emails:
                Message = x[0]
                Color = Colors[random.randint(0, len(Colors) - 1)]
                Addresse = x[1]

                for y in spec_color:
                    if (y[0] == Addresse):
                        Color = (y[1], y[2], y[3])
                draw_Message(Message.decode("utf-8"), Color, (0, 0, 0))
    except socket.error as error1:
        time.sleep(10)
        print("Socket error")
        Konto.error_log("Socket error")
    except TypeError as error1:
        draw_Message("Probleme mit der E-Mail Verbindung. Bitte die Emailadresse checken", (255, 0, 0), (0, 0, 0))
        print("Little Problem here")


from django.http import JsonResponse
from django.shortcuts import render
from .models import Article
from django.views import generic
from .forms import SearchForm
import urllib

def home(request):
    recent_blogs = Article.objects.all().order_by('-pub_date')[:3]
    return render(request, 'articles/home.html', {'recent_blogs': recent_blogs})

def all_blogs(request):
    blogs = Article.objects.all().order_by('-pub_date')
    return render(request, 'articles/all_blogs.html', {'blogs': blogs})

def search(request):
    search_form = SearchForm()
    return render(request, 'articles/search.html', {'search_form': search_form})

def search_blog(request):
    search_form = SearchForm(request.GET)
    if search_form.is_valid():
        encoded_search_term = urllib.parse.unquote(search_form.cleaned_data['search_term'])
        print(encoded_search_term)
        results = []
        articles = Article.objects.all()
        for article in articles:
            if encoded_search_term.lower() in article.title.lower() or encoded_search_term.lower() in article.summary().lower():
                json_article_object = {'id':article.id,
                                       'title': article.title,
                                       'text': article.summary(),
                                       'image_url': article.image.url,
                                       'pub_date': article.pub_date_pretty()
                                       }
                results.append(json_article_object)
        return JsonResponse({'results': results})
    else:
        return JsonResponse({'error': 'Not able to validate form'})

class DetailBlog(generic.DetailView):
    model = Article
    template_name = 'articles/detail_blog.html'


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import datetime

class Migration(migrations.Migration):

    dependencies = [
        ('spooapp', '0004_auto_20160707_2218'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='tesisler',
            name='isletme_hizmet_bitis',
        ),
        migrations.RemoveField(
            model_name='tesisler',
            name='isletme_hizmet_zaman',
        ),
        migrations.RemoveField(
            model_name='tesisler',
            name='isletme_brans',
        ),
        migrations.AddField(
            model_name='tesisler',
            name='isletme_brans',
            field=models.ManyToManyField(to='spooapp.SporDallar', max_length=255, verbose_name='İşletme Branşı', choices=[('FT', 'Futbol'), ('VB', 'Voleybol'), ('BT', 'Basketbol'), ('GK', 'Go-Kart'), ('TN', 'Tenis')]),
        ),
        migrations.AlterField(
            model_name='tesisler',
            name='isletme_kayit_tarih',
            field=models.CharField(max_length=100, verbose_name='Kayıt Tarihi', default=datetime.datetime(2016, 7, 7, 23, 5, 10, 371807)),
        ),
    ]


import re

import utils.array_utils
import utils.multiprocess_utils
from utils.id_freq_dict import IdFreqDict
import utils.tweet_keys as tk
import utils.multiprocess_utils as mu
import utils.array_utils as au
import utils.function_utils as fu
import utils.file_iterator as fi
import utils.pattern_utils as pu
import utils.ark_service_proxy as ark
import preprocess.tweet_filter as tflt
from config.configure import getcfg

def summary_files_in_path(from_path, into_path=None):
    """ Read all .json under file_path, extract tweets from them into a file under summary_path. """
    # [-13:]--hour [-13:-3]--day [-13:-5]--month,ymdh refers to the short of "year-month-date-hour"
    from_path = fi.add_sep_if_needed(from_path)
    file_ymdh_arr = pu.split_digit_arr(fi.get_parent_path(from_path)[-13:])
    if not is_target_ymdh(file_ymdh_arr):
        return

    into_file = '{}{}'.format(fi.add_sep_if_needed(into_path), '_'.join(file_ymdh_arr) + '.sum')
    fi.remove_file(into_file)
    subfiles = fi.listchildren(from_path, children_type=fi.TYPE_FILE)
    file_block = mu.split_multi_format([(from_path + subfile) for subfile in subfiles], process_num=20)
    twarr_blocks = mu.multi_process(sum_files, [(file_list, tflt.FILTER_LEVEL_LOW) for file_list in file_block])
    twarr = au.merge_array(twarr_blocks)
    if twarr:
        fu.dump_array(into_file, twarr, overwrite=True)

def sum_files(file_list, filter_level):
    res_twarr = list()
    for file in file_list:
        twarr = fu.load_twarr_from_bz2(file) if file.endswith('.bz2') else fu.load_array(file)
        twarr = tflt.filter_twarr(twarr, filter_level)
        res_twarr.extend(twarr)
    return res_twarr

def summary_files_in_path_into_blocks(from_path, into_path, file_name):
    from_path = fi.add_sep_if_needed(from_path)
    sub_files = fi.listchildren(from_path, children_type=fi.TYPE_FILE, pattern='.json$')
    into_file = fi.add_sep_if_needed(into_path) + file_name
    twarr_block = list()
    for idx, file in enumerate(sub_files):
        from_file = from_path + file
        twarr = fu.load_array_catch(from_file)
        if len(twarr) <= 0:
            continue
        twarr = tflt.filter_twarr(twarr, tflt.FILTER_LEVEL_HIGH)
        twarr_block.append(twarr)
    print(sorted([('id'+str(idx), len(twarr)) for idx, twarr in enumerate(twarr_block)], key=lambda x: x[1]))
    print('event number in total: {}'.format(len(twarr_block)))
    fu.dump_array(into_file, twarr_block)

def is_target_ymdh(ymdh_arr):
    # ymdh_arr resembles ['201X', '0X', '2X', '1X']
    year = int(ymdh_arr[0])
    month = int(ymdh_arr[1])
    date = int(ymdh_arr[2])
    hour = int(ymdh_arr[3])
    # return 4 == month and date <= 2
    # import datetime
    # tw_time = datetime.datetime.strptime('-'.join(ymdh_arr[0:3]), '%Y-%m-%d')
    # start_time = datetime.datetime.strptime('2016-06-08', '%Y-%m-%d')
    # return (tw_time - start_time).days >= 0
    return True

def get_tokens_multi(file_path):
    file_path = fi.add_sep_if_needed(file_path)
    # subfiles = au.random_array_items(fi.listchildren(file_path, children_type=fi.TYPE_FILE), 20)
    subfiles = fi.listchildren(file_path, children_type=fi.TYPE_FILE)
    file_list_block = mu.split_multi_format([(file_path + subfile) for subfile in subfiles], process_num=20)
    res_list = mu.multi_process(get_tokens, [(file_list,) for file_list in file_list_block])
    id_freq_dict, total_doc_num = IdFreqDict(), 0
    for ifd, doc_num in res_list:
        total_doc_num += doc_num
        id_freq_dict.merge_freq_from(ifd)
    print('total_doc_num', total_doc_num, 'total vocabulary_size', id_freq_dict.vocabulary_size())
    id_freq_dict.drop_words_by_condition(3)
    id_freq_dict.dump_dict(getcfg().post_dict_file)

def get_tokens(file_list):
    id_freq_dict, total_doc_num = IdFreqDict(), 0
    for file in file_list:
        twarr = fu.load_array(file)
        total_doc_num += len(twarr)
        for tw in twarr:
            tokens = re.findall(r'[a-zA-Z_#\-]{3,}', tw[tk.key_text].lower())
            real_tokens = list()
            for token in tokens:
                if len(token) >= 16:
                    real_tokens.extend(pu.segment(token))
                else:
                    real_tokens.append(token)
            for token in real_tokens:
                if (not pu.is_stop_word(token)) and pu.has_azAZ(token) and 3 <= len(token):
                    id_freq_dict.count_word(token)
    id_freq_dict.drop_words_by_condition(2)
    print(id_freq_dict.vocabulary_size())
    return id_freq_dict, total_doc_num

K_IFD, K_FILE = 'ifd', 'file'

def get_semantic_tokens_multi(file_path):
    pos_type_info = {
        ark.prop_label: {K_IFD: IdFreqDict(), K_FILE: getcfg().pre_prop_dict_file},
        ark.comm_label: {K_IFD: IdFreqDict(), K_FILE: getcfg().pre_comm_dict_file},
        ark.verb_label: {K_IFD: IdFreqDict(), K_FILE: getcfg().pre_verb_dict_file},
        ark.hstg_label: {K_IFD: IdFreqDict(), K_FILE: getcfg().pre_hstg_dict_file},
    }
    total_doc_num = 0
    file_path = fi.add_sep_if_needed(file_path)
    # subfiles = au.random_array_items(fi.listchildren(file_path, children_type=fi.TYPE_FILE), 40)
    subfiles = fi.listchildren(file_path, children_type=fi.TYPE_FILE)
    file_list_block = mu.split_multi_format([(file_path + subfile) for subfile in subfiles], process_num=20)
    res_list = mu.multi_process(get_semantic_tokens, [(file_list,) for file_list in file_list_block])
    for res_type_info, doc_num in res_list:
        total_doc_num += doc_num
        for label in res_type_info.keys():
            pos_type_info[label][K_IFD].merge_freq_from(res_type_info[label][K_IFD])
    print('total_doc_num', total_doc_num)
    for label in pos_type_info.keys():
        ifd, file_name = pos_type_info[label][K_IFD], pos_type_info[label][K_FILE]
        ifd.drop_words_by_condition(3)
        if label != ark.hstg_label:
            ifd.drop_words_by_condition(lambda word, _: word.startswith('#'))
        ifd.dump_dict(file_name)
        print('{}; vocab size:{}'.format(file_name, ifd.vocabulary_size()))

def get_semantic_tokens(file_list):
    pos_type_info = {
        ark.prop_label: {K_IFD: IdFreqDict()},
        ark.comm_label: {K_IFD: IdFreqDict()},
        ark.verb_label: {K_IFD: IdFreqDict()},
        ark.hstg_label: {K_IFD: IdFreqDict()},
    }
    total_doc_num = 0
    for file in file_list:
        twarr = ark.twarr_ark(fu.load_array(file))
        total_doc_num += len(twarr)
        pos_tokens = au.merge_array([tw[tk.key_ark] for tw in twarr])
        for pos_token in pos_tokens:
            word = pos_token[0].strip().lower()
            if len(word) <= 2 or not pu.is_valid_keyword(word):
                continue
            real_label = ark.pos_token2semantic_label(pos_token)
            if real_label:
                pos_type_info[real_label][K_IFD].count_word(word)
    return pos_type_info, total_doc_num

# sep = os.path.sep
# def load_twarr_from_bz2_list(bz2_file_list):
#     print(len(bz2_file_list))
#     twarr = list()
#     for bz2_file in bz2_file_list:
#         if not bz2_file.endswith(".bz2"):
#             continue
#         twarr.extend(load_twarr_from_bz2(bz2_file))
#     return twarr
# def load_twarr_from_bz2_multi(bz2_list, p_num=15):
#     list_per_p = fu.split_multi_format(bz2_list, min(p_num, len(bz2_list)))
#     return fu.multi_process(load_twarr_from_bz2_list, [(l, ) for l in list_per_p])
# def tw_file_date(file_or_dir):
#     if fi.is_dir(file_or_dir):
#         dir_name = file_or_dir[:file_or_dir.rfind('\\')] if file_or_dir.endswith('\\') else file_or_dir
#         timestr = dir_name[-15:]
#     elif fi.is_file(file_or_dir):
#         file_name = file_or_dir
#         timestr = fi.base_name(file_name)[:file_name.rfind('.')]
#     else:
#         raise ValueError('file_or_dir wrong format:', file_or_dir)
#     return timestr
# def list_raw_data(data_path, start_ymdh, end_ymdh):
#     """ used for the purpose of special file structure, which is organized as the tree
#         data_path/year_month/date/hour/minute.bz2
#         :return A list of sublist, where each sublist contains the data within one day """
#     ymd = [ym + sep + d + sep for ym in fi.listchildren(data_path, children_type='dir')
#            for d in fi.listchildren(data_path + ym + sep, children_type='dir')
#            if du.is_target_ymdh(pu.split_digit_arr(ym + sep + d), start_ymdh, end_ymdh)]
#     ymd_hM = [[data_path + ymd_ + h + sep + M for h in fi.listchildren(data_path + ymd_, children_type='dir')
#                for M in fi.listchildren(data_path + ymd_ + sep + h, children_type='file')] for ymd_ in ymd]
#     for idx in range(len(ymd_hM) - 1, -1, -1):
#         if not ymd_hM[idx]:
#             ymd_hM.pop(idx)
#     return ymd_hM
# # data_path = getconfig().data_path
# # start_ymdh, end_ymdh = ['2016', '03', '2'], ['2016', '03', '6']
# # bz2_lists = list_raw_data(data_path, start_ymdh, end_ymdh)
# # for bz2_list_per_day in bz2_lists:
# #     load_twarr_from_bz2_multi(bz2_list_per_day[:150], p_num=15)


# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
from feeder.feeder_kinetics import Feeder_kinetics
import os
import sys
import pickle
import argparse

import numpy as np
from numpy.lib.format import open_memmap

sys.path.append(
    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))

toolbar_width = 30

def print_toolbar(rate, annotation=''):
    # setup toolbar
    sys.stdout.write("{}[".format(annotation))
    for i in range(toolbar_width):
        if i * 1.0 / toolbar_width > rate:
            sys.stdout.write(' ')
        else:
            sys.stdout.write('-')
        sys.stdout.flush()
    sys.stdout.write(']\r')

def end_toolbar():
    sys.stdout.write("\n")

def gendata(
        data_path,
        label_path,
        data_out_path,
        label_out_path,
        num_person_in=5,  # observe the first 5 persons
        num_person_out=2,  # then choose 2 persons with the highest score
        max_frame=300):

    feeder = Feeder_kinetics(data_path=data_path,
                             label_path=label_path,
                             num_person_in=num_person_in,
                             num_person_out=num_person_out,
                             window_size=max_frame)

    sample_name = feeder.sample_name
    sample_label = []

    fp = open_memmap(data_out_path,
                     dtype='float32',
                     mode='w+',
                     shape=(len(sample_name), 3, max_frame, 18,
                            num_person_out))

    for i, s in enumerate(sample_name):
        data, label = feeder[i]
        print_toolbar(
            i * 1.0 / len(sample_name),
            '({:>5}/{:<5}) Processing data: '.format(i + 1, len(sample_name)))
        fp[i, :, 0:data.shape[1], :, :] = data
        sample_label.append(label)

    with open(label_out_path, 'wb') as f:
        pickle.dump((sample_name, list(sample_label)), f)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Kinetics-skeleton Data Converter.')
    parser.add_argument('--data_path',
                        default='data/Kinetics/kinetics-skeleton')
    parser.add_argument('--out_folder',
                        default='data/Kinetics/kinetics-skeleton')
    arg = parser.parse_args()

    part = ['train', 'val']
    for p in part:
        data_path = '{}/kinetics_{}'.format(arg.data_path, p)
        label_path = '{}/kinetics_{}_label.json'.format(arg.data_path, p)
        data_out_path = '{}/{}_data.npy'.format(arg.out_folder, p)
        label_out_path = '{}/{}_label.pkl'.format(arg.out_folder, p)

        if not os.path.exists(arg.out_folder):
            os.makedirs(arg.out_folder)
        gendata(data_path, label_path, data_out_path, label_out_path)


import os
from flask import Flask
import logging
from werkzeug.middleware.proxy_fix import ProxyFix
from .routes import bp
from .models import db, login_manager
from celery import Celery

def create_app(config={}):
    app = Flask(__name__)
    app.debug = True
    app.config.update(config)
    app.register_blueprint(bp, url_prefix='')
    db.init_app(app)
    login_manager.init_app(app)
    setLog(app)

    login_manager.login_view = 'labboxmain.routes.Login'  # redir
    app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
    # os.environ['AUTHLIB_INSECURE_TRANSPORT'] = '1'

    # redis
    celery = make_celery(app)
    return app, celery

def make_celery(app):
    celery = Celery(
        app.import_name,
        backend=app.config['celery_result_backend'],
        broker=app.config['celery_broker_url']
    )
    celery.conf.timezone = os.getenv('TZ')
    celery.conf.beat_schedule = app.config['celery_schedule']
    # celery.conf.update(app.config)

    class ContextTask(celery.Task):
        def __call__(self, *args, **kwargs):
            with app.app_context():
                return self.run(*args, **kwargs)

    celery.Task = ContextTask
    return celery

def setLog(app):
    logger = logging.getLogger('labboxmain')
    logger.setLevel(logging.DEBUG)

    # output to std
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    # output to file
    if app.config.get('logfile'):
        fh = logging.FileHandler(app.config['logfile'])
        fh.setLevel(logging.DEBUG)
        fh.setFormatter(formatter)
        logger.addHandler(fh)

    # authlib logger
    if app.debug:
        logger = logging.getLogger('authlib')
        logger.setLevel(logging.DEBUG)
        logger.addHandler(ch)


import numpy as np
import utils
import sys
import h5py

def addmappability(v,val,k):
	"""
	"""
	i=0
	while i<k:
		v.append(val)
		i+=1

def addmappability2(v,val,kbase,nbval):
	"""
	this time we are on a resoluted vec
	"""
	i=0
	while i<nbval:
		v[kbase+i]=val
		i+=1

def makeachrvec(afile,thechr,ls,sizevec,resolution):
	"""
	work only if the bed is already perfect
	"""
	thevec=np.zeros((sizevec,1))
	actualpos=np.ceil(float(ls[1])/resolution)
	print("pos actuelle ",actualpos)
	meanmap=list()
	Kmap=0
	while ls[0]==thechr:
		sizebin=float(ls[2])-float(ls[1])
		if (Kmap+sizebin)>=resolution:
			meanmap=np.array(meanmap) #easier if hardcast
			thevec[actualpos]=np.mean(meanmap)
			actualpos+=1
			if (Kmap+sizebin)==resolution:
				meanmap=list()
				Kmap=0
			else : #>
				Kmap-=resolution
				meanmap=list()
				addmappability(meanmap,float(ls[3]),sizebin-Kmap)
		else : #Kmap+sizebin<resolution
			Kmap+=sizebin
			addmappability(meanmap,float(ls[3]),sizebin)
		l=afile.readline()
		ls=l.split("\t")
	return l,thevec

def makeachrvec2(afile,thechr,ls,sizevec,resolution):
	thevec=np.zeros((sizevec,1))
	actualpos=np.ceil(float(ls[1])/resolution)
	meanmap=np.zeros((resolution,1))
	Kmeanmap=0
	while ls[0]==thechr:
		sizebin=float(ls[2])-float(ls[1])
		newpos=np.ceil(float(ls[1])/resolution)
		if newpos!=actualpos: #switch
			thevec[actualpos]=np.mean(meanmap)
			actualpos=newpos
			#restart
			meanmap=np.zeros((resolution,1))
			Kmeanmap=0
		if (Kmeanmap+sizebin)<resolution:
			addmappability2(meanmap,float(ls[3]),Kmeanmap,sizebin)
			Kmeanmap+=sizebin
		else :
			sizebin2=resolution-Kmeanmap
			addmappability2(meanmap,float(ls[3]),Kmeanmap,sizebin2)
			#push
			thevec[actualpos]=np.mean(meanmap)
			sizebin-=sizebin2
			#newinit
			while sizebin>resolution:
				thevec[actualpos]=np.mean(meanmap)
				meanmap=np.zeros((resolution,1))
				actualpos+=1
				sizebin=sizebin-resolution
				Kmeanmap=0
				#print("non?",sizebin,sizebin2,Kmeanmap)
				addmappability2(meanmap,float(ls[3]),Kmeanmap,resolution)
			if sizebin>0:
				thevec[actualpos]=np.mean(meanmap)
				meanmap=np.zeros((resolution,1))
				actualpos+=1
				sizebin=sizebin-sizebin2
				Kmeanmap=0
				#print("non!",sizebin,sizebin2,Kmeanmap)
				addmappability2(meanmap,float(ls[3]),Kmeanmap,sizebin)
		l=afile.readline()
		ls=l.split("\t")
	return l,thevec

if sys.argv[1]=="Convert":
	resolution=float(sys.argv[2])
	species=sys.argv[3]
	chrlist=utils.dictchr[species]
	chrsizedict=utils.loadchrsizedict(sys.argv[4],resolution)
	#on va commencer la partie amusante
	fin=open(sys.argv[5],'r')
	l=fin.readline()
	while l:
		ls=l.split("\t")
		if ls[0] in chrlist:
			print("chromosome : ",ls[0])
			s=utils.givematrixsize(chrsizedict,chrlist,ls[0],chrlist.index(ls[0]))
			sizemat=s[0]-s[1]
			print("taille de la matrice",sizemat)
			l,thevec=makeachrvec2(fin,ls[0],ls,sizemat,resolution)
			fh5 = h5py.File(sys.argv[6]+ls[0]+".hdf5", "w")
			fh5['data'] = thevec
			fh5.close()
		else :
			l=fin.readline()
	fin.close()
elif sys.argv[1]=="Iscontinue":
	fin=open(sys.argv[2],'r')
	l=fin.readline()
	ls=l.split("\t")
	actualchr=ls[0]
	ki=ls[1]
	kj=ls[2]
	l=fin.readline()
	kT=0
	kF=0
	while l:
		ls=l.split("\t")
		actualchr=ls[0]
		kinew=ls[1]
		kjnew=ls[2]
		while ls[0]==actualchr:
			ki=kinew
			kj=kjnew
			kinew=ls[1]
			kjnew=ls[2]
			if kinew!=kj:
				print("c'est pas vrai pour",ls[0],ki,kj,kinew,kjnew)
				kF+=1
			else:
				kT+=1
			l=fin.readline()
			ls=l.split("\t")
		l=fin.readline()
		print('c est vrai!',kT,kF)


import face_recognition_models as fm
import dlib
import cv2 as cv
import numpy as np

def vec_data(path):
    mdl = fm.pose_predictor_model_location()

    frm = fm.face_recognition_model_location()

    cnn = dlib.face_recognition_model_v1(frm)

    fd = dlib.get_frontal_face_detector()

    im = cv.imread(path)

    hi = (200,200)

    im = cv.resize(im, hi, interpolation = cv.INTER_AREA)
    df = fd(im,1)
    a = []
    if len(df)!=0:
        fpose = dlib.shape_predictor(mdl)
        pl = fpose(im,df[0])
        vec = cnn.compute_face_descriptor(im,pl)
        for z in range(len(vec)):
            a.append(vec[z])
        return a
    else:
        return a
def dis(a,b):
    arr = np.array(a)
    brr = np.array(b)
    return np.linalg.norm(arr - brr, axis=0)


from django.conf.urls import url
from django.urls import path, include
from main_store.views import *

urlpatterns = [
    path('main_store', home_store, name='main_store'),
    path('compose_mrr', mrr_compose, name='compose_mrr'),
    path('mrr_show/<mrr_no>/', mrr_show, name='mrr_show'),
    path('mrr_no_receive', mrr_no_receive, name='mrr_no_receive'),
    path('indent_box', indent_box, name='indent_box'),
    path('indent_form', indent_form, name='indent_form'),
    path('product_issue', product_issue, name='product_issue'),
    path('product_store', product_store, name='product_store'),
    path('existing_product_store', existing_product_store, name='existing_product_store'),  # product existing update.
    path('bin_card_state', bin_card_state, name='bin_card_state'),
    path('issuer_state', issuer_state, name='issuer_state'),
    path('product_name_code', product_name_code, name='product_name_code'),
    path('min_balance_list', low_balance_state, name='low_balance_state'),
    path('expire_date_list', expire_date_state, name='expire_date_state'),
    path('purchase_state', purchase_state, name='purchase_state'),
    path('material_balance_state', material_balance_state, name='material_balance_state'),
    path('statement', statement, name='statement'),
    path('data_entry_statement', data_entry_statement, name='data_entry_statement'),

    path('job_number', job_number, name='job_number'),

    # path('commercial', commercial, name='commercial'),

    # Demo code and name
    path('demo_code_input', demo_code_input, name='demo_code_input'),
    path('demo_code_show', demo_code_show, name='demo_code_show'),
    # path('duplicate_in_demo_code', duplicate_in_demo_code, name='duplicate_in_demo_code'),  # Find Duplicate in demo
    # code and Name Model.
    # path('duplicate_in_p_name_code', duplicate_in_p_name_code, name='duplicate_in_p_name_code'),  # Find Duplicate in
    # Product Name and Code model.
    path('product_name_store', product_name_store, name='product_name_store'),
    # path('demo_code_delete', demo_code_delete, name='demo_code_delete'),

    # Demo job name
    path('demo_job_name', demo_job_name, name='demo_job_name'),
    # path('demo_job_name_delete', demo_job_name_delete, name='demo_job_name_delete'),
    path('job_no_store', job_no_store, name='job_no_store'),

    path('item_group_name_change', item_group_name_change, name='item_group_name_change'),

    path('quality_receive', quality_receive, name='quality_receive'),
    path('material_quality/<quality_no>/', material_quality, name='material_quality'),
    path('quality_input', quality_input, name='quality_input'),

    path('gate_pass', gate_pass, name='gate_pass'),
    path('sr_form', sr_form, name='sr_form'),
    path('project_detail_store', project_detail_store, name='project_detail_store'),

    path('autocomplete', autocomplete, name='autocomplete'),
    path('autocomplete_job_no', autocomplete_job_no, name='autocomplete_job_no'),

    path('post_ajax', post_ajax, name='post_ajax'),
    path('ajax_search', ajax_search, name='ajax_search'),
]


# -*- coding: utf-8 -*-
"""
Created on Sat Apr 11 02:55:01 2020

@author: JANAKI
"""

import logging
import sys
import numpy as np
from keras.models import Model
from keras.layers import Input, Activation, add, Dense, Flatten
from keras.layers.convolutional import Conv2D, AveragePooling2D
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l2
from keras import backend as K

sys.setrecursionlimit(2 ** 20)
np.random.seed(2 ** 10)

def wide_basic(no_of_inputs, no_of_outputs, ch_axis, stride):
    '''First, we have conv2. For our understanding, assume that the count = 2. So, the first set of operations will be (BatchNorm, ReLU), Conv) x count stacked with shortcut connection.
    These are added. Then, the above repeats one more time: (BatchNorm, ReLU, Conv) x2 stacked with the initial Bn,ReLU set.
    Then, we move on to conv3, and so on.
    '''
    def repeat(initial_layer):
        stride_parameters = [[stride], [(1,1)]]
        for i, j in enumerate(stride_parameters):
            #only for the first iteration
            if i == 0:
                #this is only during the first iteration , count = 1
                if no_of_inputs!=no_of_outputs:
                    initial_layer = BatchNormalization(axis = ch_axis)(initial_layer)
                    initial_layer = Activation("relu")(initial_layer)
                    conv_layer = initial_layer
                else:
                    #this happens when count > 1
                    conv_layer = BatchNormalization(axis = ch_axis)(initial_layer)
                    conv_layer = Activation("relu")(conv_layer)

                #this is the conv operation which happens regardless of the count we are in
                conv_layer = Conv2D(no_of_outputs, kernel_size = (3,3), strides = j[0],
                                    padding = "same", kernel_initializer = "he_normal", kernel_regularizer=l2(0.0005),
                                    use_bias=False)(conv_layer)
                print("default conv layer visited")

                #the above cycle is done twice according to value of i and if no_of_inputs!=no_of_outputs or vice versa
            else:
                #this is for when count > 1, ie second iteration
                conv_layer = Activation("relu")(BatchNormalization(axis = ch_axis)(conv_layer))
                conv_layer = Conv2D(no_of_outputs, kernel_size = (3,3), strides = j[0], padding = "same",
                                    kernel_initializer = "he_normal", kernel_regularizer = l2(0.0005),
                                    use_bias = False)(conv_layer)

        #Time for shortcut connections to be built!
        #Here, if no_of_inputs!=no_of_outputs, a shortcut connection is built with a conv on the initial layer. Otherwise, the initial set of (Bn, ReLU) is assigned as the shortcut connection
        if no_of_inputs!=no_of_outputs:
            shortcut_connection = Conv2D(no_of_outputs, kernel_size = (1,1), strides=stride, padding="same",
                                         kernel_initializer = "he_normal", kernel_regularizer = l2(0.0005),
                                         use_bias = False, name="shortcut%d"%no_of_inputs)(initial_layer)
        else:
            shortcut_connection = initial_layer

        return add([conv_layer, shortcut_connection]) #stacking up the layer
    return repeat

def layer(ch_axis, n_input_plane, n_output_plane, count, stride):
    def f(net):
        net = wide_basic(n_input_plane, n_output_plane, ch_axis, stride)(net)
        for i in range(2, int(count + 1)):
            net = wide_basic(n_output_plane, n_output_plane, ch_axis, stride=(1, 1))(net)
        return net

    return f

def buildmodel(image_size, depth, k):

    if K.image_data_format() == "channels_first":
        logging.debug("image_dim_ordering = 'th'")
        ch_axis = 1
        input_shape = (3, image_size, image_size)
    else:
        logging.debug("image_dim_ordering = 'tf'")
        ch_axis = -1
        input_shape = (image_size, image_size, 3)

    assert ((depth - 4) % 6 == 0)
    #The number of times we have to repeat a conv block is equal to (depth - 4) / 6
    #This parameter will be applied below as 'count' while calling the function layer to build the wide residual blocks.
    n = (depth - 4) / 6

    #input layer (64, 64, 3)
    input_layer = Input(shape=input_shape)

    kernels = [16, 16*k, 32*k, 64*k]

    #conv1 layer (64, 64, 16)
    conv1 = Conv2D(filters = 16, kernel_size = (3, 3), strides = (1, 1), padding = "same", kernel_initializer = "he_normal",
                   kernel_regularizer = l2(0.0005), use_bias = False, name="Conv1")(input_layer)

    # Add wide residual blocks
    print("Adding wide residual blocks:")
    #wideblock = wide_basic()

    #Stacking the second conv layer on the first conv layer we created above. This is conv2 ()
    conv2 = layer(ch_axis, n_input_plane = kernels[0], n_output_plane = kernels[1], count = n, stride = (1, 1))(conv1)

    #conv3 layer ()
    conv3 = layer(ch_axis, n_input_plane = kernels[1], n_output_plane = kernels[2], count = n, stride = (2, 2))(conv2)

    #conv4 layer ()
    conv4 = layer(ch_axis, n_input_plane = kernels[2], n_output_plane = kernels[3], count = n, stride = (2, 2))(conv3)

    bn = BatchNormalization(axis = ch_axis)(conv4)
    relu = Activation("relu")(bn)

    #Preparing for classification
    avgpool = AveragePooling2D(pool_size = (8, 8), strides = (1, 1), padding = "same")(relu)
    flatten = Flatten()(avgpool)

    #classification layers
    pre_estimate = Dense(units = 2, kernel_initializer = "he_normal", use_bias = False, kernel_regularizer = l2(0.0005), activation="softmax")(flatten)
    predicted_age = Dense(units = 101, kernel_initializer = "he_normal", use_bias = False, kernel_regularizer = l2(0.0005), activation="softmax")(flatten)

    #building the model
    model = Model(inputs=input_layer, outputs=[pre_estimate, predicted_age])
    return model

def main():
    model = buildmodel(64, 16, 8)
    model.summary()

if __name__ == '__main__':
    main()