import random
import math
import copy

import numpy as np

from data import InferenceState
from game import Color
from game import Move
from game import Position
from model import build_model
from model import convert_inference_state_to_model_feature

from .policy import Policy

# Default path to load weights
WEIGHTS_FILE = '.build/weights.h5'

# Builds a model and loads weights.
def _build_model(config):
    num_classes = config.rows * config.columns
    input_shape = (config.rows, config.columns, 3)
    m = build_model(input_shape, num_classes)
    print("Loading model from:", WEIGHTS_FILE)
    m.load_weights(WEIGHTS_FILE)
    return m

class MCTSNode(object):

    # MCTSNode should own the board. Call site should make deepcopy if
    # necessary.
    def __init__(self, board, next_player_color, model, inject_noise=False):
        self.board = board
        self._config = board.config
        self._inject_noise_to_root = inject_noise
        self.next_player_color = next_player_color
        self.model = model

        self.legal_positions = board.legal_positions()
        if not self.legal_positions:
            raise RuntimeError("Board is full already. Unexpected.")

        # Stores children.
        self.c = {}

        # Stores statistic.
        self.total_count = 0

        self.n = {}
        self.w = {}
        self.p = {}

        # Fills in statistic.
        pred = self.model.predict(
                convert_inference_state_to_model_feature(
                    self._config,
                    InferenceState(
                        config=self._config,
                        # Just needs a view.
                        snapshot=board.snapshot(deepcopy=False),
                        next_player_color=self.next_player_color)))

        assert isinstance(pred, list)
        policy_pred, value_pred = pred
        del pred

        assert policy_pred.shape, (1, self._config.rows * self._config.columns)
        assert value_pred.shape, (1, 1)

        self.predicated_reward = float(value_pred[0][0])

        for pos in self.legal_positions:
            index = self._config.convert_position_to_index(pos)
            self.p[pos] = policy_pred[0][index]
            self.n[pos] = 0
            self.w[pos] = 0

        # We inject uniform noises.
        if inject_noise:
            a = np.random.uniform(0, 1, len(self.legal_positions))
            a = a / np.sum(a)

            for (i, pos) in enumerate(self.legal_positions):
                self.p[pos] = 0.8 * self.p[pos] + 0.2 * a[i]

    def backup(self, pos, value):
        assert pos in self.n
        self.total_count += 1
        self.n[pos] += 1
        self.w[pos] += value

    # Selects a node.
    #
    # - The algorithrm is based on current simulation results.
    # - It is used for next round of simulation.
    def select_next_pos_to_evaluate(self):
        c = 1.0
        q = {}

        sqrt_total_count = math.sqrt(self.total_count)
        for pos in self.legal_positions:
            n = self.n[pos]
            q[pos] = self.w[pos] / (n if n else 1)
            q[pos] += c * self.p[pos] * sqrt_total_count / (1.0 + self.n[pos])

        max_pos = max(q, key=lambda k: q[k])
        return max_pos

    def select_next_pos_to_play(self, debug=False, explore=False):
        q = []
        for pos in self.legal_positions:
            q.append((pos, self.n[pos]))

        if debug:
            print("")
            for i, item in enumerate(
                    sorted(q, key=lambda x: x[1], reverse=True)[:5]):
                pos = item[0]
                n = item[1]
                print("  Candidate %d: %s:  -> n (%4d) p (%f) w (%f)" % (
                    i,
                    pos,
                    n,
                    self.p[pos],
                    self.w[pos] / (n if n else 1)))
            print("")

        if not explore:
            # Select the pos with maximum visited counts.
            max_pair = max(q, key=lambda x: x[1])
            return max_pair[0]

        population, weights = zip(*q)
        max_pos = random.choices(population, weights)[0]
        return max_pos

    def simulate(self, iterations):
        _run_simulations(
                iterations=iterations,
                root_node=self,
                root_board=self.board,
                inject_noise=self._inject_noise_to_root)

# A policy based on MCTS and trained model.
#
# See AlphaGo Zero paper for details.
class MCTSPolicy(Policy):

    def __init__(self, board, color, model=None, iterations=1600, explore=False, debug=False, name=None):
        self._board = board
        self._config = board.config
        self._color = Color.of(color)
        self._model = model or _build_model(self._config)
        self._iterations = iterations
        self._explore = explore
        self._inject_noise_to_root = explore
        self._debug = debug
        self.name = name if name else "mcts_" + color

        self._root = None
        self._last_known_move_count = 0

    def next_position(self):
        if self._root is None:
            # Creates the if necessary (Just started the game)
            if self._color == Color.BLACK:
                assert len(self._board.moves) == 0
            else:
                assert self._color == Color.WHITE
                assert len(self._board.moves) == 1

            new_board = copy.deepcopy(self._board)  # make a copy
            # We inject noise only for root.
            node = MCTSNode(new_board, self._color, self._model,
                            inject_noise=self._inject_noise_to_root)
            self._root = node
        else:
            # Promot next node
            moves = self._board.moves
            last_pos = moves[-1].position  # Component's move

            new_root = self._root.c.get(last_pos)
            if new_root is None:
                # This case means that the move by component has never been
                # consided by this tree before.
                new_board = copy.deepcopy(self._board)  # make a copy
                new_root = MCTSNode(new_board, self._color, self._model,
                                    inject_noise=self._inject_noise_to_root)

            self._root = new_root

        # Simulation.
        root = self._root
        root.simulate(self._iterations)

        # Select
        explore = False
        if self._explore and len(self._board.moves) < 10:
            explore = True
        pos = root.select_next_pos_to_play(debug=self._debug, explore=explore)

        # Promote new root.
        new_root = root.c.get(pos)
        assert new_root is not None  # new_root might be `_game_is_over`.
        self._root = new_root

        return pos

# In the tree containing `MCTSNode`, some leafs are not needed as the game is
# over. `_game_is_over` is the placeholder for those so the node.c dict does not
# return `None`.
_game_is_over = object()

# Runs simulations.
#
# Write as free funciton for isolation.
def _run_simulations(iterations, root_node, root_board, inject_noise):

        for i in range(iterations):
            current_node = root_node
            current_board = root_board

            # Pairs of node and selection.
            selections = []

            # Backup reward for all nodes in `selections`.
            def backup_reward(b_reward, w_reward):
                for n, pos in selections:
                    if n.next_player_color == Color.BLACK:
                        n.backup(pos, b_reward)
                    else:
                        assert n.next_player_color == Color.WHITE
                        n.backup(pos, w_reward)

            # Keep play until game is over or new leaf is reached.
            while True:
                new_pos = current_node.select_next_pos_to_evaluate()

                selections.append((current_node, new_pos))

                new_move = Move(new_pos, current_node.next_player_color)
                new_board = copy.deepcopy(current_board)
                new_board.new_move(new_move)

                winner = new_board.winner_after_last_move()

                # Found winner.
                if winner is not None:
                    # This move will never be made so it should be None (not
                    # present) or just the placeholder (`_game_is_over`).
                    node_should_not_exist = current_node.c.get(new_pos)
                    assert (node_should_not_exist is None or
                            node_should_not_exist == _game_is_over)

                    # Writes a placehold. This is needed as `MCTSPolicy` will
                    # still promote new root node. We need an object there.
                    current_node.c[new_pos] = _game_is_over

                    if winner == Color.NA:  # Tie
                        black_player_reward = 0.0
                    else:
                        black_player_reward = (1.0
                                if winner == Color.BLACK else -1.0)

                    white_player_reward = black_player_reward * -1.0

                    backup_reward(black_player_reward, white_player_reward)
                    break  # End this iteration.

                # Expands new leaf
                if current_node.c.get(new_pos) is None:
                    next_player_color = current_node.next_player_color.reverse()
                    expanded_node = MCTSNode(
                            new_board,
                            next_player_color,
                            current_node.model,
                            inject_noise=inject_noise)

                    current_node.c[new_pos] = expanded_node

                    if next_player_color == Color.BLACK:
                        black_player_reward = expanded_node.predicated_reward
                    else:
                        black_player_reward = -1.0 * expanded_node.predicated_reward

                    white_player_reward = black_player_reward * -1.0
                    backup_reward(black_player_reward, white_player_reward)
                    break  # End this iteration.

                # Keeps playing in this iteration..
                current_board = new_board
                current_node = current_node.c.get(new_pos)


from scipy import stats
from sklearn.metrics import mean_squared_error
import numpy as np

def pearson(y, pred):
    pear = stats.pearsonr(y, pred)
    pear_value = pear[0]
    pear_p_val = pear[1]
    print("Pearson correlation is {} and related p_value is {}".format(pear_value, pear_p_val))
    return pear_value

def spearman(y, pred):
    spear = stats.spearmanr(y, pred)
    spear_value = spear[0]
    spear_p_val = spear[1]
    print("Spearman correlation is {} and related p_value is {}".format(spear_value, spear_p_val))
    return spear_value

def mse(y, pred):
    err = mean_squared_error(y, pred)
    print("Mean squared error is {}".format(err))
    return err

def squared_error(y,pred):
    errs = []
    for i in range(y.shape[0]):
        err = (y[i]-pred[i]) * (y[i]-pred[i])
        errs.append(err)
    return np.asarray(errs)


import logging
import time
import requests

logging.basicConfig()
logger = logging.getLogger('core.scrapers')
logger.setLevel(logging.DEBUG)

def get_ms_timestamp():
    return int((time.time() + 0.5) * 1000)

def scrape():
    logger.info('[ANYCOINDIRECT] Started lambda handler')

    ts = get_ms_timestamp()
    resp_buy = requests.get('https://anycoindirect.eu/en/buy/fiatpricev2?fiatCode=EUR&coinAmount=1&coinCode=BTC&paymentMethodId=1&nonce=%s&_=%s' % (ts, ts), timeout=3)
    resp_sell = requests.get('https://anycoindirect.eu/en/sell/fiatamountv2?fiatCode=EUR&coinCode=BTC&coins=1&nonce=%s&_=%s' % (ts, ts), timeout=3)
    resp_sell.raise_for_status()

    point_buy = resp_buy.json()
    point_buy = point_buy['Data']

    point_sell = resp_sell.json()
    point_sell = point_sell['Data']

    result = {
        'buy': float(point_buy['FiatPrice']),
        'sell': float(point_sell['TotalFiatPrice']),
    }

    logger.info('[ANYCOINDIRECT] Scraped pricepoint, buy: {}, sell: {}'.format(result['buy'], result['sell']))

    return result


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2016-07-04 10:00:24
# @Author  : HungChi
# @Site    :
# @File    : text_Deduplication.py
# @Software: PyCharm

import sys
import os
import collections
def bar(now,total):
    sys.stdout.write('\r')
    sys.stdout.write("[%s>%s] %s" % ('-'*int(100*now/total), ' '*int(100*(total-now)/total),str(int(100*(now+1)/total))+'%'))
    sys.stdout.flush()
def sort_by_count(d):
    #字典排序
    d = collections.OrderedDict(sorted(d.items(), key = lambda t: -t[1]))
    return d
def main():
    keys_dic = {}
    for arg in sys.argv[1:]:
        print("Input file "+arg)
        with open(arg,'rt',encoding='UTF-8') as _i_f:
            # keys_dic = dict(keys_dic,**{}.fromkeys([l.strip("\n") for l in _i_f.readlines()]))

            # for word in ['#'.join(l.strip("\n").split('#')[1:]) for l in _i_f]:
            for word in [l.strip("\n") for l in _i_f]:
                if word not in keys_dic:
                    keys_dic[word] = 0
                keys_dic[word] += 1
    # for k,v in keys_dic.items():
    #     print("dict[%s]=" % k,v)
    # keys = keys_dic.keys()
    # print(keys)
    # print(keys_dic)
    file_name = ''.join(map(str, list(os.path.splitext(arg))[0:-1]))+"_all_"+list(os.path.splitext(arg))[-1]
    # print(file_name)
    with open(file_name,'a+',encoding='UTF-8') as _o_f:
        print("output file "+file_name)
        # [_o_f.writelines(key + "," + str(value) + "\n") for key, value in sort_by_count(keys_dic).items()]
        [_o_f.writelines(key + "\n") for key, value in sort_by_count(keys_dic).items()]

if __name__ == '__main__':
    main()


#! /usr/bin/env python3

import os
import os.path
import sys
import json
import git
import common
import logging
from optparse import OptionParser

logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger("main")

def find_plan(plan_name):
    candidates = [plan_name,
                  os.path.join(os.getcwd(), "plans", plan_name),
                  os.path.join(os.getcwd(), "plans", plan_name + ".json")]
    for c in candidates:
        _logger.debug("Probe path {0} for plan file".format(c))
        if os.path.exists(c):
            _logger.debug("Open plan file at {0}".format(c))
            with open(c, 'rt') as f:
                planjson = json.load(f)
            return planjson

    return None

def main():
    parser = OptionParser(usage='usage: %prog [options] [plan]')

    optlist, args = parser.parse_args()

    if len(args) < 1:
        print('Incorrect parameters. Use option "-h" to get some help.')
        sys.exit(1)

    # Load plan file
    _logger.info("Loading plan " + args[0])
    planjson = find_plan(args[0])
    if planjson == None:
        _logger.error('The plan {0} does not exist.'.format(args[0]))
        sys.exit(1)

    settings = {}
    settings['working_dir'], settings['app_dir'] = common.set_up_workspace()

    # Load apps settings
    _logger.info('Load apps ' + json.dumps(planjson['app']))
    appSetting = planjson['app']
    settings['appsrc_type'] = appSetting['type']
    settings['app_proj'] = appSetting['project']
    if appSetting['type'] == 'github':
        _logger.info("Load test app from github")
        import git
        giturl = appSetting['uri']
        _logger.info("Github: " + giturl)
        settings['app_name'] = git.extract_repo_name(giturl)

        def app_setup():
            git.setup(giturl, settings['app_dir'])
    else:
        _logger.error('Unknown app source type ' + appSetting['type'])

    # Load cases
    _logger.info('Load test cases ' + json.dumps(planjson['cases']))
    settings['cases'] = planjson['cases']

    # Load runner
    _logger.info('Load test runner ' + json.dumps(planjson['runner']))
    settings['runner'] = planjson['runner']['module']
    def runner_setup():
        _logger.info('Setup runner')
        import_statement = 'from ' + settings['runner'] + ' import get_runner_name, run_case'
        _logger.info('Import statement: ' + import_statement)
        exec(import_statement, globals())

    # Execution!
    app_setup()
    runner_setup()

    resultFiles = [run_case(case, settings) for case in settings['cases']]
    print('Done. Results are saved in:\n' + '\n'.join(resultFiles))

if (__name__ == '__main__'):
    main()


from flask import Flask, render_template, url_for, flash, redirect
from flask_wtf import FlaskForm
from wtforms import SelectMultipleField, SubmitField, SelectField
import predictions

app = Flask(__name__)

app.config['SECRET_KEY'] = '5791628bb0b13ce0c676dfde280ba245'

class SymptomForm(FlaskForm):
    symptom1 = SelectField('Symptom1', choices=[(x,x) for x in predictions.l1])
    symptom2 = SelectField('Symptom2', choices=[(x,x) for x in predictions.l1])
    symptom3 = SelectField('Symptom3', choices=[(x,x) for x in predictions.l1])
    submit = SubmitField('Submit')

symptoms = []
@app.route("/", methods=['GET', 'POST'])
def main():
    form = SymptomForm()
    if form.validate_on_submit():
        flash('Symptoms submitted', 'success')
        value1 = dict(form.symptom1.choices).get(form.symptom1.data)
        value2 = dict(form.symptom2.choices).get(form.symptom2.data)
        value3 = dict(form.symptom3.choices).get(form.symptom3.data)
        symptoms.append(value1)
        symptoms.append(value2)
        symptoms.append(value3)
        disease = predictions.NaiveBayes(symptoms)
        return redirect(url_for('result', name =disease))
    return render_template('main.html', form=form)

@app.route("/result")
def result():
    disease = predictions.NaiveBayes(symptoms)
    return render_template("results.html", name =disease)

if __name__ == "__main__":
    app.run(debug=True)


"""Les Miserables WordCount"""

from pyspark import SparkContext

import os
import re
import sys

sc = SparkContext("local", "Les Mis Word Count")

logFile = "/Users/vboykis/Desktop/data_lake/lesmiserables*.txt"

wordcounts = sc.textFile(logFile).map( lambda x: x.replace(',',' ').replace('.',' ').replace('-',' ').lower()) \
        .flatMap(lambda x: x.split()) \
        .map(lambda x: (x, 1)) \
        .reduceByKey(lambda x,y:x+y) \
        .map(lambda x:(x[1],x[0])) \
        .sortByKey(False)

#print first 10 results
print(wordcounts.take(10))

sc.stop()


# Online Python compiler (interpreter) to run Python online.
# Write Python 3 code in this online editor and run it.
import json
import requests
from bs4 import BeautifulSoup
import smtplib
import configparser

def alert_via_email(url):

    header = {"user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"}
    email_msg = ""
    # url = "https://www.avaloncommunities.com/california/san-jose-apartments/avalon-on-the-alameda/apartments?bedroom=2BD"

    page = requests.get(url, headers=header)
    soup = BeautifulSoup(page.content, 'html.parser')
    cards = soup.find_all(class_="apartment-card")
    email_content = []
    for card in cards:
        if card.find_all(class_="special-tag"):
            text = card.find(class_="content").find(class_="availability").get_text()
            price = card.find(class_="content").find(class_="price").get_text()
            price_in_int = price.replace(',','')[1:6]
            msg = ""
            if 'Dec' in text:
                msg="available special on month december \n availability:"+text+" \n price:"+price[0:6]
            email_content.append(msg)

    if(len(email_content) > 0):
        send_msg = '\n'.join(email_content)
        print(send_msg)
        send_email(send_msg)
    else:
        send_email("nothing new today!")

def send_email(send_msg):
    config = configparser.ConfigParser()
    config.read('email_info.ini')

    sender_email = 'shravaniparsi26@gmail.com'
    receiver_email = 'vishwakt@gmail.com'
    password_for_email = 'iamthegreatpinku'
    SUBJECT = "ALERT : Price Dropped for items your are tracking"
    message = 'Subject: {}\n\n{}'.format(SUBJECT, send_msg)
    fmt = 'From: {}\r\nTo: {}\r\nSubject: {}\r\n{}'

    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.starttls()
    server.login(sender_email,password_for_email)
    print("login is green")
    server.sendmail(sender_email,receiver_email,fmt.format(receiver_email, sender_email, SUBJECT, message).encode('utf-8'))
    print("email is sent")
    server.quit()

def main():
    alert_via_email("https://www.avaloncommunities.com/california/san-jose-apartments/avalon-morrison-park/apartments?bedroom=2BD")
    alert_via_email("https://www.avaloncommunities.com/california/san-jose-apartments/avalon-on-the-alameda/apartments?bedroom=2BD")

if __name__ == '__main__':
    main()


from rest_framework.authtoken.views import ObtainAuthToken
from rest_framework.authtoken.models import Token
from rest_framework.response import Response

from django.shortcuts import get_object_or_404
from django.http import Http404

from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from rest_framework import generics

# importar modelo
from Login.models import Example2
# importar serializer
from Login.serializer import Example2Serializers

class ExampleList2(APIView):
    def get(self, request, format=None):
        print("Metodo get filter")
        queryset = Example2.objects.filter(delete=False)
        # many=True Si aplica si retorno multiples objetos
        serializer = Example2Serializers(queryset, many=True)
        return Response(serializer.data)

    def post(self, request, format=None):
        serializer = Example2Serializers(data=request.data)
        if serializer.is_valid():
            serializer.save()
            datas = serializer.data
            return Response(datas)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class CustonAuthToken(ObtainAuthToken):

    def post(self, request, * args, **kwars):
        serializer = self.serializer_class(data=request.data,
                                           context={
                                               'request': request,
                                           }
                                           )
        serializer.is_valid(raise_exception=True)
        user = serializer.validated_data['user']
        token, created = Token.objects.get_or_create(user=user)

        return Response({
            'token': token.key,
            'user_id': user.pk,
            'username': user.username
        })


# Code for converting a FontFeatures object into feaLib statements
import fontTools.feaLib.ast as feaast

def add_language_system_statements(self, ff):
    self.hoist_languages()
    if len(self.scripts_and_languages.keys()) == 0:
        return
    if len(self.scripts_and_languages.keys()) == 1:
        return
    for s, entry in self.scripts_and_languages.items():
        for l in entry:
            ff.statements.append(feaast.LanguageSystemStatement(s, l))

def asFea(self):
    return self.asFeaAST().asFea()

def asFeaAST(self):
    """Returns this font's features as a feaLib AST object, for later
  translation to AFDKO code."""
    from fontFeatures import Routine

    ff = feaast.FeatureFile()

    add_language_system_statements(self, ff)

    # Preamble
    for k in self.routines:
        assert isinstance(k, Routine)
        if not k.name:
            k.name = self.gensym("Routine_")
        pre = k.feaPreamble(self)
        for s in pre:
            ff.statements.append(s)
    for k, v in self.features.items():
        for r in v:
            pre = r.feaPreamble(self)
            for s in pre:
                ff.statements.append(s)

    for k, v in self.namedClasses.items():
        asclass = feaast.GlyphClass([feaast.GlyphName(x) for x in v])
        ff.statements.append(feaast.GlyphClassDefinition(k, asclass))

    ff.statements.append(feaast.Comment(""))

    for k in self.routines:
        if not k.inlined:
            ff.statements.append(k.asFeaAST())

    for k, v in self.features.items():
        f = feaast.FeatureBlock(k)
        for n in v:
            # If it's a routine and it's in self.routines, use a reference
            if isinstance(n, Routine) and n in self.routines:
                f.statements.append(feaast.LookupReferenceStatement(n.asFeaAST()))
            elif not isinstance(n, Routine):
                r = Routine(rules=[n], parent=self)
                if hasattr(n, "languages"):
                    r.languages = n.languages
                f.statements.append(r.asFeaAST())
            else:
                f.statements.append(n.asFeaAST())
        ff.statements.append(f)
    return ff


from eth2spec.phase0 import spec
from eth_utils import (
    to_dict, to_tuple
)
from gen_base import gen_runner, gen_suite, gen_typing
from preset_loader import loader

@to_dict
def shuffling_case(seed: spec.Bytes32, count: int):
    yield 'seed', '0x' + seed.hex()
    yield 'count', count
    yield 'shuffled', [spec.get_permuted_index(i, count, seed) for i in range(count)]

@to_tuple
def shuffling_test_cases():
    for seed in [spec.hash(spec.int_to_bytes4(seed_init_value)) for seed_init_value in range(30)]:
        for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000]:
            yield shuffling_case(seed, count)

def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
    presets = loader.load_presets(configs_path, 'minimal')
    spec.apply_constants_preset(presets)

    return ("shuffling_minimal", "core", gen_suite.render_suite(
        title="Swap-or-Not Shuffling tests with minimal config",
        summary="Swap or not shuffling, with minimally configured testing round-count",
        forks_timeline="testing",
        forks=["phase0"],
        config="minimal",
        runner="shuffling",
        handler="core",
        test_cases=shuffling_test_cases()))

def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
    presets = loader.load_presets(configs_path, 'mainnet')
    spec.apply_constants_preset(presets)

    return ("shuffling_full", "core", gen_suite.render_suite(
        title="Swap-or-Not Shuffling tests with mainnet config",
        summary="Swap or not shuffling, with normal configured (secure) mainnet round-count",
        forks_timeline="mainnet",
        forks=["phase0"],
        config="mainnet",
        runner="shuffling",
        handler="core",
        test_cases=shuffling_test_cases()))

if __name__ == "__main__":
    gen_runner.run_generator("shuffling", [mini_shuffling_suite, full_shuffling_suite])


#!/bin/env python
# coding:utf-8

# http://qiita.com/aonotas/items/8087d1bb35e0eccb63cb#_reference-ba5b9fb1cf9de5807f63

# 文ごとに学習

# refer to 15_bilstm.py

"""
bidirectional lstm
"""

import argparse
import numpy as np
import chainer.functions as F
import chainer.links as L
from chainer.training import extensions
import chainer
import random
import chainermn
from mpi4py import MPI
import json
import copy

# https://www.ianlewis.org/jp/python-add-class-method
from types import MethodType
import re
import logging

def logging_write(self, value):
    self.debug(value)

def logging_flush(self):
    pass

logger = logging.getLogger("logger")  # logger名loggerを取得
logger.write = MethodType(logging_write, logger)
logger.flush = MethodType(logging_flush, logger)
logger.setLevel(logging.DEBUG)  # loggerとしてはDEBUGで

# 双方向LSTM
class SentenceLSTM(chainer.Chain):
    def __init__(self, n_vocab=10000, n_layers=1, emb_dim=200, hidden_dim=200, use_dropout=0.1):
        super(SentenceLSTM, self).__init__(
            word_embed=L.EmbedID(n_vocab, emb_dim, ignore_label=-1),
            l1=L.NStepLSTM(n_layers=n_layers, in_size=emb_dim,
                           out_size=hidden_dim, dropout=use_dropout)
        )
        self.use_dropout = use_dropout

    def __call__(self, xs):
        hx = None
        cx = None
        x_len = [len(x) for x in xs]
        x_section = np.cumsum(x_len[:-1])
        ex = self.word_embed(F.concat(xs, axis=0))
        exs = F.split_axis(ex, x_section, 0, force_tuple=True)

        _hy, cy, ys = self.l1(hx=hx, cx=cx, xs=exs)
        return ys

class SentenceBiLSTM(chainer.Chain):
    def __init__(self, n_vocab=10000, n_layers=1, emb_dim=200, hidden_dim=200, use_dropout=0.1):
        super(SentenceBiLSTM, self).__init__(
            word_embed=L.EmbedID(n_vocab, emb_dim, ignore_label=-1),
            l1=L.NStepBiLSTM(n_layers=n_layers, in_size=emb_dim,
                             out_size=hidden_dim, dropout=use_dropout)
        )
        self.use_dropout = use_dropout

    def __call__(self, xs):
        hx = None
        cx = None
        x_len = [len(x) for x in xs]
        x_section = np.cumsum(x_len[:-1])
        ex = self.word_embed(F.concat(xs, axis=0))
        exs = F.split_axis(ex, x_section, 0, force_tuple=True)
        _hy, cy, ys = self.l1(hx=hx, cx=cx, xs=exs)
        return ys

class LSTMExecutor:
    pass

def test_model(xp,dict,modelfile,test_data_list):
    model = SentenceBiLSTM(n_vocab=len(dict), hidden_dim=len(dict))
    chainer.serializers.load_npz(modelfile, model)
    optimizer = chainer.optimizers.Adam()
    optimizer.setup(model)

    ret = []
    dict_inv = {v: k for k, v in dict.items()}
    for test_data in test_data_list:

        test_result = []
        # 1 word loop, omit last word
        for i in range(len(test_data) - 1):
            input_id = test_data[i]
            input_word = dict_inv[input_id]
            x = [chainer.Variable(xp.asarray([input_id], dtype=xp.int32))]
            # print("input word: {}".format(dict_inv[word]))
            actual_id = test_data[i + 1]
            actual_word = dict_inv[actual_id]

            # Bi-LSTMの場合の場合は双方向のLSTMのconcatが入っています。
            # http://qiita.com/aonotas/items/8e38693fb517e4e90535#hy
            predict = model(x)[0]
            predict_id = xp.argmax(predict.data)
            predict_word = dict_inv[predict_id]

            words_probability = {}
            for key in dict_inv.keys():
                if predict.data[0][key] > 0:
                    words_probability[dict_inv[key]] = predict.data[0][key]

            words_probability_sorted = sorted(words_probability.items(), key=lambda x: x[1],
                                              reverse=True)  # sort by value

            actual_probability = predict[0][actual_id]

            # for word in words:
            # print("predict word: {}".format(word))
            result = {"input_word": input_word, "predict_word": predict_word, "actual_word": actual_word,
                      "actual_probability": actual_probability, "suggestions": words_probability_sorted[0:9]}
            test_result.append(result)
        ret.append(test_result)
    return (ret)

def test(xp, args):
    with open(args.dictdata, "r") as f:
        dict = json.load(f)

    with open(args.testdata, "r") as f:
        jdata = json.load(f)

    docs = [x for x in jdata["data"]]
    revdocs = []
    for tmp in docs:
        tdoc = copy.deepcopy(tmp)
        tdoc.reverse()
        revdocs.append(tdoc)

    fw_result=test_model(xp,dict,args.modelfile+".fw",docs)
    bw_result=test_model(xp,dict,args.modelfile+".bw",revdocs)

    def list_merge(flist,blist):
        sug_dict={}
        alllist=copy.deepcopy(flist)
        alllist.extend(blist)
        for l in alllist:
            if l[0] in sug_dict:
                if sug_dict[l[0]]>l[1]:
                    continue
            sug_dict[l[0]]=l[1]

        return( sorted(sug_dict.items(), key=lambda x: x[1],reverse=True)  )# sort by value

    for fw, bw in zip(fw_result, bw_result):
        if bw is not None:
            bw.reverse()  # 逆順なので元に戻す
        # fw,bwはずれているので補正
        fw.insert(0, {"input_word": None, "predict_word": None, "actual_word": None, "actual_probability": None,
                      "suggestions": None})
        bw.append({"input_word": None, "predict_word": None, "actual_word": None, "actual_probability": None,
                   "suggestions": None})
        now_words = []
        sug_lists = []
        # bw.reverse() # 順方向へ
        for f, b in zip(fw, bw):
            actual_word = f["actual_word"]
            if actual_word is None:
                actual_word = b["actual_word"]
            if f["predict_word"] != f["actual_word"] and b["predict_word"] != b["actual_word"]:
                # check other word
                if f["actual_word"] in f["suggestions"] or b["actual_word"] in b["suggestions"]:
                    now_words.append(actual_word)
                    sug_lists.append(None)
                else:
                    # find error word
                    now_words.append("<<" + actual_word + ">>")
                    list_sug = list_merge(f["suggestions"], b["suggestions"])
                    sug_lists.append(list_sug)
                    pass
            else:
                now_words.append(actual_word)
                sug_lists.append(None)
        print("=========")
        print(now_words)
        print(sug_lists)

def train(xp, args):
    # Prepare ChainerMN communicator.
    if args.gpu:
        if args.communicator == 'naive':
            print("Error: 'naive' communicator does not support GPU.\n")
            exit(-1)
        comm = chainermn.create_communicator(args.communicator)
        device = comm.intra_rank

        chainer.cuda.get_device_from_id(device).use()
        chainer.using_config("use_cudnn", True)

    else:
        if args.communicator != 'naive':
            print('Warning: using naive communicator '
                  'because only naive supports CPU-only execution')
        comm = chainermn.create_communicator('naive')
        device = -1

    if comm.mpi_comm.rank == 0:
        print('==========================================')
        print('Num process (COMM_WORLD): {}'.format(MPI.COMM_WORLD.Get_size()))
        if args.gpu:
            print('Using GPUs')
        print('Using {} communicator'.format(args.communicator))
        print('Num Minibatch-size: {}'.format(args.batchsize))
        print('Num epoch: {}'.format(args.epoch))
        print('vocab: {}'.format(args.vocab))
        print('==========================================')

    # with open(args.dictdata,"r") as f:
    #    dict=json.load(f)
    # n_vocab=len(dict)
    n_vocab = args.vocab

    if comm.rank == 0:
        with open(args.traindata, "r") as f:
            jdata = json.load(f)
        if args.reverse:
            # http://oregengo.hatenablog.com/entry/2016/03/09/132214
            train = [np.array(x, dtype=np.int32)[::-1] for x in jdata["data"]]
        else:
            train = [np.array(x, dtype=np.int32) for x in jdata["data"]]
        train = list(filter(lambda x: len(x) >= args.minlength, train))
    else:
        train = None

    logger.debug("scatter..")
    train = chainermn.scatter_dataset(train, comm)

    logger.debug("iterator..")
    train_iter = chainer.iterators.SerialIterator(train, args.batchsize)

    logger.debug("model..")
    # モデルの準備
    if args.bidirectional:
        lstm = SentenceBiLSTM(n_vocab=n_vocab, n_layers=1, hidden_dim=n_vocab)
    else:
        lstm = SentenceLSTM(n_vocab=n_vocab, n_layers=1, hidden_dim=n_vocab)

    # このようにすることで分類タスクを簡単にかける
    # 詳しくはドキュメントを読むとよい
    def sum_softmax_cross_entropy( ys, ts):
        loss = 0
        for y, t in zip(ys, ts):
            loss += chainer.functions.softmax_cross_entropy(y, t)
        return loss

    model = L.Classifier(lstm, lossfun=sum_softmax_cross_entropy)
    if device >= 0:
        chainer.cuda.get_device_from_id(device).use()
        model.to_gpu()

    model.compute_accuracy = False
    # for param in model.params():
    #    data = param.data
    #    data[:] = np.random.uniform(-0.2, 0.2, data.shape)

    # optimizerの設定
    # Create a multi node optimizer from a standard Chainer optimizer.
    logger.debug("optimizer..")
    optimizer = chainermn.create_multi_node_optimizer(
        chainer.optimizers.Adam(), comm)
    optimizer.setup(model)

    def convert(batch, device):

        x = [chainer.Variable(xp.asarray(batch[j][:-1], dtype=xp.int32)) for j in
             range(len(batch))]
        t = [chainer.Variable(xp.asarray(batch[j][1:], dtype=xp.int32)) for j in
             range(len(batch))]
        return (x, t)

    logger.debug("updater..")
    updater = chainer.training.StandardUpdater(train_iter, optimizer, converter=convert, device=device)
    trainer = chainer.training.Trainer(updater, (args.epoch, 'epoch'), out=args.outdir)

    # Some display and output extensions are necessary only for one worker.
    # (Otherwise, there would just be repeated outputs.)
    if comm.rank == 0:
        trainer.extend(chainer.training.extensions.dump_graph('main/loss'))
        trainer.extend(chainer.training.extensions.LogReport())
        trainer.extend(chainer.training.extensions.PrintReport(
            ['epoch', 'main/loss', 'validation/main/loss',
             'main/accuracy', 'validation/main/accuracy', 'elapsed_time'], out=logger))
        trainer.extend(chainer.training.extensions.ProgressBar(out=logger))
        #trainer.extend(chainer.training.extensions.snapshot(filename='snapshot_epoch-{.updater.epoch}'))
        #trainer.extend(extensions.snapshot_object(model.predictor, filename='model_epoch-{.updater.epoch}'))

    logger.debug("trainer.run")
    trainer.run()

    suff=".fw"
    if args.reverse:
        suff=".bw"
    model.to_cpu()
    chainer.serializers.save_npz(args.modelfile+suff, model.predictor)

def main():
    # 引数の処理
    parser = argparse.ArgumentParser(description='分かち書きしたファイルからWord2Vecを作成')
    parser.add_argument('--gpu', "-g", default=False, type=bool, help='use gpu or not')
    parser.add_argument('--modelfile', default="/Volumes/DATA/data/kiji/yakyu.model", help='出力モデルファイル名')
    parser.add_argument('--dictdata', default="/Volumes/DATA/data/kiji/yakyu_dict.json", help='dictionary data file')
    parser.add_argument('--traindata', default="/Volumes/DATA/data/kiji/yakyu.json", help='入力ファイル名')
    parser.add_argument('--bidirectional', dest="bidirectional", action="store_true", help='LSTM method')
    parser.set_defaults(bidirectional=True)

    parser.add_argument('--reverse', dest="reverse", action="store_true")
    parser.set_defaults(reverse=True)
    parser.add_argument('--mode', default="test", choices=["train", "test"], help='train or test')
    parser.add_argument('--testdata', default="/Volumes/DATA/data/kiji/yakyu_test.json", help='入力ファイル名')
    parser.add_argument('--batchsize', '-b', type=int, default=10,
                        help='Number of images in each mini-batch')
    parser.add_argument('--communicator', type=str,
                        default='naive', help='Type of communicator')
    parser.add_argument('--epoch', '-e', type=int, default=20,
                        help='Number of sweeps over the dataset to train')
    parser.add_argument('--outdir', '-o', default='/Volumes/DATA/data/kiji/result', help="output directory")
    parser.add_argument('--vocab', type=int, default=23, help='Directory size')
    parser.add_argument('--minlength', type=int, default=5, help='minimum size of words in sentence ')
    parser.add_argument('--logfile', type=str, default="41_proofreading.log", help='minimum size of words in sentence ')

    args = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG, filename=args.logfile, format="%(asctime)s %(levelname)-7s %(message)s")

    random.seed(12345)

    xp = np
    if args.gpu:
        import cupy
        xp = cupy

    if args.mode == "test":
        test(xp, args)
    else:
        train(xp, args)

if __name__ == '__main__':
    main()


#!/usr/bin/python3
from accessoryFunctions.accessoryFunctions import make_path, MetadataObject, printtime
from accessoryFunctions.metadataprinter import MetadataPrinter
from spadespipeline.typingclasses import Resistance, Virulence
from sixteenS.sixteens_full import SixteenS as SixteensFull
from MLSTsippr.mlst import GeneSippr as MLSTSippr
from customsippr.customsippr import CustomGenes
from sipprverse_reporter.reports import Reports
from sipprCommon.objectprep import Objectprep
from sipprCommon.sippingmethods import Sippr
from serosippr.serosippr import SeroSippr
import MASHsippr.mash as mash
from argparse import ArgumentParser
import multiprocessing
import subprocess
import time
import os

__author__ = 'adamkoziol'

class Sipprverse(object):

    def main(self):
        """
        Run the necessary methods in the correct order
        """
        printtime('Starting {} analysis pipeline'.format(self.analysistype), self.starttime)
        # Create the objects to be used in the analyses
        objects = Objectprep(self)
        objects.objectprep()
        self.runmetadata = objects.samples
        self.threads = int(self.cpus / len(self.runmetadata.samples)) if self.cpus / len(self.runmetadata.samples) > 1 \
            else 1
        if self.genesippr:
            # Run the genesippr analyses
            self.analysistype = 'genesippr'
            self.targetpath = os.path.join(self.reffilepath, self.analysistype)
            Sippr(self, 0.90)
            # Create the reports
            self.reports = Reports(self)
            Reports.reporter(self.reports)
        if self.sixteens:
            # Run the 16S analyses
            SixteensFull(self, self.commit, self.starttime, self.homepath, 'sixteens_full', 0.985)
        if self.rmlst:
            MLSTSippr(self, self.commit, self.starttime, self.homepath, 'rMLST', 1.0, True)
        if self.resistance:
            # ResFinding
            res = Resistance(self, self.commit, self.starttime, self.homepath, 'resfinder', 0.70, False, True)
            res.main()
        if self.virulence:
            vir = Virulence(self, self.commit, self.starttime, self.homepath, 'virulence', 0.95, False, True)
            vir.reporter()
        if self.closestreference:
            self.pipeline = True
            mash.Mash(self, 'mash')
        if self.gdcs:
            # Run the GDCS analysis
            self.analysistype = 'GDCS'
            self.targetpath = os.path.join(self.reffilepath, self.analysistype)
            Sippr(self, 0.95)
            # Create the reports
            Reports.gdcsreporter(self.reports)
        if self.mlst:
            self.genus_specific()
            MLSTSippr(self, self.commit, self.starttime, self.homepath, 'MLST', 1.0, True)
        # Optionally perform serotyping
        if self.serotype:
            self.genus_specific()
            SeroSippr(self, self.commit, self.starttime, self.homepath, 'serosippr', 0.90, True)
        if self.user_genes:
            custom = CustomGenes(self)
            custom.main()
        # Print the metadata
        printer = MetadataPrinter(self)
        printer.printmetadata()

    def genus_specific(self):
        """
        For genus-specific targets, MLST and serotyping, determine if the closest refseq genus is known - i.e. if 16S
        analyses have been performed. Perform the analyses if required
        """
        # Initialise a variable to store whether the necessary analyses have already been performed
        closestrefseqgenus = False
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                try:
                    closestrefseqgenus = sample.general.closestrefseqgenus
                except KeyError:
                    pass
        # Perform the 16S analyses as required
        if not closestrefseqgenus:
            printtime('Must perform 16S analyses', self.starttime)
            # Run the 16S analyses
            SixteensFull(self, self.commit, self.starttime, self.homepath, 'sixteens_full', 0.985)

    def __init__(self, args, pipelinecommit, startingtime, scriptpath):
        """
        :param args: command line arguments
        :param pipelinecommit: pipeline commit or version
        :param startingtime: time the script was started
        :param scriptpath: home path of the script
        """
        # Initialise variables
        self.commit = str(pipelinecommit)
        self.starttime = startingtime
        self.homepath = scriptpath
        self.args = args
        # Define variables based on supplied arguments
        self.path = os.path.join(args.outputpath)
        assert os.path.isdir(self.path), u'Supplied path is not a valid directory {0!r:s}'.format(self.path)
        self.sequencepath = os.path.join(args.sequencepath)
        self.seqpath = self.sequencepath
        self.targetpath = os.path.join(args.referencefilepath)
        # ref file path is used to work with submodule code with a different naming scheme
        self.reffilepath = self.targetpath
        self.reportpath = os.path.join(self.path, 'reports')
        make_path(self.reportpath)
        assert os.path.isdir(self.targetpath), u'Target path is not a valid directory {0!r:s}' \
            .format(self.targetpath)
        # Set the custom cutoff value
        self.cutoff = args.customcutoffs
        # Use the argument for the number of threads to use, or default to the number of cpus in the system
        self.cpus = int(args.numthreads if args.numthreads else multiprocessing.cpu_count())
        self.closestreference = args.closestreference
        self.gdcs = args.gdcs
        self.genesippr = args.genesippr
        self.mlst = args.mlst
        self.resistance = args.resistance
        self.rmlst = args.rmlst
        self.serotype = args.serotype
        self.sixteens = args.sixteens
        self.virulence = args.virulence
        self.averagedepth = args.averagedepth
        try:
            self.user_genes = os.path.join(args.user_genes)
            assert os.path.isfile(self.user_genes), 'Cannot find user-supplied target file: {targets}. Please ' \
                                                    'double-check name and path of file'\
                .format(targets=self.user_genes)
        except TypeError:
            self.user_genes = args.user_genes
        # Set all the analyses to True if the full_suite option was selected
        if args.full_suite:
            self.closestreference = True
            self.gdcs = True
            self.genesippr = True
            self.mlst = True
            self.resistance = True
            self.rmlst = True
            self.serotype = True
            self.sixteens = True
            self.virulence = True
        self.reports = str()
        self.threads = int()
        self.runmetadata = MetadataObject()
        self.taxonomy = {'Escherichia': 'coli', 'Listeria': 'monocytogenes', 'Salmonella': 'enterica'}
        self.analysistype = 'GeneSippr'
        self.pipeline = False
        self.logfile = os.path.join(self.path, 'log')

if __name__ == '__main__':
    # Get the current commit of the pipeline from git
    # Extract the path of the current script from the full path + file name
    homepath = os.path.split(os.path.abspath(__file__))[0]
    # Find the commit of the script by running a command to change to the directory containing the script and run
    # a git command to return the short version of the commit hash
    commit = subprocess.Popen('cd {} && git rev-parse --short HEAD'.format(homepath),
                              shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
    # Parser for arguments
    parser = ArgumentParser(description='Performs GeneSipping on folder of FASTQ files')
    parser.add_argument('-o', '--outputpath',
                        required=True,
                        help='Path to directory in which report folder is to be created')
    parser.add_argument('-s', '--sequencepath',
                        required=True,
                        help='Path of .fastq(.gz) files to process.')
    parser.add_argument('-r', '--referencefilepath',
                        required=True,
                        help='Provide the location of the folder containing reference database')
    parser.add_argument('-a', '--averagedepth',
                        default=2,
                        help='Cutoff value for mapping depth to use when parsing BAM files.')
    parser.add_argument('-n', '--numthreads',
                        help='Number of threads. Default is the number of cores in the system')
    parser.add_argument('-c', '--customcutoffs',
                        default=0.90,
                        help='Custom cutoff values')
    parser.add_argument('-F', '--full_suite',
                        action='store_true',
                        default=False,
                        help='Perform all the built-in GeneSippr analyses (AMR, GDCS, Genesippr, MASH, MLST, '
                             'rMLST, Serotype, SixteenS, and Virulence')
    parser.add_argument('-A', '--resistance',
                        action='store_true',
                        default=False,
                        help='Perform AMR analysis on samples')
    parser.add_argument('-C', '--closestreference',
                        action='store_true',
                        default=False,
                        help='Determine closest RefSeq match with mash')
    parser.add_argument('-G', '--genesippr',
                        action='store_true',
                        default=False,
                        help='Perform GeneSippr analysis on samples')
    parser.add_argument('-M', '--mlst',
                        action='store_true',
                        default=False,
                        help='Perform MLST analysis on samples')
    parser.add_argument('-Q', '--gdcs',
                        action='store_true',
                        default=False,
                        help='Perform GDCS Quality analysis on samples')
    parser.add_argument('-R', '--rmlst',
                        action='store_true',
                        default=False,
                        help='Perform rMLST analysis on samples')
    parser.add_argument('-S', '--serotype',
                        action='store_true',
                        default=False,
                        help='Perform serotype analysis on samples determined to be Escherichia')
    parser.add_argument('-U', '--user_genes',
                        default=False,
                        help='Name and path of user provided (multi-)FASTA file of genes to run against samples')
    parser.add_argument('-V', '--virulence',
                        action='store_true',
                        default=False,
                        help='Perform virulence analysis on samples')
    parser.add_argument('-X', '--sixteens',
                        action='store_true',
                        default=False,
                        help='Perform 16S typing of samples')
    # Get the arguments into an object
    arguments = parser.parse_args()

    # Define the start time
    start = time.time()

    # Run the script
    sippr = Sipprverse(arguments, commit, start, homepath)
    sippr.main()

    # Print a bold, green exit statement
    print('\033[92m' + '\033[1m' + "\nElapsed Time: %0.2f seconds" % (time.time() - start) + '\033[0m')


import os, time, random
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
#### HOW TO USE ################################################################################################
'''
If you aren't using 64 bit linux delete the chromedriver from assets and download the correct drive for your os:
https://chromedriver.storage.googleapis.com/index.html?path=2.30/
Add the driver to the assets folder

Make sure that:
You have PIP installed
You have google chrome installed

For all OSes:
Use pip to install selenium
pip install -U selenium

Modify the settings
'''
################################################################################################################

#is linux or mac
unix = True;

#Who's profile to spam (ie https://victim.sarahah.com/)
victim = "victim"

#A random message will get picked each iteration
messages = [
"Message 1",
"Message 2",
"Message 3"
]

#author tag (leave blank for none)
author = ""

#use https ("s" = https, "" = http)
https = "s"

if unix:
	os.system("clear")

print("Yeasty Sarahah Bot v1")
print("Launching chrome automation...")
chromedriver = "assets/chromedriver"
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)

count = 0
mes = len(messages)
while 1:
	count += 1
	try:
		message = messages[random.randint(0, mes-1)]
		print("Spamming " + victim + " with " + message)
		driver.get("http" + https + "://" + victim + ".sarahah.com/")
		q = driver.find_element_by_id("Text")
		q.send_keys(message + author)
		driver.find_element_by_id("Send").click()
		time.sleep(3)
	except:
		driver.quit()
		os.system("python auto.py")


#! python3

# TODO:
# Change logo file
# insert inpute for client name and test number
# insert inputs for test standard and furnace size
# insert code that actually fucking does something

import tkinter, shutil, os
from tkinter import *
import PIL
from PIL import Image,ImageTk #pip install pillow

root = Tk()
root.geometry ('555x555')
root.title('imgeCFR')

imge=Image.open('C:\\Users\\gw444\\Downloads\\logo.png');
photo=ImageTk.PhotoImage(imge);

lab=Label(image=photo);
lab.pack();

fn=StringVar()
ln=StringVar()
var1=StringVar()
var2=StringVar()

def exit1():
    exit()
def printt(): #printt command for button
    clientName=fn.get()
    testNo=ln.get()
    testStd=var1.get()
    testScl=var2.get()
    print(f'The test code (and folder name) is {clientName}{testNo}')
    print(f'This is a {testScl} {testStd} test')

''
def makeFoldr():
    clientName=fn.get()
    testNo=ln.get()
    testStd=var1.get()
    testScl=var2.get()
    folderName = clientName + ' ' + testNo;
    folDir = ('C:\TEST REPORTS DATA PHOTOS\\' + folderName)
    os.makedirs('C:\TEST REPORTS DATA PHOTOS\\' + folderName);
    os.makedirs('C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\data');
    os.makedirs('C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\Survey Photos');
    os.makedirs('C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\Test Photos');
    os.makedirs('C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\Obsolete');
    if testScl == 'Full':
        if testStd == 'BS':
                shutil.copy('C:\Templates and Forms CURRENT\Templates for Report writing\Test Data Sheets\Data Sheets\Ful Furnace\BS Test\Data Template Master BS Ful.xlsx', 'C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\data');
        else:
                shutil.copy('C:\Templates and Forms CURRENT\Templates for Report writing\Test Data Sheets\Data Sheets\Ful Furnace\EN Test\Data Template Master EN Ful.xlsx', 'C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\data');
    else:
        if testStd == 'BS':
                shutil.copy('C:\Templates and Forms CURRENT\Templates for Report writing\Test Data Sheets\Data Sheets\Med Furnace\BS Test\Data Template Master Med BS.xlsx', 'C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\data');
        else:
                shutil.copy('C:\Templates and Forms CURRENT\Templates for Report writing\Test Data Sheets\Data Sheets\Med Furnace\EN Test\Data Template Master EN Med.xlsx', 'C:\TEST REPORTS DATA PHOTOS\\' + folderName +'\data');
    os.startfile(folDir)

##window = Tk()
##window.geometry('600x300')
##window.title('Welcome')

#Create title
label1=Label(root,text='Welcome to FolderMakerLite',fg='black',bg='white',relief='flat',font=('arial',16,'bold')).pack()
#label1.place(x=90,y=53)

#Client name: text
label2=Label(root,text='Client name:',fg='black',bg='white',relief='flat',font=('arial',10,'bold'))
label2.place(x=90,y=190)

#Client name user entry
entry1=Entry(root,textvar=fn)
entry1.place (x=240, y=190)

#Test number: text
label3=Label(root,text='Test number:',fg='black',bg='white',relief='flat',font=('arial',10,'bold'))
label3.place(x=90,y=220)

#Test number user entry
entry2=Entry(root,textvar=ln)
entry2.place (x=240, y=220)

#Test standard: text
label4=Label(root,text='Test standard:',fg='black',bg='white',relief='flat',font=('arial',10,'bold'))
label4.place(x=90,y=250)

#Test standard dropbox
list1=['BS','EN']
droplist=OptionMenu(root,var1,*list1)
var1.set('Select standard')
droplist.config(width=15)
droplist.place(x=240, y=250)

#Furnace: text
label5=Label(root,text='Furnace:',fg='black',bg='white',relief='flat',font=('arial',10,'bold'))
label5.place(x=90,y=280)

#Furnace: dropbox
list2=['Med','Full']
droplist=OptionMenu(root,var2,*list2)
var2.set('Select standard')
droplist.config(width=15)
droplist.place(x=240, y=280)

button1=Button(root,text='Produce fucking folder',fg='black',bg='red',relief='ridge',font=('arial',10,'bold'),command=makeFoldr)
button1.place(x=90,y=350)

button2=Button(root,text='Just fuck off',fg='black',bg='red',relief='ridge',font=('arial',10,'bold'), command=exit1)
button2.place(x=350,y=350)

##
##root.mainloop()


import numpy as np
import matplotlib.pyplot as plt
from utils import tf

s = tf([1, 0], 1)
"""Set-up plant and Disturbance model"""
G = 200/((10*s + 1)*(0.05*s + 1)*(0.05*s + 1))
G_d = 100/(10*s + 1)

"""The four different controllers in question"""
K0 = ((10*(10*s + 1))*(0.1*s + 1))/(s*200*(0.01*s + 1))
K1 = 0.5
K2 = 0.5*(s + 2)/s
K3 = (0.5*(s + 2)*(0.05*s + 1))/(s*(0.005*s + 1))

w = np.logspace(-2, 2, 100)
s = 1j*w

"""Closed loop tf"""
L0 = G*K0
L1 = G*K1
L2 = G*K2
L3 = G*K3

tspan = np.linspace(0, 3, 100)

"""Step response in the disturbance"""
cont0 = (1/(1+L0))*G_d
cont1 = (1/(1+L1))*G_d
cont2 = (1/(1+L2))*G_d
cont3 = (1/(1+L3))*G_d

[t, y0] = cont0.step(0, tspan)
[t, y1] = cont1.step(0, tspan)
[t, y2] = cont2.step(0, tspan)
[t, y3] = cont3.step(0, tspan)

"""Plot results"""
plt.subplot(1, 2, 1)
plt.loglog(w, np.abs(L0(s)))
plt.loglog(w, np.abs(L1(s)))
plt.loglog(w, np.abs(L2(s)))
plt.loglog(w, np.abs(L3(s)))

plt.subplot(1, 2, 2)
plt.plot(t, y0)
plt.plot(t, y1)
plt.plot(t, y2)
plt.plot(t, y3)
plt.show()


from itertools import combinations
import time

def factors(n):
    for x in xrange(2,n):
        if n%x == 0:
            return (x,) + factors(n/x)
    return (n,)

start = time.time()

i = 2
triang = 3
cond = True

while cond:
    div = 2 # 1 and triang counted
    fac = list(factors(triang))
    # print(triang)
    # print(fac)
    for j in xrange(len(fac)):
        b = set(combinations(fac,j+1))
        # c = list(combinations(fac, j+1))
        div += len(b)
    if div >= 500:
        print(triang)
        cond = False
    else:
        i = i+1
        triang += i

end = time.time()
print(end-start)

"""
second method: shiiiit
"""

# y = 1
# div1 = 1
# div2 = 1

# def factors2(n):
#     global y, div1, div2
#     for x in range(2,n):
#         if n%x == 0:
#             if y != x:
#                 print(x)
#                 y = x
#                 div2 = div1
#                 div1 = 1
#                 print("div2 " + str(div2))
#             else:
#                 div1 += 1
#             return (x,) + factors2(n/x)
#     # print(n)
#     return (n,)

# a = factors2(600)
# print(a)

# div = dict()

# def factors3(n):
#     global div
#     for x in range(2,n):
#         if n%x == 0:
#             try:
#                 div[x] += 1
#             except:
#                 div[x] = 1
#             return (x,) + factors3(n/x)
#     try:
#         div[n] += 1
#     except:
#         div[n] = 1
#     return (n,)

# a = factors3(144)


from collections import namedtuple

# the order of the keys of this dictionary
FunctionalGroup = namedtuple("FunctionalGroup", ["resName", "donor", "acceptor", "atoms"])
FunctionalGroup.__doc__ = """Store hydrogen bonding information about a
functional group as well as information that can be used to position it
in three-space.

Attributes
----------
resName : str
    Name of the functional group.
donor : bool
    True if the functional group can be a donor in a hydrogen bond.
acceptor : bool
    True if the functional group can be an acceptor in a hydrogen bond.
atoms : list of str
    List of three atom names that are used to construct a coordinate
    frame to describe the position of the functional group in three-space.
"""

functional_groups = {
    "OH_": FunctionalGroup("hydroxide", True, True, ["CV", "OH", "HH"]),
    "G__": FunctionalGroup("guanidinium", True, False, ["CZ", "NH1", "NH2"]),
    "I__": FunctionalGroup("imidazole", True, True, ["ND1", "CD2", "NE2"]),
    # imidazole tautomer
    "ID_": FunctionalGroup("imidazole_D", True, True, ["ND1", "CD2", "NE2"]),
    "A__": FunctionalGroup("amine", True, False, ["NZ", "1HZ", "2HZ"]),
    "C__": FunctionalGroup("carboxylate", False, True, ["CD", "OE1", "OE2"]),
    "CA_": FunctionalGroup("carboxamide", True, True, ["CG", "OD1", "ND2"])
}

ResInfo = namedtuple("ResInfo", ["grp", "atoms"])
ResInfo.__doc__ = """Store functional group information about an amino
acid as well as information that can be used to position it in
three-space.

Attributes
----------
grp : str
    Name of a functional group.
atoms : list of str
    List of three atom names that are used to construct a coordinate
    frame to describe the position of the functional group of the amino
    acid in three-space.
"""

def resolve_rays(groups):
    """Expand list of interaction groups into the appropriate ray
    selection methods.

    Parameters
    ----------
    groups : list of str
        Interaction groups to search in.

    Returns
    -------
    list of tuple of str, function
        Names of groups and the corresponding ray selectors for the
        groups.
    """

    from . import (
        bidentate, network, catpi
    )
    ray_bins = {
        "bidentate": [
            ("sc_sc", bidentate.sc_sc_rays),
            ("sc_scbb", bidentate.sc_scbb_rays),
            ("sc_bb", bidentate.sc_bb_rays)
        ],
        "network": [
            ("acceptor_acceptor", network.acceptor_acceptor_rays),
            ("acceptor_donor", network.donor_acceptor_rays),
            ("donor_acceptor", network.donor_acceptor_rays),
            ("donor_donor", network.donor_donor_rays)
        ],
        "cation-pi": [("cation-pi", catpi.rays)]
    }

    ray_selectors = [ ]

    for group in groups:
        for k in ray_bins:
            for v in ray_bins[k]:
                if (group == "all" or group in [k, v[0]]):
                    ray_selectors.append(v)

    return ray_selectors


import os
import re
import cv2
import json
import platform

from subprocess import Popen, PIPE, STDOUT
from barez import *

def _get_command_path(command):
    if platform.system() == 'Windows':
        command = command + '.exe'
    return command

def _pdf_info(pdf_path):
    command = [_get_command_path("pdfinfo"), pdf_path]
    proc = Popen(command, stdout=PIPE, stderr=PIPE)
    out, err = proc.communicate()

    page_count = int(re.search(r'Pages:\s+(\d+)',
                      out.decode("utf8", "ignore")).group(1))

    pdf_info = []
    for i in range(page_count):
        cmd = [_get_command_path("pdfinfo"), '-f',
               str(i + 1), '-l', str(i + 1), pdf_path]
        proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()
        # Page size: 792 x 1224 pts
        page_size = int(re.search('Page.*size:\s+(\d+)',
                        out.decode("utf8", "ignore")).group(1))
        pdf_info.append((str(i+1),  page_size))
    return pdf_info

def textline2ocr(file_path='./input/GED00264650_03_001_PROJECT DOCUMENTS_S5_30 PDFA.pdf',
                 path_out='debug', thread_num=12):
    """
    file_path: path of pdf file
    path_out: path out of image(s)
    thread_num: number of thread
    output is jpeg file
    """
    assert 'pdf' in file_path
    base_name = file_path.split('/')[-1][:-4]
    page_info = _pdf_info(file_path)

    large_page = [(pnum, psize) for (pnum, psize) in page_info if psize > 3000]
    small_page = [(pnum, psize) for (pnum, psize) in page_info if psize <= 3000]

    processes = []
    proc_cnt = 0

    for page_num, page_size in small_page + large_page:
        if (page_num, page_size) in large_page:
            dpi = 150
        else:
            dpi = 300

        img = cv2.imread('./input/GED00264650_03_001_PROJECT DOCUMENTS_S5_30 PDFA/images/{}.jpg'.format(int(page_num) - 1))

        command = ['pdftotext', '-r', str(dpi), '-f', page_num,
                   '-l', page_num, '-bbox-layout', file_path]
        p = Popen(command, stdout=PIPE)
        stdout, _ = p.communicate()

        # open html file
        html_file_path = file_path[:-3] + 'html'
        html_file = open(html_file_path, 'r')

        # <line xMin="3835.000000" yMin="2381.166667" xMax="5008.999500" yMax="2418.704167">
        html_data = ''.join(html_file.readlines())

        line_pattern = re.compile(r"<line xMin=\"([\d.]+)\" yMin=\"([\d.]+)\" xMax=\"([\d.]+)\" yMax=\"([\d.]+)\">")
        bbox = line_pattern.findall(html_data)

        json_out = {
            "imageData": None,
            "imagePath": "{}.jpg".format(int(page_num) - 1),
            "lineColor": [0, 255, 0, 128],
            "fillColor": [254, 0, 0, 128],
            "shapes": [],
            "flags": {}}

        # get the list of text line
        for xMin, yMin,xMax, yMax in bbox:
            top = int(float(xMin)) - 2
            left = int(float(yMin)) - 2
            width = int(float(xMax)) - top  + 4
            height = int(float(yMax)) - left + 4

            command = ['pdftotext', '-r', str(dpi), '-f', page_num,
                       '-l', page_num, '-x', str(top), "-y",
                       str(left), "-W", str(width), "-H",
                       str(height), file_path, "-"]
            p = Popen(command, stdout=PIPE)
            stdout, _ = p.communicate()

            ocr_value = stdout.decode('utf-8')

            json_out["shapes"].append({
                "label": ocr_value,
                "points": [
                    [top, left],
                    [top + width, left],
                    [top + width, left + height],
                    [top, left + height]
                ],
                "line_color": None,
                "fill_color": None
            })

            cv2.putText(img,
                        ocr_value,
                        (top - 5, left - 5),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.5, (0,0,255), 2)
        cv2.imwrite("./visualize/{}.jpg".format(int(page_num) - 1), img)

        json.dump(json_out, open("./debug/{}.json".format(int(page_num) - 1), 'w'))

if __name__ == "__main__":
    textline2ocr()


from sklearn.base import BaseEstimator
import numpy as np

__all__ = [
    '_StagedClassifier',
    '_StagedRegressor',
    '_cat_staged_predict',
    '_xgb_staged_predict',
    '_lgb_staged_predict',
    '_get_scores',
]

class _StagedClassifier(BaseEstimator):

    _estimator_type = 'classifier'

    def predict_proba(self, X):
        return X

    def predict(self, X):
        y = X[:, 1] > 0.5
        y = y.astype(int)
        return y

class _StagedRegressor(BaseEstimator):

    _estimator_type = 'regressor'

    def predict(self, X):
        return X

def _xgb_staged_predict(estimator, X, max_iter=0, step=1):

    booster = estimator.get_booster()

    leafs = estimator.apply(X, max_iter)
    M = leafs.shape[0]
    N = leafs.shape[1]

    trees = np.mgrid[0:M, 0:N][1]
    return None

def _lgb_staged_predict(estimator, X, max_iter=0, step=1):

    booster = estimator.booster_

    max_iter = max_iter if max_iter else booster.num_trees()

    leafs = booster.predict(X, pred_leaf=True, num_iteration=max_iter)
    M = leafs.shape[0]
    N = leafs.shape[1]

    trees = np.mgrid[0:M, 0:N][1]

    mapper = {}
    for i in range(estimator.n_estimators):
        for j in range(estimator.num_leaves):
            mapper[i, j] = booster.get_leaf_output(i, j)

    preds = np.vectorize(lambda i, j: mapper[i, j])(trees, leafs).T
    preds = preds.cumsum(axis=0)[np.arange(step, max_iter+step, step)-1]

    if estimator._estimator_type == 'classifier':
        preds = sigmoid(preds)

        for pred in preds:
            yield np.vstack([1-pred, pred]).T

    elif estimator._estimator_type == 'regressor':
        for pred in preds:
            yield pred

def _cat_staged_predict(estimator, X, max_iter=0, step=1):

    if estimator._estimator_type == 'classifier':
        return estimator.staged_predict_proba(X, ntree_end=max_iter, eval_period=step)

    elif estimator._estimator_type == 'regressor':
        return estimator.staged_predict(X, ntree_end=max_iter, eval_period=step)

def _get_scores(estimator, generator, predictor, trn, val, X, y,
                scorer, max_iter, step, train_score):

    stages = np.arange(step, max_iter+step, step)

    trn_scores = []
    val_scores = []

    if train_score:
        X_trn, y_trn = X.iloc[trn], y.iloc[trn]
        S_trn = generator(estimator, X_trn, max_iter, step)

        for _ in stages:
            trn_scores.append(scorer(predictor, next(S_trn), y_trn))

    if True:
        X_val, y_val = X.iloc[val], y.iloc[val]
        S_val = generator(estimator, X_val, max_iter, step)

        for _ in stages:
            val_scores.append(scorer(predictor, next(S_val), y_val))

    return trn_scores, val_scores

def sigmoid(x):
    return 1 / (1 + np.exp(-x))


# -*- coding: utf-8 -*-
"""
Created on Wed Sep 23 11:48:03 2015

@author: Vera
"""

import os
import sys
import numpy as np
import matplotlib.pyplot as plt
import fnmatch

mobility =  np.empty(15)
mlin = np.empty(15)
i = 0
C_LG = 1e-7 # assumption from DEME-TFSI on MoS2
dimensions = {}
dimensions['sd'] = [25.0,5.0]
dimensions['s1'] = [13.0,1.0]
dimensions['s2'] = [13.0,3.5]
dimensions['s3'] = [16.3,3.5]
dimensions['s4'] = [8.2,1.0]
dimensions['d1'] = [13.0,3.5]
dimensions['d2'] = [21.0,1.0]
dimensions['d3'] = [16.3,1.0]
dimensions['d4'] = [8.2,3.5]
dimensions['12'] = [13.0,2.0]
dimensions['34'] = [8.2,2.0]
dimensions['14'] = [0.5,8.5]
dimensions['23'] = [0.5,8.5]

mobility = {}

source_dir = 'C:/Users/Vera/surfdrive/Werk/Science/Projects/[QDs]Hallbar_and_liquid_gate/measurements_square_lattices/Liquid_gate/9_17_150828_63/devE/'
for file in os.listdir(source_dir):
    if fnmatch.fnmatch(file, '*.npy'):
        filename =  file
        try:
            W,L = dimensions[filename[2]+filename[1]]

        except KeyError:
            W,L = dimensions[filename[1]+filename[2]]

        R = np.load(source_dir + file)
        I_sd = 0.01/R
        plt.figure()
        plt.scatter(np.linspace(0,-0.500, 6), I_sd*1e9)
        fit = np.polyfit(np.linspace(0,-0.500, 6), I_sd[0:6], 1)
        mlin[i] = fit[0]
        p = np.poly1d(fit)
        plt.plot(np.linspace(0, -0.5,100), p(np.linspace(-0.1, -0.5,100))*1e9)
        mobility[filename[1:3]] = np.abs(mlin[i]*(L/W)*(1/(0.01*C_LG)))

        i=i+1


"""
Keras models
By: Xiaochi (George) Li
Dec.2018

Unfortunately, non of them can work.
Last Error:FailedPreconditionError: Attempting to use uninitialized value training/TFOptimizer/beta1_power
"""
import tensorflow as tf
from keras import layers
from keras import optimizers
from keras.applications import inception_v3
from keras.applications import mobilenet
from keras.models import Model
from keras import backend as K
from keras import metrics

def first_model():
    """
    This model just makes sure that the code can work
    :return:
        model: keras model object
    """
    model = tf.keras.Sequential([
        layers.Conv2D(64, kernel_size=3, activation='relu', input_shape=(100,100,3)),
        layers.Conv2D(32, kernel_size=3, activation='relu'),
        layers.Flatten(),
        layers.Dense(7, activation='softmax')
    ])

    model.compile(optimizer=tf.train.AdadeltaOptimizer(0.001),
                  loss='categorical_crossentropy',
                 metrics=[tf.keras.metrics.categorical_accuracy])

    return model

def ft_inception_v3():
    """
    Use pretrained VGG16 as bottom

    :return:
    """
    base_model = inception_v3.InceptionV3(include_top=False, weights='imagenet', input_shape=(299, 299, 3))
    base_out = base_model.output
    x = layers.Flatten()(base_out)
    # 'Concatenate' object has no attribute 'outbound_nodes'
    x = layers.Dense(4096, activation='relu')(x)  # Can't connect
    x = layers.Dense(1024, activation='relu')(x)
    x = layers.Dense(256, activation='relu')(x)
    x = layers.Dense(64, activation='relu')(x)
    x = layers.Dense(16, activation='relu')(x)
    predictions = layers.Dense(7, activation='softmax')(x)

    model = Model(inputs=base_model.input, outputs=predictions)
    for layer in base_model.layers():
        layer.trainable = False

    model.compile(optimizer=tf.train.AdadeltaOptimizer(0.001),
                  loss='categorical_crossentropy',
                  metrics=[metrics.categorical_accuracy])

    return model

def ft_mobile_net():
    """
    Use pretrained mobile net as bottom

    :return:
    """
    # input = layers.Input((224,224,3), name='RGB')
    base_model = mobilenet.MobileNet(include_top=False, weights='imagenet', input_shape=(224, 224,3))
    base_out = base_model.output
    x = layers.Flatten()(base_out)
    x = layers.Dense(1024, activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros', name='Dense1024')(x)
    x = layers.Dense(256, activation='relu',kernel_initializer='random_uniform', bias_initializer='zeros', name='Dense256')(x)
    x = layers.Dense(64, activation='relu',kernel_initializer='random_uniform', bias_initializer='zeros', name='Dense64')(x)
    x = layers.Dense(16, activation='relu',kernel_initializer='random_uniform', bias_initializer='zeros', name='Dense16')(x)
    predictions = layers.Dense(7, activation='softmax',kernel_initializer='random_uniform', bias_initializer='zeros')(x)

    model = Model(inputs=base_model.input, outputs=predictions)
    for layer in base_model.layers:
        layer.trainable = False
    sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    with tf.Session() as sess:
        adam = tf.train.AdamOptimizer()
        print(adam.variables())
        sess.run(tf.variables_initializer(adam.variables()))
        model.compile(optimizer=adam,
                  loss='categorical_crossentropy',
                  metrics=[metrics.categorical_accuracy])
        # model.compile(optimizer=sgd,
        #               loss='categorical_crossentropy',
        #               metrics=[tf.keras.metrics.categorical_accuracy])
        K.set_session(tf.Session(graph=model.output.graph))
        init = K.tf.global_variables_initializer()
        K.get_session().run(init)

    # tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value training/TFOptimizer/beta1_power
        return model


from django.conf.urls import url

from . import views

app_name = 'deployments'
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^task-(?P<task_id>[0-9]+)/$', views.task_view, name='task_view'),
    url(r'^task-add/$', views.task_add, name='task_add'),
    url(r'^component/(?P<component>[\w-]+)/$', views.component_view, name='component_view'),
]


from django.db import migrations
import mptt
import mptt.managers

def rebuild_mptt(apps, schema_editor):
    manager = mptt.managers.TreeManager()
    InventoryItem = apps.get_model('dcim', 'InventoryItem')
    manager.model = InventoryItem
    mptt.register(InventoryItem)
    manager.contribute_to_class(InventoryItem, 'objects')
    manager.rebuild()

class Migration(migrations.Migration):

    dependencies = [
        ('dcim', '0118_inventoryitem_mptt'),
    ]

    operations = [
        migrations.RunPython(
            code=rebuild_mptt,
            reverse_code=migrations.RunPython.noop
        ),
    ]


__author__='棒棒糖'
from openpyxl import load_workbook
if __name__=='__main__':
    print('读取已经存在的excel文件')
    wb=load_workbook('excel示例.xlsx')
    #获取所有的sheet名，
    sheets=wb.get_sheet_names()
    print(type(sheets))
    #遍历sheets，并打印
    for sh in sheets:
        print('读取工作簿名称：',sh)
    #获取要读取的sheet
    ws=wb.get_sheet_by_name(sheets[0])
    #读取Sheet A1 ， A2, B2, C2单元格内容
    A1=ws['A1'].value
    print('A1=',A1)
    for index in ('A2','B2','C2'):
        print(index,'=',ws[index].value)
    #读取空值的单元格, openpyxl对于空值的单元格，返回None
    F1=ws['F1'].value
    print('F1=',F1)

第一期/上海-棒棒糖/第二次任务-每日代码练习/2017-1/1-16/read_excel_test.py


#! /usr/bin/python
import rospy
import cv2
import sys
import numpy as np
from sensor_msgs.msg import Image as KinectImage
from cv_bridge import CvBridge
from math import sqrt, atan2, sin, cos, exp, degrees
import termios, fcntl, sys, os, rospy, roslib
from std_msgs.msg import String
from geometry_msgs.msg import Twist, Point
from matplotlib import pyplot as plt
import message_filters
from corobot_common.msg import Pose, Goal
from sensor_msgs.msg import Image, CameraInfo
import sensor_msgs.point_cloud2 as pc2
from sensor_msgs.msg import LaserScan
from laser_geometry import LaserProjection
import rospy
from nav_msgs.msg import Odometry
from tf.transformations import euler_from_quaternion, quaternion_from_euler
import math

# Range for which Kinect v1 worked
CAMERA_RANGE = 48

class Polar:  # commands
    __slots__ = ['d', 'a']

    def __init__(self, distance, angle):
        self.d = distance
        self.a = angle

    def __str__(self):
        return '(d={},a={})'.format(self.d, self.a)

    def valid(self):
        return self.d > 0

class Robot:

    def __init__(self, tracker):
        self.bridge = CvBridge()  # OpenCV API
        self.image = None  # Start with no image
        self.depth_image = None  # Start with no image
        self.tracker = tracker  # tacker OBJECT
        self.tracker_success = False  # tacker read flag set to False
        self.tracker_bbox = None  # No initial bounding box
        self.tracker_init = False
        self.count_num_linear = -1
        self.count_num_angular = -1

        self.prev_center_x = None
        self.prev_center_y = None
        self.prev_area = None
        self.controller = Controller()
        self.rgb_image_queue = []
        self.d_image_queue = []
        self.object_centers_queue = []
        self.bouding_box_queue = []
        self.pose = None
        self.scan = None
        self.laser_projector = LaserProjection()
        self.detecting = True
        self.disable_continuous_motion = False
        self.wp_publisher = rospy.Publisher('waypoints', Point)
        self.screens_count = 0
        # Polar points
        self.cache = []
        self.last_success_depth = None
        self.roll = self.pitch = self.yaw = 0.0
        self.target = 90
        self.kp = 0.5

    def histogram_equalize(self, img):
        b, g, r = cv2.split(img)
        red = cv2.equalizeHist(r)
        green = cv2.equalizeHist(g)
        blue = cv2.equalizeHist(b)
        return cv2.merge((blue, green, red))

    def save_images(self, rgb_image, depth_image):
        self.image = self.bridge.imgmsg_to_cv2(rgb_image, "passthrough")
        self.image = self.histogram_equalize(self.image)
        self.depth_image = self.bridge.imgmsg_to_cv2(depth_image, "32FC1")

    def convertRobotCoorToGlobalCoor(self, polarPoint):
        sp = Pose()
        sp.x = self.pose.x + polarPoint.d * cos(self.pose.theta + math.radians(polarPoint.a))
        sp.y = self.pose.y + polarPoint.d * sin(self.pose.theta + math.radians(polarPoint.a))
        sp.theta = 0
        return sp

    def main_orchestrator(self, rgb_image, depth_image):
        # Solve all of perception here...
        # get both rgb and depth image and save
        self.save_images(rgb_image, depth_image)
        self.tracker_success, self.tracker_bbox = self.tracker.update(self.image)
        if (self.tracker_init):
            if self.tracker_success:
                curr_center_x = int(self.tracker_bbox[0] + (self.tracker_bbox[2] // 2))
                curr_center_y = int(self.tracker_bbox[1] + (self.tracker_bbox[3] // 2))
                temp_d = np.array(self.depth_image, dtype=np.float32)
                temp_i = np.array(self.image, dtype=np.float32)

                if( self.tracker_bbox[0] + (self.tracker_bbox[2] // 2) >= temp_i.shape[1] ):
                    cv2.putText(self.image, "Tracking Failure", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                    return
                if( self.tracker_bbox[1] + (self.tracker_bbox[3] // 2) >= temp_i.shape[0] ):
                    cv2.putText(self.image, "Tracking Failure", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                    return

                depth = temp_d[curr_center_y][curr_center_x]

                calculated_depth = depth / 1000.0

                if( temp_i.shape[0] > temp_i.shape[1] ):
                    max_dimension = temp_i.shape[0] * 1.0
                else:
                    max_dimension = temp_i.shape[1] * 1.0

                actual_center = max_dimension // 2

                difference = actual_center - curr_center_x

                angle_per_pixel = CAMERA_RANGE / max_dimension
                total_angle = angle_per_pixel * difference
                destination = self.convertRobotCoorToGlobalCoor(Polar(calculated_depth, total_angle))

                if abs(destination.y - self.pose.y) <= 0.099 and abs(destination.x - self.pose.x) <= 0.099:
                    pass
                else:
                    if self.detecting or not self.disable_continuous_motion:
                        self.wp_publisher.publish(Point(destination.x, destination.y, 0))
                self.detecting = False

                p1 = (int(self.tracker_bbox[0]), int(self.tracker_bbox[1]))
                p2 = (int(self.tracker_bbox[0] + self.tracker_bbox[2]),
                      int(self.tracker_bbox[1] + self.tracker_bbox[3]))
                cv2.rectangle(self.image, p1, p2, (255, 0, 0), 2, 1)

                if self.image is not None:
                    print("in image Tracking")
                    cv2.putText(self.image, "Tracking ...", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                else:
                    cv2.putText(self.image, "Failure ...", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                    rospy.loginfo("Nothing to visualize!")

            else:
                print("Tracking Failure")
                pass

    def waypointsReached(self, point):
        print("reached" + str(point))
        self.detecting = True

    # to select Object
    def selectROI(self, raw_image):
        # convert raw image to cv2 image format
        self.image = self.bridge.imgmsg_to_cv2(raw_image, "passthrough")
        if not self.tracker_init:
            # initialize parameters for robot to start moving
            self.controller.start_moving()
            # get tracker/bounding box
            self.tracker_bbox = cv2.selectROI(self.image, False)
            # boolean variable indicating if object was found
            self.tracker_success = self.tracker.init(self.image, self.tracker_bbox)
            # tracker initialized
            self.tracker_init = True

    def reduce_covariance(self, cov):
        """Convert a flat 6x6 covariance matrix into a flat 3x3."""
        """ 6 variances are for linear x, y, z and rotate x, y, z """
        return (cov[0], cov[1], cov[5],
                cov[6], cov[7], cov[11],
                cov[30], cov[31], cov[35])

    def odom_to_pose(self, odom):
        """Utility function to convert an Odometry message into a Pose message."""
        pose = Pose()
        pose.header = odom.header
        pose.x = odom.pose.pose.position.x
        pose.y = odom.pose.pose.position.y
        qz = odom.pose.pose.orientation.z
        qw = odom.pose.pose.orientation.w
        pose.theta = atan2(2 * qw * qz, 1 - 2 * qz * qz)
        pose.cov = self.reduce_covariance(odom.pose.covariance)
        return pose

    # Pose -> None
    def poseCallback(self, odom):
        self.pose = self.odom_to_pose(odom)
        print(self.pose.theta)

def main():
    if len(sys.argv) != 2:
        raise AssertionError("Usage: rosrun project_name tracker.py tracker_type")

    # Create the tracker
    tracker = getattr(cv2, "Tracker{}_create".format(sys.argv[1]))()

    # Initialize node
    rospy.init_node("sy4923_tracker", anonymous=False)
    robot = Robot(tracker)  # create the robot object

    # to get robots current pose
    rospy.Subscriber('odom', Odometry, robot.poseCallback)

    # Synchronous callbacks of rgb and depth images
    image_rgb_sub = message_filters.Subscriber('/camera/rgb/image_color', KinectImage)
    image_depth_sub = message_filters.Subscriber('/camera/depth_registered/image_raw', KinectImage)
    ts = message_filters.ApproximateTimeSynchronizer([image_rgb_sub, image_depth_sub],
                                                     queue_size=2, slop=0.1)
    ts.registerCallback(robot.main_orchestrator)

    # Callback to select ROI
    rospy.Subscriber("/camera/rgb/image_color", KinectImage, robot.selectROI, queue_size=1)

    rospy.Subscriber('waypoints_reached', Point, robot.waypointsReached)
    rospy.Subscriber('waypoints_failed', Point, robot.waypointsReached)

    print("Done")
    rospy.spin()

class Controller:
    __slots__ = ['commandVelocityPublisher', 'wp_publisher']

    def __init__(self):
        self.commandVelocityPublisher = rospy.Publisher('mobile_base/commands/velocity', Twist)
        rospy.Subscriber('handle_twist', Twist, self.handle_twist)

    def handle_twist(self, twist_callback):
        self.commandVelocityPublisher.publish(twist_callback)

    def start_moving(self):
        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        oldflags = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK)
        try:
            self.twist = Twist()
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
            fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)

        pass

    def move_Left_By(self, amount):
        print('move left {} m/s'.format(amount))
        self.twist.angular.z = amount
        # self.commandVelocityPublisher.publish(self.twist)

    def move_Right_By(self, amount):
        print('move right {} m/s'.format(amount))
        self.twist.angular.z = -amount
        # self.commandVelocityPublisher.publish(self.twist)

    def move_Forward_By(self, amount):
        print('move forward {} m/s'.format(amount))
        self.twist.linear.x = amount
        # self.commandVelocityPublisher.publish(self.twist)

    def move_Backward_By(self, amount):
        print('move backward {} m/s'.format(amount))
        self.twist.linear.x = -amount
        # self.commandVelocityPublisher.publish(self.twist)

    def test(self):
        p = Point(0.0, 0.5, 0.0)
        p.x = 1900

        while (True):
            self.wp_publisher.publish(p)

    def run(self):

        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        oldflags = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK)

        speed = .5

        try:
            while True:
                twist = Twist()
                try:
                    c = sys.stdin.read(1)
                    if c == '\x1b':
                        c = sys.stdin.read(2)
                    if c == '[A' or c == 'w':
                        print('move forward {} m/s'.format(speed))
                        twist.linear.x = speed
                    elif c == 's' or c == '[B':
                        print('move backward {} m/s'.format(-speed))
                        twist.linear.x = -speed
                    elif c == '[C' or c == 'd':
                        print('rotate right {} m/s'.format(speed))
                        twist.angular.z = -speed
                    elif c == '[D' or c == 'a':
                        print('rotate left {} m/s'.format(speed))
                        twist.angular.z = speed
                    elif c == '-':
                        speed = max(.1, speed - .1)
                        print('decreasing linear speed to ', speed)
                    elif c == '+':
                        speed = min(.5, speed + .1)
                        print('increasing linear speed to ', speed)
                    elif c == 'q':
                        return
                    else:
                        print('Controls:')
                        print('          w or up    : move forward')
                        print('          a or left  : rotate counterclockwise')
                        print('          d or right : rotate clockwise')
                        print('          -          : decrease linear speed by .1 m/s (.1 m/s minimum)')
                        print('          +          : increase linear speed by .1 m/s (.5 m/s maximum)')
                        print('          q          : quit')
                    # more seamless with multiple requests
                    self.commandVelocityPublisher.publish(twist)
                    self.commandVelocityPublisher.publish(twist)
                    self.commandVelocityPublisher.publish(twist)
                except IOError:
                    pass
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
            fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)

if __name__ == "__main__":
    try:
        main()
    except rospy.ROSInterruptException:
        rospy.loginfo("Thank you for using the tracker! Exiting ...")


from google.cloud import language_v1
from google.cloud.language_v1 import enums
import os

os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = input('Provide location of GCloud Credentials: ')

def get_entities_sentiments(text):
    extract = language_v1.LanguageServiceClient().analyze_entity_sentiment({'content': text,
                                                'type': enums.Document.Type.PLAIN_TEXT,
                                                'language': 'en'},
                                            encoding_type=enums.EncodingType.UTF8)
    entities = []
    for entity in extract.entities:
        #only want entities that are likely to be people
        if enums.Entity.Type(entity.type).name not in ['PERSON', 'PROPER']:
            continue

        entities.append([entity.name,
                        entity.salience,
                        entity.sentiment.score,
                        entity.sentiment.magnitude])
    return(entities)

def get_subreddit_names(path_to_file):
    with open(path_to_file) as f:
        names = f.read().splitlines()
    return names

for subreddit in get_subreddit_names('data\\subreddits\\candidate_subreddits.txt'):
    print('Analyzing sentiment from comments in subreddit r/%s.' %subreddit)

    with open('data\\sentiment data\\Sentiment Data_%s_verbose.txt' %subreddit, 'a') as dest_file_verbose:
        with open('data\\sentiment data\\Sentiment Data_%s_succinct.txt' %subreddit, 'a') as dest_file_succinct:
            with open('data\\comments\\Comments_%s.txt' %subreddit) as f:
                for line in f.readlines():
                    try:
                        line_split = line.split('\t')
                        user = line_split[2]
                        if len(line_split)>=3:
                            comment = line_split[3]
                            entities = get_entities_sentiments(comment)

                            dest_file_verbose.write(user + '\t' + comment + '\t' + str(entities) + '\n')
                            for entity in entities:
                                dest_file_succinct.write(str(entity) + '\n')
                    except:
                        continue


from tkinter import *
from PIL import ImageTk,Image
from tkinter import messagebox
import pyodbc

def return_book():

    all_bids = []
    bid = book_Info1.get()

    extract_bid = "SELECT bid FROM "+book_Table
    print(extract_bid)
    try:
        cursor.execute(extract_bid)
        for i in cursor.fetchall():
            all_bids.append(i[0])
        print("all Bids",all_bids)

        if bid in all_bids:
            check_Avialable = "SELECT status FROM "+book_Table+" WHERE bid = '"+bid+"'"
            cursor.execute(check_Avialable)

            for i in cursor.fetchall():
                check = i[0]
            if check == "issued":
                status = True
            else:
                status = False

        else:
            messagebox.showinfo('Error','Book ID not Found')

    except:
        messagebox.showinfo("Error","Can't Fatch Book IS's")

    delete_SQL = "DELETE FROM "+issue_Table+" WHERE bid = '"+bid+"'"

    update_Status = "UPDATE "+book_Table+" SET status = 'available' where bid ='"+bid+"'"
    try:
        if bid in all_bids and status == True:
            cursor.execute(delete_SQL)
            conn.commit()
            cursor.execute(update_Status)
            conn.commit()
            messagebox.showinfo("Sucess","Book returned  Sucessfully")
            root.destroy()
        else:
            all_bids.clear()
            messagebox.showinfo("Message","Book Available in Library")
            root.destroy()
            return
    except:
        messagebox.showinfo("Search Error","Book Id not Available,Try Again")

    all_bids.clear()

def return_Book():
    global book_Info1,book_Info2,book_Info3,book_Info4,conn,cursor,book_Table,root,Canvas,issue_Table

    root = Tk()
    root.title("Library")
    root.minsize(width=450,height=450)
    root.geometry('700x700')

    conn = pyodbc.connect(
        'Driver={SQL Server};'
        'Server=LAPTOP-9O99F71M\SQLEXPRESS;'
        'Database=library;'
        'Trusted_Connection=yes;'
        )

    cursor = conn.cursor()

    # name of the table here
    book_Table = 'books'
    issue_Table = "books_issued"

    Canvas1 = Canvas(root)
    Canvas1.config(bg='black')
    Canvas1.pack(expand=True,fill=BOTH)

    heading_Frame1 = Frame(root,bg='#ff6e40',bd=5,relief = RAISED,cursor='target')
    heading_Frame1.place(relx=0.25,rely=0.1,relwidth=0.5,relheight=0.13)

    heading_Label = Label(heading_Frame1,text='Return Books',bg='black',fg='red',font=('Courier',15))
    heading_Label.place(relx=0,rely=0,relwidth=1,relheight=1)

    label_Frame = Label(root,bg='black',relief = RAISED)
    label_Frame.place(relx=0.1,rely=0.4,relwidth=0.8,relheight=0.5)

    label_Frame1 = Label(label_Frame,text="Book ID :- ",bg='black',fg='white')
    label_Frame1.place(relx=0.05,rely=0.5)

    book_Info1 = Entry(label_Frame)
    book_Info1.place(relx=0.3,rely=0.5,relwidth=0.62)

    # Button for Sbumit
    Submit_button = Button(root,text="RETURN",bg='#d1ccc0',fg='black',command=return_book)
    Submit_button.place(relx=0.28,rely=0.9,relwidth=0.18,relheight=0.08)

    # Button for Quit
    quit_button = Button(root,text="Quit",bg='#f7f1e3',fg='black',command = root.destroy)
    quit_button.place(relx=0.5,rely=0.9,relwidth=0.18,relheight=0.08)

    root.mainloop()


from apf.metrics import GenericMetricsProducer
from apf.metrics import DateTimeEncoder
from confluent_kafka import Producer
from apf.core import get_class

import json

class KafkaMetricsProducer(GenericMetricsProducer):
    """Write metrics in a Kafka Topic.

    Useful for high-throughput distributed metrics, a complex architecture can be build using
    Apache Kafka as a queue and writing the metrics inside a time series data store, for example
    Prometheus, InfluxDB or Elasticsearch.

    Parameters
    ----------
    config : dict.
        Parameters passed to the producer.

        - PARAMS: Parameters passed to :class:`apf.producer.KafkaProducer`.
        - TOPIC: List of topics to produce, for example ['metrics'].

    producer : apf.producers.GenericProducer
        An apf producer, by default is :class:`apf.producer.KafkaProducer`.

    """

    def __init__(self, config, producer=None):
        super().__init__(config)
        self.config = config
        if producer is not None:
            self.producer = producer
        else:
            self.producer = Producer(self.config["PARAMS"])
        self.time_encoder = self.config.get("TIME_ENCODER_CLASS", DateTimeEncoder)
        self.dynamic_topic = False
        if self.config.get("TOPIC"):
            self.logger.info(f'Producing metrics to {self.config["TOPIC"]}')
            self.topic = [self.config["TOPIC"]]
        elif self.config.get("TOPIC_STRATEGY"):
            self.dynamic_topic = True
            TopicStrategy = get_class(self.config["TOPIC_STRATEGY"]["CLASS"])
            self.topic_strategy = TopicStrategy(
                **self.config["TOPIC_STRATEGY"]["PARAMS"]
            )
            self.topic = self.topic_strategy.get_topics()
            self.logger.info(f'Using {self.config["TOPIC_STRATEGY"]}')
            self.logger.info(f"Producing to {self.topic}")

    def send_metrics(self, metrics):
        metrics = json.dumps(metrics, cls=self.time_encoder).encode("utf-8")

        if self.config.get("TOPIC_STRATEGY"):
            self.topic = self.topic_strategy.get_topics()
        for topic in self.topic:
            try:
                self.producer.produce(topic, metrics)
            except BufferError as e:
                self.logger.info(f"Error producing metrics: {e}")
                self.logger.info("Calling poll to empty queue and producing again")
                self.producer.poll(1)
                self.producer.produce(topic, metrics)

    def __del__(self):
        self.logger.info("Waiting to produce last messages")
        self.producer.flush()