import argparse
import os
import re
import json
import pdb

from transner import Transner
from doc2txt import doc2txt

from tools import annotator, aggregator, generator

from sentence_transformers.cross_encoder import CrossEncoder

import importlib
sutime_mod = importlib.import_module("python-sutime.sutime")

class PathwayGenerator():
    def __init__(self,
                file_path,
                pilot,
                service,
                use_cuda=False,
                cuda_device=-1,
                annotation_model='en',
                section_split_model='/home/easyrights/Documents/pgr/section_split/models/training_unfolding_structure-2020-12-22_11-07-07_distilroberta-base/'):

        ''' PathwayGenerator object constructor

        Args:
            path (str): path of the file from which the pathway is generated.
            pilot (str): name of the pilot.
            service (str): name of the service considered.
            use_cuda (bool): flag to use gpu model.
            cuda_device (int, optional): Id of the gpu device to use. Defaults to -1.
            annotation_model (str, optional): The name of the annotation model
            section_split_model (str, optional): The name of the section splitter model
        '''

        assert file_path is not None, "A file path is required"

        languages = {
            'Larissa': 'el',
            'Birmingham': 'en',
            'Malaga': 'es',
            'Palermo': 'it'
        }

        self.path = file_path
        if os.path.splitext(self.path)[-1] == '.txt':
            self.converted_file = doc2txt.purge_urls(open(self.path, 'r').read(), os.path.splitext(self.path)[0])
        self.use_cuda = use_cuda
        self.cuda_device = cuda_device
        self.language = languages[pilot]
        # TODO: language detection param?
        if annotation_model is None:
            self.annotation_model = Transner(pretrained_model='bert_uncased_base_easyrights_v0.1', use_cuda=use_cuda, cuda_device=cuda_device, language_detection=True, threshold=0.85, args={"use_multiprocessing": False})
        else:
            self.annotation_model = Transner(pretrained_model='bert_uncased_'+annotation_model, use_cuda=use_cuda, cuda_device=cuda_device, language_detection=True, threshold=0.85, args={"use_multiprocessing": False})

        self.section_split_model = CrossEncoder(section_split_model, num_labels=1)

        self.annotation_metadata = metadata = pilot + ' - ' + service + ' - ' + os.path.basename(self.path)

        self.generation_metadata = pilot + ' - ' + service + ' - ' + os.path.basename(self.path) + ' - '

    def to_list(self):
        element_list = [] # Make an empty list

        for element in re.split('[.\n]', self.converted_file):
            stripped_element = element.strip()
            if stripped_element != '':
                element_list.append(stripped_element) #Append to list the striped element

        return element_list

    def do_convert(self):
        self.converted_file = doc2txt.convert_to_txt(self.path)
        return self.converted_file

    def do_split(self, threshold=0.5):
        sentence_list = self.to_list()

        scores = []
        for i in range(0, len(sentence_list)-1):
            current_sentence = sentence_list[i]
            next_sentence = sentence_list[i+1]

            score = self.section_split_model.predict([current_sentence, next_sentence])
            scores.append(score)

        sections = []
        section_text = []
        section_text.append(sentence_list[0])
        for i in range(0, len(scores)):
            if scores[i] >= threshold:
                section_text.append(sentence_list[i+1])
            else:
                sections.append(section_text)
                section_text = []
                section_text.append(sentence_list[i+1])
        sections.append(section_text)

        print(sections)

        return sections

    def do_annotate(self, sentence_list):
        self.ner_dict = self.annotation_model.ner(sentence_list, apply_regex=True)
        if self.language in ['es', 'en']:
            self.ner_dict = self.annotate_sutime(self.ner_dict)
        else:
            self.ner_dict = self.annotation_model.find_dates(self.ner_dict)

        self.ner_dict = annotator.aggregate_dict(self.ner_dict)

        self.ner_dict['entities'] = sorted(self.ner_dict['entities'], key=lambda ner: ner['start_offset'])

        self.ner_dict = annotator.resolve_uri_entities(self.ner_dict, self.path)

        return self.ner_dict

    def do_generate(self):
        if os.path.splitext(self.path)[-1] == '.json':
            self.ner_dict = json.load(open(self.path, 'r'))
        aggregated_ner_dict = aggregator.aggregate_entities(self.ner_dict)

        json_pathway = generator.generate(aggregated_ner_dict)
        mapped_entities = json.loads(json_pathway)

        dict_pathway = json.load(open("tools/dict_pathway.json", 'r'))

        self.pathway = {}

        for key, sub_types in dict_pathway.items():
            self.pathway[key] = {}
            for sub_type in sub_types:
                self.pathway[key][sub_type] = []

        for entity in mapped_entities:
            self.pathway[self.keys_of_value(dict_pathway, entity['step'])][entity['step']].append(entity)

        #todo: remove return because we can read the value in the pgr object
        return self.pathway

    def export_annotation_to_doccano(self, add_confidence=False):
        filename = os.path.splitext(self.path)[0]

        doccano_dict = {}
        doccano_dict['text'] = self.ner_dict['text']
        doccano_dict['labels'] = []

        doccano_dict['meta'] = self.annotation_metadata

        for item in self.ner_dict['entities']:
            if add_confidence:
                doccano_dict['labels'].append([item['start_offset'], item['end_offset'], item['type'], item['confidence']])
            else:
                doccano_dict['labels'].append([item['start_offset'], item['end_offset'], item['type']])

        file_out = open(filename +'_ner.jsonl', 'w', encoding='utf-8')
        file_out.write(json.dumps(doccano_dict))
        file_out.write('\n')

        return doccano_dict, filename +'_ner.jsonl'

    def export_generation_to_doccano(self, pathway=None):
        dict_translations = json.load(open("tools/dict_translations.json", 'r'))

        filename = os.path.splitext(self.path)[0]
        pathway_jsonl = []

        for key in pathway:
            tmp_dict = {"text": '', "labels": [], "meta": ''}
            tmp_dict["text"] = key

            for step, step_dict in pathway[key].items():
                tmp_dict["meta"] = self.generation_metadata + key
                for sub_type, entities in step_dict.items():
                    label = dict_translations[self.language][step] + ' - ' + dict_translations[self.language][sub_type] + ': '
                    if len(entities) == 0:
                        label = label + '-'
                        tmp_dict['labels'].append(label)
                    else:
                        for entity in entities:
                            label = label + entity['entity'].strip() + ' , '

                        tmp_dict['labels'].append(label[:-2].strip())

            pathway_jsonl.append(tmp_dict)

        file_out = open(filename + '_pathway.jsonl', 'w', encoding='utf-8')

        return_string = ''

        for element in pathway_jsonl:
            string_element = str(json.dumps(element, ensure_ascii=False))
            file_out.write(string_element)
            file_out.write('\n')

            return_string = return_string + string_element + '\n'

        return return_string, filename + '_pathway.jsonl'

    def keys_of_value(self, dct, value):
        for k in dct:
            if isinstance(dct[k], list):
                if value in dct[k]:
                    return k
            else:
                if value == dct[k]:
                    return k

    def annotate_sutime(self, ner_dict):
        for item in ner_dict:
            text = item['sentence']
            jar_files = os.path.join('python-sutime/', 'jars')
            sutime = sutime_mod.SUTime(jars=jar_files, mark_time_ranges=True)

            json = sutime.parse(text)

            time_type = self.annotation_model.check_opening_time(item['entities'])

            for item_sutime in json:
                if not self.annotation_model.find_overlap(item['entities'], item_sutime['start'], item_sutime['end']):
                    item['entities'].append({'type': time_type, 'value': item_sutime['text'], 'confidence': 0.85, 'offset': item_sutime['start']})

        return ner_dict

    def sections_to_doccano(self, sections):
        count, step = 0, 1
        doccano_dict = {'text': '', 'labels': []}

        for section in sections:
            initial_count, final_count = count, 0

            for sentence in section:
                doccano_dict['text'] = doccano_dict['text'] + sentence + '.\n'
                final_count = final_count + len(sentence) + 2

            doccano_dict['labels'].append([initial_count, initial_count+final_count-1, 'Step'+str(step)])
            step = step + 1
            count = initial_count+final_count

        return doccano_dict

def main(path=None, empty=False, convert=True, pilot='', service=''):
    pgr = PathwayGenerator(file_path=path, pilot=pilot, service=service, use_cuda=False, cuda_device=0)
    converted_file = pgr.do_convert()
    sections = pgr.do_split()
    full_ner_dict = {}
    count = 1
    for section in sections:
        pgr.annotation_model.reset_preprocesser()
        ner_dict = pgr.do_annotate(section)
        pathway = pgr.do_generate()
        label = 'Step'+str(count)
        full_ner_dict[label] = pathway
        count = count + 1
    pathway_dict, pathway_path = pgr.export_generation_to_doccano(full_ner_dict)
    print(pathway_dict)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '-f',
        '--path',
        help='List of files to be converted before transner',
        required=True
    )

    parser.add_argument(
        '-e',
        '--empty',
        help='Specify if the jsonl output for doccano needs to be empty. Default is false.',
        required=False
    )
    parser.add_argument(
        '-c',
        '--convert',
        help='Specify if you need the file conversion. Default is true.',
        required=False
    )

    parser.add_argument(
        '-p',
        '--pilot',
        help='Specify pilot.',
        required=True
    )

    parser.add_argument(
        '-s',
        '--service',
        help='Specify service.',
        required=True
    )
    args = parser.parse_args()

    main(path=args.path, empty=args.empty, convert=args.convert, pilot=args.pilot, service=args.service)


__all__ = ["send_positions", "send_commands", "ServerConnectionError"]

import json
import socket

HOSTNAME = "raspberrypi.local"

def send_positions(data: dict) -> None:
    send(data, 35007)

def send_commands(data: dict) -> None:
    send(data, 35008)

def send(data: dict, port: int) -> None:
    """Send data to server by serializing the provided dict and sending it."""
    serialized_data = json.dumps(data)

    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
        try:
            sock.connect((HOSTNAME, port))
        except socket.gaierror as e:
            raise ServerConnectionError(
                f'Address-related error connecting to server.\nCheck server is on and running.\n\nLib says: "{e.__class__.__module__}.{e.__class__.__qualname__}: {e}"'
            )
        except socket.error as e:
            raise ServerConnectionError(
                f'General connection error.\nCheck server is on and running.\n\nLib says: "{e.__class__.__module__}.{e.__class__.__qualname__}: {e}"'
            )
        else:
            sock.sendall(bytes(serialized_data + "\n", "utf-8"))

class ServerConnectionError(Exception):
    def __init__(self, message: str):
        self.message = message


# -*- coding:utf-8 -*-
# Author: ssdcxy
# Date: 2020-03-20 10:48:23
# LastEditTime: 2020-03-20 11:38:54
# LastEditors: ssdcxy
# Description: 排序链表
# FilePath: /arithmetic_oj/LeetCode/P0148.py

import json
from typing import List

# Definition for singly-linked list.
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:
    def sortList(self, head: ListNode) -> ListNode:
        if not head or not head.next: return head
        slow, fast = head, head.next
        while fast and fast.next:
            slow = slow.next
            fast = fast.next.next
        mid, slow.next = slow.next, None
        left, right = self.sortList(head), self.sortList(mid)
        h = res = ListNode(0)
        while left and right:
            if left.val < right.val:
                h.next = left
                left = left.next
            else:
                h.next = right
                right = right.next
            h = h.next
        h.next = left if left else right
        return res.next

def stringToIntegerList(input):
    return json.loads(input)

def stringToListNode(input):
    # Generate list from the input
    numbers = stringToIntegerList(input)

    # Now convert that list into linked list
    dummyRoot = ListNode(0)
    ptr = dummyRoot
    for number in numbers:
        ptr.next = ListNode(number)
        ptr = ptr.next

    ptr = dummyRoot.next
    return ptr

def listNodeToString(node):
    if not node:
        return "[]"

    result = ""
    while node:
        result += str(node.val) + ", "
        node = node.next
    return "[" + result[:-2] + "]"

def main():
    import sys
    import io
    def readlines():
        for line in io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8'):
            yield line.strip('\n')

    lines = readlines()
    while True:
        try:
            line = next(lines)
            head = stringToListNode(line);

            ret = Solution().sortList(head)

            out = listNodeToString(ret);
            print(out)
        except StopIteration:
            break

if __name__ == '__main__':
    main()


"""Normalized position-wise feed-forward module for MEGA block."""

import torch

class NormalizedPositionwiseFeedForward(torch.nn.Module):
    """NormalizedPositionFeedForward module definition.

    Args:
        size: Input/Output size.
        hidden_size: Hidden size.
        normalization: Normalization module.
        activation: Activation function.
        dropout_rate: Dropout rate.

    """

    def __init__(
        self,
        size: int,
        hidden_size: int,
        normalization: torch.nn.Module = torch.nn.LayerNorm,
        activation: torch.nn.Module = torch.nn.ReLU,
        dropout_rate: float = 0.0,
    ) -> None:
        """Construct an NormalizedPositionwiseFeedForward object."""
        super().__init__()
        self.linear1 = torch.nn.Linear(size, hidden_size)
        self.linear2 = torch.nn.Linear(hidden_size, size)

        self.normalization = normalization
        self.activation = activation

        self.dropout = torch.nn.Dropout(p=dropout_rate)
        self.hidden_dropout = torch.nn.Dropout(p=dropout_rate)

        self.reset_parameters()

    def reset_parameters(self, val: float = 0.0, std: float = 0.02) -> None:
        """Reset module parameters.

        Args:
            val: Initialization value.
            std: Standard deviation.

        """
        torch.nn.init.normal_(self.linear1.weight, mean=val, std=std)
        torch.nn.init.constant_(self.linear1.bias, val)

        torch.nn.init.normal_(self.linear2.weight, mean=val, std=std)
        torch.nn.init.constant_(self.linear2.bias, val)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """Compute feed-forward module.

        Args:
            x: NormalizedPositionwiseFeedForward input sequences. (B, L, size)

        Returns:
            x: NormalizedPositionwiseFeedForward output sequences. (B, L, size)

        """
        residual = x

        x = self.hidden_dropout(self.activation(self.linear1(x)))
        x = self.dropout(self.linear2(x))

        x = x + residual

        x = self.normalization(x)

        return x


# coding=utf-8
from __future__ import unicode_literals

import os
import re
import sys
import json
import codecs
import datetime

PY2 = sys.version_info[0] == 2

JS_PATTERN = re.compile(r'<!-- build -->(.*)<!-- endbuild -->',
                        re.IGNORECASE | re.MULTILINE | re.DOTALL)
JS_SRC_PATTERN = re.compile(r'src=\"(.*?)\"')

def freeze_js(html_content):
    """

    :param html_content:
    :return:
    """
    matches = JS_PATTERN.finditer(html_content)

    if not matches:
        return html_content

    for match in reversed(tuple(matches)):
        # JS file block
        src_matches = JS_SRC_PATTERN.findall(match.group(1))

        js_content = ""
        for src in src_matches:
            src = src.strip()
            src = src.split('/')
            file_path = os.path.join(
                get_resource_dir('templates'), *src)

            with codecs.open(file_path, "r", "utf-8") as f:
                js_content += f.read() + '\n'
        # Replace matched string with inline JS
        fmt = '<script type="text/javascript">{}</script>'
        js_content = fmt.format(js_content)
        html_content = (html_content[:match.start()] + js_content +
                        html_content[match.end():])

    return html_content

def get_resource_dir(*paths):
    """

    :param path:
    :return:
    """
    current_path = os.path.dirname(__file__)
    resource_path = os.path.join(current_path, *paths)
    return resource_path

def write_utf8_html_file(file_name, html_content):
    """

    :param file_name:
    :param html_content:
    :return:
    """
    if PY2:
        html = html_content.encode('utf-8')
        with open(file_name, "w+") as fout:
            fout.write(html)
    else:
        with open(file_name, "w+", encoding="utf-8") as fout:
            fout.write(html_content)

class UnknownTypeEncoder(json.JSONEncoder):
    """
    UnknownTypeEncoder`类用于处理数据的编码，使其能够被正常的序列化
    """
    def default(self, obj):
        if isinstance(obj, (datetime.datetime, datetime.date)):
            return obj.isoformat()
        else:
            # Pandas and Numpy lists
            try:
                return obj.astype(float).tolist()
            except Exception:
                try:
                    return obj.astype(str).tolist()
                except Exception:
                    return json.JSONEncoder.default(self, obj)

def json_dumps(data, indent=0):
    """ json 序列化编码处理

    :param data: 字典数据
    :param indent: 缩进量
    """
    return json.dumps(data, indent=indent, cls=UnknownTypeEncoder)


# -*- coding: utf-8 -*-
"""
Created on Tue May 19 10:45:15 2020

@author: Kyle
"""
import pandas as pd
import numpy as np
from scipy.sparse import diags
from scipy.stats import norm
N=1000

def DGP1(N,delta):
    k=100
    rho=0.5
    d = np.array([rho*np.ones(k-1),np.ones(k),rho*np.ones(k-1)])
    offset = [-1,0,1]
    x_cov = diags(d,offset).toarray()
    x_mean = np.ones(k)
    X = np.random.multivariate_normal(x_mean,x_cov,N)
    epsilon = np.random.multivariate_normal([0,0],[[1,0],[0,1]],size=N)
    theta = np.array([1 for l in list(range(1,(k+1)))])
    theta = theta.reshape(k,1)
    p = norm.cdf((X@theta-(theta.T@x_mean))/(theta.T@x_cov@theta))
    T = np.random.binomial(1,p)

    theta2 = theta+delta

    beta = [1 + (1*delta)/2, 1 + (2*delta)/2]

    Y1 = beta[0]*T + X@theta + epsilon[:,0,None]
    Y2 = beta[1]*T + X@theta2 + epsilon[:,1,None]
    columns = ['Y1','Y2','T'] + ['x'+str(a) for a in range(0,k)]
    data = pd.DataFrame(np.column_stack((Y1,Y2,T,X)),columns = columns)

    return data


#!/usr/bin/env python
# coding: utf-8

#
#
#
# *This notebook contains an excerpt from the [Python Data Science Handbook](http://shop.oreilly.com/product/0636920034919.do) by Jake VanderPlas; the content is available [on GitHub](https://github.com/jakevdp/PythonDataScienceHandbook).*
#
# *The text is released under the [CC-BY-NC-ND license](https://creativecommons.org/licenses/by-nc-nd/3.0/us/legalcode), and code is released under the [MIT license](https://opensource.org/licenses/MIT). If you find this content useful, please consider supporting the work by [buying the book](http://shop.oreilly.com/product/0636920034919.do)!*

#
# < [Histograms, Binnings, and Density](04.05-Histograms-and-Binnings.ipynb) | [Contents](Index.ipynb) | [Customizing Colorbars](04.07-Customizing-Colorbars.ipynb) >
#
#
#

# # Customizing Plot Legends

# Plot legends give meaning to a visualization, assigning meaning to the various plot elements.
# We previously saw how to create a simple legend; here we'll take a look at customizing the placement and aesthetics of the legend in Matplotlib.
#
# The simplest legend can be created with the ``plt.legend()`` command, which automatically creates a legend for any labeled plot elements:

# In[1]:

import matplotlib.pyplot as plt
plt.style.use('classic')

# In[2]:

get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np

# In[3]:

x = np.linspace(0, 10, 1000)
fig, ax = plt.subplots()
ax.plot(x, np.sin(x), '-b', label='Sine')
ax.plot(x, np.cos(x), '--r', label='Cosine')
ax.axis('equal')
leg = ax.legend();

# But there are many ways we might want to customize such a legend.
# For example, we can specify the location and turn off the frame:

# In[4]:

ax.legend(loc='upper left', frameon=False)
fig

# We can use the ``ncol`` command to specify the number of columns in the legend:

# In[5]:

ax.legend(frameon=False, loc='lower center', ncol=2)
fig

# We can use a rounded box (``fancybox``) or add a shadow, change the transparency (alpha value) of the frame, or change the padding around the text:

# In[6]:

ax.legend(fancybox=True, framealpha=1, shadow=True, borderpad=1)
fig

# For more information on available legend options, see the ``plt.legend`` docstring.

# ## Choosing Elements for the Legend
#
# As we have already seen, the legend includes all labeled elements by default.
# If this is not what is desired, we can fine-tune which elements and labels appear in the legend by using the objects returned by plot commands.
# The ``plt.plot()`` command is able to create multiple lines at once, and returns a list of created line instances.
# Passing any of these to ``plt.legend()`` will tell it which to identify, along with the labels we'd like to specify:

# In[7]:

y = np.sin(x[:, np.newaxis] + np.pi * np.arange(0, 2, 0.5))
lines = plt.plot(x, y)

# lines is a list of plt.Line2D instances
plt.legend(lines[:2], ['first', 'second']);

# I generally find in practice that it is clearer to use the first method, applying labels to the plot elements you'd like to show on the legend:

# In[8]:

plt.plot(x, y[:, 0], label='first')
plt.plot(x, y[:, 1], label='second')
plt.plot(x, y[:, 2:])
plt.legend(framealpha=1, frameon=True);

# Notice that by default, the legend ignores all elements without a ``label`` attribute set.

# ## Legend for Size of Points
#
# Sometimes the legend defaults are not sufficient for the given visualization.
# For example, perhaps you're be using the size of points to mark certain features of the data, and want to create a legend reflecting this.
# Here is an example where we'll use the size of points to indicate populations of California cities.
# We'd like a legend that specifies the scale of the sizes of the points, and we'll accomplish this by plotting some labeled data with no entries:

# In[9]:

import pandas as pd
cities = pd.read_csv('data/california_cities.csv')

# Extract the data we're interested in
lat, lon = cities['latd'], cities['longd']
population, area = cities['population_total'], cities['area_total_km2']

# Scatter the points, using size and color but no label
plt.scatter(lon, lat, label=None,
            c=np.log10(population), cmap='viridis',
            s=area, linewidth=0, alpha=0.5)
plt.axis(aspect='equal')
plt.xlabel('longitude')
plt.ylabel('latitude')
plt.colorbar(label='log$_{10}$(population)')
plt.clim(3, 7)

# Here we create a legend:
# we'll plot empty lists with the desired size and label
for area in [100, 300, 500]:
    plt.scatter([], [], c='k', alpha=0.3, s=area,
                label=str(area) + ' km$^2$')
plt.legend(scatterpoints=1, frameon=False, labelspacing=1, title='City Area')

plt.title('California Cities: Area and Population');

# The legend will always reference some object that is on the plot, so if we'd like to display a particular shape we need to plot it.
# In this case, the objects we want (gray circles) are not on the plot, so we fake them by plotting empty lists.
# Notice too that the legend only lists plot elements that have a label specified.
#
# By plotting empty lists, we create labeled plot objects which are picked up by the legend, and now our legend tells us some useful information.
# This strategy can be useful for creating more sophisticated visualizations.
#
# Finally, note that for geographic data like this, it would be clearer if we could show state boundaries or other map-specific elements.
# For this, an excellent choice of tool is Matplotlib's Basemap addon toolkit, which we'll explore in [Geographic Data with Basemap](04.13-Geographic-Data-With-Basemap.ipynb).

# ## Multiple Legends
#
# Sometimes when designing a plot you'd like to add multiple legends to the same axes.
# Unfortunately, Matplotlib does not make this easy: via the standard ``legend`` interface, it is only possible to create a single legend for the entire plot.
# If you try to create a second legend using ``plt.legend()`` or ``ax.legend()``, it will simply override the first one.
# We can work around this by creating a new legend artist from scratch, and then using the lower-level ``ax.add_artist()`` method to manually add the second artist to the plot:

# In[10]:

fig, ax = plt.subplots()

lines = []
styles = ['-', '--', '-.', ':']
x = np.linspace(0, 10, 1000)

for i in range(4):
    lines += ax.plot(x, np.sin(x - i * np.pi / 2),
                     styles[i], color='black')
ax.axis('equal')

# specify the lines and labels of the first legend
ax.legend(lines[:2], ['line A', 'line B'],
          loc='upper right', frameon=False)

# Create the second legend and add the artist manually.
from matplotlib.legend import Legend
leg = Legend(ax, lines[2:], ['line C', 'line D'],
             loc='lower right', frameon=False)
ax.add_artist(leg);

# This is a peek into the low-level artist objects that comprise any Matplotlib plot.
# If you examine the source code of ``ax.legend()`` (recall that you can do this with within the IPython notebook using ``ax.legend??``) you'll see that the function simply consists of some logic to create a suitable ``Legend`` artist, which is then saved in the ``legend_`` attribute and added to the figure when the plot is drawn.

#
# < [Histograms, Binnings, and Density](04.05-Histograms-and-Binnings.ipynb) | [Contents](Index.ipynb) | [Customizing Colorbars](04.07-Customizing-Colorbars.ipynb) >
#
#
#


## func related to lstm model
import matplotlib.pyplot as plt
import numpy as np
from keras import regularizers
from keras.callbacks import Callback
from keras.layers import (Dense, concatenate, Embedding, Bidirectional,
                          LSTM, Reshape)
from keras.models import Model, Input
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer as keras_Tokenizer
from sklearn.metrics import classification_report
from sklearn.metrics import f1_score, precision_score, recall_score
from sklearn.metrics import precision_recall_fscore_support
from sklearn.model_selection import StratifiedKFold
import util
import preprocessing

## plotting train and test plot
def training_plot(history):
	# Plot training & validation accuracy values
	plt.plot(history.history['acc'])
	plt.plot(history.history['val_acc'])
	plt.title('Model accuracy')
	plt.ylabel('Accuracy')
	plt.xlabel('Epoch')
	plt.legend(['Train', 'Validation'], loc='upper left')
	plt.show()

	# Plot training & validation loss values
	plt.plot(history.history['loss'])
	plt.plot(history.history['val_loss'])
	plt.title('Model loss')
	plt.ylabel('Loss')
	plt.xlabel('Epoch')
	plt.legend(['Train', 'Test'], loc='upper left')
	plt.show()

## prepare the tokenizer
# @return encoded data, max len , vocalb size and embeddign matrix
def prepare_lstm_data(data, dimension):
	max_len = util.get_max_length(data)
	if max_len > 60:
		max_len = 60
	print("max_length", max_len)
	keras_tkzr = fit_tokenizer(data)
	vocalb_size = len(keras_tkzr.word_index) + 1
	print("vocalb", vocalb_size)

	## embedding matrix
	print("creating glove embeddign matrix")
	embedding_matrix = util.get_embedding_matrix(vocalb_size, dimension, util.embedding_file,
	                                             keras_tkzr)  ## tokenizer contains the vocalb info
	## encoding the docs
	print("encoding the data")
	X = get_encoded_data(data, keras_tkzr, max_len)

	print("X-train", X.shape)

	return (X, keras_tkzr, max_len, vocalb_size, embedding_matrix)

## calculated the lstm prediction
def cal_lstm_pred(test_data, model, keras_tkzr, max_len):
	## encoding the test data
	encoded_docs = keras_tkzr.texts_to_sequences(test_data["tweetText"])
	X_test = (pad_sequences(encoded_docs, maxlen=max_len, padding='post'))
	## calculate the model predictions
	temp = model.predict(X_test)
	y_pred = [np.argmax(value) for value in temp]  ## sigmoid
	return y_pred

# calculated combined lstm predictions
# !! deprecated
def cal_comb_lstm_pred(test_data,model,keras_tkzr, max_len):
	encoded_docs = keras_tkzr.texts_to_sequences(test_data["tweetText"])
	X_test = (pad_sequences(encoded_docs, maxlen=max_len, padding='post'))
	X_test_user, _ = preprocessing.prepare_user_features(test_data)
	## calculate the model predictions
	temp = model.predict([X_test,X_test_user])
	y_pred = [np.argmax(value) for value in temp]  ## sigmoid
	return y_pred

## handle two different inputs and then concatenate them (user and text features)
## return lstm mdoel with input = [words_in,user_in]
def create_model(max_len, vocalb_size, dimension, embedding_matrix):
	## handle text features..
	input = Input(shape=(max_len,))
	emb_word = Embedding(vocalb_size, dimension, weights=[embedding_matrix], input_length=max_len)(input)
	lstm_word = Bidirectional(LSTM(100, return_sequences=False, dropout=0.50, kernel_regularizer=regularizers.l2(0.01)),
	                          merge_mode='concat')(emb_word)

	# modelR = SpatialDropout1D(0.1)(modelR)
	output = Dense(2, activation='sigmoid')(lstm_word)
	model = Model(input, output)
	# sgd = SGD(lr=0.1, momentum=0.9, decay=0.0, nesterov=False)
	print("compiling the model")
	model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
	print(model.summary())
	return model

## handle two different inputs and then concatenate them (user and text features)
## return lstm mdoel with input = [words_in,user_in]
def create_model_comb(max_len, user_feature_len, vocalb_size, dimension, embedding_matrix):
	## handle text features..
	words_in = Input(shape=(max_len,))
	emb_word = Embedding(vocalb_size, dimension, weights=[embedding_matrix], input_length=max_len)(words_in)
	lstm_word = Bidirectional(LSTM(100, return_sequences=False, dropout=0.50, kernel_regularizer=regularizers.l2(0.01)),
	                          merge_mode='concat')(emb_word)
	lstm_word = Dense(user_feature_len, activation='relu')(lstm_word)

	## takes the user features as input
	user_input = Input(shape=(user_feature_len,))

	## concatenate both of the features
	modelR = concatenate([lstm_word, user_input])
	# modelR = SpatialDropout1D(0.1)(modelR)
	output = Dense(2, activation='softmax')(modelR)
	model = Model([words_in, user_input], output)
	model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
	return model

def create_model_seq(max_len, window, vocab_size, dimension, embedding_matrix):
	max_len = max_len
	n_words = vocab_size
	Dimension = 100
	input = Input(shape=(window, max_len,))
	model = Embedding(n_words, dimension, weights=[embedding_matrix], input_length=(window, max_len,))(input)
	model = Reshape(target_shape=(window, (max_len * Dimension)))(model)
	# model =  Bidirectional (LSTM (100,return_sequences=True,dropout=0.25),merge_mode='concat')(model)
	model = Bidirectional(LSTM(100, return_sequences=False, dropout=0.25), merge_mode='concat')(model)
	output = Dense(2, activation='sigmoid')(model)
	model = Model(input, output)
	# sgd = SGD(lr=0.1, momentum=0.9, decay=0.0, nesterov=False)
	print("compiling the model")
	model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
	print(model.summary())
	return model

## callback function for overriding epoch end etc.
class Metrics(Callback):
	def on_train_begin(self, logs={}):
		self.val_f1s = []
		self.val_recalls = []
		self.val_precisions = []

	def on_epoch_end(self, epoch, logs={}):
		val_predict = (np.asarray(self.model.predict(self.validation_data[0]))).round()
		val_predict = np.array([np.argmax(value) for value in val_predict])
		val_targ = self.validation_data[1]
		_val_f1 = f1_score(val_targ, val_predict)
		_val_recall = recall_score(val_targ, val_predict)
		_val_precision = precision_score(val_targ, val_predict)
		self.val_f1s.append(_val_f1)
		self.val_recalls.append(_val_recall)
		self.val_precisions.append(_val_precision)
		print("— val_f1: %f — val_precision: %f — val_recall %f" % (_val_f1, _val_precision, _val_recall))
		print('  Classification Report:\n', classification_report(val_targ, val_predict), '\n')
		return

def get_encoded_data(data, keras_tkzr, max_len):
	## encoding the docs
	encoded_docs = keras_tkzr.texts_to_sequences(data)
	X = (pad_sequences(encoded_docs, maxlen=max_len, padding='post'))
	return X

def fit_tokenizer(data):
	print("preparing the tokenizer")
	keras_tkzr = keras_Tokenizer()
	keras_tkzr.fit_on_texts(data)
	return keras_tkzr

# gives the encoded and oversampled data
## prepare the tokenizer
def split_data_text(data, Y, dimension,option="over"):
	max_len = util.get_max_length(data)
	if max_len > 60:
		max_len = 60
	print("max_length", max_len)
	keras_tkzr = fit_tokenizer(data)
	vocalb_size = len(keras_tkzr.word_index) + 1
	print("vocalb", vocalb_size)

	## embedding matrix
	print("creating glove embeddign matrix")
	embedding_matrix = util.get_embedding_matrix(vocalb_size, dimension, util.embedding_file,
	                                             keras_tkzr)  ## tokenizer contains the vocalb info
	## encoding the docs
	print("encoding the data")
	X_train = get_encoded_data(data, keras_tkzr, max_len)

	print("X-train", X_train.shape)

	## either of the options for under and over sampling
	if (option == "over"):
		X_train, Y_train = util.get_oversample(X_train, Y)

	return (X_train, Y_train, max_len, vocalb_size, embedding_matrix)

## return cross_val mean score for each class
def get_cross_val_score(model, X, Y, n_splits, epoch=5):
	skf = StratifiedKFold(n_splits=n_splits, shuffle=False)
	scores = []
	for fold, (train, test) in enumerate(skf.split(X, Y)):
		history = model.fit(X[train], Y[train], validation_split=0.25, nb_epoch=epoch,
		                    verbose=1, batch_size=32, class_weight=None, )
		training_plot(history)

		## prediction
		temp = model.predict(X[test])
		y_pred = [np.argmax(value) for value in temp]  ## sigmoid
		f1 = precision_recall_fscore_support(Y[test], y_pred, average=None)[2]
		print("fold =", fold)
		print('  Classification Report:\n', classification_report(Y[test], y_pred), '\n')
		scores.append(f1)
	score1 = np.mean([ele[0] for ele in scores])
	score2 = np.mean([ele[1] for ele in scores])     ## mean of the 5 splits

	print("*************")
	print(score1, score2)
	return (score1, score2)


###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

import json
import logging

from django.core.exceptions import PermissionDenied, ValidationError
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, HttpResponseBadRequest, HttpResponse
from django.shortcuts import render

from amcat.models import CodingJob, Project, CodedArticle, STATUS_COMPLETE, \
    STATUS_INPROGRESS
from amcat.models.authorisation import ROLE_PROJECT_ADMIN
from annotator.utils.validator import validate

log = logging.getLogger(__name__)

def index(request, project_id, codingjob_id):
    """returns the HTML for the main annotator page"""
    return render(request, "annotator/codingjob.html", {
        'codingjob': CodingJob.objects.get(id=codingjob_id),
        'project': Project.objects.get(id=project_id),
        'coder': request.user,
        'annotator': True
    })

def save(request, project_id, codingjob_id, coded_article_id):
    """
    Big fat warning: we don't do server side validation for the codingvalues. We
    do check if the codingjob and logged in user correspond, but it's the users
    responsibilty to send correct data (we don't care!).
    """
    coded_article = CodedArticle.objects.select_related("codingjob__project", "codingjob", "article").get(
        id=coded_article_id)

    # sanity checks
    if coded_article.codingjob.project_id != int(project_id):
        raise PermissionDenied(
            "Given codingjob ({coded_article.codingjob}) does not belong to project ({coded_article.codingjob.project})!".format(
                **locals()))
    if coded_article.codingjob_id != int(codingjob_id):
        raise PermissionDenied(
            "CodedArticle has codingjob_id={coded_article.codingjob_id} but {codingjob_id} given in url!")

    if coded_article.codingjob.coder_id != request.user.id:
        # the user is not the assigned coder. Is s/he project admin?
        if not coded_article.codingjob.project.has_role(request.user, ROLE_PROJECT_ADMIN):
            raise PermissionDenied("Only {request.user} or project admins can edit this codingjob.".format(**locals()))

    try:
        codings = json.loads(request.body.decode())
    except ValueError:
        return HttpResponseBadRequest("Invalid JSON in POST body")

    coded_article.status_id = codings["coded_article"]["status_id"]
    coded_article.comments = codings["coded_article"]["comments"]

    coded_article.save()

    new_coding_objects, new_coding_values = coded_article.replace_codings(codings["codings"])

    status = {
        "saved_codings": len(new_coding_objects),
        "saved_values": len(new_coding_values),
        "status": coded_article.status.id,
        "error": None
    }

    if coded_article.status_id == STATUS_COMPLETE:
        try:
            validate(coded_article)
        except ValidationError as e:
            coded_article.status_id = STATUS_INPROGRESS
            coded_article.save()

            status["status"] = STATUS_INPROGRESS
            status["error"] = {"message": [suberror.message for suberror in e.error_list]}
            status["error"]["fields"] = getattr(e, "fields", None)

    return HttpResponse(status=201, content=json.dumps(status))

def redirect(request, codingjob_id):
    cj = CodingJob.objects.get(id=codingjob_id)
    return HttpResponseRedirect(reverse("annotator:annotator-codingjob", kwargs={
        "codingjob_id": codingjob_id, "project_id": cj.project_id
    }))


import torch
import numpy as np
from random import shuffle
from sklearn.utils import shuffle as shuffle_ds
def rbf(sigma=1):
    def rbf_kernel(x1,x2,sigma):
        X12norm = torch.sum(x1**2,1,keepdims=True)-2*x1@x2.T+torch.sum(x2**2,1,keepdims=True).T
        return torch.exp(-X12norm/(2*sigma**2))
    return lambda x1,x2: rbf_kernel(x1,x2,sigma)

def poly(n=3):
    return lambda x1,x2: (x1 @ x2.T)**n

def grpf(sigma, d):
    return lambda x1,x2: ((d + 2*rbf(sigma)(x1,x2))/(2 + d))**(d+1)

class svm_model_torch:
    def __init__(self, m, n_class, device="cpu"):
        self.device = device
        self.n_svm = n_class * (n_class - 1)//2
        self.m = m # number of samples
        self.n_class = n_class
        self.blacklist = [set() for i in range(self.n_svm)]

        # multiplier
        self.a = torch.zeros((self.n_svm,self.m), device=self.device) # SMO works only when a is initialized to 0
        # bias
        self.b = torch.zeros((self.n_svm,1), device=self.device)

        # kernel function  should input x [n,d] y [m,d] output [n,m]
        # Example of poly kernel: lambda x,y:  torch.matmul(x,y.T)**2
        self.kernel = lambda x,y:  torch.matmul(x,y.T)

        # Binary setting for every SVM,
        # Mij says the SVMj should give
        # Mij label to sample with class i
        self.lookup_matrix=torch.zeros((self.n_class, self.n_svm), device=self.device)

        # The two classes SVMi concerns,
        # lookup_class[i]=[pos, neg]
        self.lookup_class=torch.zeros((self.n_svm, 2), device=self.device)

        k=0
        for i in range(n_class-1):
            for j in range(i+1,n_class):
                self.lookup_class[k, 0]=i
                self.lookup_class[k, 1]=j
                k += 1

        for i in range(n_class):
            for j in range(self.n_svm):
                if i == self.lookup_class[j,0] or i == self.lookup_class[j,1]:
                    if self.lookup_class[j, 0]==i:
                        self.lookup_matrix[i,j]=1.0
                    else:
                        self.lookup_matrix[i,j]=-1.0

    def fit(self, x_np, y_multiclass_np, C, iterations=1, kernel=rbf(1)):
        x_np, y_multiclass_np = shuffle_ds(x_np,y_multiclass_np)
        self.C = C # box constraint
        # use SMO algorithm to fit
        x = torch.from_numpy(x_np).float() if not torch.is_tensor(x_np) else x_np
        x = x.to(self.device)
        self.x = x.to(self.device)

        y_multiclass = torch.from_numpy(y_multiclass_np).view(-1,1) if not torch.is_tensor(y_multiclass_np) else y_multiclass_np
        y_multiclass=y_multiclass.view(-1)
        self.y_matrix = torch.stack([self.cast(y_multiclass, k) for k in range(self.n_svm)],0).to(self.device)
        self.kernel = kernel
        a = self.a
        b = self.b
        for iteration in range(iterations):
            print("Iteration: ",iteration)
            for k in range(self.n_svm):
                y = self.y_matrix[k, :].view(-1).tolist()
                index = [i for i in range(len(y)) if y[i]!=0]
                shuffle(index)
                traverse = []
                if index is not None:
                    traverse = [i for i in range(0, len(index)-1, 2)]
                    if len(index)>2:
                         traverse += [len(index)-2]
                for i in traverse:
                    if str(index[i])+str(index[i+1]) not in self.blacklist[k]:
                        y1 = y[index[i]]
                        y2 = y[index[i+1]]
                        x1 = x[index[i],:].view(1,-1)
                        x2 = x[index[i+1],:].view(1,-1)
                        a1_old = a[k,index[i]].clone()
                        a2_old = a[k,index[i+1]].clone()

                        if y1 != y2:
                            H = max(min(self.C, (self.C + a2_old-a1_old).item()),0)
                            L = min(max(0, (a2_old-a1_old).item()),self.C)
                        else:
                            H = max(min(self.C, (a2_old + a1_old).item()),0)
                            L = min(max(0, (a2_old + a1_old - self.C).item()),self.C)
                        E1 =  self.g_k(k, x1) - y1
                        E2 =  self.g_k(k, x2) - y2
                        a2_new = torch.clamp(a2_old + y2 * (E1-E2)/self.kernel(x1 - x2,x1 - x2), min=L, max=H)
                        a[k,index[i+1]] = a2_new

                        a1_new = a1_old - y1 * y2 * (a2_new - a2_old)
                        a[k, index[i]] = a1_new

                        b_old = b[k,0]
                        K11 = self.kernel(x1,x1)
                        K12 = self.kernel(x1,x2)
                        K22 = self.kernel(x2,x2)
                        b1_new = b_old - E1 + (a1_old-a1_new)*y1*K11+(a2_old-a2_new)*y2*K12
                        b2_new = b_old - E2 + (a1_old-a1_new)*y1*K12+(a2_old-a2_new)*y2*K22
                        if (0<a1_new) and (a1_new<self.C):
                            b[k,0] = b1_new
                        if (0<a2_new) and (a2_new<self.C):
                            b[k,0] = b2_new
                        if ((a1_new == 0) or (a1_new ==self.C)) and ((a2_new == 0) or (a2_new==self.C)) and (L!=H):
                            b[k,0] = (b1_new + b2_new)/2
                        if b_old == b[k,0] and a[k,index[i]] == a1_old and a[k,index[i+1]] == a2_old:
                            self.blacklist[k].add(str(index[i]) + str(index[i+1]))
                            self.blacklist[k].add(str(index[i+1]) + str(index[i]))

    def predict(self,x_np):
        xp = torch.from_numpy(x_np) if not torch.is_tensor(x_np) else x_np
        xp = xp.float().to(self.device)
        k_predicts = (self.y_matrix.to(self.device) * self.a) @ self.kernel(xp,self.x).T  + self.b
        result = torch.argmax(self.lookup_matrix @ k_predicts,axis=0)
        return result.to("cpu").numpy()

    def cast(self, y, k):
        # cast the multiclass label of dataset to
        # the pos/neg (with 0) where pos/neg are what SVMk concerns
        return (y==self.lookup_class[k, 0]).float() - (y==self.lookup_class[k, 1]).float()

    def wTx(self,k,xi):
        # The prediction of SVMk without bias, w^T @ xi
        y = self.y_matrix[k, :].reshape((-1,1))
        a = self.a[k,:].view(-1,1)
        wTx0 =  self.kernel(xi, self.x) @ (y * a)
        return wTx0

    def g_k(self,k,xi):
        # The prediction of SVMk, xi[1,d]
        return self.wTx(k,xi) + self.b[k,0].view(1,1)

    def get_avg_pct_spt_vec(self):
        # the average percentage of support vectors,
        # test error shouldn't be greater than it if traing converge
        return torch.sum((0.0<self.a) & (self.a<self.C)).float().item()/(self.n_svm*self.m)


from kivy.app import App
from kivy.uix.button import Button
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.label import Label

class Teste(App):
    def build(self):
        box = BoxLayout(orientation='vertical')
        botão = Button(text='É Foda bixo',font_size=30,on_release=self.incrementar)
        self.texto = Label(text='1',font_size=30)
        box.add_widget(botão)
        box.add_widget(self.texto)
        return box
    def incrementar(self,botão):
        botão.text = 'pressionado'
        self.texto.text = str(int(self.texto.text)+1)

Teste().run()


import pygame
import sys
from midi_fxns import *
import random

# Initialize Pygame
pygame.init()

# Set the window dimensions
WINDOW_WIDTH = 640
WINDOW_HEIGHT = 480

# Create the Pygame window
screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
screen.fill((255,255,255))

font = pygame.font.SysFont("Arial", 24)

maj_min = {"up":"Major", "down":"Minor", "left":"Perfect", "right":"Perfect", "space":"Perfect"}
notes = {
    "0": 0,
    "1": 1,
    "2": 2,
    "3": 3,
    "4": 4,
    "5": 5,
    "6": 6,
    "7": 7,
    "8": 8
}
interval_names = {
    1: "Unison",
    2: "Second",
    3: "Third",
    4: "Fourth",
    5: "Fifth",
    6: "Sixth",
    7: "Seventh",
    8: "Octave"
}
def calc_interval(note1,note2,in_qual,in_name):
    interval = abs(note2 - note1)
    in_name = interval_names[in_name]

    interval_quality_map = {
        0: "Perfect",
        1: "Minor",
        2: "Major",
        3: "Minor",
        4: "Major",
        5: "Perfect",
        6: "Tritone",
        7: "Perfect",
        8: "Minor",
        9: "Major",
        10: "Minor",
        11: "Major",
        12: "Perfect"
    }

    interval_name_map = {
        0: "Unison",
        1: "Second",
        2: "Second",
        3: "Third",
        4: "Third",
        5: "Fourth",
        6: "Tritone",
        7: "Fifth",
        8: "Sixth",
        9: "Sixth",
        10: "Seventh",
        11: "Seventh",
        12: "Octave"
    }

    true_qual = interval_quality_map[interval]
    true_name = interval_name_map[interval]

    if interval_quality_map[interval] == "Tritone":
        if in_qual == "Major" and in_name == "Fourth":
            return (True,"Tritone")
        if in_qual == "Minor" and in_name == "Fifth":
            return (True,"Tritone")
        else:
            return (False,"Tritone")
    else:
        if in_qual == true_qual and in_name == true_name:
            return (True,f"{true_qual} {true_name}")
        else:
            return (False,f"{true_qual} {true_name}")

def play_notes_game():
    # Set the range of playable notes
    LOW_NOTE = 48  # C3
    HIGH_NOTE = 96  # C7

    # Generate two random notes within an octave of each other
    note1 = 48#random.randint(LOW_NOTE, HIGH_NOTE)
    note2 = 54#note1 + random.randint(-12,12)

    # Play the two notes back to back
    play_notes([note1])
    play_notes([note2])

    # Wait for the player to press the keys
    maj_min_pressed = None
    num_pressed = None
    while not (maj_min_pressed and num_pressed):
        for event in pygame.event.get():
            if event.type == pygame.KEYDOWN:
                key = pygame.key.name(event.key)
                if key in maj_min:
                    maj_min_pressed = maj_min[key]
                    print(maj_min_pressed)
                elif key in notes:
                    num_pressed = notes[key]
                    print(num_pressed)

            # Check for quit event
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

    result = calc_interval(note1,note2,maj_min_pressed,num_pressed)
    if result[0] == True:
        play_notes([note1,note2])
        text_surface = font.render(result[1], True, (0, 0, 0))
        screen.blit(text_surface, (100, 100))
        pygame.display.update()
    else:
        bad_int = f"{maj_min_pressed} {interval_names[num_pressed]}"
        #bad_int = '\u0336' + '\u0336'.join(bad_int)
        #font.set_strikethrough(True)
        text_surface1 = font.render(bad_int, True, (255, 0, 0))
        #font.set_strikethrough(False)
        text_surface2 = font.render(result[1], True, (0, 0, 0))
        screen.blit(text_surface1, (100, 100))
        screen.blit(text_surface2, (100, 200))

# Run the game loop
while True:
    # Handle events
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()

    # Clear the screen
    screen.fill((255, 255, 255))

    play_notes_game()

    # Update the screen
    pygame.display.flip()


from utils.utils import printProgress
import os
import ujson as json
from utils.corpus_utils import unbag_corpus

class NavigationDataLoader:
    def __init__(self, path, seq_len, max_replays=None, continuous=False, with_print=False):
        self.seq_len = seq_len

        actions = []
        replay_files = os.listdir(path)
        N = len(replay_files)
        if max_replays is None:
            max_replays = N
        i = 0
        self.n_replays = 0
        for fname in replay_files:
            if i >= max_replays:
                break
            if with_print:
                printProgress(i, N)
            file_path = os.path.join(path, fname)
            if not os.path.isdir(file_path):
                with open(file_path) as f:
                    replay_actions = json.load(f)
                actions.append(replay_actions)
                self.n_replays += 1
            i += 1
        for i in range(len(actions)):
            actions[i] = self.divide_into_composite(actions[i], continuous)
        self.actions = actions
        if with_print:
            print('Data loaded.')

    def divide_into_composite(self, actions: list, continuous=False):
        composite_actions = []
        ind = 0
        actions = ''.join(actions)
        while ind < len(actions):
            curr_word = actions[ind: ind + self.seq_len]
            if (len(curr_word) < seq_len):
                break
            composite_actions.append(curr_word)
            if continuous:
                ind += 1
            else:
                ind += self.seq_len
        return composite_actions

if __name__ == "__main__":
    replay_path = 'Navigation/action_corpus/player1/'
    embedding_dims = [3, 5, 10]
    seq_lens = [2, 3, 4]
    windows = [2, 3, 5]
    bag_of_words = True

    from gensim.test.utils import get_tmpfile
    from gensim.models import Word2Vec

    path_w2vec = get_tmpfile("act2vec.model")

    for seq_len in seq_lens:
        DL = NavigationDataLoader(replay_path, seq_len, continuous=True, with_print=True)
        orig_corpus = DL.actions
        for window in windows:
            for embedding_dim in embedding_dims:
                if not bag_of_words:
                    print('Unbagging corpus...')
                    corpus = unbag_corpus(orig_corpus, window=window)
                else:
                    corpus = orig_corpus

                print('Building Action2Vec model with window size {} and embedding dim {}'.format(window, embedding_dim))

                model = Word2Vec(corpus, size=embedding_dim, window=window, min_count=1, workers=4)

                vocab = list(model.wv.vocab)

                # print("Training word2vec model")
                model.train(corpus, total_examples=len(corpus), epochs=50)

                desc = "act2vec_dataset:navigation" + \
                       "_dim:" + str(embedding_dim) + \
                       "_win:" + str(window) + \
                       "_seq_len:" + str(seq_len)

                if bag_of_words:
                    desc += "_bagofwords"

                model.save("Navigation/embeddings/embeddings/" + desc + ".model")


import pytest

from rssant_feedlib import FeedResponseBuilder
from rssant_feedlib.response_file import FeedResponseFile

def _build_simple_response(status):
    builder = FeedResponseBuilder()
    builder.url('https://www.example.com/feed.xml')
    builder.status(status)
    builder.headers({
        'etag': '5e8c43f8-4d269b',
        'content-type': 'application/xml',
    })
    builder.content('''
        <?xml version="1.0" encoding="utf-8"?>
        <feed xmlns="http://www.w3.org/2005/Atom">
        <title>V2EX - 技术</title>
    '''.encode('utf-8'))
    response = builder.build()
    return response

def _build_no_content_response(status):
    builder = FeedResponseBuilder()
    builder.url('https://www.example.com/feed.xml')
    builder.status(status)
    response = builder.build()
    return response

_builder_funcs = {
    'simple': _build_simple_response,
    'no_content': _build_no_content_response,
}

@pytest.mark.parametrize('builder, status', [
    ('simple', 200),
    ('simple', 500),
    ('no_content', 204),
    ('no_content', -200),
])
def test_response_file(tmp_path, builder, status):
    response = _builder_funcs[builder](status)
    file = FeedResponseFile(tmp_path / 'test_response_file')
    file.write(response)
    file2 = FeedResponseFile(file.filepath)
    response2 = file2.read()
    assert repr(response) == repr(response2)


# pypi에 있는 라이브러리를 설치하면 이런식으로 쓸 수 있다.
from requests import get

websites = (
    "google.com",
    "airbnb.com",
    "https://twitter.com",
    "facebook.com",
    "https://youtube.com"
)

results = {}

for website in websites:
    if not website.startswith("https://"):
        website = f"https://{website}"
    response = get(website)
    status_code = response.status_code

    # 로그인 성공 등
    if status_code == 200:
        results[website] = "OK"
    # 회원가입 성공 / 글 작성 성공 등
    elif status_code == 201:
        results[website] = "Created"
    # 필수로 요구하는 Query Parameter를 누락했을 때 등
    elif status_code == 400:
        results[website] = "Bad Request"
    # 로그인하지 않은 상태에서 자신의 사용자 정보 요청 등
    elif status_code == 401:
        results[website] = "Unauthorized"
    # 로그인했지만 관리자용 API를 요청한 경우
    elif status_code == 403:
        results[website] = "Forobidden"
    # 존재하지 않는 Route에 요청을 보냈을 때
    elif status_code == 404:
        results[website] = "Not Found"
    # 어떤 리소스에 대해서 수정/삭제를 막고 가져오기만 허용하고 싶을 때
    elif status_code == 405:
        results[website] = "Method Not Allowed"
    # 중복 회원가입 요청이 발생했을 때
    elif status_code == 409:
        results[website] = "Conflict"
    # 전달받은 파일의 크기가 정의한 값보다 클 때
    elif status_code == 413:
        results[website] = "Payload Too Large"
    # 데이터베이스 오류, 예외처리하지 않은 오류 발생
    elif status_code == 500:
        results[website] = "Bad Gateway"
    # 기타
    else:
        results[website] = "ETC"

print(results)

# Response 옆에 숫자는 아래에 html status 코드를 보면 확인할 수 있다.
# https://developer.mozilla.org/ko/docs/Web/HTTP/Status


import numpy as np
import h5py
import mcubes
from mayavi import mlab
#from itertools import count
from mayavi.api import OffScreenEngine
import trimesh
import pickle
from skimage import measure
from skimage import morphology

# input_model_file  = 'CA1_ssmall_model.lm'
# input_morpho_file = "CA1_ssmall.h5"

input_model_file = "lms/_model.lm"
input_morpho_file = "CA1_small.h5"
input_label_file = "lm_annot/labels.hdf5"
output_file = 'morph_domains.png'

with open('lm_annot/list.pickle','rb') as f:
    list = pickle.load(f)

with h5py.File(input_label_file,'r') as f:
    labels = f['dendrite'][()]

print('labels.shape: ', labels.shape)

ids = np.unique(labels)
ids = ids[1:-2]
print('ids: ', ids)

with h5py.File(input_morpho_file,'r') as f:
    dendrite     = f['dendrite'][()]
    memb_v    = f['membrane vertices'][()]
    memb_f    = f['membrane faces'][()]
    PSD_ids   = f['PSD ids in membrane faces'][()]
    mito_v    = f['mitochondrion vertices'][()]
    mito_f    = f['mitochondrion faces'][()]
    pitch = f['unit length per voxel (um)'][()]

print('dendrite.shape: ', dendrite.shape)

dendrite = dendrite.astype(np.bool) ^ (labels > 0).astype(np.bool)
memb_v, memb_f, normals, values = measure.marching_cubes(dendrite, 0.5, spacing=(1,1,1))

trimesh.constants.tol.merge = 1e-7
mesh = trimesh.Trimesh(vertices=memb_v, faces=memb_f)
mesh_smooth = trimesh.smoothing.filter_laplacian(mesh, iterations=5)
memb_v = mesh_smooth.vertices
memb_f = mesh_smooth.faces

#memb_v = memb_v / pitch
mito_v = mito_v / pitch
print('pitch: ', pitch)

# Plot surface mesh
mlab.figure(bgcolor=(1.0,1.0,1.0), size=(400,741))
mlab.view(-180, 90, 700, [120.0, 120.0, 120.0 ])
mlab.triangular_mesh(mito_v[:,0] , mito_v[:,1] , mito_v[:,2] , mito_f, color=(1.0,1.0,0.6), opacity=0.6)
mlab.triangular_mesh(memb_v[:,0], memb_v[:,1], memb_v[:,2], memb_f, color=(0.7,0.7,0.7)  , opacity=0.3)
#mlab.triangular_mesh(memb_v[:,0], memb_v[:,1], memb_v[:,2], memb_f[PSD_ids,:], color=(1,0,0)  , opacity=0.3)

for id in ids:
    l = (labels == id).astype(np.bool)
    v, f, normals, values = measure.marching_cubes(l, 0.5, spacing=(1,1,1))
    mesh = trimesh.Trimesh(vertices=v, faces=f)
    mesh_smooth = trimesh.smoothing.filter_laplacian(mesh, iterations=5)
    v = mesh_smooth.vertices
    f = mesh_smooth.faces
    c = [x for x in list['list'] if x['id'] == id]
    r = c[0]['r']/256.0
    g = c[0]['g']/256.0
    b = c[0]['b']/256.0
    mlab.triangular_mesh(v[:,0], v[:,1], v[:,2], f, color=(r,g,b)  , opacity=0.3)

#
xvnum,yvnum,zvnum = dendrite.shape
Zoff = 0
xvnum = xvnum / 2
voxel_length = 50
length = voxel_length*pitch
text = "{0:.1f} um".format(length)
mlab.plot3d( [xvnum,xvnum],[0,voxel_length],[Zoff,Zoff],color=(0.7,0.7,0.7),tube_radius=2.5)
mlab.text3d( xvnum, voxel_length, Zoff-30, text, scale=10,color=(0.2,0.2,0.2))
# 0.02 um x 100

mlab.savefig(output_file)
mlab.show()


# -*- coding: utf-8 -*-

import argparse
import numpy as np
import os
from PIL import Image
import AdvGenerator
import random
import math
from Candidate import Candidate
from AdversarialExample import AdversarialExample
from operator import itemgetter, attrgetter

def main(args):
    import warnings
    warnings.simplefilter(action='ignore', category=FutureWarning)
    warnings.simplefilter(action='ignore', category=UserWarning)

    # suppress tensorflow error output
    import tensorflow as tf
    tf.logging.set_verbosity(tf.logging.ERROR)
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    from keras.datasets import cifar10
    from keras.models import load_model
    from keras.utils import to_categorical
    from keras.applications.resnet50 import preprocess_input, decode_predictions

    input_shape = (32, 32, 3)
    num_classes = 10
    class_labels = {
        0: 'airplane',
        1: 'automobile',
        2: 'bird',
        3: 'cat',
        4: 'deer',
        5: 'dog',
        6: 'frog',
        7: 'horse',
        8: 'ship',
        9: 'truck',
    }

    # load CIFAR-10 dataset
    (x_train, y_train), (x_test, y_test) = cifar10.load_data()

    # scale images from 0-255 to 0-1 and add channel dimension
    x_train = np.float32(x_train) / 255
    x_test = np.float32(x_test) / 255
    print(f'Input shape: {x_train.shape}')

    # convert label into a one-hot encoding
    y_train = to_categorical(y_train, num_classes)
    y_test = to_categorical(y_test, num_classes)
    print(f'Output shape: {y_train.shape}')

    # load models, target DNN model to attack
    file_path = 'cifar10_dnn.h5'
    print(f'Loading {file_path}...')
    model = load_model(file_path)

    # models to test for transferability
    file_path = 'alexnetmodel.h5'
    print(f'Loading {file_path}...')
    model1 = load_model(file_path)

    file_path = 'cifar10_dnn_target.h5'
    print(f'Loading {file_path}...')
    alexnet = load_model(file_path)
    # model.summary()

    starting_index = 2019

    resnetTransCount = 0
    alexnetTransCount = 0
    total = 0
    i = 0
    # for i in range(5):
    while total <= 30:
        i += 1

        print("Testing transferability property for testset[" + str(starting_index + i) + "].")
        image = x_test[starting_index + i]

        x = np.expand_dims(image, 0)
        groundtruth = np.argmax(y_test[starting_index + i])
        basemodelprediction = np.argmax(model.predict(x))
        testmodelprediction = np.argmax(model1.predict(x))
        alexprediction = np.argmax(alexnet.predict(x))

        # print(f'groundtruth: {groundtruth}')
        # print(f'basemodelprediction: {basemodelprediction}')
        # print(f'testmodelprediction: {testmodelprediction}')
        # print(f'alexprediction: {alexprediction}')

        if groundtruth == basemodelprediction and groundtruth == testmodelprediction and groundtruth == alexprediction:

            # generate adversarial example here.
            total += 1

            advExample, generation = AdvGenerator.parallelGAhighConf(populationSize=15, generation=1000, inputImage=image, model=model,
                                                             y_truth=groundtruth, IMAGEDIMENSION=32)

            img = Image.fromarray(np.uint8(advExample.reshape(input_shape) * 255), 'RGB').resize(
                (128, 128))

            img.show(title="test")

            advExample = np.expand_dims(advExample, 0)
            print(f'fail prediction: {model.predict(advExample)}')

            if np.argmax(model.predict(advExample)) == np.argmax(model1.predict(advExample)):
                resnetTransCount += 1

            if np.argmax(model.predict(advExample)) == np.argmax(alexnet.predict(advExample)):
                alexnetTransCount += 1

            print("\nadv example ")
            print(f'groundtruthMOD: {groundtruth}')
            print(f'basemodelpredictionMOD: {model.predict(advExample)}')
            print(f'testmodelpredictionMOD: {model1.predict(advExample)}')
            print(f'alexpredictionMOD: {alexnet.predict(advExample)}')

        print("\n\n\n")

    print(f'Total number of images compared: {total}')
    print(f'Number of adversarial examples transferred to second resnet: {resnetTransCount}')
    print(f'Percentage: {resnetTransCount/total}')
    print(f'Number of adversarial examples transferred : {alexnetTransCount}')
    print(f'Percentage: {alexnetTransCount/total}')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    main(parser.parse_args())


from django.contrib import admin
from .models import Resolution

# Register your models here.

@admin.register(Resolution)
class AdminResolution(admin.ModelAdmin):
    list_display = ('id', 'number', 'start_date', 'end_date', 'document', 'years', )
    search_fields = ('number', )
    list_filter = ('number', )


# -*- coding: utf-8 -*-
# @Time    : 2018/7/31 16:07
from cnki import env
import logging
# import pyodbc
import pymssql
from time import sleep

class Sql(object):
    def __init__(self):
        self.open()
        super().__init__()
    def close(self):
        self.cursor_two.close()
        self.connects_two.close()

    def open(self):
        self.connects_two = pymssql.connect(env.path, env.name, env.password, env.sql_name,charset='utf8')
        self.cursor_two = self.connects_two.cursor()

    def open_two(self):
        connects_two = pymssql.connect(env.path, env.name, env.password, env.sql_name,charset='utf8')
        cursor_two = connects_two.cursor()
        return connects_two,cursor_two

    def select_two(self, sql):
        try:
            self.cursor_two.execute(sql)
            result = self.cursor_two.fetchone()
            return result[0]
        except Exception as e:
            print(e)
            # logging.error("error " + str(e))
            # logging.error("error "+sql)
    def inset_fulltext(self,sql,nr):
        try:
            self.cursor_two.execute(sql,nr)
            self.connects_two.commit()
            print("插入数据库")
            return True
        except Exception as e:
            if str(e).find("2627")>-1:
                return True
            print(e)
            return False
    def execute_two(self,sql,nr,connects_two,cursor_two):
        try:
            cursor_two.execute(sql,nr)
            connects_two.commit()
            print("插入数据库")
            return True
        except Exception as e:
            if str(e).find("2627")>-1:
                return True
            print(e)
            return False
            # logging.error(e)
            # logging.error("error "+sql)
    def creat_fulltext(self):
        sql = "CREATE TABLE [dbo].[%s] (\
                [url] nvarchar(900) NOT NULL ,\
                [filename] nvarchar(200) NOT NULL ,\
                [path] nvarchar(200) NOT NULL \
                );\
                ALTER TABLE [dbo].[%s] ADD PRIMARY KEY ([url]);"%(env.quanwen_name,env.quanwen_name)
        try:
            self.cursor_two.execute(sql)
            self.connects_two.commit()
            print("新建表")
        except Exception as e:
            print(str(e))
            print("if 2714 表已经有了")
        self.close()
        self.open()

    def creat_table(self):
        sql =  "CREATE TABLE [dbo].[%s] (\
                [article_uid] nvarchar(200) NOT NULL ,\
                [article_type] nvarchar(200) NOT NULL ,\
                [article_url] nvarchar(1000) NOT NULL ,\
                [relationship] nvarchar(200) NULL ,\
                [refer_url] nvarchar(900) NULL ,\
                [refer_type] nvarchar(200) NULL ,\
                [refer_text] nvarchar(MAX) NULL ,\
                [refer_level] nvarchar(10) NULL \
                );\
                CREATE TABLE [dbo].[%s] (\
                [题名] nvarchar(MAX) NULL ,\
                [作者] nvarchar(255) NULL ,\
                [学校] nvarchar(255) NULL ,\
                [摘要] nvarchar(MAX) NULL ,\
                [关键词] nvarchar(MAX) NULL ,\
                [导师] nvarchar(255) NULL ,\
                [分类号] nvarchar(255) NULL ,\
                [url] nvarchar(255) NOT NULL ,\
                [insertTime] nvarchar(255) NULL \
                );\
                ALTER TABLE [dbo].[%s] ADD PRIMARY KEY ([url]);"%(env.refer_name,env.yuanshuju_name,env.yuanshuju_name)
        try:
            self.cursor_two.execute(sql)
            self.connects_two.commit()
            print("新建表")
        except Exception as e:
            print(str(e))
            print("if 2714 表已经有了")
        self.close()
        # self.open()


from dataclasses import dataclass
from os import stat, walk
from os.path import exists, join, basename
from typing import List, BinaryIO, Tuple

from asura.common.enums import ChunkType, ArchiveType
from asura.common.error import ParsingError
from asura.common.mio import PackIO
from asura.common.models.archive import BaseArchive, FolderArchive, ZbbArchive
from asura.common.models.chunks import BaseChunk
from asura.common.factories import ChunkUnpacker, ArchiveParser, initialize_factories

# The default root
DEFAULT_ROOT_DIR = "unpack"
# The path relative to root, containing cached decompressed archives
DECOMPRESSED_DIR = "decompressed"
# The path relative to the archive, containing chunk information
DUMP_DIR = "archives"

@dataclass
class UnpackOptions:
    output_directory: str = None
    # Appended to output directory,
    output_name: str = None

    def get_output_directory(self) -> str:
        return self.output_directory or DEFAULT_ROOT_DIR

    def get_named_output_directory(self) -> str:
        return self.get_output_directory() if self.output_name is None else join(self.get_output_directory(),
                                                                                self.output_name)

    def create_decompressed_cache_path(self, name: str = None) -> str:
        if name is None:
            return join(self.get_named_output_directory(), DECOMPRESSED_DIR)
        else:
            return join(self.get_named_output_directory(), DECOMPRESSED_DIR, name)

    def create_path(self, name: str = None) -> str:
        if name is None:
            return join(self.get_named_output_directory(), DUMP_DIR)
        else:
            return join(self.get_named_output_directory(), DUMP_DIR, name)

    included_chunks: List[ChunkType] = None
    excluded_chunks: List[ChunkType] = None

    included_archives: List[ArchiveType] = None
    excluded_archives: List[ArchiveType] = None

    cache_decompressed: bool = True
    use_cached_decompressed: bool = True
    unpack_decompressed: bool = True

    overwrite_chunks: bool = False
    strict_archive: bool = False

    def get_print_str_parts(self) -> List[str]:
        def list_opts(n, l: List):
            if l is None:
                return None
            else:
                return f"{n}: {', '.join([str(p) for p in l])}"

        def bool_opts(n: str, b: bool):
            if b is not None:
                return f"{n}: {'enabled' if b else 'disabled'}"
            else:
                return None

        parts = [
            list_opts("included_chunks", self.included_chunks),
            list_opts("excluded_chunks", self.excluded_chunks),

            list_opts("included_archives", self.included_archives),
            list_opts("excluded_archives", self.excluded_archives),

            bool_opts("cache_decompressed", self.cache_decompressed),
            bool_opts("use_cached_decompressed", self.use_cached_decompressed),
            bool_opts("unpack_decompressed", self.unpack_decompressed),
            bool_opts("overwrite_chunks", self.overwrite_chunks),
            bool_opts("strict_archive", self.strict_archive)
        ]
        return [s for s in parts if s is not None]

def unpack_chunk(chunk: BaseChunk, chunk_name: str, options: UnpackOptions = None) -> bool:
    options = options or UnpackOptions()
    chunk_path = options.create_path(chunk_name)
    print(f"\t\t\t{chunk_path}")
    return ChunkUnpacker.unpack(chunk, chunk_path, options.overwrite_chunks)

def write_meta(name: str, options: UnpackOptions = None):
    options = options or UnpackOptions()
    archive_path = options.create_path(name)
    PackIO.make_parent_dirs(archive_path)
    with open(archive_path + PackIO.ARCHIVE_INFO_EXT, "w"):
        pass

def unpack_archive(archive: BaseArchive, archive_name: str, stream: BinaryIO = None,
                   options: UnpackOptions = None) -> Tuple[bool, int, int]:
    options = options or UnpackOptions()
    if archive is None:
        return False, -1, -1

    if type(archive) == BaseArchive:
        print(f"Unimplimented ~ {archive.type}")
        return False, -1, -1

    if options.included_archives is not None and archive.type not in options.included_archives:
        return False, -1, -1
    if options.excluded_archives is not None and archive.type in options.included_archives:
        return False, -1, -1

    if isinstance(archive, FolderArchive):
        # Avoid loading chunks into memory for large archives
        written = 0
        total = 0
        try:
            write_meta(archive_name)
            for i, chunk in enumerate(archive.load_chunk_by_chunk(stream, options.included_chunks)):
                chunk_path = join(archive_name, f"Chunk {i}")
                if unpack_chunk(chunk, chunk_path, options):
                    written += 1
                total += 1
            return True, written, total
        except ParsingError as e:
            print(archive_name, e)
            return False, written, total
    elif isinstance(archive, ZbbArchive):
        cache_path = options.create_decompressed_cache_path(archive_name)
        if exists(cache_path) and stat(cache_path).st_size == archive.size and options.use_cached_decompressed:
            if options.unpack_decompressed:
                with open(cache_path, "rb") as cached:
                    is_archive, success, unpacked, total = unpack_stream(cached, archive_name, options)
                    if is_archive:
                        return success, unpacked, total
                    else:
                        return False, -1, -1
            else:
                return False, -1, -1
        elif options.cache_decompressed:
            PackIO.make_parent_dirs(cache_path)
            with open(cache_path, "w+b") as cached:
                archive.decompress_to_stream(stream, cached)
                if options.unpack_decompressed:
                    cached.seek(0)
                    is_archive, success, unpacked, total = unpack_stream(cached, archive_name, options)
                    if is_archive:
                        return success, unpacked, total
                    else:
                        return False, -1, -1
        else:
            if options.unpack_decompressed:
                decompressed_archive = archive.decompress(stream)
                return unpack_archive(decompressed_archive, archive_name, stream, options)

    return False, -1, -1

def unpack_stream(stream: BinaryIO, stream_name: str, options: UnpackOptions = None) -> Tuple[bool, bool, int, int]:
    try:
        archive = ArchiveParser.parse(stream)
    except ParsingError:
        if not options.strict_archive:
            return False, False, -1, -1
        else:
            raise
    success, unpacked, total = unpack_archive(archive, stream_name, stream, options)
    return True, success, unpacked, total

def unpack_file(path: str, file_name: str, options: UnpackOptions = None) -> \
        Tuple[bool, bool, int, int]:
    with open(path, "rb") as stream:
        return unpack_stream(stream, file_name, options)

def unpack_directory(search_dir: str, options: UnpackOptions = None) -> Tuple[
    int, int, int, int]:  # Archive_Unpacked, Archive Total, Chunks Unpacked, Chunks Total
    print(f"Unpacking '{search_dir}'")
    unpacked_archives = 0
    total_archives = 0
    unpacked_chunks = 0
    total_chunks = 0
    for root, _, files in walk(search_dir):
        for file in files:
            file_path = join(root, file)
            name = file_path.replace(search_dir, "").lstrip("\\/")
            print(f"\t...\\{name}")
            is_archive, success, unpacked, total = unpack_file(file_path, name, options=options)
            if is_archive:
                total_archives += 1
                if success:
                    unpacked_archives += 1
                    unpacked_chunks += unpacked
                    total_chunks += total_chunks
    return unpacked_archives, total_archives, unpacked_chunks, total_chunks

#

if __name__ == "__main__":
    initialize_factories()
    launcher_root = r"G:\Clients\Steam\Launcher"
    steam_root = r"C:\Program Files (x86)\Steam"
    eg_root = fr"steamapps\common\Evil Genius 2"

    se_group_root = r"E:\Downloads\sniper elite"
    # se1_root = join(se_group_root, "Sniper Elite")
    # se2_root = join(se_group_root, "Sniper Elite V2")
    # se3_root = join(se_group_root, "Sniper Elite 3")
    # se4_root = join(se_group_root, "Sniper Elite 4")

    eg_roots = [
        join(launcher_root, eg_root),
        join(steam_root, eg_root)
    ]
    eg_files = [
        (
            r"Misc\packages\required",
            [
             "furniture.asr",
             "furniture_content.asr",
             "furniture_content.asr.pc.sounds",
             "furniture_content.ts"
            ]
        )
    ]
    my_options = UnpackOptions(overwrite_chunks=False, output_name=basename(eg_root))
    DO_EG_FILES = True
    DO_EG_DIRS = False
    print("Options:")
    for part in my_options.get_print_str_parts():
        print("\t", part)

    if DO_EG_FILES:
        for root in eg_roots:
            if exists(root):
                for part_root, parts in eg_files:
                    for part in parts:
                        full_path = join(root, part_root, part)
                        print(full_path)
                        unpack_file(full_path, file_name=full_path.replace(root,"").lstrip("\\/"), options=my_options)
            else:
                print(f"Cant dump '{root}', doesn't exist.")
    if DO_EG_DIRS:
        for root in eg_roots:
            if exists(root):
                unpack_directory(root, options=my_options)
                break
            else:
                print(f"Cant dump '{root}', doesn't exist.")
    se_roots = [
        # se1_root, se2_root, se3_root, se4_root
    ]
    # for root in se_roots:
    #     if exists(root):
    #         unpack_directory(root, unpack_name=basename(root))
    #     else:
    #         print(f"Cant dump '{root}', doesn't exist.")


from django.urls import path,include
from rest_framework import routers
from .views import RegisterUserViewSet, UserLoginViewSet, PostViewSet

router = routers.DefaultRouter()
router.register(r'posts', PostViewSet)

urlpatterns=[
     path('register/', RegisterUserViewSet.as_view({'post': 'create'}), name='register-user'),
     path('login',UserLoginViewSet.as_view({'post':'create'}),name='user-login'),
     path('', include(router.urls)),

]


from itertools import permutations

def check(n, A):
    for i in range(len(A)):
        m = list(map(int, list(str(A[i][0]))))
        s, b = 0, 0
        for j in range(3):
            if n[j] == m[j]:
                s += 1
            elif n[j] in m:
                b += 1
        if s != A[i][1] or b != A[i][2]:
            return False
    return True

def solve(A):
    cnt = 0
    perm = permutations([i for i in range(1, 10)], 3)
    for p in perm:
        n = list(p)
        if check(n, A):
            cnt += 1
    return cnt

A = [list(map(int, input().split())) for _ in range(int(input()))]
print(solve(A))


import os
import sys
import pickle as pkl
import numpy as np
sys.path.append("/home/wyshi/simulator")
log_temp_dir = "/home/xweiwang/test/systest/dialog{}/dialogs_pkl/"

len_list = []

import scipy.stats

def mean_confidence_interval(data, confidence=0.95):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return m, h

for i in range(1, 8):
    len_list_tmp = []
    for filename in os.listdir(log_temp_dir.format(i)):
        if (not filename.endswith('incomplete.pkl')) and (filename.endswith("pkl")):
            print(log_temp_dir.format(i)+filename)
            with open(log_temp_dir.format(i)+filename, "rb") as fh:
                c = pkl.load(fh)
                len_list_tmp.append(len(c['dialog']))
    len_list.append(len_list_tmp)

for i in range(7):
    m, ci = mean_confidence_interval(len_list[i])
    print("{}: mean, {}; ci, ({})".format(i+1, m, ci))


"""Pipeline model.

A word count example using a threadpool, a sub-pipeline, and a mapped
pipeline to count words in multiple files across multiple threads while
consuming a very small amount of memory:

    from concurrent.futures import ThreadPoolExecutor
    import operator as op

    from tinyflow import ops, MapPipeline, Pipeline

    # Compute word count for a single file
    wordcount = MapPipeline() \
        | ops.cat() \
        | ops.methodcaller('lower') \
        | ops.methodcaller('split') \
        | ops.filter() \
        | ops.flatten() \
        | ops.counter()

    # Summarize stats across all files
    aggregate_wordcount = Pipeline() \
        | ops.flatten() \
        | ops.reduce_by_key(op.iadd, op.itemgetter(0), op.itemgetter(1))

    # Distributes the process across threads, aggregates, sorts, and
    # determines the 10 most frequent words.
    pipeline = Pipeline() \
       | ops.map(wordcount, pool='thread') \
       | aggregate_wordcount \
       | ops.sort(op.itemgetter(1), reverse=True) \
       | ops.take(10)

    # Execute the pipeline and give it access to 4 threads.
    infiles = ('LICENSE.txt' for _ in range(2))
    with ThreadPoolExecutor(4) as threads:
        for item in pipeline(infiles, thread_pool=threads):
            print(item)
"""

from .exceptions import NoPool, NotAnOperation
from .ops import Operation

__all__ = ['MapPipeline', 'Pipeline']

class Pipeline(object):

    """A ``tinyflow`` pipeline model.  Subclass to attach your own custom
    ``__init__()`` init.

        from tinyflow import ops, Pipeline

        pipeline = Pipeline() \
            | ops.map(lambda x: x ** 2)

        for item in pipeline(range(10)):
            pass

    Pipelines can also be treated as operations:

        from tinyflow import ops, Pipeline

        square_pipeline = Pipeline() \
            | ops.map(lambda x: x ** 2)

        pipeline = Pipeline() \
            | ops.map(lambda x: x - 1) \
            | square_pipeline

        for item in pipeline(range(10)):
            pass

    Attributes
    ----------
    operations : tuple
        Instances of ``tinyflow.ops.Operation()`` that will be used to process
        data.
    thread_pool : concurrent.futures.ThreadPoolExecutor or None
        Will be ``None`` unless a thread pool was passed to
        ``Pipeline.__call__()``.
    process_pool : concurrent.futures.ProcessPoolExecutor or None
        Will be ``None`` unless a process pool was passed to
        ``Pipeline.__call__()``.
    """

    @property
    def operations(self):
        return getattr(self, '_operations', tuple())

    @property
    def thread_pool(self):
        pool = getattr(self, '_thread_pool', None)
        if pool is None:
            raise NoPool(
                "An operation requested a thread pool but {!r} did not "
                "receive one.".format(self))
        return pool

    @property
    def process_pool(self):
        pool = getattr(self, '_process_pool', None)
        if pool is None:
            raise NoPool(
                "An operation requested a process pool but {!r} did not "
                "receive one.".format(self))
        return pool

    def close(self):
        """Override if to teardown a pipeline in ``Pipeline.__exit__()``."""
        pass

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def __or__(self, other):

        """Add a ``tinyflow.ops.Operation()`` to the pipeline."""

        if not isinstance(other, (Operation, Pipeline)):
            raise NotAnOperation(
                "Expected an 'Operation()', not: {}".format(other))
        other.pipeline = self

        # Enforce immutability when calling 'Pipeline.operations'
        self._operations = tuple(list(self.operations) + [other])

        return self

    __ior__ = __or__

    def __call__(self, data, process_pool=None, thread_pool=None):

        """Stream data through the pipeline.

        Parameters
        ----------
        data : object
            Input data.  Most operations expect an iterable, but some
            do not.
        process_pool : None or concurrent.futures.ProcessPoolExecutor
            A process pool that individual operations can use if needed.
        thread_pool : None or concurrent.futures.ThreadPoolExecutor
            A thread pool that individual operations can use if needed.
        """

        data = iter(data)

        self._process_pool = process_pool
        self._thread_pool = thread_pool

        for op in self.operations:
            # Ensure downstream nodes get an ambiguous iterator and not
            # something like a list that they get hooked on abusing.
            data = iter(op(data))

        return data

class MapPipeline(Pipeline):

    """Like ``Pipeline()`` except ``__call__()`` expects a single object
    instead of an iterable.  This is to enable mapping entire pipelines
    across a stream controlled by a different pipeline.  For instance,
    this produces a word count derived from multiple files across multiple
    threads:

        from tinyflow import MapPipeline, ops, Pipeline

        wordcount = MapPipeline() \
            | ops.cat() \
            | ops.methodcaller('lower') \
            | ops.methodcaller('split') \
            | ops.filter() \
            | ops.flatten() \
            | ops.counter()

        pipeline = Pipeline() \
            | ops.map(wordcount, pool='thread') \
            | ops.flatten() \
            | ops.reduce_by_key(op.iadd, op.itemgetter(0), op.itemgetter(1)) \

        for word, count in pipeline(<infiles>):
            pass
    """

    def __call__(self, data, *args, **kwargs):
        return iter(super(MapPipeline, self).__call__(
            iter([data]), *args, **kwargs))


import requests
import json
from application.api.stock.StockSchema import stock_list_schema
from application.api.stock.StockPrice import StockPrice
from application.api.stock.Stock import Stock
import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

####### region Crawler
def get_stock_index(symbol):
    pass
    # url = f'https://svr4.fireant.vn/api/Data/Companies/CompanyInfo?symbol={symbol}'
    # response = requests.get(url=url)
    # if response.ok:
    #     result = json.loads(response.content)
    new_item = {
        'symbol': symbol
        # 'symbol': result.get('Symbol', None),
        # 'company_name': result.get('CompanyName', None)
    }
    Stock.create(**new_item)
    # else:
    #     print('not ok')

def get_all_stock_indices():
    stock_sql = Stock.get_all()
    stock = stock_list_schema.dump(stock_sql)
    result = []
    for item in stock:
        result.append(item.get('symbol'))
    return result

def get_all_stock_indices_online():
    url = 'https://svr4.fireant.vn/api/Data/Finance/AllLastestFinancialInfo'
    response = requests.get(url=url)
    if response.ok:
        print('ok')
        # sql = f"delete from public.stock where 1=1;"
        # Stock.execute(sql)
        data = json.loads(response.content)
        result = []
        for item in data:
            result.append(item['Symbol'])
            # get_stock_index(item['Symbol'])
        return result
    else:
        print('get_all_stock_indices not ok')

def get_crawl_url(stock_index, start_date, end_date):
    return f'https://svr1.fireant.vn/api/Data/Companies/HistoricalQuotes?symbol={stock_index}&startDate={start_date}&endDate={end_date}';

def crawl(stock_index, start_date='2021-02-05', end_date='2021-07-04'):
    url = get_crawl_url(stock_index=stock_index, start_date=start_date, end_date=end_date)
    response = requests.get(url=url)
    if response.ok:
        print('ok')
        result = json.loads(response.content)
        sql = f"delete from public.stock_price where symbol = '{stock_index}' and stock_date between '{start_date}' and '{end_date}';"
        StockPrice.execute(sql)
        for item in result:
            new_item = {
                'symbol': item.get('Symbol', None),
                'stock_date': item.get('Date', None),
                'currency_unit': 'VND',
                'open_price': item.get('PriceOpen', None),
                'high_price': item.get('PriceHigh', None),
                'low_price': item.get('PriceLow', None),
                'close_price': item.get('PriceClose', None),
                'volume': item.get('Volume', None),
                'market_cap': item.get('MarketCap', None)
            }
            StockPrice.create(**new_item)
    else:
        print('not ok')

def crawl_all_list():
    # list_stock = Stock.get_all()
    list_stock = get_all_stock_indices()
    result = []
    list_stock.sort()
    for item in list_stock:
        result.append(item)
        crawl(item)
    return result

def crawl_at_a_date(stock_index, date):
    url = get_crawl_url(stock_index=stock_index, start_date=date, end_date=date)
    print(f"==crawl: {stock_index} at {date} via {url}")
    response = requests.get(url=url)
    if response.ok:
        result = json.loads(response.content)
        sql = f"delete from public.stock_price where symbol = '{stock_index}' and stock_date = '{date}';"
        # print(sql)
        # print(result)
        StockPrice.execute(sql)
        for item in result:
            new_item = {
                'symbol': item.get('Symbol', None),
                'stock_date': item.get('Date', None),
                'currency_unit': 'VND',
                'open_price': item.get('PriceOpen', None),
                'high_price': item.get('PriceHigh', None),
                'low_price': item.get('PriceLow', None),
                'close_price': item.get('PriceClose', None),
                'volume': item.get('Volume', None),
                'market_cap': item.get('MarketCap', None)
            }
            StockPrice.create(**new_item)
    else:
        print('not ok')

def crawl_all_list_at_a_date(date):
    # list_stock = Stock.get_all()
    print('crawl_all_list_at_a_date')
    try:
        list_stock = get_all_stock_indices()
        result = []
        list_stock.sort()
        for item in list_stock:
            result.append(item)
            crawl_at_a_date(item, date)
        return result
    except Exception as e:
        logger.exception(e)

####### endregion Crawler

####### region insert all to table

####### endregion insert all to table

####### endregion calculate and insert ema200, sma200, ema50, ema20, rsi14, macd to table stock

####### endregion event observer schedualing crawl and insert ema, rsi to table stock

####### endregion backtest: entry when, exit when, stop loss when

####### endregion pip or % calculator


import numpy as np
import pandas as pd
import gc
from sklearn import preprocessing
import matplotlib.pyplot as plt
import seaborn as sns

sns.set(font_scale=1.5)

from sklearn import neighbors
from sklearn.preprocessing import OneHotEncoder

from sklearn.preprocessing import LabelEncoder

#function to deal with variables that are actually string/categories
def zoningcode2int( df, target ):
    storenull = df[ target ].isnull()
    enc = LabelEncoder( )
    df[ target ] = df[ target ].astype( str )

    # print('fit and transform')
    df[ target ]= enc.fit_transform( df[ target ].values )
    # print( 'num of categories: ', enc.classes_.shape  )
    df.loc[ storenull, target ] = np.nan
    # print('recover the nan value')
    return enc

def fillna_knn(df, base, target, fraction=1, threshold=10, n_neighbors = 10):
    assert isinstance(base, list) or isinstance(base, np.ndarray) and isinstance(target, str)
    whole = [target] + base

    miss = df[target].isnull()
    notmiss = ~miss
    nummiss = miss.sum()

    if nummiss != 0:
        enc = OneHotEncoder()
        X_target = df.loc[notmiss, whole].sample(frac=fraction)

        enc.fit(X_target[target].unique().reshape((-1, 1)))

        Y = enc.transform(X_target[target].values.reshape((-1, 1))).toarray()
        X = X_target[base]

        # print('fitting')
        clf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform')
        clf.fit(X, Y)

        # print('the shape of active features: ', enc.active_features_.shape)

        # print('perdicting')
        Z = clf.predict(df.loc[miss, base])

        numunperdicted = Z[:, 0].sum()
        if numunperdicted / nummiss * 100 < threshold:
            # print('writing result to df')
            df.loc[miss, target] = np.dot(Z, enc.active_features_)
            # print('num of unperdictable data: ', numunperdicted)
            return enc
        else:
            # print('out of threshold: {}% > {}%'.format(numunperdicted / nummiss * 100, threshold))
            th = 0

def impute_geo_info(df_train,df_test):

    df_train.dropna(axis=0, subset=['latitude', 'longitude'], inplace=True)
    df_test.dropna(axis=0, subset=['latitude', 'longitude'], inplace=True)

    fillna_knn( df = df_train,
                  base = [ 'latitude', 'longitude' ] ,
                  target = 'regionidzip', fraction = 0.1 )

    fillna_knn(df=df_train,
               base=['latitude', 'longitude'],
               target='regionidcity', fraction=0.15)

    fillna_knn(df=df_train,
               base=['latitude', 'longitude'],
               target='lotsizesquarefeet', fraction=0.15, n_neighbors=1)

    fillna_knn(df=df_train,
               base=['latitude', 'longitude'],
               target='yearbuilt', fraction=0.15, n_neighbors=1)

    zoningcode2int(df=df_train,
                   target='propertyzoningdesc')

    fillna_knn(df=df_train,
               base=['latitude', 'longitude'],
               target='propertyzoningdesc', fraction=0.15, n_neighbors=1)

    zoningcode2int(df=df_train,
                   target='propertycountylandusecode')
    fillna_knn(df=df_train,
               base=['latitude', 'longitude'],
               target='propertycountylandusecode', fraction=0.15, n_neighbors=1)

    # test set

    fillna_knn( df = df_test,
                  base = [ 'latitude', 'longitude' ] ,
                  target = 'regionidzip', fraction = 0.1 )

    fillna_knn(df=df_test,
               base=['latitude', 'longitude'],
               target='regionidcity', fraction=0.15)

    fillna_knn(df=df_test,
               base=['latitude', 'longitude'],
               target='lotsizesquarefeet', fraction=0.15, n_neighbors=1)

    fillna_knn(df=df_test,
               base=['latitude', 'longitude'],
               target='yearbuilt', fraction=0.15, n_neighbors=1)

    zoningcode2int(df=df_test,
                   target='propertyzoningdesc')

    fillna_knn(df=df_test,
               base=['latitude', 'longitude'],
               target='propertyzoningdesc', fraction=0.15, n_neighbors=1)

    zoningcode2int(df=df_test,
                   target='propertycountylandusecode')
    fillna_knn(df=df_test,
               base=['latitude', 'longitude'],
               target='propertycountylandusecode', fraction=0.15, n_neighbors=1)

    return df_train,df_test


# -*- coding: UTF-8 -*-
import requests
from bs4 import BeautifulSoup

def get_all_list(url):
    r = requests.get(url)
    r.encoding = 'GBK'
    soup = BeautifulSoup(r.text,'html.parser')
    table = soup.select("table")
    ser_and_nov_fir = []
    for i in table:
        the_class = i.get("class")
        try:
            if the_class[1] == 'series':
                ser_and_nov_fir.append([0, i])
            elif the_class[1] == 'novel':
                ser_and_nov_fir.append([1, i])
        except:
            pass
    #print(ser_and_nov_fir)
    ser_and_nov_sec = []
    for i in ser_and_nov_fir:
        if not i[0]:
            ser_and_nov_sec.append([0, i[1].get_text(strip=True)])
            #print(i)
        else:
            #i[1]
            a = i[1].select('a')[0]
            novelid = a.get("href")[7:]
            name = a.get_text(strip=True)
            tr = i[1].select('td')[1:]
            dic = {}
            for td in tr:
                td_str = td.get_text(strip=True)
                if td_str[:2] == "类型":
                    type_ = td_str[3:].split("-")
                elif td_str[:2] == "进度":
                    ing = td_str[3:]
                elif td_str[:2] == "字数":
                    num = td_str[3:]
                elif td_str[:2] == "积分":
                    integral = td_str[3:]
                elif td_str[:2] == "收藏":
                    collect = "".join(td_str[3:].split(","))
            dic["novelid"] = novelid
            dic["书名"] = name
            dic["类型"] = type_
            dic["进度"] = ing
            dic["字数"] = int(num)
            dic["积分"] = integral
            dic["收藏"] = int(collect)
            ser_and_nov_sec.append([1, dic])
    #print(ser_and_nov_sec)
    return ser_and_nov_sec

if __name__ == "__main__":
    print("本程序用于获取某个作者的所有作品ID。")
    #author_id = "38144"
    author_id = input("请输入专栏id:")
    url = "https://m.jjwxc.net/wapauthor/" + author_id
    all_list = get_all_list(url)
    #is_all = int(input("是否需要作者的所有小说id？（需要输入“1”，需要额外筛选的输入“0”）"))
    is_all = 1
    if is_all:
        novelid = []
        for a_id in all_list:
            if a_id[0]:
                novelid.append(a_id[1]['novelid'])
        print(novelid)
    '''
    elif not is_all:
        select = input("请输入筛选选项（“作者自分类”、“类型”、“进度”、“字数”、“收藏”中选择输入，如需多个请在中间加入半角逗号“,”）：")
        selects = select.split(",")
        if len(selects) >= 2:
            UorN = input("交集请输入“1”，并集请输入“0”")
        i = 0
        for j in selects:
            eval("select_" + str(i)) = select()
    else:
        print("请输入“0”或“1”。")
    '''