import json
from .processing_utils import flatten_list
from collections import OrderedDict
from typing import *
from pathlib import Path
import os
import re

def get_summaries(directory: Path, docset_id: Optional[str] = None):
    eval_files = OrderedDict()
    pattern = re.compile(f'{docset_id}\.\w') if docset_id else r'D10\d\d-A\.M\.100\.\w\.\w'
    for filename in os.listdir(directory):
        if re.search(pattern, filename):
            with open(os.path.join(directory, filename), 'r', encoding='cp1252') as summary:
                eval_files[filename] = summary.read()
    return eval_files

def docset_loader(
        datafile: Path,
        docset: str,
        sentences_are_documents: Optional[bool] = False,
        merge_sentences_to_doc: Optional[bool] = False
    ) -> Tuple[List[List[str]], Dict[str, int]]:
    '''
    Read a data file and obtain the document set as a list of tokenized strings
    Args:
        - docset: The document set to extract
        - datafile: The datafile to extract the document set from
        - sentences_are_documents: Whether to consider sentences as documents. If True,
          the function returns a list of sentences for all docs in the docset. If False,
          the function returns a list of tokenized documents for all docs in the docset.
        - merge_sentences_to_doc: Whether to merge the sentences in a doc into a single string
    Returns:
        - Tuple: a list of all documents in the dataset and indices to reference those documents
    '''
    with open(datafile, 'r') as dfilestream:
        data = json.load(dfilestream)
    data = data[docset]
    documents, indexes = [], OrderedDict()
    for i, (doc_id, document) in enumerate(data.items()):
        sentences = []
        for sent in document[-1]:
            if isinstance(sent[0], list):
                sentences.extend(sent)
            else:
                sentences.append(sent)
        if sentences_are_documents:
            for j in range(len(sentences)):
                indexes[doc_id + "." + str(j)] = sentences[j]
            documents.extend(sentences)
        else:
            documents.append(flatten_list(sentences))
            indexes[doc_id] = i
    return documents, indexes

def dataset_loader(
        datafile: Path,
        sentences_are_documents: Optional[bool] = False,
        return_dict: Optional[bool] = False
    ) -> Tuple[List[List[str]], Dict[str, int]]:
    '''
    Read the whole data file as a list of documents. Docsets are ignored. This function
    is useful for calculating TFIDF on a training data
    Args:
        - datafile: The datafile to extract the document set from
        - sentences_are_documents: Whether to consider sentences as documents. If True,
          the function returns a list of sentences for all docs in the docset. If False,
          the function returns a list of tokenized documents for all docs in the docset.
        - sent_tokenize: Whether to return the documents as a dictionary of docsets to documents
    Returns:
        - Tuple: a list of all documents in the dataset and indices to reference those documents
    '''
    with open(datafile, 'r') as dfilestream:
        data = json.load(dfilestream)
    if return_dict:
        alldocuments = OrderedDict()
        for docset_id, docset in data.items():
            for doc_id, document in docset.items():
                sentences = []
                for sent in document[-1]:
                    if isinstance(sent[0], list):
                        sentences.extend(sent)
                    else:
                        sentences.append(sent)
                if sentences_are_documents:
                    for j, d in enumerate(sentences):
                        index_key = docset_id + "." + str(j)
                        alldocuments[index_key] = d
                else:
                    index_key = docset_id + "." + doc_id
                    alldocuments[index_key] = flatten_list(sentences)
        indexes = None
    else:
        alldocuments, indexes = [], OrderedDict()
        for docset_id, docset in data.items():
            for i, (doc_id, document) in enumerate(docset.items()):
                sentences = []
                for sent in document[-1]:
                    if isinstance(sent[0], list):
                        sentences.extend(sent)
                    else:
                        sentences.append(sent)
                if sentences_are_documents:
                    for j in range(len(sentences)):
                        index_key = docset_id + "." + str(j)
                        indexes[index_key] = len(alldocuments) + j
                    alldocuments.extend(sentences)
                else:
                    alldocuments.append(flatten_list(sentences))
                    index_key = docset_id + "." + doc_id
                    indexes[index_key] = i
    return alldocuments, indexes


"""
Ok, second attempt. To recap, we need to take a pointcloud and
    1. Determine the k most populated pillars
    2. For each point
        a. determine if in pillar
        b. if so, add to pillar centroid
        c. and if space, add to pillar tensor
"""
import torch
import numpy as np
from numba import jit

def pillarize(pcl, pos, metadata, max_npoints=100, max_npillars=10000):
    """
    sanitize inputs and pass to numba
    """
    #first center the pointcloud
    pcl_center = pcl - pos

    #filter extraneous points
    ox = -metadata['length_x']/2.
    oy = -metadata['length_y']/2.
    lx = metadata['length_x']
    ly = metadata['length_y']
    res = metadata['resolution']
    nx = int(lx / res)
    ny = int(ly / res)
    width = max(nx, ny)
    mask = (pcl_center[:, 0] >= ox) & (pcl_center[:, 0] < ox + lx) & (pcl_center[:, 1] >= oy) & (pcl_center[:, 1] < oy + ly)
    pcl_center = pcl_center[mask]

    #bin points into cells
    grid_xs = ((pcl_center[:, 0] - ox) / res).astype(np.int32)
    grid_ys = ((pcl_center[:, 1] - oy) / res).astype(np.int32)
    grid_hashes = grid_xs * width + grid_ys
    cnts = np.bincount(grid_hashes)
    topk_grid_hashes = np.argsort(cnts)[-max_npillars:]

    pillars = np.ones((max_npillars, max_npoints, 3)) * 0.
    pillar_cnts = np.zeros(max_npillars).astype(np.int32)
    pillar_idxs = np.stack([
        topk_grid_hashes // width,
        topk_grid_hashes % width
    ], axis=-1)

    grid_hash_mapping = -np.ones(cnts.shape[0]).astype(np.int32)
    grid_hash_mapping[topk_grid_hashes] = np.arange(topk_grid_hashes.shape[0])

    pillarize_kernel(
        pcl_center,
        pillars,
        pillar_cnts,
        grid_hash_mapping,
        ox,
        oy,
        nx,
        ny,
        res,
        max_npoints
    )

    #SAM TODO: move decoration to the numba part
    pillar_centroids = pillars.sum(axis=1) / np.expand_dims(pillar_cnts, axis=-1)
    centroid_diffs = pillars - np.expand_dims(pillar_centroids, axis=1)

    pillar_centers = np.stack([
        pillar_idxs[:, 0] * res + ox + res/2.,
        pillar_idxs[:, 1] * res + oy + res/2.,
    ], axis=-1)
    pillar_diffs = pillars[:, :, :2] - np.expand_dims(pillar_centers, axis=1)

    decorated_points = np.concatenate([
        pillars,
        centroid_diffs,
        pillar_diffs
    ], axis=-1)

    k = np.arange(max_npoints).reshape(1, -1)
    k2 = np.tile(k, (max_npillars, 1))
    empty_mask = k2 >= np.expand_dims(pillar_cnts, axis=-1)
    decorated_points[empty_mask] = 1e6

    return torch.tensor(decorated_points).float(), torch.tensor(pillar_idxs).long()

@jit(nopython=True, fastmath=True)
def pillarize_kernel(
                        pcl,
                        pillars,
                        pillar_cnt,
                        grid_hash_mapping,
                        ox,
                        oy,
                        nx,
                        ny,
                        res,
                        max_npoints
                    ):
    """
    speed this part up with numba.
    Args:
        pcl: [N x 3] array of points to pillarize
        pillars: [B x P x 3] accumulator of points
        pillar_cnt: [B] array to keep track of num pts
        grid_hash_mapping:
    """
    width = max(nx, ny)
    for ii, point in enumerate(pcl):
        gx = int((point[0] - ox) / res)
        gy = int((point[1] - oy) / res)
        pt_hash = gx * width + gy
        if grid_hash_mapping[pt_hash] != -1:
            pillar_idx = grid_hash_mapping[pt_hash]
            if pillar_cnt[pillar_idx] < max_npoints:
                pillars[pillar_idx, pillar_cnt[pillar_idx]] = point
                pillar_cnt[pillar_idx] += 1

def plot_pillars(pillars, pillar_idxs, metadata):
    fig = plt.figure(figsize = (18, 9))
    ax0 = fig.add_subplot(1, 2, 1, projection='3d')
    ax1 = fig.add_subplot(1, 2, 2)
    res = metadata['resolution']
    nx = int(metadata['length_x'] / res)
    ny = int(metadata['length_y'] / res)
    pillar_grid = np.zeros([nx, ny])
    pts = []
    for pillar, pidx in zip(pillars, pillar_idxs):
        valid_mask = np.all(np.isfinite(pillar), axis=-1)
        subpillar = pillar[valid_mask]
        pts.append(subpillar)
        pillar_grid[pidx[0], pidx[1]] = subpillar.shape[0]

    allpts = np.concatenate(pts, axis=0)
    ax0.scatter(allpts[:, 0], allpts[:, 1], allpts[:, 2], s=1., alpha=0.2, c=allpts[:, 2]/allpts[:, 2].max(), cmap='rainbow')
    ax1.imshow(pillar_grid.T, origin='lower')
    plt.show()

if __name__ == '__main__':
    import time
    import rosbag
    import ros_numpy
    from sensor_msgs.msg import PointCloud2
    import matplotlib.pyplot as plt
    from mpl_toolkits import mplot3d

    bag_fp = '/home/striest/Desktop/datasets/yamaha_maxent_irl/big_gridmaps_pointclouds/rosbags_debug/2022-07-10-13-48-09_1.bag'
    bag = rosbag.Bag(bag_fp, 'r')
    pcl_topic = '/merged_pointcloud'
    pose_topic = '/integrated_to_init'
    pose = None
    times = []
    cnt = 0

    metadata = {
        'origin':None,
        'length_x':120.0,
        'length_y':120.0,
        'resolution':0.5
    }

    for topic, msg, t in bag.read_messages(topics=[pcl_topic, pose_topic]):
        if topic == pose_topic:
            pose = msg.pose

        if topic == pcl_topic and pose is not None:
            if (cnt % 100) == 0:
                #dumb rosbag hack
                msg2 = PointCloud2(
                    header = msg.header,
                    height = msg.height,
                    width = msg.width,
                    fields = msg.fields,
                    is_bigendian = msg.is_bigendian,
                    point_step = msg.point_step,
                    row_step = msg.row_step,
                    data = msg.data,
                    is_dense = msg.is_dense
                )
                pcl = ros_numpy.numpify(msg2)
                pcl_tensor = np.stack([pcl[k] for k in 'xyz'], axis=-1)
                pos_tensor = np.array([pose.pose.position.x, pose.pose.position.y, pose.pose.position.z])
                t1 = time.time()
                pillars, pillar_idxs = pillarize(pcl_tensor, pos_tensor, metadata, max_npillars=10000, max_npoints=128)
                t2 = time.time()
                times.append(t2-t1)

                plot_pillars(pillars, pillar_idxs, metadata)
                break

    times = np.array(times)
    print('Avg pillarize time: {:.4f}s'.format(times.mean()))


from django.urls import path
from MVT_APP.views import *
from authentication_app.views import *
from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    path("inicio/", homepage, name="homepage"),
    path("Libros/", libros, name="libros"),
    path("Libros/nuevo", libros_crear, name="libro-crear"),
    path("autores/", autores, name="autores"),
    path("autores/nuevo", autores_crear, name="autores-crear"),
    path("editoriales/", editoriales, name="editoriales"),
    path("editoriales/nuevo", editoriales_crear, name="editoriales-crear"),
    path("editoriales/buscar/resultados/", resultados_busqueda_editoriales, name="editorial_resultados"),
    path("libros/buscar/resultados/", resultados_busqueda_libros, name="libro_resultados"),
    path("autores/buscar/resultados/", resultados_busqueda_autores, name="autor_resultados"),
    path("accounts/login/", iniciar_sesion, name="auth-login"),
    path("accounts/signup/", registrar_usuario, name="auth-register"),
    path("logout/", cerrar_sesion, name="auth-logout"),
    path("accounts/profile/editar/", editar_perfil, name="auth-editar-perfil"),
    path("accounts/profile/avatar/", agregar_avatar, name="auth-avatar"),
    path("editoriales/borrar/<pk>", EditorialDelete.as_view(), name="editorial-delete"),
    path("editoriales/actualizar/<pk>", EditorialEdit.as_view(), name="editorial-edit"),
    path("editoriales/listar/", EditorialList.as_view(), name="editorial-list"),
    path("editoriales/detalle/<pk>", EditorialDetail.as_view(), name="editorial-detail"),
    path("libros/borrar/<pk>", LibroDelete.as_view(), name="libro-delete"),
    path("libros/actualizar/<pk>", LibroEdit.as_view(), name="libro-edit"),
    path("libros/listar/", LibroList.as_view(), name="libro-list"),
    path("libros/detalle/<pk>", LibroDetail.as_view(), name="libro-detail"),
    path("autores/borrar/<pk>", AutorDelete.as_view(), name="autor-delete"),
    path("autores/actualizar/<pk>", AutorEdit.as_view(), name="autor-edit"),
    path("autores/listar/", AutorList.as_view(), name="autor-list"),
    path("autores/detalle/<pk>", AutorDetail.as_view(), name="autor-detail"),
    path("about/", sobre_nosotros, name="about"),

]
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


#1
import openpyxl
from openpyxl import Workbook
wb = openpyxl.load_workbook('task10_2.xlsx')
for sh in wb.sheetnames:
    ws = wb[sh]
    print(ws.title, '--------------------')
    for i in range(1, ws.max_row +1):
        for j in range(1, ws.max_column +1):
            print(i+1, j+1, ws.cell(i, j).value)
#2
import openpyxl
from openpyxl import Workbook
wb = openpyxl.load_workbook('task10_2.xlsx')
sheets = wb.sheetnames
sheet = wb[sheets[2]]
tes = set()
dict = {}
for i in range(sheet.max_row):
    tes.add(sheet.cell(row = i+1, column = 1).value)
lst = list(tes)
for n in range(len(lst)):
    dict.setdefault(lst[n], 0)
for i in range(sheet.max_row):
    s = sheet.cell(row = i + 1, column = 1).value
    dict[s] = dict[s] + sheet.cell(row=i + 1, column=2).value
sheet.append(['ИТОГО', sum(dict.values())])
sheet = wb[sheets[3]]
sheet.append({1:'111', 2:'222', 3:'333'})
wb.save('task10_2.xlsx')
#3
import time
x = 1000000
for i in range(0, x*10, x):
    t0 = time.time()
    su = 0
    for j in range(i):
        su += j
    t1 = time.time()
    print(i, t1 - t0)
#4
import time
d = {}
for i in range(11):
    if i % 2 == 0:
        def sl2():
            t0 = time.time()
            time.sleep(2)
            t1 = time.time()
            return (t1 - t0)
        d.setdefault(i, sl2())
    else:
        def sl3():
            t0 = time.time()
            time.sleep(3)
            t1 = time.time()
            return (t1 - t0)
        d.setdefault(i, sl3())
for i in range(len(d)):
    print(i, d[i])
print(sum(d.values()))
#5
import datetime
from datetime import date
import locale
locale.setlocale(locale.LC_ALL, "ru")
birthday = date(1995, 1, 11)
print('Название месяца:', birthday.strftime('%B'))
print('Название дня недели:', birthday.strftime('%A'))
print('Год:', birthday.strftime('%Y'))
print('Месяц:', birthday.strftime('%m'))
print('День:', birthday.strftime('%d'))
#6
import calendar
year = int(input('Введите год: '))
d = {0:0, 1:0, 2:0, 3:0, 4:0, 5:0, 6:0}
for month in range(1, 13):
    for day in range(1, calendar.monthrange(year, month)[1] + 1):
        print(year, month, day)
        print(calendar.weekday(year, month, day))
        dd = calendar.weekday(year, month, day)
        d[dd] += 1
print(d)

Lessons/Lesson 11/Lesson 11 (archive).py


# ********** ap27 6.24.2020 **********
# Granville will filter all outgoing requests
# based on keywords
# log the request and eventually mail a report
# based on https://docs.mitmproxy.org/stable/addons-scripting/
# ************************************

from mitmproxy import ctx
from mitmproxy import http
import logging
import os

# initiate logging
my_path = os.path.abspath(os.path.dirname(__file__))
alertlog = os.path.join(my_path, './logs/alert.log')
logging.basicConfig(filename=alertlog, format='%(asctime)s : %(message)s', level=logging.INFO)

# https://github.com/mitmproxy/mitmproxy/issues/237
# We don't want console to explode with flows, set max to 50 and code below clears flows
view = ctx.master.addons.get("view")
MAX_STORE_COUNT = 50

# filtering is done here - log and send artificial response back
# -> None means this function returns nothing (perhaps similar to void in Java)
def request(flow: http.HTTPFlow) -> None:

	# calls function below
	clearFlows()

	if "jenna" in flow.request.pretty_url:

		logging.info(str(flow.request.pretty_url))
		#flow.request.host = "0.0.0.0"
		#flow.kill()
		# This sends a reply from the proxy immediately
		flow.response = http.HTTPResponse.make(200,b"stop, not so fast yall...",{"Content-Type": "text/html"})

def clearFlows():

	# Use the below code to clear the flows every 50
	if view.store_count() >= MAX_STORE_COUNT:

		view.clear() # Same as hitting 'z' on console to clear


'''
Created on Jul 28, 2015

upload the CCLE metadata into CloudSQL

Copyright 2015, Institute for Systems Biology.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

@author: michael
'''
from datetime import date
import json
import logging
import sys
import traceback

# from isbcgc_metadata_cloudsql_model import ISBCGC_metadata_database_helper
from util import create_log
from util import import_module

def getlog(configFileName):
    try:
        with open(configFileName) as configFile:
            config = json.load(configFile)
        log_dir = str(date.today()).replace('-', '_') + '_' + config['log_dir_tag'] + '/'
        log_name = create_log(log_dir, 'create_ccle_metadata')
        log = logging.getLogger(log_name)
    except Exception as e:
        traceback.print_exc(5)
        raise e
    return config, log

def checkliteral(fields, value, name2index):
    if "'" == value[0]:
        return value[1:-1]
    else:
        return fields[name2index[value]]

def choice(fields, choices, name2index):
    '''
        choices = [fieldname1, value, fieldname2]
        choice1 = fields[name2index[fieldname1]]
        choice2 = fields[name2index[fieldname2]]
        if choice1 == value:
            return choice2
        else:
            return choice1
    '''
    value = fields[name2index[choices[0]]]
    if value == choices[1]:
        return fields[name2index[choices[2]]]
    else:
        return value

def mapit(config, fields, mappings, name2index):
    '''
        mappings = [fieldname, mapname]:
        key = fields[name2index[fieldname]]
        return config[mapname][key]
            or
        mappings = [fieldname1!fieldname2, mapname]:
        key1 = fields[name2index[fieldname1]]
        key2 = fields[name2index[fieldname2]]
        return config[mapname][key1][key2]
    '''
    mapkeys = mappings[0].split('!')
    if 1 == len(mapkeys):
        return config[mappings[1]][fields[name2index[mapkeys[0]]]]
    else:
        return config[mappings[1]][fields[name2index[mapkeys[0]]]][fields[name2index[mapkeys[1]]]]

def concat(config, fields, pieces, name2index):
    retVal = ''
    for piece in pieces:
        if 1 < len(piece.split('%')):
            # nested map
            retVal += mapit(config, fields, piece.split('%')[1:], name2index)
        else:
            retVal += checkliteral(fields, piece, name2index)
    return retVal

def substr(config, fields, pieces, name2index):
    return checkliteral(fields, pieces[0], name2index)[int(pieces[1]):int(pieces[2])]

def create_row(config, mapname, fields, name2index):
    retVal = []
    for key, _ in config[mapname]:
        mapping = key.split(':')
        if 1 == len(mapping):
            retVal += [checkliteral(fields, mapping[0], name2index)]
        elif 'choice' == mapping[0]:
            retVal += [choice(fields, mapping[1:], name2index)]
        elif 'map' == mapping[0]:
            retVal += [mapit(config, fields, mapping[1:], name2index)]
        elif 'concat' == mapping[0]:
            retVal += [concat(config, fields, mapping[1:], name2index)]
        elif 'substr' == mapping[0]:
            retVal += [substr(config, fields, mapping[1:], name2index)]
    return tuple(retVal)

def create_insert_columns(mapping):
    retVal = []
    for (_, value) in mapping:
        retVal += [value]
    return retVal

def parse_metadata(config, log):
    clinical_index = 0
    biospeciman_index = 1
    data_index = 2
    samples_index = 3
    try:
        log.info('\tbegin parse %s' % (config['input_file']))
        count = 0
        with open(config['input_file'], 'r') as input_file:
            #red the header line and map the indices
            header = input_file.readline()
            count += 1
            name2index = {}
            for index, name in enumerate(header.split('\t')):
                name2index[name.strip()] = index

            table_columns = [
                create_insert_columns(config['clinical_column_mapping']),
                create_insert_columns(config['biospecimen_column_mapping']),
                create_insert_columns(config['data_column_mapping']),
                create_insert_columns(config['samples_column_mapping'])
            ]
            table_rows = [set(), set(), set(), set()]
            for line in input_file:
                if 0 == count % 128:
                    log.info('\t\tprocessed %s rows' % (count))
                count += 1
                fields = [field.strip() for field in line.split('\t')]
                table_rows[clinical_index].add(create_row(config, 'clinical_column_mapping', fields, name2index))
                table_rows[biospeciman_index].add(create_row(config, 'biospecimen_column_mapping', fields, name2index))
                table_rows[data_index].add(create_row(config, 'data_column_mapping', fields, name2index))
                table_rows[samples_index].add(create_row(config, 'samples_column_mapping', fields, name2index))
        log.info('\tend parse %s.  processed %s total rows' % (config['input_file'], count))
    except Exception as e:
        log.exception('problem parsing on line %s' % (count))
        raise e
    return table_columns, table_rows

def insert_metadata(config, table_columns, table_rows, log):
    try:
        datastore = import_module(config['database_module'])
        tables = ['metadata_clinical', 'metadata_biospecimen', 'metadata_data', 'metadata_samples']
        for index in range(len(tables)):
            datastore.ISBCGC_database_helper.column_insert(config, list(table_rows[index]), tables[index], table_columns[index], log)

    except Exception as e:
        log.exception('problem saving metadata to the database')
        raise e

def main(configFileName):
    config, log = getlog(configFileName)
    log.info('begin create ccle metadata')
    table_columns, table_rows = parse_metadata(config, log)
    insert_metadata(config, table_columns, table_rows, log)
    log.info('end create metadata data dictionary')

if __name__ == '__main__':
    main(sys.argv[1])


from unittest.mock import Mock, patch

import pytest

from deep_translator.deepl import DeeplTranslator
from deep_translator.exceptions import AuthorizationException

@patch("deep_translator.deepl.requests")
def test_simple_translation(mock_requests):
    translator = DeeplTranslator(
        api_key="imagine-this-is-an-valid-api-key", source="en", target="es"
    )
    # Set the request response mock.
    mock_response = Mock()
    mock_response.status_code = 200
    mock_response.json.return_value = {"translations": [{"text": "hola"}]}
    mock_requests.get.return_value = mock_response
    translation = translator.translate("hello")
    assert translation == "hola"

@patch("deep_translator.deepl.requests.get")
def test_wrong_api_key(mock_requests):
    translator = DeeplTranslator(
        api_key="this-is-a-wrong-api-key!", source="en", target="es"
    )
    # Set the response status_code only.
    mock_requests.return_value = Mock(status_code=403)
    with pytest.raises(AuthorizationException):
        translator.translate("Hello")


# -*- coding: utf-8 -*-
from encodings.utf_8 import encode
from bs4 import BeautifulSoup
import requests

def find_date(tag, classe, url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, "html.parser")
    arr = []
    if tag == 'div':
        for t_tag in soup.find_all(tag, classe):
            arr.append(t_tag.text)
    return arr

list =find_date('div',{'class':'fEDvV'},"https://www.tripadvisor.fr/Attraction_Review-g297683-d317329-Reviews-Taj_Mahal-Agra_Agra_District_Uttar_Pradesh.html")
print(list)


""":class:`domain_intel.awis.actions.TrafficHistory` unit test cases.

"""
import os
import json
import pytest

import domain_intel.awis.actions

def test_awis_traffichistory_init():
    """Initialise a domain_intel.awis.actions.TrafficHistory object.
    """
    # When I initialise an Alexa Web Information object
    awis = domain_intel.awis.actions.TrafficHistory()

    # I should get a domain_intel.awis.actions.TrafficHistory instance
    msg = 'Object not a domain_intel.awis.actions.TrafficHistory instance'
    assert isinstance(awis, domain_intel.awis.actions.TrafficHistory), msg

@pytest.mark.usefixtures('docker_compose', 'kafka_ready')
def test_add_domains(add_traffic_domains):
    """Load of TrafficHistory domains into Kafka topic.
    """
    msg = 'Load of TrafficHistory domains into Kafka error'
    assert add_traffic_domains == 100, msg

@pytest.mark.usefixtures('docker_compose', 'kafka_ready')
def test_parse_raw_traffichistory(parse_raw_traffichistory):
    """Parse raw Alexa TrafficHistory action as JSON.
    """
    msg = 'Re-load of raw TrafficHistory Alexa JSON into Kafka error'
    assert parse_raw_traffichistory == (20123, 100, 100), msg

def test_flatten_single_traffichistory_xml():
    """Flatten a single TrafficHistory XML response from Alexa AWIS.
    """
    # Given an Alexa AWIS TrafficHistory XML response
    with open(os.path.join('domain_intel',
                           'test',
                           'files',
                           'samples',
                           'single_traffichistory_result.xml')) as _fh:
        source_xml = _fh.read().rstrip()

    # when I flatten the XML
    awis = domain_intel.awis.actions.TrafficHistory()
    received = json.loads(awis.flatten_xml(source_xml))

    # then I should receive a list of JSON-encode domains
    msg = 'Flattened domain should produce JSON-encoded domain list'
    expected = '2017-06-01'
    assert received['TrafficHistory']['Start']['$'] == expected, msg

def test_flatten_empty_traffichistory_xml():
    """Flatten an empty TrafficHistory XML response from Alexa AWIS.
    """
    # Given an Alexa AWIS TrafficHistory XML response
    with open(os.path.join('domain_intel',
                           'test',
                           'files',
                           'samples',
                           'single_traffichistory_empty.xml')) as _fh:
        source_xml = _fh.read().rstrip()

    # when I flatten the XML
    awis = domain_intel.awis.actions.TrafficHistory()
    received = json.loads(awis.flatten_xml(source_xml))

    # then I should receive a list of JSON-encode domains
    msg = 'Flattened domain should produce JSON-encoded domain list'
    expected = '2017-07-01'
    assert received['TrafficHistory']['Start']['$'] == expected, msg

def test_flatten_errored_traffichistory_xml():
    """Flatten an errored TrafficHistory XML response from Alexa AWIS.
    """
    # Given an Alexa AWIS TrafficHistory XML response
    with open(os.path.join('domain_intel',
                           'test',
                           'files',
                           'samples',
                           'single_traffichistory_alexa_error.xml')) as _fh:
        source_xml = _fh.read().rstrip()

    # when I flatten the XML
    awis = domain_intel.awis.actions.TrafficHistory()
    received = awis.flatten_xml(source_xml)

    # then I should receive a list of JSON-encode domains
    msg = 'Flattened domain should produce JSON-encoded domain list'
    assert received is None, msg

@pytest.mark.usefixtures('docker_compose',
                         'kafka_ready',
                         'parse_raw_traffichistory')
def test_flatten_traffic(flatten_traffic):
    """Flatten Alexa TrafficHistory XML and publish.
    """
    msg = 'Flattening TrafficHistory Alexa XML back into Kafka error'
    assert flatten_traffic == (100, 100), msg

@pytest.mark.usefixtures('docker_compose',
                         'kafka_ready',
                         'parse_raw_urlinfo',
                         'flatten_domains',
                         'init_arango',
                         'persist_domains',
                         'parse_raw_traffichistory',
                         'flatten_traffic')
def test_persist(persist_traffichistory):
    """Persist TrafficHistory to the database.
    """
    # ... then I should receive a record count if domains stored
    msg = 'Count against the "traffic" collection incorrect'
    assert persist_traffichistory == (100, 100), msg


import csv
import numpy as np
from keras import backend as K
samples = {}
ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+ =<>()[]{}"
MAX_LENGTH =1014

import numpy as np
from six.moves import cPickle
class Alphabet:
    characters = '''abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}\n'''
    def __init__(self):
        self.lookup = np.zeros(256, dtype = np.int32)
        for i, c in enumerate(self.characters):
            self.lookup[ord(c)] = i + 1

    def transform(self, s):
        ''' Return the index of characters in ``s``, for characters in alphabet, return index from 1,
        from characters out of alphabet, return index 0, transform upper case to lower case
        Args:
            s (string)
        Return:
            array(dtype = np.int32) of same length of ``s``
        '''
        arr = np.frombuffer(s.lower(), dtype=np.uint8)
        return np.take(self.lookup, arr)

    @property
    def vocab_size(self):
        return len(self.characters)

def onehot(arr, dims):
    return np.take(np.identity(dims), arr, axis = 0)

def load_data(filepath, max_document_length):
    alphabet = Alphabet()
    with open(filepath, 'rb') as fi:
        texts, labels = cPickle.load(fi)
        y = onehot(labels, 2)
        x = np.zeros([len(texts), max_document_length], dtype=np.int32)
        for i, text in enumerate(texts):
            text_arr = alphabet.transform(text[:max_document_length])
            x[i, : len(text_arr)] = text_arr
        return x, y

x, y = load_data('../data/aclImdb/chr-imdb-train.pkl', MAX_LENGTH)

np.random.seed(10)
shuffle_indices = np.random.permutation(np.arange(len(y)))
x_shuffled = x[shuffle_indices]
y_shuffled = y[shuffle_indices]
# Split train/test set
n_dev_samples = 1000
# TODO: Create a fuckin' correct cross validation procedure
x_train, x_dev = x_shuffled[:-n_dev_samples], x_shuffled[-n_dev_samples:]
y_train, y_dev = y_shuffled[:-n_dev_samples], y_shuffled[-n_dev_samples:]
print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev)))

import model_keras
import model
import tensorflow as tf
import os
import time
import datetime
def batch_iter(x, y, batch_size, num_epochs, shuffle=True):
    """
    Generates a batch iterator for a dataset.
    """
    # data = np.array(data)
    data_size = len(x)
    num_batches_per_epoch = int(data_size/batch_size) + 1
    for epoch in range(num_epochs):
        print("In epoch >> " + str(epoch + 1))
        print("num batches per epoch is: " + str(num_batches_per_epoch))
        # Shuffle the data at each epoch
        if shuffle:
            shuffle_indices = np.random.permutation(np.arange(data_size))
            x_shuffled = x[shuffle_indices]
            y_shuffled = y[shuffle_indices]
        else:
            x_shuffled = x
            y_shuffled = y
        for batch_num in range(num_batches_per_epoch-1):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            x_batch = x_shuffled[start_index:end_index]
            y_batch = y_shuffled[start_index:end_index]
            batch = list(zip(x_batch, y_batch))
            yield batch

import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--num_epochs', type=int, default=20)
parser.add_argument('--gpu', type=str, default="")
CharCNN = model.CharCNN
args = parser.parse_args()

os.environ['CUDA_VISIBLE_DEVICES']=args.gpu
batch_size = args.batch_size
num_epochs = args.num_epochs

with tf.Graph().as_default():
    cnn = CharCNN(num_classes=2)
    session_conf = tf.ConfigProto(
        log_device_placement=False)
    session_conf.gpu_options.per_process_gpu_memory_fraction = 0.5
    sess = tf.Session(config=session_conf)
    with sess.as_default():
        # Define Training procedure
        global_step = tf.Variable(0, name="global_step", trainable=False)
        optimizer = tf.train.AdamOptimizer(1e-3)
        grads_and_vars = optimizer.compute_gradients(cnn.loss)
        # print('updates oprs = {}'.format(len(cnn.updates.updates)))
        if hasattr(cnn, 'updates'):
            with tf.control_dependencies(cnn.updates):
                train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
        else:
            train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)

        # Keep track of gradient values and sparsity (optional)
        grad_summaries = []
        for g, v in grads_and_vars:
            if g is not None:
                grad_hist_summary = tf.summary.histogram(
                    "{}/grad/hist".format(v.name), g)
                sparsity_summary = tf.summary.scalar(
                    "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
                grad_summaries.append(grad_hist_summary)
                grad_summaries.append(sparsity_summary)
        grad_summaries_merged = tf.summary.merge(grad_summaries)

        # Output directory for models and summaries
        timestamp = str(int(time.time()))
        out_dir = os.path.abspath(os.path.join(
            os.path.curdir, "runs_new", timestamp))
        print("Writing to {}\n".format(out_dir))

        # Summaries for loss and accuracy
        loss_summary = tf.summary.scalar("loss", cnn.loss)
        acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)

        # Train Summaries
        train_summary_op = tf.summary.merge(
            [loss_summary, acc_summary, grad_summaries_merged])
        train_summary_dir = os.path.join(out_dir, "summaries", "train")
        train_summary_writer = tf.summary.FileWriter(
            train_summary_dir, sess.graph)

        # Dev summaries
        dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
        dev_summary_dir = os.path.join(out_dir, "summaries", "dev")
        dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)

        # Checkpoint directory. Tensorflow assumes this directory already
        # exists, so we need to create it.
        checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
        checkpoint_prefix = os.path.join(checkpoint_dir, "model")
        if not os.path.exists(checkpoint_dir):
            os.makedirs(checkpoint_dir)
        saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES))

        # Initialize all variables
        sess.run(tf.variables_initializer(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)))
        def train_step(x_batch, y_batch):
            """
            A single training step
            """
            x_batch = np.array(x_batch)
            feed_dict = {
              cnn.input_x: x_batch,
              cnn.input_y: y_batch,
              cnn.training:1,
              cnn.dropout_keep_prob: 0.5,
              K.learning_phase() : 1
            }

            _, step, summaries, loss, accuracy = sess.run(
                [train_op, global_step, train_summary_op,
                    cnn.loss, cnn.accuracy],
                feed_dict)
            time_str = datetime.datetime.now().isoformat()
            # write fewer training summaries, to keep events file from
            # growing so big.
            if step % (evaluate_every / 2) == 0:
                print("{}: step {}, loss {:g}, acc {:g}".format(
                    time_str, step, loss, accuracy))
                train_summary_writer.add_summary(summaries, step)
        def dev_step(x_batch, y_batch, writer=None):
            dev_size = len(x_batch)
            max_batch_size = 500
            num_batches = dev_size/max_batch_size
            acc = []
            losses = []
            #print("Number of batches in dev set is " + str(num_batches))
            for i in range(num_batches):
                x_batch_dev = x_batch[i * max_batch_size:(i + 1) * max_batch_size]
                y_batch_dev = y_batch[i * max_batch_size: (i + 1) * max_batch_size]
                feed_dict = {
                  cnn.input_x: x_batch_dev,
                  cnn.input_y: y_batch_dev,
                  cnn.training:0,
                  cnn.dropout_keep_prob: 1.0,
                  K.learning_phase() : 0
                }
                step, summaries, loss, accuracy = sess.run(
                    [global_step, dev_summary_op, cnn.loss, cnn.accuracy],
                    feed_dict)
                acc.append(accuracy)
                losses.append(loss)
                time_str = datetime.datetime.now().isoformat()
                #print("batch " + str(i + 1) + " in dev >>" +" {}: loss {:g}, acc {:g}".format(time_str, loss, accuracy))
                if writer:
                    writer.add_summary(summaries, step)
            print("Mean accuracy=" + str(sum(acc)/len(acc)))
            print("Mean loss=" + str(sum(losses)/len(losses)))
        # just for epoch counting
        num_batches_per_epoch = int(len(x_train)/batch_size) + 1
        # Generate batches
        batches = batch_iter(x_train, y_train,batch_size, num_epochs)
        evaluate_every = 300
        checkpoint_every =900
        # Training loop. For each batch...
        for batch in batches:
            x_batch, y_batch = zip(*batch)
            train_step(x_batch, y_batch)
            current_step = tf.train.global_step(sess, global_step)
            if current_step % evaluate_every == 0:
                print("\nEvaluation:")
                dev_step(x_dev, y_dev, writer=dev_summary_writer)
                print("Epoch: {}".format(
                    int(current_step / num_batches_per_epoch)))
                print("")
            if current_step % checkpoint_every == 0:
                path = saver.save(
                    sess, checkpoint_prefix, global_step=current_step)
                print("Saved model checkpoint to {}\n".format(path))


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from sparse_operation_kit import kit_lib
from tensorflow.distribute import MirroredStrategy
from tensorflow.distribute import get_replica_context
from tensorflow.distribute import has_strategy, get_strategy
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.ops import check_ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import resource_variable_ops
import sys, os

CommToolSet = set(["Strategy", "MPI", "Horovod", "OneDevice"])
def get_global_replica_id(comm_tool=None):
    def _strategy():
        replica_ctx = get_replica_context()
        return replica_ctx.replica_id_in_sync_group

    def _MPI():
        return int(os.getenv("OMPI_COMM_WORLD_RANK"))

    def _Horovod():
        import horovod.tensorflow as hvd
        return hvd.local_rank()

    def _OneDevice():
        return 0

    if comm_tool is None:
        raise RuntimeError("SparseOperationKit can only works with "
                        "tf.distribute.Strategy, MPI, Horovod or single GPU.")

    if comm_tool not in CommToolSet:
        raise RuntimeError("SparseOperationKit only works with tf.distribute.Strategy, "
                        "MPI, Horovod or single GPU. But got %s" %comm_tool)

    if "Strategy" == comm_tool:
        return _strategy()
    elif "MPI" == comm_tool:
        return _MPI()
    elif "Horovod" == comm_tool:
        return _Horovod()
    elif "OneDevice" == comm_tool:
        return _OneDevice()

def _get_comm_tool():
    if "horovod.tensorflow" in sys.modules:
        return "Horovod"
    elif has_strategy():
        return "Strategy"
    elif os.getenv("OMPI_COMM_WORLD_SIZE") is not None:
        return "MPI"
    else:
        return "OneDevice"

def _get_embedding_variable_attr(embedding_variable, attr):
    if not isinstance(attr, str):
        raise ValueError("attr must be a string, but got {}".format(type(attr)))
    if hasattr(embedding_variable, "values"):
        return getattr(embedding_variable.values[0], attr)
    else:
        return getattr(embedding_variable, attr)

def embedding_lookup(embedding_variable, values, training=True, dynamic_input=False):
    """
    This function is a wrapper of SOK's dense forward propagation.
    """
    embedding_layer = embedding_variable.embedding_layer

    resource_variable_ops.variable_accessed(embedding_variable)

    comm_tool = _get_comm_tool()

    return kit_lib.plugin_dense_fprop(embedding_variable._handle,
                                      embedding_layer.handle,
                                      values=values,
                                      global_replica_id=get_global_replica_id(comm_tool),
                                      training=training,
                                      unique_op_name=embedding_variable.name,
                                      dynamic_input=dynamic_input,
                                      dtype=embedding_variable.dtype)

def embedding_lookup_sparse(embedding_variable, sp_ids, slot_num, training=True):
    """
    This function is a wrapper of SOK's sparse forward propagation.
    """
    if not isinstance(sp_ids, sparse_tensor.SparseTensor):
        raise TypeError("sp_ids must be SparseTensor")

    values = sp_ids.values
    indices = check_ops.ensure_shape(sp_ids.indices, shape=(None, 2))
    row_indices = array_ops.transpose(indices, perm=[1, 0])[0]

    embedding_layer = embedding_variable.embedding_layer

    resource_variable_ops.variable_accessed(embedding_variable)

    comm_tool = _get_comm_tool()

    return kit_lib.plugin_sparse_fprop(embedding_variable._handle,
                                       embedding_layer.handle,
                                       values, row_indices,
                                       get_global_replica_id(comm_tool),
                                       slot_num=slot_num,
                                       training=training,
                                       unique_op_name=embedding_variable.name,
                                       dtype=embedding_variable.dtype)


import mne
import numpy as np
import scipy.io
import matplotlib.pyplot as plt
import numpy as np
#设置通道名
biosemi_montage = mne.channels.make_standard_montage('biosemi32')
#生成数据

datafile1='D:\\Project\\paper4\\统计\\06Topography\\results\\hc_offsets.mat'
hc_offsets = scipy.io.loadmat(datafile1)
hc_offsets = hc_offsets .get('hc_offsets')
#创建info对象
info = mne.create_info(ch_names=biosemi_montage.ch_names, sfreq=250.,ch_types='eeg')
#创建evokeds对象 1
evoked1= mne.EvokedArray(hc_offsets, info)
#evokeds设置通道1
evoked1.set_montage(biosemi_montage)
print(evoked1.data)

datafile12='D:\\Project\\paper4\\统计\\06Topography\\results\\pdoff_offsets.mat'
pdoff_offsets = scipy.io.loadmat(datafile12)
pdoff_offsets = pdoff_offsets .get('pdoff_offsets')
#创建info对象
info = mne.create_info(ch_names=biosemi_montage.ch_names, sfreq=250.,ch_types='eeg')
#创建evokeds对象 2
evoked2= mne.EvokedArray(pdoff_offsets, info)
#evokeds设置通道2
evoked2.set_montage(biosemi_montage)
print(evoked2.data)

datafile13='D:\\Project\\paper4\\统计\\06Topography\\results\\pdon_offsets.mat'
pdon_offsets = scipy.io.loadmat(datafile13)
pdon_offsets = pdon_offsets .get('pdon_offsets')
#创建info对象
info = mne.create_info(ch_names=biosemi_montage.ch_names, sfreq=250.,ch_types='eeg')
#创建evokeds对象 3
evoked3= mne.EvokedArray(pdon_offsets, info)
#evokeds设置通道3
evoked3.set_montage(biosemi_montage)
print(evoked3.data)

#画图
fig, ax = plt.subplots(ncols=3, figsize=(8, 4), gridspec_kw=dict(top=0.9),dpi=300,
                       sharex=True, sharey=True)
#mne.viz.plot_topomap(evoked1.data[:, 0], evoked1.info, axes=ax[0], show=False, cmap='RdBu_r', contours='6', image_interp='bilinear', res=1200)

#mne.viz.plot_topomap(evoked2.data[:, 0], evoked2.info, axes=ax[1], show=False, cmap='RdBu_r', contours='6', res=1200)

# add titles
ax[0].set_xlabel('\n'+'HC')
ax[1].set_xlabel('\n'+'PD_OFF')
ax[2].set_xlabel('\n'+'PD_ON')

#将64通道的weight映射到脑电头皮地形中，xy /= 5.3是为了将头皮轮廓扩大到和地形图一样大

im, cn = mne.viz.plot_topomap(evoked1.data[:, 0], evoked1.info, axes=ax[0], show=False, cmap='RdBu_r', contours='6', image_interp='bilinear', res=300)

im1, cn = mne.viz.plot_topomap(evoked2.data[:, 0], evoked2.info, axes=ax[1], show=False, cmap='RdBu_r', contours='6', image_interp='bilinear', res=300)

im2, cn = mne.viz.plot_topomap(evoked3.data[:, 0], evoked3.info, axes=ax[2], show=False, cmap='RdBu_r', contours='6', image_interp='bilinear', res=300)

plt.colorbar(im, ax=[ax[0], ax[1], ax[2]], fraction=0.02, label="Offsets")

#plt.colorbar(im1, ax=ax[1])
plt.show()


import julia
import numpy as np

JULIA_UTILS_PATH = "/home/claudio/postdoc/Castle/castle/julia_utils.jl"

jl = julia.Julia(compiled_modules=False)
from julia import Main
Main.using("JuLIP: Chemistry.AtomicNumber, Atoms")
Main.using("ACE: rpi_basis")
Main.using(
    """IPFitting.Data: read_Atoms, read_energy,
     read_forces, read_virial, read_configtype""")
Main.using("IPFitting: Dat")
Main.include(JULIA_UTILS_PATH)

matrix_indices_in_voigt_notation = [
    (0, 0),
    (1, 1),
    (2, 2),
    (1, 2),
    (0, 2),
    (0, 1),
]

def frame_to_julia_at(frame, energy_name="energy",
                      force_name="force", virial_name="virial"):
    at = Main.read_Atoms(frame)
    E = Main.read_energy(frame, energy_name)
    F = Main.read_forces(frame, force_name)
    V = Main.read_virial(frame, virial_name)
    cf = Main.read_configtype(frame)
    if cf is None:
        cf = 'unknown'
    at_ = Main.Dat(at, cf, E=E, F=F, V=V)
    return at_

def descriptors_from_at(basis, at):
    X, X_std, dX_dr, dX_ds = Main.extract_info_frame(basis, at.at)
    return (X.T, X_std.T, np.transpose(np.array(dX_dr), (1, 2, 0)),
            np.transpose(np.array(dX_ds), (1, 2, 0)))

def get_basis(N, maxdeg, rcut, species, r0=1.0,
              rin=1.0, constants=False):
    basis = Main.rpi_basis(N=N, r0=r0,
                        maxdeg=maxdeg, rcut=rcut,
                        species=species,
                        rin=rin, constants=constants)
    return basis, Main.length(basis)

def descriptors_from_frame(basis, frame, species,
                           energy_name="energy",
                           force_name="force",
                           virial_name="virial"):

    at = frame_to_julia_at(frame, energy_name,
                           force_name, virial_name)
    X, X_std, dX_dr, dX_ds = descriptors_from_at(basis, at)
    dX_ds = dX_ds[matrix_indices_in_voigt_notation, :][:, 0, 0, :]
    X, dX_dr, dX_ds = add_onebody_term(frame, species, X, dX_dr, dX_ds)
    return X, X_std, dX_dr, dX_ds

def descriptors_from_frame_no_forces(basis, frame, species, energy_name="energy"):
    at = frame_to_julia_at(frame, energy_name)
    X, X_std = Main.sum_descriptor(basis, at.at)
    X, _, __ = add_onebody_term(frame, species, X)
    return X, X_std

def local_descriptors_from_frame(basis, frame, species,
                           energy_name="energy",
                           force_name="force",
                           virial_name="virial"):
    at = frame_to_julia_at(frame, energy_name,
                           force_name, virial_name)
    X = Main.environment_descriptor(basis, at.at)
    dX_dr, dX_ds = Main.environment_d_descriptor(basis, at.at)
    dX_ds = dX_ds[matrix_indices_in_voigt_notation, :][:, 0, 0, :]
    dX_dr = np.array(dX_dr)
    X, dX_dr, dX_ds = add_onebody_local_term(frame, species, X, dX_dr, dX_ds)
    return  X, dX_dr, dX_ds

def local_descriptors_from_frame_no_forces(basis, frame, species, energy_name="energy"):
    at = frame_to_julia_at(frame, energy_name)
    X = Main.environment_descriptor(basis, at.at)
    X, _, __ = add_onebody_local_term(frame, species, X)
    return X

def add_onebody_term(frame, species, X, dX_dr=None, dX_ds=None):
    at_ns = frame.get_atomic_numbers()
    unique, counts = np.unique(at_ns, return_counts=True)
    unique_counts = dict(zip(unique, counts))
    tally = np.zeros(len(species))
    for i, s in enumerate(species):
        # Fix to prevent error messages when not all declared species are present in a frame
        try:
            tally[i] = unique_counts[s]
        except KeyError:
            pass
    if dX_dr is not None:
        dX_dr = np.concatenate([dX_dr, np.zeros((dX_dr.shape[0], 3, len(tally)))], axis = -1)
    if dX_ds is not None:
        dX_ds = np.concatenate([dX_ds, np.zeros((6, len(tally)))], axis = -1)
    X = np.concatenate([X, tally])
    return X, dX_dr, dX_ds

def add_onebody_local_term(frame, species, X, dX_dr=None, dX_ds=None):
    at_ns = frame.get_atomic_numbers()
    tally = np.zeros((len(at_ns), len(species)))
    for i in np.arange(len(at_ns)):
        # Fix to prevent error messages when not all declared species are present in a frame
        try:
            tally[i, :] = (species == at_ns[i]).astype('float')
        except KeyError:
            pass
    X = np.concatenate([X, tally], axis = -1)
    if dX_dr is not None:
        dX_dr = np.concatenate([dX_dr, np.zeros((dX_dr.shape[0], dX_dr.shape[1], len(species), 3))], axis = -2)
    if dX_ds is not None:
        dX_ds = np.concatenate([dX_ds, np.zeros((6, len(species)))], axis = -1)
    return X, dX_dr, dX_ds


# -*- coding: utf-8 -*-
import unittest
from flask import Flask
from todos import app
from todos.models.Todo import Todo
from todos.database import db
import tempfile, os
import json

class TestTodos(unittest.TestCase):
    def setUp(self):
        self.db_fd,app.config['DATABASE'] = tempfile.mkstemp()
        app.config['TESTING'] = True
        self.app = app.test_client()
        db.drop_all()
        db.create_all()

        " Populate the db with some todos "
        todo1 = Todo(title='do stuff')
        todo2 = Todo(title='do more stuff')
        todo3 = Todo(title='do even more stuff')
        db.session.add(todo1)
        db.session.add(todo2)
        db.session.add(todo3)
        db.session.commit()

    def tearDown(self):
        os.close(self.db_fd)
        os.unlink(app.config['DATABASE'])

    def test_get(self):
        right_resp=dict()
        right_resp["todos"] = []
        right_resp["todos"].append({"done": False, "id": 1,"order": None, "title": "do stuff"})
        right_resp["todos"].append({"done": False, "id": 2,"order": None, "title": "do more stuff"})
        right_resp["todos"].append({"done": False, "id": 3,"order": None, "title": "do even more stuff"})

        response = self.app.get('/api/')
        response_dict = json.loads(response.data)

        self.assertEqual(right_resp,response_dict)

    def test_add_todo(self):
        data = dict(title="new todo item")
        response = self.app.post('/api/', data=json.dumps(data),
        content_type='application/json')
        response_dict = json.loads(response.data)

        right_resp=dict()
        right_resp['title'] = 'new todo item';
        right_resp['done'] =  False;
        right_resp['order'] = None;
        right_resp['id'] = 4;

        self.assertEqual(right_resp,response_dict)

    def test_update_todo(self):
        data = dict()
        data['title'] = 'new title';
        data['done'] =  False;
        data['order'] = 1;

        todo_testid = 1
        response = self.app.put('/api/'+str(todo_testid), data=json.dumps(data),
        content_type='application/json')
        response_dict = json.loads(response.data)

        self.assertEqual(dict(response='OK'),response_dict)

        todos = Todo()
        todo = todos.get(todo_testid)
        self.assertEqual(todo['title'],data['title'])
        self.assertEqual(todo['order'],data['order'])

if __name__ == '__main__':
    unittest.main()


import requests
import json

import eodnharvester.history as history
import eodnharvester.settings as settings
import eodnharvester.auth as auth

from eodnharvester.product import Product

class Entity(object):
    def __init__(self, **kwargs):
        self.entity_id = kwargs.get("entityId", None)

        self.populated = False
        self.log = history.Record()

        self.metadata = {}
        self.products = []

        self.metadata["acquisitionDate"] = kwargs.get("acquisitionDate", None)
        self.metadata["startTime"]       = kwargs.get("startTime", None)
        self.metadata["endTime"]         = kwargs.get("endTime", None)
        self.metadata["browseUrl"]       = kwargs.get("browseUrl", None)
        self.metadata["downloadUrl"]     = kwargs.get("downloadUrl", None)
        self.metadata["lowerLeftCoordinate"]  = kwargs.get("lowerLeftCoordinate", None)
        self.metadata["upperRightCoordinate"] = kwargs.get("upperRightCoordinate", None)
        self.metadata["upperLeftCoordinate"]  = kwargs.get("upperLeftCoordinate", None)
        self.metadata["lowerRightCoordinate"] = kwargs.get("lowerRightCoordinate", None)
        self.metadata["datasetName"] = kwargs.get("datasetName", settings.DATASET_NAME)
        self.metadata["node"]        = kwargs.get("node", settings.NODE)

    def GetProducts(self):
        logger = history.GetLogger()
        apiKey = auth.login(self.log)

        self.products = []
        response = ""
        url = "http://{usgs_host}/inventory/json/{request_code}"

        sceneRequest = {
            "datasetName": self.metadata["datasetName"],
            "apiKey":      apiKey,
            "node":        self.metadata["node"],
            "entityIds":   [self.entity_id]
        }
        scene_url = url.format(usgs_host    = settings.USGS_HOST,
                               request_code = "downloadoptions")

        try:
            logger.info("Getting information on {product} products".format(product = self.entity_id))
            logger.debug("{url}?jsonRequest={params}".format(url = scene_url, params = json.dumps(sceneRequest)))
            response = requests.get(scene_url, params = { 'jsonRequest': json.dumps(sceneRequest) }, timeout = settings.TIMEOUT)
            response = response.json()
            logger.debug(response)
        except requests.exceptions.RequestException as exp:
            error = "Failed to get scene metadata - {exp}".format(exp = exp)
            logger.error(error)
            self.log.error(self.entity_id, error)
            auth.logout(self.log)
            return False
        except ValueError as exp:
            error = "Error while decoding scene json - {exp}".format(exp = exp)
            logger.error(error)
            self.log.error(self.entity_id, error)
            auth.logout(self.log)
            return False
        except Exception as exp:
            error = "Unknown error while getting scene metadata - {exp}".format(exp = exp)
            logger.error(error)
            self.log.error(self.entity_id, error)
            auth.logout(self.log)
            return False

        for entity in response["data"][0]["downloadOptions"]:
            product = Product(self.entity_id, entity["downloadCode"], entity["filesize"], self.metadata)
            self.products.append(product)

        auth.logout(self.log)
        self.populated = True
        return True

    def __iter__(self):
        return _entity_iter(self)

class _entity_iter(object):
    def __init__(self, entity):
        self.entity = entity
        self.index = 0

    def __iter__(self):
        return self

    # Python2.x backwards compatibility
    def next(self):
        return self.__next__()

    def __next__(self):
        try:
            if not self.entity.populated:
                self.entity.GetProducts()

            product = self.entity.products[self.index]
            self.index += 1
        except Exception as exp:
            raise StopIteration()

        return product


#import dependencies
import numpy as np   #for matrix math
import cPickle as pickle  #to save/load model
import gym

#hyperparameters
H = 200 #number of nodes in the hidden layer
batch_size = 10
learning_rate = 1e-4
gamma = 0.99 #discount factor
decay_rate = 0.99 #for RMS Prop Optimiser for Gradient Descent
resume = False #to resume from previous checkpoint or not

#initialise : init model
D = 80*80 #input dimension
if resume:
    model = pickle.load(open('model.v','rb'))
else:
    model = {}
    #xavier initialisation of weights
    model['W1'] = np.random.randn(H,D)*np.sqrt(2.0/D)
    model['W2'] = np.random.randn(H)*np.sqrt(2.0/H)
grad_buffer = {k: np.zeros_like(v) for k,v in model.iteritems()} #to store our gradients which can be summed up over a batch
rmsprop_cache = {k: np.zeros_like(v) for k,v in model.iteritems()} #to store the value of rms prop formula

#activation function
def sigmoid(x):
    return 1.0/(1.0+np.exp(-x))   #adding non linearing + squashing

def relu(x):
    x[x<0] = 0
    return x

#preprocessing function
def prepro(I): #where I is the single frame of the game as the input
    """ prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector """
    #the values below have been precomputed through trail and error by OpenAI team members
    I = I[35:195] #cropping the image frame to an extent where it contains on the paddles and ball and area between them
    I = I[::2,::2,0] #downsample by the factor of 2 and take only the R of the RGB channel.Therefore, now 2D frame
    I[I==144] = 0 #erase background type 1
    I[I==109] = 0 #erase background type 2
    I[I!=0] = 1 #everything else(other than paddles and ball) set to 1
    return I.astype('float').ravel() #flattening to 1D

def discount_rewards(r):
    """ take 1D float array of rewards and compute discounted reward """
    discount_r = np.zeros_like(r)
    running_add = 0 #addition of rewards
    for t in reversed(xrange(0,r.size)):
        if r[t] != 0: #episode ends
            running_add = 0
        running_add = gamma*running_add+r[t]
        discount_r[t] = running_add
    return discount_r

def policy_forward(x):
    h = np.dot(model['W1'],x)
    h = relu(h)
    logit = np.dot(model['W2'],h)
    p = sigmoid(logit)
    return p,h   #probability of action 2(i.e. UP) and hidden layer state i.e. hidden state

def policy_backward(arr_hidden_state,gradient_logp,observation_values):
    """ backward pass """
    #arr_hidden_state is array of intermediate hidden states  shape [200x1]
    #gradient_logp is the loss value [1x1]
    dW2 = np.dot(arr_hidden_state.T,gradient_logp).ravel()  # [200x1].[1x1] => [200x1] =>flatten=>[1x200]
    dh = np.outer(gradient_logp,model['W2']) # [1x1]outer[1x200] => [1x200]
    dh = relu(dh) #[1x200]
    dW1 = np.dot(dh.T,observation_values)  #[200x1].[1x6400] => [200x6400]
    return {'W1':dW1,'W2':dW2}

#implementation details
env = gym.make('Pong-v0')
observation = env.reset()
prev_x = None #prev frame value in order to compute the difference between current and previous frame
#as discussed frames are static and the difference is used to capture the motion
#Intially None because there's no previous frame if the current frame is the 1st frame of the game
episode_hidden_layer_values, episode_observations, episode_gradient_log_ps, episode_rewards = [], [], [], []
running_reward = None
reward_sum = 0
episode_number = 0

#begin training
while True:
    env.render()
    #get the input and preprocess it
    cur_x = prepro(observation)
    #get the frame difference which would be the input to the network
    if prev_x is None:
        prev_x = np.zeros(D)
    x = cur_x - prev_x
    prev_x = cur_x

    #forward propagation of the policy network
    #sample an action from the returned probability
    aprob, h = policy_forward(x)
    #stochastic part
    if np.random.uniform() < aprob:
        action = 2
    else:
        action = 3

    episode_observations.append(x) #record observation
    episode_hidden_layer_values.append(h) #record hidden state
    if action == 2:
        y = 1
    else:
        y = 0

    episode_gradient_log_ps.append(y-aprob) #record the gradient

    #new step in the environment
    observation,reward,done,info = env.step(action)
    reward_sum+=reward #for advantage purpose
    episode_rewards.append(reward) #record the reward

    if done:  #if the episode is over
        episode_number+=1

        #stack inputs,hidden_states,actions,gradients_logp,rewards for the episode
        arr_hidden_state = np.vstack(episode_hidden_layer_values)
        gradient_logp = np.vstack(episode_gradient_log_ps)
        observation_values = np.vstack(episode_observations)
        reward_values = np.vstack(episode_rewards)

        #reset the memory arrays
        episode_hidden_layer_values, episode_observations, episode_gradient_log_ps, episode_rewards = [], [], [], []

        #discounted reward computation
        discounted_episoderewards = discount_rewards(reward_values)
        #normalise discounted_episoderewards i.e. we obtain Advantage
        discounted_episoderewards = (discounted_episoderewards - np.mean(discounted_episoderewards))/np.std(discounted_episoderewards)

        #modulate the gradient with the advantage
        gradient_logp *= discounted_episoderewards

        grad = policy_backward(arr_hidden_state,gradient_logp,observation_values)

        #summing the gradients over the batch size
        for layer in model:
            grad_buffer[layer]+=grad[layer]

        #perform RMS prop to update weights after every 10 episodes
        if episode_number % batch_size == 0:
            epsilon = 1e-5
            for weight in model.keys():
                g = grad_buffer[weight] #gradient
                rmsprop_cache[weight] = decay_rate*rmsprop_cache[weight]+(1-decay_rate)*g**2
                model[weight]+=learning_rate*g/(np.sqrt(rmsprop_cache[weight]) + epsilon)
                grad_buffer[weight] = np.zeros_like(model[weight])

        if running_reward is None:
            running_reward = reward_sum
        else:
            running_reward = running_reward*learning_rate+reward_sum*(1-learning_rate)

        print('Episode Reward : {}, Running Mean Award : {}'.format(reward_sum,running_reward))
        if episode_number % 100 == 0:
            pickle.dump(model,open('model.v','wb'))

        reward_sum = 0
        prev_x = None
        observation = env.reset() #resetting the environment since episode has ended

    if reward != 0: #if reward is either +1 or -1 i.e. an episode has ended
        print("Episode {} ended with reward {}".format(episode_number,reward))


import pandas as pd
from numerize import numerize

data = {'TOP10': ['United States','China','Japan','Germany','India','United Kingdom','France','Italy','Brazil','Canada']}
df = pd.DataFrame(data)

print(df)

upperCaseNames = [x.upper() for x in df["TOP10"]]
print(upperCaseNames)

forwardAndReverse = [(x.upper(),x.upper()[::-1]) for x in df["TOP10"]]
print(forwardAndReverse)

data = {'Country Name': ['United States','China','Japan','Germany','India','United Kingdom','France','Italy','Brazil','Canada'],
        'GDP 2021 Est. Trillions':[22675271000000,16642318000000,5378136000000,4319286000000,3124650000000,3049704000000,2938271000000,2106287000000,1883487000000,1806707000000]}
df = pd.DataFrame(data, index = [1,2,3,4,5,6,7,8,9,10])
print(df)

df["Country Name"] = [x.upper() for x in df["Country Name"]]
print(df)

df["GDP 2021 Est. Trillions"] = [ numerize.numerize(x) for x in df["GDP 2021 Est. Trillions"]]
print(df)


from rest_framework import permissions
from rest_framework.authtoken.models import Token

from api.models import User

class AdminAuthentication(permissions.BasePermission):

    def has_permission(self, request, view):
        try:
            token = request.headers['token']
        except:
            token = ''

        token_query = Token.objects.filter(key=request.headers['token']).first()
        user = token_query.user
        return user.is_admin


import tensorflow as tf
import numpy as np
from tensorflow.keras import layers
import pandas
import tensorflow_datasets as tfds
import cv2
import tensorflow.keras.backend as K

cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath="checkpoints/1", save_freq='epoch', verbose=1)

idConv = {0: 0, 1: 217, 2: 482, 3: 491, 4: 497, 5: 566, 6: 569, 7: 571, 8: 574, 9: 701}
convertId = lambda x: idConv[x]
counter = 0
def resize(im, size):
    global counter
    counter += 1
    print(counter)
    #https://jdhao.github.io/2017/11/06/resize-image-to-square-with-padding/
    old_size = im.shape[:2] # old_size is in (height, width) format

    ratio = float(size)/max(old_size)
    new_size = tuple([int(x*ratio) for x in old_size])

    # new_size should be in (width, height) format

    im = cv2.resize(im, (new_size[1], new_size[0]))

    delta_w = size - new_size[1]
    delta_h = size - new_size[0]
    top, bottom = delta_h//2, delta_h-(delta_h//2)
    left, right = delta_w//2, delta_w-(delta_w//2)

    color = [0, 0, 0]
    new_im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT,
        value=color)
    return new_im

ds = tfds.load(name="imagenette/320px-v2", split="validation")
ds = tfds.as_numpy(ds)
dsX = np.array([resize(x['image'], 224) / 255 for x in ds])
classifier = tf.keras.applications.VGG19()
print(classifier)

def lossFunc(targetIdx, vgg, a):
    target = np.zeros(1000)
    target[targetIdx] = 1
    targetTensor = K.constant(target)
    def loss(y_true, y_pred):
        return a * tf.keras.losses.MSE(y_true, y_pred) + (1-a) * tf.keras.losses.categorical_crossentropy(targetTensor, vgg(tf.keras.applications.vgg19.preprocess_input(y_pred))[0])

    return loss

model = tf.keras.Sequential(layers=[
    layers.InputLayer(input_shape=(224, 224, 3)),
    layers.Conv2D(256, 5, strides=1, activation="relu"),
    layers.Conv2D(256, 5, strides=1, activation="relu"),
    layers.MaxPool2D(),

    layers.Conv2D(128, 5, strides=1, activation="relu"),
    layers.Conv2D(128, 5, strides=1, activation="relu"),
    layers.MaxPool2D(),

    layers.Conv2D(64, 5, strides=1, activation="relu"),
    layers.Conv2D(64, 5, strides=1, activation="relu"),
    layers.MaxPool2D(),

    layers.Conv2D(16, 3, strides=1, activation="relu"),
    layers.Conv2D(16, 3, strides=1, activation="relu"),

    layers.Conv2DTranspose(16, 3, strides=1, activation="relu", padding='valid'),
    layers.Conv2DTranspose(16, 3, strides=1, activation="relu", padding='valid'),
    layers.UpSampling2D(),

    layers.Conv2DTranspose(64, 5, strides=1, activation="relu", padding='valid'),
    layers.Conv2DTranspose(64, 5, strides=1, activation="relu", padding='valid'),
    layers.UpSampling2D(),

    layers.Conv2DTranspose(128, 5, strides=1, activation="relu", padding='valid'),
    layers.Conv2DTranspose(128, 5, strides=1, activation="relu", padding='valid'),
    layers.UpSampling2D(),

    layers.Conv2DTranspose(256, 5, strides=1, activation="relu", padding='valid'),
    layers.Conv2DTranspose(256, 5, strides=1, activation="relu", padding='valid'),

    layers.Conv2D(3, kernel_size=1, activation='sigmoid', padding='valid')
])

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=.0001), loss=lossFunc(417, classifier, 1))
print(model.summary())

model.fit(x=dsX, y=dsX, batch_size=8, callbacks=[cp_callback], epochs=1000)


import typing as tp

from logging import getLogger
from scalecodec.types import GenericCall, GenericExtrinsic
from substrateinterface import Keypair, SubstrateInterface, ExtrinsicReceipt
from substrateinterface.exceptions import ExtrinsicFailedException

from .account import Account
from ..decorators import check_socket_opened
from ..exceptions import NoPrivateKeyException
from ..types import QueryParams, TypeRegistryTyping, RWSParamsTyping

logger = getLogger(__name__)

class ServiceFunctions:
    """
    Class for custom queries, extrinsics and RPC calls to Robonomics parachain network.
    """

    def __init__(
        self,
        account: Account,
        wait_for_inclusion: bool = True,
        return_block_num: bool = False,
        rws_sub_owner: tp.Optional[str] = None,
    ):
        """
        Assign Account dataclass parameters and create an empty interface attribute for a decorator.

        :param account: Account dataclass with ``seed``, ``remote_ws`` and node ``type_registry``.
        :param wait_for_inclusion: Whether wait for a transaction to included in block. You will get the hash anyway.
        :param return_block_num: If set to True, any executed extrinsic function will return a tuple of form
            ``(<extrinsic_hash>, <block_number-idx>)``. ONLY WORKS WHEN ``wait_for_inclusion`` IS SET TO TRUE.
        :param rws_sub_owner: Subscription owner address. If passed, all extrinsics will be executed via RWS
            subscriptions.

        """
        self.remote_ws: str = account.remote_ws
        self.type_registry: TypeRegistryTyping = account.type_registry
        self.keypair: Keypair = account.keypair
        self.interface: tp.Optional[SubstrateInterface] = None
        self.wait_for_inclusion: bool = wait_for_inclusion
        self.return_block_num: bool = return_block_num
        self.rws_sub_owner: tp.Optional[str] = rws_sub_owner

    @check_socket_opened
    def chainstate_query(
        self,
        module: str,
        storage_function: str,
        params: QueryParams = None,
        block_hash: tp.Optional[str] = None,
        subscription_handler: tp.Optional[callable] = None,
    ) -> tp.Any:
        """
        Create custom queries to fetch data from the Chainstate. Module names and storage functions, as well as required
        parameters are available at https://parachain.robonomics.network/#/chainstate.

        :param module: Chainstate module.
        :param storage_function: Storage function.
        :param params: Query parameters. None if no parameters. Include in list, if several.
        :param block_hash: Retrieves data as of passed block hash.
        :param subscription_handler: Callback function that processes the updates of the storage query subscription.
            The workflow is the same as in substrateinterface lib. Calling method with this parameter blocks current
            thread! Example of subscription handler:
            https://github.com/polkascan/py-substrate-interface#storage-subscriptions

        :return: Output of the query in any form.

        """

        logger.info(f"Performing query {module}.{storage_function}")
        return self.interface.query(
            module,
            storage_function,
            [params] if params is not None else None,
            block_hash=block_hash,
            subscription_handler=subscription_handler,
        ).value

    @check_socket_opened
    def extrinsic(
        self,
        call_module: str,
        call_function: str,
        params: tp.Optional[tp.Dict[str, tp.Any]] = None,
        nonce: tp.Optional[int] = None,
    ) -> tp.Union[str, tp.Tuple[str, str]]:
        """
        Create an extrinsic, sign&submit it. Module names and functions, as well as required parameters are available
        at https://parachain.robonomics.network/#/extrinsics.

        :param call_module: Call module from extrinsic tab on portal.
        :param call_function: Call function from extrinsic tab on portal.
        :param params: Call parameters as a dictionary. ``None`` for no parameters.
        :param nonce: Transaction nonce, defined automatically if None. Due to the feature of substrate-interface lib,
            to create an extrinsic with incremented nonce, pass account's current nonce. See
            https://github.com/polkascan/py-substrate-interface/blob/85a52b1c8f22e81277907f82d807210747c6c583/substrateinterface/base.py#L1535
            for example.

        :return: A tuple of form ``(<extrinsic_hash>, <block_number-idx>)`` if ``return_block_num`` and
            ``wait_for_inclusion`` in ``__init__`` were set to ``True``. String ``<extrinsic_hash>`` otherwise.
        """

        if not self.keypair:
            raise NoPrivateKeyException("No seed was provided, unable to use extrinsics.")

        if not self.rws_sub_owner:
            logger.info(f"Creating a call {call_module}:{call_function}")
            call: GenericCall = self.interface.compose_call(
                call_module=call_module,
                call_function=call_function,
                call_params=params or None,
            )
        else:
            logger.info(f"Creating an RWS call {call_module}:{call_function}")

            rws_params: RWSParamsTyping = {
                "subscription_id": self.rws_sub_owner,
                "call": {
                    "call_module": call_module,
                    "call_function": call_function,
                    "call_args": params,
                },
            }

            call: GenericCall = self.interface.compose_call(
                call_module="RWS", call_function="call", call_params=rws_params
            )

        logger.info("Creating extrinsic")
        extrinsic: GenericExtrinsic = self.interface.create_signed_extrinsic(
            call=call, keypair=self.keypair, nonce=nonce
        )

        logger.info("Submitting extrinsic")
        receipt: ExtrinsicReceipt = self.interface.submit_extrinsic(
            extrinsic, wait_for_inclusion=self.wait_for_inclusion
        )

        logger.info(f"Extrinsic {receipt.extrinsic_hash} for RPC {call_module}:{call_function} submitted.")

        if self.wait_for_inclusion:

            if not receipt.is_success:
                raise ExtrinsicFailedException(receipt.error_message)

            block_num: int = self.interface.get_block_number(receipt.block_hash)
            logger.info(f"Extrinsic included in block {block_num}")

            if self.return_block_num:
                return receipt.extrinsic_hash, f"{block_num}-{receipt.extrinsic_idx}"

            else:
                return receipt.extrinsic_hash

        else:
            return receipt.extrinsic_hash

    @check_socket_opened
    def rpc_request(
        self,
        method: str,
        params: tp.Optional[tp.List[str]],
        result_handler: tp.Optional[tp.Callable],
    ) -> tp.Dict[str, tp.Any]:
        """
        Method that handles the actual RPC request to the Substrate node. The other implemented functions eventually
        use this method to perform the request.

        :param method: Method of the ``JSONRPC`` request.
        :param params: A list containing the parameters of the ``JSONRPC`` request.
        :param result_handler: Callback function that processes the result received from the node.

        :return: Result of the request.

        """

        return self.interface.rpc_request(method, params, result_handler)

    @check_socket_opened
    def subscribe_block_headers(self, callback: callable) -> dict:
        """
        Get chain head block headers.

        :return: Chain head block headers.

        """

        return self.interface.subscribe_block_headers(subscription_handler=callback)


# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import r2_score

# Generic Settings
dataset = pd.read_csv(r'D:\Learning\Udemy ML\MLPracticeProjects\Classification\Classification Evaluation Tool\Data.csv')
display_outcome = False # shows actual ouput vs predicted output
np.set_printoptions(precision=2) # sets property for output arrays
class color: # class for print in particular format
   CYAN = '\033[96m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

# Allocating x and y paramters from the dataset
x = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, random_state = 0)

# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
x_test = sc.transform(x_test)

########################################## Logistic Regression
print(color.BOLD + color.RED + 'Logistic Regression Start -------------------' + color.END)

# Training
from sklearn.linear_model import LogisticRegression
logistic_classifier = LogisticRegression(random_state = 0)
logistic_classifier.fit(x_train, y_train)

# Predict
logistic_y_pred = logistic_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((logistic_y_pred.reshape(len(logistic_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, logistic_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, logistic_y_pred) * 100, '%')

print(color.BOLD + color.RED + 'Logistic Regression End ---------------------' + color.END)
##########################################
########################################## K-NN Classifier
print(color.BOLD + color.BLUE + 'K-NN Classifier Start -------------------' + color.END)

# Training
from sklearn.neighbors import KNeighborsClassifier
knn_classifier = KNeighborsClassifier(n_neighbors = 5, metric = 'minkowski', p = 2) # default parameters
knn_classifier.fit(x_train, y_train)

# Predict
knn_y_pred = knn_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((knn_y_pred.reshape(len(knn_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, knn_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, knn_y_pred) * 100, '%')

print(color.BOLD + color.BLUE + 'K-NN Classifier End ---------------------' + color.END)
##########################################
########################################## SVM Classifier
print(color.BOLD + color.CYAN + 'SVM Classifier Start -------------------' + color.END)

# Training
from sklearn.svm import SVC
svm_classifier = SVC(kernel = 'linear', random_state = 0)
svm_classifier.fit(x_train, y_train)

# Predict
svm_y_pred = svm_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((svm_y_pred.reshape(len(svm_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, svm_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, svm_y_pred) * 100, '%')

print(color.BOLD + color.CYAN + 'SVM Classifier End ---------------------' + color.END)
##########################################
########################################## Kernel SVM Classifier
print(color.BOLD + color.CYAN + 'Kernel SVM Classifier Start -------------------' + color.END)

# Training
from sklearn.svm import SVC
ksvm_classifier = SVC(kernel = 'rbf', random_state = 0)
ksvm_classifier.fit(x_train, y_train)

# Predict
ksvm_y_pred = ksvm_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((ksvm_y_pred.reshape(len(ksvm_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, ksvm_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, ksvm_y_pred) * 100, '%')

print(color.BOLD + color.CYAN + 'Kernel SVM Classifier End ---------------------' + color.END)
##########################################
########################################## Naive Bayes Classifier
print(color.BOLD + color.GREEN + 'Naive Bayes Classifier Start -------------------' + color.END)

# Training
from sklearn.naive_bayes import GaussianNB
nb_classifier = GaussianNB()
nb_classifier.fit(x_train, y_train)

# Predict
nb_y_pred = nb_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((nb_y_pred.reshape(len(nb_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, nb_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, nb_y_pred) * 100, '%')

print(color.BOLD + color.GREEN + 'Naive Bayes Classifier End ---------------------' + color.END)
##########################################
########################################## Decision Tree Classifier
print(color.BOLD + color.YELLOW + 'Decision Tree Classifier Start -------------------' + color.END)

# Training
from sklearn.tree import DecisionTreeClassifier
dt_classifier = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)
dt_classifier.fit(x_train, y_train)

# Predict
dt_y_pred = dt_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((dt_y_pred.reshape(len(dt_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, dt_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, dt_y_pred) * 100, '%')

print(color.BOLD + color.YELLOW + 'Decision Tree Classifier End ---------------------' + color.END)
##########################################
########################################## Random Forest Classifier
print(color.BOLD + color.YELLOW + 'Random Forest Classifier Start -------------------' + color.END)

# Training
from sklearn.ensemble import RandomForestClassifier
rf_classifier = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0)
rf_classifier.fit(x_train, y_train)

# Predict
rf_y_pred = rf_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((rf_y_pred.reshape(len(rf_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, rf_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, rf_y_pred) * 100, '%')

print(color.BOLD + color.YELLOW + 'Random Forest Classifier End ---------------------' + color.END)
##########################################


import json
import imageio
import numpy as np

FILES = ["EasyStandard.dat"]
RATIO = 16  # length of a beat in BeatWalls units
SCALE = 0.05
THICKNESS = 0.1
SCROLL_DIST = 8
LETTER_SPACING = 2
DIV_WALL_WIDTH = 0.2
DIV_WALL_HEIGHT = 5

def load_font():
    alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ,'#1234567890-"
    font_img = imageio.imread("font.png")
    img_dim = font_img.shape[:2]

    # Separate letters
    letter_offsets = []
    letter_widths = []
    start = 0
    for i in range(1, img_dim[1]):
        if font_img[0, i, 2] > 100:
            if i > start:
                letter_offsets.append(start)
                letter_widths.append(i - start)
            start = i + 1
    assert len(letter_offsets) == len(alphabet)

    # Convert the letters to walls (1 pixel = 1 unit)
    font = {}
    for k in range(len(alphabet)):
        letter = alphabet[k]
        letter_img = font_img[
            :, letter_offsets[k]:letter_offsets[k] + letter_widths[k], :3]
        letter_matrix = (
            (letter_img[:, :, 0]
                + letter_img[:, :, 1]
                + letter_img[:, :, 2]) > 100)
        letter_walls = []  # list of x, y, w, h
        while True:
            h_lines = []
            v_lines = []
            # Horizontal lines
            for i in range(img_dim[0]):
                start_j = None
                for j in range(letter_widths[k]):
                    if letter_matrix[i, j] and start_j is None:
                        start_j = j
                    if not letter_matrix[i, j] and start_j is not None:
                        h_lines.append((j - start_j, i, start_j))
                        start_j = None
                if start_j is not None:
                    h_lines.append(
                        (letter_widths[k] - start_j, i, start_j))
            # Vertical lines
            for j in range(letter_widths[k]):
                start_i = None
                for i in range(img_dim[0]):
                    if letter_matrix[i, j] and start_i is None:
                        start_i = i
                    if not letter_matrix[i, j] and start_i is not None:
                        v_lines.append((i - start_i, start_i, j))
                        start_i = None
                if start_i is not None:
                    v_lines.append(
                        (img_dim[0] - start_i, start_i, j))
            # Choose the longest line, add it and clear the pixels
            h_lines.sort(key=lambda x: x[0], reverse=True)
            v_lines.sort(key=lambda x: x[0], reverse=True)
            if not h_lines and not v_lines:
                break
            elif not h_lines or v_lines[0][0] >= h_lines[0][0]:
                h, i, j = v_lines[0]
                letter_walls.append((j, img_dim[0] - i - h, 1, h))
                for r in range(i, i + h):
                    letter_matrix[r, j] = False
            else:
                w, i, j = h_lines[0]
                letter_walls.append((j, img_dim[0] - i - 1, w, 1))
                for c in range(j, j + w):
                    letter_matrix[i, c] = False
        font[letter] = (letter_widths[k], letter_walls)

    font[" "] = (5, [])

    return (img_dim[0], font)

def add_text(walls, font, text):
    text_width = sum(font[letter][0] for letter in text["content"])
    text_width += LETTER_SPACING * (len(text["content"]) - 1)

    offset = text["offset"] if "offset" in text else 0
    if text["type"] in {"ground", "air"}:
        offset -= text_width // 2

    for letter in text["content"]:
        for letter_wall in font[letter][1]:
            if text["type"] == "ground":
                pos = [SCALE * (letter_wall[0] + offset),
                        0,
                        text["time"] + SCALE * letter_wall[1] / RATIO]
                dim = [SCALE * letter_wall[2],
                        THICKNESS,
                        SCALE * letter_wall[3] / RATIO]
            elif text["type"] == "air":
                pos = [SCALE * (letter_wall[0] + offset),
                        SCALE * (letter_wall[1] + text["height"]),
                        text["time"]]
                dim = [SCALE * letter_wall[2],
                        SCALE * letter_wall[3],
                        THICKNESS / RATIO]
            elif text["type"] == "sidescroll":
                pos = [-SCROLL_DIST,
                        SCALE * (letter_wall[1] + text["height"]),
                        text["time"] + SCALE * (letter_wall[0] + offset) / RATIO]
                dim = [THICKNESS,
                        SCALE * letter_wall[3],
                        SCALE * letter_wall[2] / RATIO]
            walls.append({
                "_time": pos[2],
                "_lineIndex": 0,
                "_type": 0,
                "_duration": dim[2],
                "_width": 0,
                "_customData": {
                    "_position": pos[:2],
                    "_scale": dim[:2],
                    "_rotation": 90 if text["type"] == "sidescroll" else text["angle"] if "angle" in text else 0,
                    "_localRotation": [0, 0, 0],
                }
            })
        offset += font[letter][0] + LETTER_SPACING

def main():
    with open("text.json", encoding='raw_unicode_escape') as json_file:
        content_json = json.loads(
            json_file.read().encode('raw_unicode_escape').decode())

    _, font = load_font()

    walls = []
    notes = []

    for text in content_json["text"]:
        add_text(walls, font, text)

    for choice in content_json["choices"]:
        walls.append({
            "_time": choice["time"],
            "_lineIndex": 0,
            "_type": 0,
            "_duration": 2.1,
            "_width": 0,
            "_customData": {
                "_position": [-DIV_WALL_WIDTH / 2, 0],
                "_scale": [DIV_WALL_WIDTH, DIV_WALL_HEIGHT],
                "_rotation": 0,
                "_localRotation": [0, 0, 0],
            }
        })
        add_text(walls, font, {
            "type": "air",
            "content": "bravo",
            "time": choice["time"] + 2,
            "height": 60,
            "offset": -50 if choice["side"] == "left" else 50
        })
        add_text(walls, font, {
            "type": "air",
            "content": "wrong",
            "time": choice["time"] + 2,
            "height": 60,
            "offset": -50 if choice["side"] == "right" else 50
        })
        notes.append({
            "_time": choice["time"] + 2,
            "_lineIndex": 0 if choice["side"] == "left" else 3,
            "_lineLayer": 0,
            "_type": 1,
            "_cutDirection": 1
        })

    walls.sort(key=lambda x: x["_time"])

    for filename in FILES:
        with open(filename, "r") as json_file:
            song_json = json.load(json_file)

        song_json["_obstacles"] = walls
        song_json["_notes"] = notes

        with open(filename, "w") as json_file:
            json.dump(song_json, json_file)

main()


import copy
import time

import torch
import torch.nn as nn
import utils

from .impl import iterative_unlearn

def discretize(x):
    return torch.round(x * 255) / 255

def FGSM_perturb(x, y, model=None, bound=None, criterion=None):
    device = model.parameters().__next__().device
    model.zero_grad()
    x_adv = x.detach().clone().requires_grad_(True).to(device)

    pred = model(x_adv)
    loss = criterion(pred, y)
    loss.backward()

    grad_sign = x_adv.grad.data.detach().sign()
    x_adv = x_adv + grad_sign * bound
    x_adv = discretize(torch.clamp(x_adv, 0.0, 1.0))

    return x_adv.detach()

@iterative_unlearn
def boundary_shrink_iter(
    data_loaders, model, criterion, optimizer, epoch, args, mask=None, test_model=None
):
    assert test_model is not None

    bound = 0.1  # hard coding in the paper

    train_loader = data_loaders["forget"]
    print(len(train_loader))
    losses = utils.AverageMeter()
    top1 = utils.AverageMeter()

    # switch to train mode
    model.train()

    start = time.time()

    for i, (image, target) in enumerate(train_loader):
        if epoch < args.warmup:
            utils.warmup_lr(
                epoch, i + 1, optimizer, one_epoch_step=len(train_loader), args=args
            )

        image = image.cuda()
        target = target.cuda()

        test_model.eval()
        image_adv = FGSM_perturb(
            image, target, model=test_model, bound=bound, criterion=criterion
        )

        adv_outputs = test_model(image_adv)
        adv_label = torch.argmax(adv_outputs, dim=1)

        # compute output
        output_clean = model(image)
        loss = criterion(output_clean, adv_label)

        optimizer.zero_grad()
        loss.backward()

        if mask:
            for name, param in model.named_parameters():
                if param.grad is not None:
                    param.grad *= mask[name]
                    # print(mask[name])

        optimizer.step()

        output = output_clean.float()
        loss = loss.float()
        # measure accuracy and record loss
        prec1 = utils.accuracy(output.data, target)[0]

        losses.update(loss.item(), image.size(0))
        top1.update(prec1.item(), image.size(0))

        if (i + 1) % args.print_freq == 0:
            end = time.time()
            print(
                "Epoch: [{0}][{1}/{2}]\t"
                "Loss {loss.val:.4f} ({loss.avg:.4f})\t"
                "Accuracy {top1.val:.3f} ({top1.avg:.3f})\t"
                "Time {3:.2f}".format(
                    epoch, i, len(train_loader), end - start, loss=losses, top1=top1
                )
            )
            start = time.time()

    print("train_accuracy {top1.avg:.3f}".format(top1=top1))

    return top1.avg

def boundary_shrink(data_loaders, model: nn.Module, criterion, args, mask=None):
    device = model.parameters().__next__().device
    test_model = copy.deepcopy(model).to(device)
    return boundary_shrink_iter(
        data_loaders, model, criterion, args, mask, test_model=test_model
    )


# -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Set up the ACL to access the API."""

import flask
import keystoneclient.middleware.auth_token as auth_token
from oslo.config import cfg

from kwstandby.openstack.common import policy

_ENFORCER = None
OPT_GROUP_NAME = 'keystone_authtoken'

def register_opts(conf):
    """Registers keystoneclient middleware options."""
    conf.register_opts(auth_token.opts,
                       group=OPT_GROUP_NAME,
                       )
    auth_token.CONF = conf

register_opts(cfg.CONF)

def install(app, conf):
    """Installs ACL check on application."""
    app.wsgi_app = auth_token.AuthProtocol(app.wsgi_app,
                                           conf=dict(conf.get(OPT_GROUP_NAME)))
    app.before_request(check)

def check():
    """Checks application access."""
    headers = flask.request.headers
    global _ENFORCER
    if not _ENFORCER:
        _ENFORCER = policy.Enforcer()
    if not _ENFORCER.enforce('context_is_admin',
                             {},
                             {'roles': headers.get('X-Roles', "").split(",")}):
        return "Access denied", 401


import os
import sys;
from pathlib import Path

sys.path.extend('../')
import networkx as nx
from src.graph_stats import GraphStats
from src.parallel import parallel_async
from src.utils import load_pickle, get_imt_input_directory, walker, ColorPrint, get_imt_output_directory

def stats_computation(dataset, model, trial, filename, stats):
    path = Path(get_imt_output_directory()) / 'pickles' / dataset / model / filename
    graph_list = load_pickle(path)
    assert isinstance(graph_list, list), f'Expected type "list" and got type {type(graph_list)}.'
    assert all(isinstance(g, nx.Graph) for g in graph_list), f'Expected a list of nx.Graph and got disappointed instead.'

    # ColorPrint.print_orange(f'{filename} has length {len(graph_list)}')

    for idx, G in enumerate(graph_list):
        gs_obj = GraphStats(graph=G, dataset=dataset, model=model, trial=trial, iteration=idx)
        gs_obj.write_stats_jsons(stats=stats)

    return None

if __name__ == '__main__':
    stat = ['pagerank', 'degree_dist', 'pgd_graphlet_counts', 'laplacian_eigenvalues', 'b_matrix']#[-1:]
    datasets, models, trials, filenames = walker()
    #datasets, models, trials, filenames = ['eucore']*10, ['BTER']*10, [str(x) for x in range(1, 11)], [f'list_20_{x}.pkl.gz' for x in range(1, 11)]

    args = [(dataset, model, trial, filename, stat)
            for dataset, model, trial, filename in zip(datasets, models, trials, filenames)
            if dataset != 'enron']

    parallel_async(stats_computation, args, num_workers=8)


from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from news.views import IndexView,MessageViewHome
from django.views.generic import TemplateView
from exceptionsapp.views import error_404,error_500,error_403,error_400

handler404 = 'exceptionsapp.views.error_404'
handler403 = 'exceptionsapp.views.error_403'
handler400 = 'exceptionsapp.views.error_400'
handler400 = 'exceptionsapp.views.error_500'

urlpatterns = [
    path("", include("contracts.urls")),
    path("user/", include("users.urls")),
    path("", include("news.urls")),
    path("", include("company.urls")),
    path("", include("construction.urls")),
    path("", include("dam.urls")),
    path("", include("damknowledge.urls")),

    # home
    path("", IndexView.as_view(), name="index"),
    path('message/<slug:slug>/', MessageViewHome.as_view(), name='message'),
    #list of apps
    path('admin/', admin.site.urls),
    path("", include('django.contrib.auth.urls')),
    #external package
    path('ckeditor/', include('ckeditor_uploader.urls')),

]

urlpatterns += staticfiles_urlpatterns()

urlpatterns += static(settings.STATIC_URL,document_root = settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)