# -*- encoding: utf-8 -*-

import random, math, time
import Data
from GA import GA
from matplotlib import pyplot as plt

class TSP(object):
    def __init__(self, city_coordinate, distance_matrix, aLifeCount = 400):
        self.city_coordinate = city_coordinate
        self.distance_matrix = distance_matrix
        self.lifeCount = aLifeCount
        self.every_gen_best = []
        self.ga = GA(aCrossRate = 0.8,
              aMutationRate = 0.2,
              aLifeCount = self.lifeCount,   # the number of individual population
              aGeneLength = len(self.city_coordinate),
              aMatchFun = self.matchFun())

    # calculate the distance between cities
    def distance(self, solution):
        total_distance = 0.0
        for (city1,city2) in zip(solution[:-1], solution[1:]):
            total_distance += self.distance_matrix[city1][city2]
        total_distance += self.distance_matrix[solution[-1]][solution[0]]
        return total_distance

    # the fitness
    def matchFun(self):
        return lambda life: 1.0 / self.distance(life.gene)

    def run(self, n = 0):
        iter = n
        while n > 0:
            self.ga.next()
            distance = self.distance(self.ga.best.gene)
            self.every_gen_best.append(distance)
            if n%100 == 0:
                print ("The generation %d"%(iter-n))
#             print (("Generation times %d : distance %f") % (self.ga.generation, distance))
#             print self.ga.best.gene
            n -= 1
        print ("The end generation time %d, the best distance %f"%(self.ga.generation, distance))
#         print "The city order:",
#         print self.ga.best.gene
        total_distance = self.distance(self.ga.best.gene)
        city_result = [city_index + 1 for city_index in self.ga.best.gene]
        return city_result, total_distance

def showResult(result, city_coordinate):
    x_coordination = []
    y_coordination = []
    # draw the result
    for point in result:
        x_coordination.append(city_coordinate[point-1][0])
        y_coordination.append(city_coordinate[point-1][1])
    x_coordination.append(city_coordinate[result[0]-1][0]) # end to start
    y_coordination.append(city_coordinate[result[0]-1][1])

    plt.scatter(x_coordination,y_coordination)
    plt.plot(x_coordination,y_coordination)
    # draw the city mark
    for index in result:
        plt.text(city_coordinate[index-1][0], city_coordinate[index-1][1], index)
    plt.text(city_coordinate[result[0]-1][0], city_coordinate[result[0]-1][1], '     Start')   #start
    plt.text(city_coordinate[result[-1]-1][0], city_coordinate[result[-1]-1][1], '    End')   #end
    plt.xlabel('City x coordination')
    plt.ylabel("City y coordination")
    plt.title("The traveling map by GA")
    plt.show()

if __name__ == '__main__':
    cost_matrix, city_list = Data.fromTSPFile("eil51.tsp")
    cost_matrix_list = cost_matrix.matrix
    # Create an initial solution
    number_of_cities = len(city_list)
    city_coordinate = []

    for city in city_list:
        city_coordinate.append((city[1],city[2]))
    time_start = time.time()
    time_end = time.time()
    # solve the TSP by GA
    tsp = TSP(city_coordinate, cost_matrix_list)
    city_result, total_distance = tsp.run(200)
    time_end = time.time()
    print ('Time cost %s s.' % (time_end-time_start))
    print ('City solution:%s'% city_result)
    print ('The traveling total distance: %s' % total_distance)
    showResult(city_result, city_coordinate)


from pyspark import SparkConf
from pyspark import SparkContext

cnf = SparkConf().setAppName("demo01").setMaster("local")
sc = SparkContext(conf=cnf)

stopwords = ('the', 'of', 'or', 'and', 'to', 'any', 'in', 'this', 'a')

file = sc.textFile("/home/sunbeam/hadoop-2.7.3/LICENSE.txt")
result = file.map(lambda line: line.lower())\
    .flatMap(lambda line:line.split())\
    .filter(lambda word: word not in stopwords)\
    .map(lambda word: (word,1))\
    .reduceByKey(lambda a,c: a + c)\
    .sortBy(lambda wordone: wordone[1], ascending=False)\
    .take(10)
print(result)

sc.stop()


from mesa import Agent
from rog_rl.agent_state import AgentState

class DiseaseSimAgent(Agent):  # noqa
    """
    DiseaseSimAgent
    """
    pos = None
    moore = True
    prob_agent_movement = 0.0

    def __init__(self, unique_id, model,  prob_agent_movement=0.0, moore=True):
        """
        Customize the agent
        """
        self.unique_id = unique_id
        super().__init__(unique_id, model)

        self._is_infection_scheduled = False
        self.prob_agent_movement = prob_agent_movement
        self.moore = moore

        # self.state = self.random.choice([x for x in AgentState])
        # Holds the state transition plan, with timestep as the key
        self.state_transition_plan = {}

        # Default State for Agents
        self.state = AgentState.SUSCEPTIBLE

    def step(self):
        self.random_move()
        self.process_state_transitions()

    def set_state(self, new_state: AgentState):
        previous_state = self.state
        self.state = new_state

        # Update Agent State Registry in Scheduler
        self.model.schedule.update_agent_state_in_registry(
            self, previous_state=previous_state)

        # Update Global Observation in model observation buffer
        self.model.observation[self.pos[0], self.pos[1], :] = 0
        self.model.observation[self.pos[0], self.pos[1], self.state.value] = 1

    def process_state_transitions(self):
        try:
            _event = self.state_transition_plan[self.model.schedule.steps]
            assert self.state == _event.previous_state, \
                "Mismatch in state during state_transition"
            self.set_state(_event.new_state)
            _event.mark_as_executed()
        except KeyError:
            """
            If not state transition plan exists for the said timestep
            then ignore
            """
            pass

    def trigger_infection(self, prob_infection=1.0):
        """
        Attempts to trigger an infection, and if infection is triggered,
        then it returns True, else returns False.
        """
        if self._is_infection_scheduled:
            return False
        else:
            if self.random.random() < prob_infection:
                # Prepare a disease plan
                disease_plan = \
                    self.model.disease_planner.get_disease_plan(
                        base_timestep=self.model.schedule.steps)
                for _agent_event in disease_plan:
                    # Check if a state transition plan is already present
                    # for the said timestep
                    try:
                        foo = self.state_transition_plan[_agent_event.update_timestep]  # noqa
                        raise Exception(
                            "Attempt to assign multiple state transition plans for the same timestep")  # noqa
                    except KeyError:
                        pass
                    # Mark the state transition plan for the said timestep
                    self.state_transition_plan[_agent_event.update_timestep] = _agent_event  # noqa
                self._is_infection_scheduled = True
                return True
            else:
                return False

    def move_to(self, new_position):
        """
        Move the agent to a new location on the grid
        and do other associated house keeping tasks
            - Update global observation in model
        """
        # Clear up global observation cache in model at the previous coord
        self.model.observation[self.pos[0], self.pos[1], :] = 0
        # Move Agent in Grid
        self.model.grid.move_agent(self, new_position)
        # Add a new entry in the global observation cache for the new position
        self.model.observation[self.pos[0], self.pos[1], self.state.value] = 1

    def random_move(self):
        if self.random.random() < self.prob_agent_movement:
            # Find empty cells in neighborhood
            empty_cells_in_neighborhood = []
            for x, y in self.model.grid.iter_neighborhood(
                    pos=self.pos, moore=self.moore, include_center=False, radius=1):  # noqa
                if (x, y) in self.model.grid.empties:
                    empty_cells_in_neighborhood.append((x, y))
            # If empty cells are availabel - move to a randomly chosen one
            if len(empty_cells_in_neighborhood) > 0:
                new_position = self.random.choice(empty_cells_in_neighborhood)
                # Move to a randomly selected empty cell in the neighborhood

                self.move_to(new_position)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
角色管理
"""
from django.shortcuts import render, redirect, HttpResponse
from django.urls import reverse
from django.http import JsonResponse
from rbac import models
from rbac.forms.role import RoleModelForm
from django.views.decorators.csrf import csrf_exempt

def role_list(request):
    """
    角色列表
    :param request:
    :return:
    """
    role_queryset = models.Role.objects.all()

    return render(request, 'rbac/role_list.html', {'roles': role_queryset})

def role_add(request):
    """
    添加角色
    :param request:
    :return:
    """
    if request.method == 'GET':
        form = RoleModelForm()
        return render(request, 'rbac/change.html', {'form': form})

    form = RoleModelForm(data=request.POST)
    if form.is_valid():
        form.save()
        return redirect(reverse('rbac:role_list'))

    return render(request, 'rbac/change.html', {'form': form})

def role_edit(request, pk):
    """
    编辑角色
    :param request:
    :param pk: 要修改的角色ID
    :return:
    """
    obj = models.Role.objects.filter(id=pk).first()
    if not obj:
        return HttpResponse('角色不存在')
    if request.method == 'GET':
        form = RoleModelForm(instance=obj)
        return render(request, 'rbac/change.html', {'form': form})

    form = RoleModelForm(instance=obj, data=request.POST)
    if form.is_valid():
        form.save()
        return redirect(reverse('rbac:role_list'))

    return render(request, 'rbac/change.html', {'form': form})

# 跳转删除需要修改url接收pk
# def role_del(request, pk):
#     """
#     删除角色
#     :param request:
#     :param pk:
#     :return:
#     """
#     origin_url = reverse('rbac:role_list')
#     if request.method == 'GET':
#         return render(request, 'rbac/delete.html', {'cancel': origin_url})
#
#     models.Role.objects.filter(id=pk).delete()
#     return redirect(origin_url)

# 弹出框删除使用该函数，并且修改此函数的url为不接收pk
@csrf_exempt
def role_del(request):
    """
    删除角色v2.0
    :return:
    """
    status = {"msg": None}
    pk = request.POST.get("pk")
    print(pk)
    if pk:
        models.Role.objects.filter(id=pk).delete()
        status["msg"] = True
    else:
        return render(request, "rbac/error-404.html")

    return JsonResponse(status)


#!/usr/bin/env python3

import argparse
import os
import re
import subprocess

VERSION_PATTERN = '\d+\.\d+\.\d+'

parser = argparse.ArgumentParser(
    description="Update files containing hard-coded aws-crt-java version numbers.")
parser.add_argument('version', nargs='?',
                    help='version to use (i.e. "0.1.2"). default: automatically detect latest version')
parser.add_argument('--check-consistency', action='store_true',
                    help='Exit with error if version is inconsistent between files')
parser.add_argument('--update_samples', action='store_true',
                    help="Update the SDK samples to the latest SDK release OR to the passed in version")
parser.add_argument('--update_sdk_text', action='store_true',
                    help="Update the SDK text (readme, etc) to the latest SDK release OR to the passed in version")
args = parser.parse_args()

def main():
    if args.check_consistency:
        # we'll find the version from the first file we scan
        args.version = None
    else:
        if args.version is None:
            args.version = get_latest_github_version("https://github.com/awslabs/aws-crt-java.git")
            print(f'Latest CRT version: {args.version}')

        if re.fullmatch(VERSION_PATTERN, args.version) is None:
            exit(f'Invalid version: "{args.version}". Must look like "0.1.2"')

    os.chdir(os.path.dirname(os.path.abspath(__file__)))

    if args.update_samples == True:
        update_samples()
    elif args.update_sdk_text == True:
        sdk_version = get_latest_github_version("https://github.com/aws/aws-iot-device-sdk-java-v2.git")
        print (f"Latest SDK version: {sdk_version}")
        update(filepath='README.md',
            preceded_by=r'<artifactId>aws-iot-device-sdk</artifactId>\s*<version>',
            followed_by=r'</version>',
            force_version=sdk_version)
        update(filepath='README.md',
            preceded_by=r"Replace `",
            followed_by=r"` in `<version>.*</version>` with the latest release version for the SDK.",
            force_version=sdk_version)
        update(filepath='README.md',
            preceded_by=r"Replace .* in `<version>",
            followed_by=r"</version>` with the latest release version for the SDK.",
            force_version=sdk_version)
    else:
        update(filepath='sdk/pom.xml',
            preceded_by=r'<artifactId>aws-crt</artifactId>\s*<version>',
            followed_by=r'</version>')
        update(filepath='README.md',
            preceded_by=r'--branch v',
            followed_by=r' .*aws-crt-java.git')
        update(filepath='README.md',
            preceded_by=r"implementation 'software.amazon.awssdk.crt:android:",
            followed_by=r"'")
        update(filepath='android/iotdevicesdk/build.gradle',
            preceded_by=r"api 'software.amazon.awssdk.crt:aws-crt-android:",
            followed_by=r"'")
        update(filepath='README.md',
            preceded_by=r'Use the latest version of the CRT here instead of ".*',
            followed_by=r'"')
        update(filepath='README.md',
            preceded_by=r"Replace `",
            followed_by=r"` in .* with the latest version of the CRT.")
        update(filepath='README.md',
            preceded_by=r"Replace .* in `software.amazon.awssdk.crt:android:",
            followed_by=r"` with the latest version of the CRT.")

def update(*, filepath, preceded_by, followed_by, force_version=None):
    """
    Args:
        filepath: File containing hard-coded CRT version numbers.
        preceded_by: Regex pattern for text preceding the CRT version number.
        followed_by: Regex pattern for text following the CRT version number.
    """
    with open(filepath, 'r+') as f:
        txt_old = f.read()

        full_pattern = rf'({preceded_by})({VERSION_PATTERN})({followed_by})'
        full_replacement = rf'\g<1>{args.version}\g<3>'

        if force_version != None:
            full_replacement = rf'\g<1>{force_version}\g<3>'

        matches = re.findall(full_pattern, txt_old)
        if len(matches) == 0:
            exit(f'Version not found in: {filepath}\n' +
                 f'Preceded by: "{preceded_by}"')

        if args.check_consistency and force_version == None:
            # in --check-consistency mode we remember the version from the first
            # file we scan, and then ensure all subsequent files use that version too
            for match in matches:
                found_version = match[1]
                if args.version is None:
                    print(f'Found version {found_version} in: {filepath}')
                    args.version = found_version
                elif found_version != args.version:
                    exit(f'Found different version {found_version} in: {filepath}')
        else:
            # running in normal mode, update the file
            txt_new = re.sub(full_pattern, full_replacement, txt_old)
            f.seek(0)
            f.write(txt_new)
            f.truncate()

def update_samples():
    sdk_version = get_latest_github_version("https://github.com/aws/aws-iot-device-sdk-java-v2.git")
    print (f"Latest SDK version: {sdk_version}")

    sample_folders = [x[0] for x in os.walk("samples")]
    for sample_folder in sample_folders:
        sample_files = os.walk(sample_folder).__next__()[2]
        for file in sample_files:
            if file.endswith(".xml"):
                update(filepath=sample_folder + "/" + file,
                    preceded_by=r'<artifactId>aws-iot-device-sdk</artifactId>\s*<version>',
                    followed_by=r'</version>',
                    force_version=sdk_version)

def get_latest_github_version(github_repo="https://github.com/awslabs/aws-crt-java.git"):
    repo = github_repo
    cmd = ['git', 'ls-remote', '--tags', repo]
    results = subprocess.run(cmd, check=True, capture_output=True, text=True)

    latest_str = None
    latest_nums = None
    for line in results.stdout.splitlines():
        # line looks like: "e18f041a0c8d17189f2eae2a32f16e0a7a3f0f1c refs/tags/v0.5.18"
        pattern = r'[a-f0-9]+\s+refs/tags/v((\d+)\.(\d+)\.(\d+))'
        match = re.fullmatch(pattern, line)
        if not match:
            continue

        # we only want the latest version
        # convert to tuple of numbers so we can compare
        version_str = match.group(1)
        version_nums = [int(match.group(x)) for x in range(3, 5)]
        if latest_nums is None or (version_nums > latest_nums):
            latest_nums = version_nums
            latest_str = version_str

    return latest_str

if __name__ == '__main__':
    main()


# ============================================================
#   File        : 运行这个.py
#   Author      : zmdsjtu@163.com
#   Created date: 2021/12/28 21:08
#   Description : Just run this script
# ============================================================

from utils.segmentation_mediapipe import InputData, InitSegmentation, ShowResult

# 初始化输入源, file支持数字（相机）以及视频文件路径，图片路径或文件夹路径
input_data = InputData("test/hand_tracking.mp4")  # input_data = InputData("test/imgs")
bg_data = InputData("test/imgs/background", repeat=True, repeat_step= 15)
selfie_segmentation = InitSegmentation(model=0)
# 获取图像以及结果的generator
run_pose_result = selfie_segmentation.run_segmentation(input_data.get_next_img())
# 显示结果， ESC退出，图片模式按任意键继续
ShowResult(input_data.wait_key).show_result(run_pose_result, bg_data.get_next_img())


import unittest
import paramunittest
import readConfig as readConfig
from common import Log as Log
from common import common
from common import configHttp as ConfigHttp

addressList_xls = common.get_xls("userCase.xlsx", "getAddressList")
localReadConfig = readConfig.ReadConfig()
configHttp = ConfigHttp.ConfigHttp()
info = {}

@paramunittest.parametrized(*addressList_xls)
class GetAddressList(unittest.TestCase):
    def setParameters(self, case_name, method, url, code, msg):
        """
        set params
        :param case_name:
        :param method:
        :param url:
        :param code:
        :param msg:
        :return:
        """
        self.case_name = str(case_name)
        self.url = str(url)
        self.method = str(method)
        self.code = str(code)
        self.msg = str(msg)
        self.info = None

    def description(self):
        self.case_name

    def setUp(self):
        self.log = Log.MyLog.get_log()
        self.logger = self.log.get_logger()

    def testGetAddressList(self):
        # set url
        configHttp.set_url(self.url)

        # get visitor token
        token = localReadConfig.get_headers("TOKEN_U")

        # set headers
        header = {"token": str(token)}
        configHttp.set_headers(header)

        # test interface
        if self.method == 'get':
            self.return_json = configHttp.get()
        elif self.method == 'post':
            self.return_json = configHttp.post()
        else:
            self.logger.info("No this interface's method:" + self.method)
        # check result
        self.checkResult()

    def tearDown(self):
        pass

    def checkResult(self):
        self.info = self.return_json.json()
        if self.info['code'] == self.code:
            self.assertEqual(self.info['msg'], self.msg)
            self.log.build_OK_line(self.case_name, self.code, self.msg)
        else:
            self.log.build_NG_line(self.case_name, self.info['code'], self.info['msg'])


from enum import Enum
from os import path

class AccessType(Enum):
    # Enumeration of different access types

    def __str__(self):
        return self.name

    PRIVATE = 0
    REMOTE = 1
    OFF_CHIP = 2

class Stats:
    # Class to track the statistics of the cache simulator.
    def __init__(self, verbose=False):
        self.cycles = 0
        self.verbose = verbose
        self.cycle_dict = {AccessType.PRIVATE: [],
                           AccessType.REMOTE: [],
                           AccessType.OFF_CHIP: []}
        self.invalidations_sent = 0
        self.replacement_writebacks = 0
        self.coherence_writebacks = 0
        self.access_type = AccessType.PRIVATE

    def hit_rate(self):
        return len(self.cycle_dict[AccessType.PRIVATE]) / len(self.cycle_dict[AccessType.PRIVATE] +
                                                              self.cycle_dict[AccessType.REMOTE] +
                                                              self.cycle_dict[AccessType.OFF_CHIP])

    def reset(self):
        self.cycles = 0
        self.access_type = AccessType.PRIVATE
        if self.verbose:
            print("\n")

    def save_stats(self):
        self.cycle_dict[self.access_type].append(self.cycles)

    def cache_probe(self):
        # Tag and state access.
        if self.verbose:
            print("Cache probe. (1)")
        self.cycles += 1

    def cache_access(self):
        # Read or write.
        if self.verbose:
            print("Cache access. (1)")
        self.cycles += 1

    def sram_access(self):
        if self.verbose:
            print("SRAM access. (1)")
        self.cycles += 1

    def directory_access(self):
        if self.verbose:
            print("Directory access. (1)")
        self.cycles += 1

    def hop_between_processors(self):
        if self.verbose:
            print("Hop between processors. (3)")
        self.cycles += 3

    def hop_between_processor_and_directory(self):
        if self.verbose:
            print("Hop between processor and directory. (5)")
        self.cycles += 5

    def memory_access_latency(self):
        if self.verbose:
            print("Access memory. (15)")
        self.cycles += 15

    def final_stats(self, filename, to_file=False):
        private_accesses = len(self.cycle_dict.get(AccessType.PRIVATE))
        remote_accesses = len(self.cycle_dict[AccessType.REMOTE])
        off_chip_accesses = len(self.cycle_dict[AccessType.OFF_CHIP])
        total_accesses = private_accesses + remote_accesses + off_chip_accesses
        replacement_writebacks = self.replacement_writebacks
        coherence_writebacks = self.coherence_writebacks
        invalidations_sent = self.invalidations_sent
        private_access_latency = (sum(self.cycle_dict[AccessType.PRIVATE]) / private_accesses) if private_accesses > 0 else 0
        remote_access_latency = (sum(self.cycle_dict[AccessType.REMOTE]) / remote_accesses) if remote_accesses > 0 else 0
        off_chip_access_latency = (sum(self.cycle_dict[AccessType.OFF_CHIP]) / off_chip_accesses) if off_chip_accesses > 0 else 0
        total_latency = sum(self.cycle_dict[AccessType.PRIVATE]) + sum(self.cycle_dict[AccessType.REMOTE]) + sum(self.cycle_dict[AccessType.OFF_CHIP])
        average_latency = total_latency / total_accesses

        st = "Private-accesses: {}\nRemote-accesses: {}\nOff-chip-accesses: {}\nTotal-accesses: {}" \
               "\nReplacement-writebacks: {}\nCoherence-writebacks: {}\nInvalidations-sent: {}\nAverage-latency: {}" \
               "\nPriv-average-latency: {}\nRem-average-latency: {}\nOff-chip-average-latency: {}" \
               "\nTotal-latency: {}".format(private_accesses, remote_accesses, off_chip_accesses, total_accesses,
                                            replacement_writebacks, coherence_writebacks, invalidations_sent,
                                            average_latency, private_access_latency, remote_access_latency,
                                            off_chip_access_latency, total_latency)

        if to_file:
            outname = 'out_{}'.format(filename)
            outpath = path.join('./out-files', outname)
            with open(outpath, 'w') as f:
                f.write(st)
            print("File {} written with these stats:\n".format(outpath))
        return st


#!/usr/bin/env python3

"""
test code for the object_canvas

Note: Testing image generation is hard.  So for now, this mostly just
      tests that the rendering function runs.
      And during development, you can look at the resulting files.

      One could store "properly" rendered results for future tests to
      check against.
"""

# import os
import pathlib
import object_canvas as oc

SAVE_ALL=True  # save all the temp files?

def render_to_file(canvas, filename="test_image.png", save=False):
    """
    utility to render a canvas to a file

    :param filename: name of file to render to it will be put in a test_images dir.

    :param remove=True: whether to remove the file after rendering.
    """
    path = pathlib.Path("test_images")
    path.mkdir(exist_ok=True)
    path /= filename
    canvas.render(str(path))
    assert path.is_file()
    if not (SAVE_ALL or save):
        path.unlink()

def test_init():
    canvas = oc.ObjectCanvas()

    assert canvas

def test_backgound():
    canvas = oc.ObjectCanvas(background='blue')
    render_to_file(canvas, "blue_background.png")

def test_polyline():
    """
    can we draw a polyline?
    """
    canvas = oc.ObjectCanvas()
    points = ((10, 10),  # this should be a triangle
              (10, 400),
              (400, 10),
              (10, 10),
              )

    pl = oc.PolyLine(points)
    canvas.add_object(pl)
    render_to_file(canvas, "polyline.png")

def test_circle():
    canvas = oc.ObjectCanvas()
    center = (100, 100)
    diameter = 75
    for line_width in range(1, 5):
        c = oc.Circle(center,
                      diameter,
                      line_color="red",
                      fill_color="blue",
                      line_width=line_width,
                      )
        canvas.add_object(c)
        center = (center[0] + 50, center[0] + 50)
    render_to_file(canvas, "circle.png")


import json
from time import sleep
from channels.generic.websocket import WebsocketConsumer
from .models import Monitor

class WSConsumer(WebsocketConsumer):    # Create WebSocket Server

    def connect(self):  # Connect WebSocket
        queryset = Monitor.objects.latest('id')     # Get the recentest data from database

        self.accept()
        values = json.dumps({   # Convert to JSON format
            "temperature": queryset.temperature,
            "humity": queryset.humity,
        })
        print(values)
        self.send(values)
        sleep(1)

    def receive(self, text_data):   # Create a response,for the constant requests(real time) of components from frontend

        if text_data == "NEEDNEWDATA":

            queryset = Monitor.objects.latest('id')
            values = json.dumps({
                "temperature": queryset.temperature,
                "humity": queryset.humity,
            })
            self.send(values)


import numpy as np
import pymc3 as pm
import pandas as pd
import pickle
import wandb
import yaml
import warnings
from sklearn.cluster import k_means
from scipy.special import softmax, logsumexp, loggamma
from sklearn.metrics.pairwise import cosine_similarity

# constants
C=32
M=3
P=2

# labels
idx96 = pd.MultiIndex.from_tuples([
            ('C>A', 'ACA'), ('C>A', 'ACC'), ('C>A', 'ACG'), ('C>A', 'ACT'),
            ('C>A', 'CCA'), ('C>A', 'CCC'), ('C>A', 'CCG'), ('C>A', 'CCT'),
            ('C>A', 'GCA'), ('C>A', 'GCC'), ('C>A', 'GCG'), ('C>A', 'GCT'),
            ('C>A', 'TCA'), ('C>A', 'TCC'), ('C>A', 'TCG'), ('C>A', 'TCT'),
            ('C>G', 'ACA'), ('C>G', 'ACC'), ('C>G', 'ACG'), ('C>G', 'ACT'),
            ('C>G', 'CCA'), ('C>G', 'CCC'), ('C>G', 'CCG'), ('C>G', 'CCT'),
            ('C>G', 'GCA'), ('C>G', 'GCC'), ('C>G', 'GCG'), ('C>G', 'GCT'),
            ('C>G', 'TCA'), ('C>G', 'TCC'), ('C>G', 'TCG'), ('C>G', 'TCT'),
            ('C>T', 'ACA'), ('C>T', 'ACC'), ('C>T', 'ACG'), ('C>T', 'ACT'),
            ('C>T', 'CCA'), ('C>T', 'CCC'), ('C>T', 'CCG'), ('C>T', 'CCT'),
            ('C>T', 'GCA'), ('C>T', 'GCC'), ('C>T', 'GCG'), ('C>T', 'GCT'),
            ('C>T', 'TCA'), ('C>T', 'TCC'), ('C>T', 'TCG'), ('C>T', 'TCT'),
            ('T>A', 'ATA'), ('T>A', 'ATC'), ('T>A', 'ATG'), ('T>A', 'ATT'),
            ('T>A', 'CTA'), ('T>A', 'CTC'), ('T>A', 'CTG'), ('T>A', 'CTT'),
            ('T>A', 'GTA'), ('T>A', 'GTC'), ('T>A', 'GTG'), ('T>A', 'GTT'),
            ('T>A', 'TTA'), ('T>A', 'TTC'), ('T>A', 'TTG'), ('T>A', 'TTT'),
            ('T>C', 'ATA'), ('T>C', 'ATC'), ('T>C', 'ATG'), ('T>C', 'ATT'),
            ('T>C', 'CTA'), ('T>C', 'CTC'), ('T>C', 'CTG'), ('T>C', 'CTT'),
            ('T>C', 'GTA'), ('T>C', 'GTC'), ('T>C', 'GTG'), ('T>C', 'GTT'),
            ('T>C', 'TTA'), ('T>C', 'TTC'), ('T>C', 'TTG'), ('T>C', 'TTT'),
            ('T>G', 'ATA'), ('T>G', 'ATC'), ('T>G', 'ATG'), ('T>G', 'ATT'),
            ('T>G', 'CTA'), ('T>G', 'CTC'), ('T>G', 'CTG'), ('T>G', 'CTT'),
            ('T>G', 'GTA'), ('T>G', 'GTC'), ('T>G', 'GTG'), ('T>G', 'GTT'),
            ('T>G', 'TTA'), ('T>G', 'TTC'), ('T>G', 'TTG'), ('T>G', 'TTT')],
            names=['Type', 'Subtype'])

mut96 = ['A[C>A]A', 'A[C>A]C', 'A[C>A]G', 'A[C>A]T', 'C[C>A]A', 'C[C>A]C', 'C[C>A]G', 'C[C>A]T',
         'G[C>A]A', 'G[C>A]C', 'G[C>A]G', 'G[C>A]T', 'T[C>A]A', 'T[C>A]C', 'T[C>A]G', 'T[C>A]T',
         'A[C>G]A', 'A[C>G]C', 'A[C>G]G', 'A[C>G]T', 'C[C>G]A', 'C[C>G]C', 'C[C>G]G', 'C[C>G]T',
         'G[C>G]A', 'G[C>G]C', 'G[C>G]G', 'G[C>G]T', 'T[C>G]A', 'T[C>G]C', 'T[C>G]G', 'T[C>G]T',
         'A[C>T]A', 'A[C>T]C', 'A[C>T]G', 'A[C>T]T', 'C[C>T]A', 'C[C>T]C', 'C[C>T]G', 'C[C>T]T',
         'G[C>T]A', 'G[C>T]C', 'G[C>T]G', 'G[C>T]T', 'T[C>T]A', 'T[C>T]C', 'T[C>T]G', 'T[C>T]T',
         'A[T>A]A', 'A[T>A]C', 'A[T>A]G', 'A[T>A]T', 'C[T>A]A', 'C[T>A]C', 'C[T>A]G', 'C[T>A]T',
         'G[T>A]A', 'G[T>A]C', 'G[T>A]G', 'G[T>A]T', 'T[T>A]A', 'T[T>A]C', 'T[T>A]G', 'T[T>A]T',
         'A[T>C]A', 'A[T>C]C', 'A[T>C]G', 'A[T>C]T', 'C[T>C]A', 'C[T>C]C', 'C[T>C]G', 'C[T>C]T',
         'G[T>C]A', 'G[T>C]C', 'G[T>C]G', 'G[T>C]T', 'T[T>C]A', 'T[T>C]C', 'T[T>C]G', 'T[T>C]T',
         'A[T>G]A', 'A[T>G]C', 'A[T>G]G', 'A[T>G]T', 'C[T>G]A', 'C[T>G]C', 'C[T>G]G', 'C[T>G]T',
         'G[T>G]A', 'G[T>G]C', 'G[T>G]G', 'G[T>G]T', 'T[T>G]A', 'T[T>G]C', 'T[T>G]G', 'T[T>G]T']

mut32 = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
         'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT',
         'ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
         'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']

mut16 = ['A_A', 'A_C', 'A_G', 'A_T', 'C_A', 'C_C', 'C_G', 'C_T',
         'G_A', 'G_C', 'G_G', 'G_T', 'T_A', 'T_C', 'T_G', 'T_T']

mut6 = ['C>A','C>G','C>T','T>A','T>C','T>G']

def load_config(config_fp):

    # load the yaml file
    with open(config_fp, 'r') as f:
        config = yaml.safe_load(f)
        print(f"Loaded configuration file {config_fp}")

    # remove any parameters not applicable to selected data source
    ds = config.pop('dataset')
    ds[ds['dataset_sel']].update({'dataset_sel': ds['dataset_sel']})

    # update dataset args to subsetted list
    config.update({'dataset': ds[ds['dataset_sel']]})

    ## handle seeding
    #config['dataset'].update({'data_rng': np.random.default_rng(config['dataset']['data_seed'])})
    #config['model'].update({'model_rng': np.random.default_rng(config['model']['model_seed'])})

    return config['dataset'], config['model'], config['pymc3']

def detect_naming_style(fp):

    # first column must have type at minimum
    df = pd.read_csv(fp, index_col=0, sep = None, engine = 'python')
    naming_style = 'unrecognized'

    # check if index is type style
    if df.index.isin(mut96).any(): naming_style = 'type'

    # check if index is type/subtype style
    else:
        df = df.reset_index()
        df = df.set_index(list(df.columns[0:2]))
        if df.index.isin(idx96).any(): naming_style = 'type/subtype'

    assert naming_style == 'type' or naming_style == 'type/subtype', \
            'Mutation type naming style could not be identified.\n'\
            '\tExpected either two column type/subtype (ex. C>A,ACA) or\n'\
            '\tsingle column type (ex A[C>A]A). See examples at COSMIC database.'

    return naming_style

def load_sigs(fp):

    naming_style = detect_naming_style(fp)

    if naming_style == 'type':
        # read in sigs
        sigs = pd.read_csv(fp, index_col = 0, sep = None, engine = 'python').reindex(mut96)
        # sanity check for matching mut96, should have no NA
        sel = (~sigs.isnull()).all(axis = 1)
        assert sel.all(), f'invalid signature definitions: null entry for types {list(sigs.index[~sel])}'
        # convert to pcawg convention
        sigs = sigs.set_index(idx96)

    elif naming_style == 'type/subtype':
        # read in sigs
        sigs = pd.read_csv(fp, index_col = (0,1), header=0).reindex(idx96)
        # sanity check for idx, should have no NA
        sel = (~sigs.isnull()).all(axis = 1)
        assert sel.all(), f'invalid signature definitions: null entry for types {list(sigs.index[~sel])}'

    # check colsums are 1
    sel = np.isclose(sigs.sum(axis=0), 1)
    assert sel.all(), f'invalid signature definitions: does not sum to 1 in columns {list(sigs.columns[~sel])}'

    assert all(sigs.index == idx96) or all(sigs.index == mut96), 'signature defintions failed to be read correctly'

    # force Jx96 and mut96 convention
    sigs = sigs.T
    sigs.columns = mut96

    return sigs

def load_counts(counts_fp):
    counts = pd.read_csv(counts_fp, index_col = 0, header = 0)[mut96]
    assert counts.ndim == 2, 'Mutation counts failed to load. Check column names are mutation type (ex. A[C>A]A). See COSMIC database for more.'
    assert counts.shape[1] == 96, f'Expected 96 mutation types, got {counts.shape[1]}'
    return counts

def subset_samples(dataset, annotation, annotation_subset, sel_idx = 0):
    # subset sample ids by matching to annotation_subset
    # expect annotation_subset to be pd dataframe with ids as index

    if annotation_subset is None:
        return dataset, annotation

    # stop string being auto cast to list
    if type(annotation_subset) == str:
        annotation_subset = [annotation_subset]

    if annotation.ndim > 2:
        warnings.warn(f"More than one annotation is available per sample, selection index {sel_idx}", UserWarning)

    # annotation ids should match sample ids
    assert dataset.index.isin(annotation.index).any(), 'No sample ID matches found in dataset for the provided annotation'

    # reoder annotation (with gaps) to match dataset
    annotation = annotation.reindex(dataset.index)

    # partial matches allowed
    sel = np.fromiter((map(any, zip(*[annotation[annotation.columns[sel_idx]].str.contains(x) for x in annotation_subset] ))), dtype = bool)

    # type should appear in the type column of the lookup
    assert sel.any(), 'Cancer type subsetting yielded no selection. Check keywords?'

    dataset = dataset.loc[annotation.index[sel]]
    annotation = annotation.loc[annotation.index[sel]]
    return dataset, annotation

def save_checkpoint(fp, model, trace, dataset_args, model_args, pymc3_args, run_id):
    with open(f'{fp}', 'wb') as buff:
        pickle.dump({'model': model, 'trace': trace, 'dataset_args': dataset_args,
                     'model_args': model_args, 'pymc3_args': pymc3_args, 'run_id': run_id}, buff)
    print(f'checkpoint saved to {fp}')

def load_checkpoint(fn):
    with open(fn, 'rb') as buff:
        data = pickle.load(buff)
        print(f'checkpoint loaded from {fn}')
        wandb.init(id=data['run_id'], resume='allow')
        return data['model'], data['trace'], data['dataset_args'], data['model_args'], data['pymc3_args'], data['run_id']

def load_dataset(dataset_sel, counts_fp=None, annotation_fp=None, annotation_subset=None, seed=None,
                 data_seed = None, sig_defs_fp=None, sim_S=None, sim_N=None, sim_I=None, sim_tau_hyperprior=None,
                 sim_J=None, sim_K=None, sim_alpha_bias=None, sim_psi_bias=None, sim_gamma_bias=None, sim_beta_bias=None):
    # load counts, or simulated data - as specified by dataset_sel
    # seed -> rng as per https://albertcthomas.github.io/good-practices-random-number-generators/

    if dataset_sel == 'load_counts':
        dataset = load_counts(counts_fp)
        annotation = pd.read_csv(annotation_fp, index_col = 0, header = 0)
        dataset, annotation = subset_samples(dataset, annotation, annotation_subset)
        return dataset, annotation

    elif dataset_sel == 'sim_from_sigs':
        sig_defs = load_sigs(sig_defs_fp)
        dataset, sim_params = sim_from_sigs(sig_defs, sim_tau_hyperprior, sim_S, sim_N, sim_I, seed)
        return dataset, sim_params

    elif dataset_sel == 'sim_parametric':
        dataset, sim_params = sim_parametric(sim_J,sim_K,sim_S,sim_N,sim_alpha_bias,sim_psi_bias,sim_gamma_bias,sim_beta_bias,seed)
        return dataset, sim_params

    else:
        assert False, 'dataset selection not recognized'

def load_datasets(dataset_args):
    yargs = dataset_args.copy()
    ca = [load_dataset(counts_fp = j[0], annotation_fp = j[1], **yargs) for j in zip(yargs.pop('counts_fp'), yargs.pop('annotation_fp'))]
    counts = pd.concat([c[0] for c in ca ])
    annotation = pd.concat([a[1] for a in ca])
    return counts, annotation

def split_by_count(data, fraction, rng):
    # assumes data is a pandas df

    frac1 = data.apply(lambda r: rng.choice( np.repeat(np.arange(96), r), int(fraction*r.sum()), replace = False) , axis = 1)
    frac1 = pd.DataFrame(frac1.apply(lambda r: np.histogram(r, bins=96, range=(0, 96))[0]).to_list(),
                         index = data.index, columns = data.columns)
    #frac1 = (data * fraction).astype(int)
    #frac1 = frac1.apply(lambda r: np.histogram(r, bins=96, range=(0, 96))[0], axis = 1, result_type='expand')
    #frac1.columns = data.columns
    frac2 = data - frac1
    assert all(frac2 >= 0) and all(frac1 >= 0)
    assert np.all(data == frac1 + frac2), 'Splitting failed'
    return frac1, frac2

def split_by_S(data, fraction, rng):
    # assumes data is a pandas df with an index
    frac1 = data.sample(frac=fraction, random_state=np.random.RandomState(rng.bit_generator))
    frac2 = data.drop(frac1.index)
    return frac1, frac2

def split_data(counts, S_frac = 0.9, c_frac = 0.8, rng=np.random.default_rng()):
    # get train/val/test split
    trn, tst = split_by_S(counts, S_frac, rng)
    trn, val = split_by_count(trn, c_frac, rng)
    tst1, tst2 = split_by_count(tst, c_frac, rng)
    return trn, val, tst1, tst2

def get_tau(phi, eta):
    assert len(phi.shape) == 2 and len(eta.shape) == 3
    tau =  np.einsum('jpc,kpm->jkpmc', phi.reshape((-1,2,16)), eta).reshape((-1,96))
    return tau

def get_phi(sigs):
    # for each signature in sigs, get the corresponding phi
    wrapped = sigs.reshape(-1, 2, 3, 16)
    phi = wrapped.sum(2).reshape(-1,32)
    return phi

def get_eta(sigs):
    # for each signature in sigs, get the corresponding eta (kxpxm)
    wrapped = sigs.reshape(-1, 6, 16)
    eta = wrapped.sum(2).reshape(-1,3)
    # notrmalize such that etaC and etaT sum to 1 respectively.
    eta = (eta/eta.sum(1)[:,None]).reshape(-1,P,M)
    return eta

def flatten_eta(eta): # eta pkm -> kc
    warnings.warn('Eta no longer constructed as pkm - use reshape instead', DeprecationWarning)
    return np.moveaxis(eta,0,1).reshape(-1, 6)

def alr(x, e=1e-12):
    '''
    additive log ratio
    x is a NxD matrix
    >>> x = np.array([.1, .3, .4, .2])
    >>> alr(x)
    array([ 1.09861229,  1.38629436,  0.69314718])
    '''
    # add small value for stability in log
    x = x + e
    return (np.log(x) - np.log(x[...,-1]).reshape(-1,1))[:,0:-1]

def alr_inv(y):
    '''
    inverse alr transform
    y is a Nx(D-1) matrix
    >>> x = np.array([.1, .3, .4, .2])
    >>> alr_inv(alr(x))
    array([ 0.1,  0.3,  0.4,  0.2])
    '''
    if y.ndim == 1: y = y.reshape(1,-1)
    return softmax(np.hstack([y, np.zeros((y.shape[0], 1)) ]), axis = 1)

def kmeans_alr(data, nsig, rng=np.random.default_rng()):
    #https://github.com/scikit-learn/scikit-learn/issues/16988#issuecomment-817375063
    km = k_means(alr(data), nsig, random_state=np.random.RandomState(rng.bit_generator))
    return alr_inv(km[0])

def alp_B(data, B):
    return (data * np.log(B)).sum() / data.sum()

def mult_ll(x, p):
    # x and p should both be same dimensions; Sx96
    return loggamma(x.sum(1) + 1) - loggamma(x+1).sum(1) + (x * np.log(p)).sum(1)

def lap_B(data, Bs):
    # Bs should be shape DxSx96 where D is the number of posterior samples
    # use logsumexp for stability
    assert Bs.ndim == 3, 'expected multiple trials for B'
    return logsumexp(np.vstack([mult_ll(data, B) for B in Bs]).sum(1)) - np.log(Bs.shape[0])

def profile_sigs(sigs, refsigs, thresh = 0.9, refidx = None):
    # return mapping of sigs to similar refsigs

    if isinstance(refsigs, pd.core.frame.DataFrame) and refsigs.index is not None:
        refidx = refsigs.index
    elif refidx is None:
        refidx = pd.Index([f'refsig_{i}' for i in range(0,refsigs.shape[0])])

    sim = cosine_similarity(sigs, refsigs)
    closest_dist = np.max(sim, axis=1)
    closest = refidx[np.argmax(sim, axis=1)]
    above_thresh = [str(refidx[x].to_numpy()) for x in sim > thresh]

    df=pd.DataFrame.from_dict({'inferred signature': [f'sig_{i}' for i in range(0,sigs.shape[0])],
                               'closest reference signature':closest,
                               'dist to closest': closest_dist,
                               f'reference signatures with >{thresh} similarity': above_thresh
                              })
    df.index = [f'sig_{i}' for i in range(0,sigs.shape[0])]
    return df


from __future__ import print_function
import numpy as np
import os
import sys

from keras.models import load_model
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer
from keras.utils.np_utils import to_categorical

from parse_data import get_data, get_mapping, get_one_hot_vectors, \
    normalize_vectors, clean_credit, get_pos_freqs, \
    remove_stop_words

import var

def test_model():
    '''
    Evaluates a saved model against eval and test sets. The trained model
    folder path should be the first argument. All saved models inside
    that folder will be evaluated.
    '''
    paths = None
    if len(sys.argv) < 2:
        raise Exception(
            "Must specify the path to the folder of the saved model to test")
    model_path = sys.argv[1]
    folder = ''

    if os.path.isdir(model_path):
        paths = [
            os.path.join(
                model_path,
                f) for f in os.listdir(model_path) if os.path.isfile(
                os.path.join(
                    model_path,
                    f)) and f.endswith(".hdf5")]
        folder = model_path
    elif os.path.isfile(model_path):
        paths = [model_path]

    print()
    print("List of files to test")
    print(paths)
    print()
    model_results = []

    for i, path in enumerate(paths):
        print()
        print("Testing file " + str(i + 1) + " out of " + str(len(paths)))

        model = load_model(path)

        # Recreate the input tokenizer
        train_labels, train_sentences, train_subjects, train_party, train_credit = get_data(
            var.TRAINING_DATA_PATH)
        if var.NO_STOP_WORDS:
            train_sentences = remove_stop_words(train_sentences)
        train_party = [[party] for party in train_party]
        tokenizer = Tokenizer(num_words=var.MAX_NUM_WORDS)
        tokenizer.fit_on_texts(train_sentences)

        # Get the val input via tokenizer and val labels
        val_labels, val_sentences, val_subjects, val_party, val_credit = get_data(
            var.VALIDATION_DATA_PATH)
        if var.NO_STOP_WORDS:
            val_sentences = remove_stop_words(val_sentences)
        val_party = [[party] for party in val_party]
        val_sequences = tokenizer.texts_to_sequences(val_sentences)
        x_val = pad_sequences(val_sequences, maxlen=var.MAX_SEQUENCE_LENGTH)
        y_val = to_categorical(np.asarray(
            [var.LABEL_MAPPING[label] for label in val_labels]))

        # Get the test input via tokenizer and test labels
        test_labels, test_sentences, test_subjects, test_party, test_credit = get_data(
            var.TEST_DATA_PATH)
        if var.NO_STOP_WORDS:
            test_sentences = remove_stop_words(test_sentences)
        test_party = [[party] for party in test_party]
        test_sequences = tokenizer.texts_to_sequences(test_sentences)
        x_test = pad_sequences(test_sequences, maxlen=var.MAX_SEQUENCE_LENGTH)
        y_test = to_categorical(np.asarray(
            [var.LABEL_MAPPING[label] for label in test_labels]))

        x_test_pos = np.asarray(get_pos_freqs(test_sentences))
        x_val_pos = np.asarray(get_pos_freqs(val_sentences))

        var.SUBJECT_MAPPING = get_mapping(train_subjects)
        x_test_subjects = np.asarray(
            get_one_hot_vectors(
                test_subjects,
                var.NUM_SUBJECTS,
                var.SUBJECT_MAPPING))
        x_val_subjects = np.asarray(
            get_one_hot_vectors(
                val_subjects,
                var.NUM_SUBJECTS,
                var.SUBJECT_MAPPING))

        var.PARTY_MAPPING = get_mapping(train_party)
        x_test_party = np.asarray(
            get_one_hot_vectors(
                test_party,
                var.NUM_PARTIES,
                var.PARTY_MAPPING))
        x_val_party = np.asarray(
            get_one_hot_vectors(
                val_party,
                var.NUM_PARTIES,
                var.PARTY_MAPPING))

        test_credit = clean_credit(test_labels, test_credit)
        val_credit = clean_credit(val_labels, val_credit)
        x_test_credit = np.asarray(normalize_vectors(test_credit))
        x_val_credit = np.asarray(normalize_vectors(val_credit))

        test_input = [x_test]
        val_input = [x_val]
        if var.USE_SUBJECTS:
            test_input.append(x_test_subjects)
            val_input.append(x_val_subjects)
        if var.USE_PARTY:
            test_input.append(x_test_party)
            val_input.append(x_val_party)
        if var.USE_CREDIT:
            test_input.append(x_test_credit)
            val_input.append(x_val_credit)
        if var.USE_POS:
            test_input.append(x_test_pos)
            val_input.append(x_val_pos)

        test_score = model.evaluate(
            test_input, y_test, batch_size=var.BATCH_SIZE)
        val_score = model.evaluate(val_input, y_val, batch_size=var.BATCH_SIZE)

        print()
        print("model = " + str(path))
        print(
            "val loss = %0.4f, val acc = %0.4f" %
            (val_score[0], val_score[1]))
        print(
            "test loss = %0.4f, test acc = %0.4f" %
            (test_score[0], test_score[1]))
        model_results.append(
            (os.path.basename(path), round(
                val_score[0], 4), round(
                val_score[1], 4), round(
                test_score[0], 4), round(
                    test_score[1], 4)))

    average_val_loss = round(
        sum([info[1] for info in model_results]) / len(model_results), 4)
    average_val_acc = round(
        sum([info[2] for info in model_results]) / len(model_results), 4)
    average_test_loss = round(
        sum([info[3] for info in model_results]) / len(model_results), 4)
    average_test_acc = round(
        sum([info[4] for info in model_results]) / len(model_results), 4)

    best_val = sorted(model_results, key=lambda x: x[2], reverse=True)[:5]
    best_test = sorted(model_results, key=lambda x: x[4], reverse=True)[:5]

    with open(os.path.join(folder, "stats.txt"), "w") as f:
        f.write("Num models = " + str(len(model_results)) + "\n")
        f.write("Average val_loss = " + str(average_val_loss) + "\n")
        f.write("Average val_acc = " + str(average_val_acc) + "\n")
        f.write("Average test_loss = " + str(average_test_loss) + "\n")
        f.write("Average test_acc = " + str(average_test_acc) + "\n")
        f.write("\n")
        f.write("Top 5 validation accuracies:\n")
        for model in best_val:
            f.write("\t" + str(model[2]) + "\t" + str(model[0]) + "\n")
        f.write("\n")
        f.write("Top 5 test accuracies:\n")
        for model in best_test:
            f.write("\t" + str(model[4]) + "\t" + str(model[0]) + "\n")
        f.write("\n")
        f.write("Results for all models:\n")
        for model in sorted(model_results, key=lambda x: x[0]):
            f.write("\t" + str(model) + "\n")

    print()
    print("---------------------- OVERALL STATISTICS -----------------------")
    print()
    with open(os.path.join(folder, "stats.txt"), "r") as f:
        print(f.read())

if __name__ == '__main__':
    test_model()


"""name URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from ui_app import views

urlpatterns = [
    #path de ui_app
    path('', views.home, name="home"),
    path('abuse/', views.abuse, name='abuse'),
    path('account/', views.account, name='account'),
    path('activities/', views.activities, name='activities'),
    path('adverts/', views.adverts, name='adverts'),
    path('emails/', views.emails, name='emails'),
    path('features/', views.features, name='features'),
    path('info_pages/', views.info_pages, name='info_pages'),
    path('languages/', views.languages, name='languages'),
    path('media/', views.media, name='media'),
    path('messenger/', views.messenger, name='messenger'),
    path('modals/', views.modals, name='modals'),
    path('payments/', views.payments, name='payments'),
    path('payouts/', views.payouts, name='payouts'),
    path('post/', views.post, name='post'),
    path('reels/', views.reels, name='reels'),
    path('timeline/', views.timeline, name='timeline'),
    path('user/', views.user, name='user'),
    #path del admin
    path('admin/', admin.site.urls),
]


from database import Database
from module.post import Posts
import datetime
import uuid

class Users(object):

    def __init__(self,author="none",user_id=uuid.uuid4().hex,discription="none"):
        self.author = author
        self.user_id = user_id
        self.discription = discription
        self.save_to_db()

    def new_post(self,title="none",date=datetime.datetime.utcnow(),content="none"):
        date = datetime.datetime.strptime(date,"%d%m%Y")
        post = Posts(title=title,date=date,user_id=self.user_id,content=content)

    def save_to_db(self):
        retJson = self.json()
        Database.insert("users",retJson)

    def json(self):
        return {
            "author": self.author,
            "user_id": self.user_id,
            "discription": self.discription
        }


import os
import numpy as np
import h5py
import librosa
import soundfile as sf

from scipy.signal import decimate

from matplotlib import pyplot as plt

# ----------------------------------------------------------------------------

def load_h5(h5_path):
    # load training data
    with h5py.File(h5_path, 'r') as hf:
        print('List of arrays in input file:', list(hf.keys()))
        X = np.array(hf.get('data'))
        Y = np.array(hf.get('label'))
        print('Shape of X:', X.shape)
        print('Shape of Y:', Y.shape)

    return X, Y
# ttyadd: create dirs if not exist
def check_dir(path):
    if os.path.exists(path) is False:
        os.makedirs(path)

# ttyadd: stolen from https://github.com/deepakbaby/se_relativisticgan
def reconstruct_wav(wavmat, stride_factor=0.5):
    """
    Reconstructs the audiofile from sliced matrix wavmat
    """
    # shape [frames, len]
    window_length = wavmat.shape[1]
    window_stride = int(stride_factor * window_length)
    wav_length = (wavmat.shape[0] - 1) * window_stride + window_length
    wav_recon = np.zeros((1, wav_length))
    # print ("wav recon shape " + str(wav_recon.shape))
    for k in range(wavmat.shape[0]):
        wav_beg = k * window_stride
        wav_end = wav_beg + window_length
        wav_recon[0, wav_beg:wav_end] += wavmat[k, :]

    # now compute the scaling factor for multiple instances
    noverlap = int(np.ceil(1 / stride_factor))
    scale_ = (1 / float(noverlap)) * np.ones((1, wav_length))
    for s in range(noverlap - 1):
        s_beg = s * window_stride
        s_end = s_beg + window_stride
        scale_[0, s_beg:s_end] = 1 / (s + 1)
        scale_[0, -s_beg - 1: -s_end:-1] = 1 / (s + 1)

    return wav_recon * scale_

def upsample_wav(wav, args, model):

    # load signal
    x_hr, fs = librosa.load(wav, sr=args.sr)

    x_lr_t = decimate(x_hr, args.r)

    # pad to mutliple of patch size to ensure model runs over entire sample
    x_hr = np.pad(x_hr, (0, args.patch_size - (x_hr.shape[0] % args.patch_size)), 'constant', constant_values=(0,0))

    # downscale signal
    x_lr = decimate(x_hr, args.r)

    # ttyadd: generate patches --> predict --> ola
    if args.ola == 'true':# Mabey not, it seems like worse
        print("++++++++++++++++++++ola++++++++++++++++++++")
        d_lr = int(2048//2)
        s = int(1024//2)
        lr_patches = []
        max_i = len(x_lr) - d_lr + 1
        for i in range(0, max_i, s):
            i_lr = i
            lr_patch = np.array(x_lr[i_lr: i_lr + d_lr])
            lr_patches.append(lr_patch)
        lr_patches = np.array(lr_patches).flatten()
        x_sp, P = model.predict(lr_patches.reshape((1,len(lr_patches),1)))
        x_pr = P.flatten()
        pr_mat = x_pr.reshape((-1,1024))
        x_pr = reconstruct_wav(pr_mat, stride_factor=0.5)
        x_pr = x_pr.flatten()
    # original method
    else:
        x_sp, P = model.predict(x_lr.reshape((1, len(x_lr), 1)))
        # print(np.shape(P))
        x_pr = P.flatten()

    # save the file
    # ttyadd: wav outname
    outname = args.logname+'/eval'+wav[wav.find('Corpus')+6:]+ '.' + args.out_label
    check_dir(os.path.abspath(os.path.dirname(outname)))
    #outname = wav + '.' + args.out_label
    #sf.write(outname + '.lr.wav', x_lr_t, int(fs / args.r))
    #sf.write(outname + '.sp.wav', x_sp, fs) # ttyadd
    #sf.write(outname + '.hr.wav', x_hr, fs)
    sf.write(outname + '.pr.wav', x_pr, fs)

    # save the spectrum
    #S = get_spectrum(x_pr, n_fft=2048)
    #save_spectrum(S, outfile=outname + '.pr.png')
    #S = get_spectrum(x_hr, n_fft=2048)
    #save_spectrum(S, outfile=outname + '.hr.png')
    #S = get_spectrum(x_lr, n_fft=int(2048/args.r))
    #save_spectrum(S, outfile=outname + '.lr.png')

# ----------------------------------------------------------------------------

def get_spectrum(x, n_fft=2048):
    S = librosa.stft(x, n_fft)
    p = np.angle(S)
    S = np.log1p(np.abs(S))
    return S

def save_spectrum(S, lim=800, outfile='spectrogram.png'):
    plt.imshow(S.T, aspect=10)
    # plt.xlim([0,lim])
    plt.tight_layout()
    plt.savefig(outfile)


from django.contrib import admin
from edc_fieldsets.fieldlist import Insert
from edc_fieldsets.fieldsets_modeladmin_mixin import FormLabel
from edc_model_admin import audit_fieldset_tuple

from ...admin_site import flourish_facet_admin
from ...forms import FacetChildMedicalHistoryForm
from ...models import FacetChildMedicalHistory
from ..modeladmin_mixins import CrfModelAdminMixin

@admin.register(FacetChildMedicalHistory, site=flourish_facet_admin)
class FacetChildMedicalHistoryAdmin(CrfModelAdminMixin, admin.ModelAdmin):

    form = FacetChildMedicalHistoryForm

    list_display = (
        'facet_visit', 'chronic_since')
    list_filter = ('chronic_since',)

    fieldsets = (
        (None, {
            'fields': [
                'facet_visit',
                'report_datetime',
                'chronic_since',
                'child_chronic',
                'child_chronic_other',
                'current_illness',
                'current_symptoms',
                'current_symptoms_other',
                'symptoms_start_date',
                'seen_at_local_clinic',
                'currently_taking_medications',
                'current_medications',
                'current_medications_other',
                'duration_of_medications',

            ]}
         ), audit_fieldset_tuple)

    radio_fields = {'chronic_since': admin.VERTICAL,
                    'med_history_changed': admin.VERTICAL, }

    filter_horizontal = ('child_chronic',)

    custom_form_labels = [
        FormLabel(
            field='med_history_changed',
            label=('Since the last scheduled visit in {previous}, has any of '
                   'your medical history changed?'),
            previous_appointment=True)
    ]

    quartely_schedules = ['child_a_sec_qt_schedule1', 'child_a_quart_schedule1',
                          'child_b_sec_qt_schedule1', 'child_b_quart_schedule1',
                          'child_c_sec_qt_schedule1', 'child_c_quart_schedule1',
                          'child_pool_schedule1', 'child_a_fu_schedule1',
                          'child_b_fu_schedule1', 'child_c_fu_schedule1',
                          'child_a_fu_qt_schedule1', 'child_b_fu_qt_schedule1',
                          'child_c_fu_qt_schedule1']

    radio_fields = {'chronic_since': admin.VERTICAL,
                    'currently_taking_medications': admin.VERTICAL,
                    'duration_of_medications': admin.VERTICAL,
                    'current_illness': admin.VERTICAL,
                    'seen_at_local_clinic': admin.VERTICAL,
                    'med_history_changed': admin.VERTICAL}

    filter_horizontal = (
        'child_chronic', 'current_symptoms', 'current_medications')

    conditional_fieldlists = {}
    for schedule in quartely_schedules:
        conditional_fieldlists.update(
            {schedule: Insert('med_history_changed', after='report_datetime')})

    def get_form(self, request, obj=None, *args, **kwargs):
        form = super().get_form(request, *args, **kwargs)
        form.previous_instance = self.get_previous_instance(request)
        return form


# setup.py file

from setuptools import find_packages, setup

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(
    name="lamdata_dspt6_rawalk",  # the name that you will install via pip
    version="1.0.1",
    author="Kush Rawal",
    author_email="kush-rawal@lambdastudents.com",
    description="2 helper utility functions",
    long_description=long_description,
    long_description_content_type="text/markdown",
    # required if using a md file for long desc
    # license="MIT",
    url="https://github.com/rawalk/Lambdata-rawalk",
    # keywords="",
    packages=find_packages()  # ["my_lambdata"]
)


import io
import zipfile

from pydantic import BaseModel, HttpUrl, ValidationError, field_validator

from web_resource_watchdog import Config

class CreateWebResource(BaseModel):
    """Pydantic schema for web resource model."""

    full_url: HttpUrl

class ZipFileValidator(BaseModel):
    """Pydantic schema for zip-file validation."""

    zip_file: bytes

    @field_validator("zip_file")
    def validate_zip_file(cls, value):
        """Validate the uploaded zip file.

        Parameters:
            value (bytes): The uploaded zip file as bytes.

        Returns:
            io.BytesIO: A BytesIO object representing
            the validated zip buffer.

        Raises:
            ValidationError: If the uploaded file is not a valid
            zip archive.
        """
        zip_buffer = io.BytesIO(value)
        try:
            with zipfile.ZipFile(zip_buffer, "r") as zip_ref:
                zip_ref.testzip()
            return zip_buffer
        except zipfile.BadZipFile:
            raise ValidationError(
                "Invalid zip file",
            )

def allowed_file(filename):
    """Check file extension in allowed extensions."""
    return ("." in filename) and (
        filename.rsplit(".", 1)[1].lower()
        in Config.ALLOWED_ARCHIVES_EXTENSIONS
    )


# Written By: Nick Evans
# Purpose: Lightweight script to append timestamp to the beginning of .xisf image files to enable pixinsight blink tool to load images in the correct order.
# Usage: Either
# 1) place this script in the directory with your .xisf files you wish to rename, and run it or
# 2) run this script from commandline with argument to the path to where the files are. Example: $ python3 timestamp_files.py /c/images/

import time
import os
from pathlib import Path
import sys

## Get Path to process
if(sys.argv[1] != ""):
    # get path via command line argument
    path = Path(sys.argv[1])
else:
    # get path from where the script is ran from
    path = Path(os.getcwd())

### Insert path validation code / error catching here...

# Update user
print('------------------------------------')
print('Input path = ', path)
print('Detected %s files. Processing... ' %len(os.listdir(path)))
print('------------------------------------')

# Process Files
for x in os.listdir(path):
    if x.endswith(".xisf"): #could update with different file extension - perhaps via another argument
        f_path = os.path.join(path, x)

        # Obtaining the modified time (in seconds) of the file/folder
        t = os.path.getmtime(f_path)

        # Converting the time to an epoch string
        t_str = time.ctime(t)

        # Converting the string to a time object
        t_obj = time.strptime(t_str)

        # Transforming the time object to a timestamp to ISO 8601 format
        form_t = time.strftime("%Y-%m-%d %H:%M:%S", t_obj)

        # Modified Letter Colon " " (U+A789)
        form_t = form_t.replace(":", "꞉")

        # Renaming the filename to its timestamp
        os.rename(f_path, os.path.split(f_path)[0] + '/' + form_t +'_'+ x)
        print(os.path.split(f_path)[0] + '/' + form_t +'_'+ x) #+ os.path.splitext(f_path)[1]) # for the extension
print('------------------------------------')
print('Processing Complete.')


import argparse
from flask import Flask, jsonify, request
import socket
from pathlib import Path
from platform import system as OS

from syncFiles.syncFiles import check_sum

DEBUG = True
currentIP = socket.gethostbyname(socket.gethostname())

ap = argparse.ArgumentParser(description='Run a deamon that checks sums of sent files.',
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)

ap.add_argument('storage_folder_path',
                help='Path to the folder where the data is supposed to be moved.',
                type=Path)
ap.add_argument('--host',
                help="Host's IP.",
                default=currentIP)
ap.add_argument('--port',
                help='Port to listen to.',
                default=9001,
                type=int)
ap = ap.parse_args()
app  = Flask(__name__)

@app.route('/')
def index():
    return f'checkSummer operating.\nIP: {ap.host} PORT: {ap.port}\nDATA FOLDER:{ap.storage_folder_path}'

@app.route('/check', methods=['POST'])
def get_project_id():
    if request.data:
        print(request.data)
        print(ap.storage_folder_path)
        relative_path, check_sum_kwds = request.get_json()
        local_abs_file_path = ap.storage_folder_path/relative_path
        print(local_abs_file_path)
        cs = check_sum(local_abs_file_path, **check_sum_kwds)
        print(cs)
        return jsonify(cs)

if __name__ == '__main__':
    app.run(debug=DEBUG,
            host=ap.host,
            port=ap.port,
            threaded=True)


from django.shortcuts import render
# from rest_framework import Response
from rest_framework.response import Response
from .models import Customer
from rest_framework.decorators import api_view
from .serializers import CustomerSerializer
# Create your views here.
from PIL import Image
from .S3Service import S3
from .imagga import Imagga_Request
from .tasks import check_request, send_mail_submit, send_mail_resubmit
from .Messages import *
from .mailgun import send_mail
from ipware import get_client_ip
@api_view(['GET'])
def ping(request):
    return Response({"PING": "PONG"})

@api_view(['POST'])
def submit_request(request):

    try:
        client_ip, _ = get_client_ip(request)
        customer_requests = Customer.objects.filter(national_id = request.data['national_id'])
        if len(customer_requests) != 0:
            for req in customer_requests:
                if req.state != "R":
                    return Response({"message": DUPLICATE}, status=400)

            customer_requests[0].state = "P"
            customer_requests[0].save()
            send_mail_resubmit(customer_requests[0])
            check_request.delay(customer_requests[0].email,
                                customer_requests[0].last_name,
                                customer_requests[0].img1.name.split('/')[-1],
                                customer_requests[0].img2.name.split('/')[-1])

            return Response({"message": RESUBMIT}, status=200)
        new_customer = Customer(email=request.data['email'],
                                last_name=request.data['last_name'],
                                national_id=request.data['national_id'],
                                img1=request.data['img1'],
                                img2=request.data['img2'],
                                user_ip=client_ip, state="P")
        new_customer.save()
        s3 = S3()

        s3.insert_object(new_customer.img1.name)
        s3.insert_object(new_customer.img2.name)

        send_mail_submit.delay(new_customer.email,
                               new_customer.last_name
                               )
        check_request.delay(new_customer.email,
                            new_customer.last_name,
                            new_customer.img1.name.split('/')[-1],
                            new_customer.img2.name.split('/')[-1]
                            )
        return Response({"message": SUBMIT}, status=200)
    except Exception as exc:
        new_customer.delete()
        return Response({"message": str(exc)}, status=500)

@api_view(['POST'])
def get_status(request):
    try:
        client_ip, _ = get_client_ip(request)

        national_id = request.data['national_id']
        customer = Customer.objects.get(national_id=national_id)
        if client_ip != customer.user_ip:
            return Response({"message": ACCESS_DENIED}, status=403)
        message = ''
        _s = customer.state
        if _s == "P":
            message = PENDING
        elif _s == "r":
            message = REJECTED
        else:
            message = APPROVED
        return Response({"details": message}, status=200)
    except Exception as exc:
        return Response({"message": str(exc)}, status=500)

@api_view(['GET'])
def testing(request):
    try:
        return Response({"message": "there is no test here"}, status=200)
    except Exception as exc:
        print(exc)
        return Response({"message": str(exc)}, status=500)

@api_view(['GET'])
def echo_ip(request):
    _ip, _ = get_client_ip(request)
    return Response({"Message": _ip})


# -*- coding: utf-8 -*-
#harvest theses from TCD, Dublin
#FS: 2020-09-02
#FS: 2023-03-09

import getopt
import sys
import os
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
import re
import ejlmod3
import time

years = [ejlmod3.year(backwards=1), ejlmod3.year()]
rpp = 50
pages = 20
numberofrecords = rpp*pages
skipalreadyharvested = True

publisher = 'TCD, Dublin'
jnlfilename = 'THESES-TrinityCollegeDublin-%s' % (ejlmod3.stampoftoday())

boringschools = ['Biochemistry & Immunology', 'Business', 'Chemistry',
                 'Creative Arts', 'Dental Sciences', 'Religion',
                 'Ecumenics', 'Education', 'Engineering', 'English',
                 'Histories & Humanities', 'Lang, Lit', 'Law',
                 'Linguistic Speech & Comm Sci', 'Medicine', 'Natural Sciences',
                 'Nursing & Midwifery', 'Pharmacy & Pharma', 'Psychology',
                 'Social Sciences & Philosophy', 'Social Work & Social Policy']

if skipalreadyharvested:
    alreadyharvested = ejlmod3.getalreadyharvested(jnlfilename)

hdls = []

prerecs = []
for year in years:
    for section in ['76240', '221', '171']:
        realpages = pages
        for page in range(pages):
            if page*rpp < numberofrecords:
                tocurl = 'http://www.tara.tcd.ie/handle/2262/' + section + '/discover?rpp=' + str(rpp) + '&page=' + str(page+1) + '&group_by=none&etal=0&filtertype_0=dateIssued&filtertype_1=type&filter_0=[' + str(year) + '+TO+' + str(year) + ']&filter_relational_operator_1=equals&filter_1=Thesis&filter_relational_operator_0=equals'
                ejlmod3.printprogress("=", [[year], [section], [page+1, realpages]])
                try:
                    tocpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(tocurl), features="lxml")
                    time.sleep(3)
                except:
                    print("retry %s in 180 seconds" % (tocurl))
                    time.sleep(180)
                    tocpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(tocurl), features="lxml")
                if page == 0:
                    for p in tocpage.body.find_all('p', attrs = {'class' : 'pagination-info'}):
                        if re.search('\d of \d+', p.text):
                            numberofrecords = int(re.sub('.*of (\d+).*', r'\1', p.text.strip()))
                            print('  %i theses in query' % (numberofrecords))
                            realpages = (numberofrecords-1) // rpp + 1
                for rec in ejlmod3.getdspacerecs(tocpage, 'http://www.tara.tcd.ie'):
                    if rec['hdl'] in hdls or rec['hdl'] in ['2262/82940']:
                        print('  HDL:%s already from other collection' % (rec['hdl']))
                    elif skipalreadyharvested and rec['hdl'] in alreadyharvested:
                        print('  HDL:%s already in backup' % (rec['hdl']))
                    else:
                        prerecs.append(rec)
                        hdls.append(rec['hdl'])
        print('  %4i records so far' % (len(prerecs)))

i = 0
recs = []
for rec in prerecs:
    keepit = True
    i += 1
    ejlmod3.printprogress("-", [[year], [i, len(prerecs)], [rec['link']], [len(recs)]])
    try:
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['link'] + '?show=full'), features="lxml")
        time.sleep(3)
    except:
        print("retry %s in 180 seconds" % (rec['link'] + '?show=full'))
        time.sleep(180)
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['link']), features="lxml")
    ejlmod3.metatagcheck(rec, artpage, ['citation_authors', 'DCTERMS.issued', 'DCTERMS.abstract',
                                        'DC.subject'])
    for meta in artpage.head.find_all('meta', attrs = {'name' : 'DC.publisher'}):
        if re.search('School of', meta['content']):
            rec['school'] = re.sub('.*School of (.*?)\..*', r'\1', meta['content'])
        else:
            rec['note'].append(meta['content'])
    if 'school' in list(rec.keys()):
        if rec['school'] == 'Mathematics':
            rec['autaff'][-1].append('Trinity Coll., Dublin')
            rec['fc'] = 'm'
        elif rec['school'] in boringschools:
            print('   skip School of %s' % (rec['school']))
            keepit = False
        else:
            rec['note'].append(rec['school'])
    if len(rec['autaff'][-1]) == 1:
        rec['autaff'][-1].append(publisher)
    #license
    ejlmod3.globallicensesearch(rec, artpage)
    #fulltext
    for div in artpage.body.find_all('div', attrs = {'class' : 'simple-item-view-download'}):
        for a in div.find_all('a'):
            if 'license' in list(rec.keys()):
                rec['FFT'] = a['href']
            else:
                rec['hidden'] = a['href']
    #supervisor
    for tr in artpage.body.find_all('tr', attrs = {'class' : 'ds-table-row'}):
        tdt = ''
        for td in tr.find_all('td'):
            if td.has_attr('label-cell'):
                tdt = td.text.strip()
            else:
                if tdt == 'dc.contributor.advisor':
                    rec['supervisor'].append(td.text.strip())

    if keepit:
        recs.append(rec)
        ejlmod3.printrecsummary(rec)
    else:
        ejlmod3.adduninterestingDOI(rec['hdl'])

ejlmod3.writenewXML(recs, publisher, jnlfilename)


import networkx as nx
import numpy as np
import matplotlib.pyplot as plt
from networkx.drawing.nx_agraph import graphviz_layout

class GRA():

    def __init__(self):
        self.G = nx.Graph()
        self.ind = 0

    def addNodes(self, nodes=([[("act","val","totVal")]], [])):

        # G.add_nodes_from(range(0,len(nodes)))
        (layers, anchors) = nodes
        layer = 0
        for n in layers:
            layer += 1

            xpos = 0
            for node in n:

                self.G.add_node(self.ind, pos=(xpos,layer), label="{} {} {}".format(node[0],node[1],node[2]))
                self.G.add_edge(anchors[layer]+10*layer, self.ind)
                self.ind += 1
                xpos += 1

    def plotGraph(self):
        pos = nx.get_node_attributes(self.G,'pos')
        labels = nx.get_node_attributes(self.G,'label')
        nx.draw(self.G,pos,labels)


''' Module used to work with .json roster files. '''

import json

def add_roster():
    ''' Enter student names to create a period's roster. '''
    period_name = input('Which period? ')
    roster = []
    print('Type exit to quit.')
    while True:
        name = input('Student? ')
        if name == 'exit':
            break
        roster.append(name)

    filename = f'{period_name.title()}.json'
    with open(filename, 'w') as f:
        json.dump(roster,f)
    print(f'Congratulations, you added roster {period_name.title()}.')

def load_roster(period = 'second'):
    ''' Load and return a period's roster. '''
    period_name = period
    filename = f'{period_name.title()}.json'
    with open(filename) as f:
        roster = json.load(f)
    return roster

def edit_roster():
    ''' Edit a pre-existing roster. '''
    period_name = input('Which period? ')
    filename = f'{period_name.title()}.json'
    with open(filename) as f:
        roster = json.load(f)
    print(roster)
    change_from = input('Which name would you like to change? ' )
    change_to = input(f'What would you like to change {change_from} to? ')
    roster[roster.index(change_from)] = change_to
    with open(filename, 'w') as g:
        json.dump(roster,g)
    print(f'Congratulations, you changed {change_from} to {change_to}.')

def update_roster():
    ''' Re-write roster from given list. '''


from django.urls import path
from . import views

urlpatterns = [
    path('',views.home,name='home'),
    path('search', views.search, name="search"),
    path('compatibility', views.compatibility, name="comp"),
    path('register', views.register, name="register"),
    path('sms', views.sms, name="sms"),
    path('login', views.login, name="login"),
    path('mydata', views.mydata, name="mydata"),
    path('emergency', views.emergency, name="emergency"),

]


from flask import Blueprint
from flask_restful import Api
from utils.output import output_json

from . import article, channel, collection
news_bp = Blueprint('news', __name__)
news_api = Api(news_bp, catch_all_404s=True)
news_api.representation('application/json')(output_json)

news_api.add_resource(article.ArticleListResource, '/v1_0/articles',
                      endpoint='Articles')

news_api.add_resource(article.ArticleResource, '/v1_0/articles/<int(min=1):article_id>',
                      endpoint='Article')

news_api.add_resource(channel.ChannelListResource, '/v1_0/channels',
                      endpoint='Channels')

# 收藏文章
news_api.add_resource(collection.CollectionListResource, '/v1_0/article/collections',
                      endpoint='ArticleCollections')

# 取消收藏文章
news_api.add_resource(collection.CollectionResource, '/v1_0/article/collections/<int(min=1):target>',
                      endpoint='ArticleCollection')


#Usage: python create_descriptor.py  <list of training images>

import sys
import os
import cv2

from utility_functions import *

trainLst = sys.argv[1]

trainFile = open (trainLst)

for i in trainFile:
    i = i[:-1] #remove new line character
    img = cv2.imread(i)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    sift = cv2.SIFT_create()
    keypoints, descriptors = sift.detectAndCompute(gray, None)
    print(len(keypoints), end=" ")
    print("keypoints detected")
    descriptorFileName = i[:-3] + 'decp'
    saveDescriptorListFile(descriptors, descriptorFileName)

trainFile.close()


import sys
#sys.stdin = open("input.txt", "rt")

##### 큐 자료구조 활용 #####
## deque 라는 자료구조를 통해 적용
## deque : 앞뒤 둘다 뺐다 넣었다 할 수 있음
## appendleft, popleft / append, pop
from collections import deque

n, k = map(int, input().split())
dq = list(range(1, n+1))
dq = deque(dq)

while dq:
    for _ in range(k-1):
        cur = dq.popleft()
        dq.append(cur)
    dq.popleft()
    if len(dq) == 1:
        print(dq[0])
        dq.popleft() # break 할 수도 있지만 비워버려서 while문 조건 만족시켜서 끝냄

'''
# 코드 설명 안보고 개념만 보고 푼 풀이
n, k = map(int, input().split())

pr = [x for x in range(1, n+1)]

#print(pr)
lst = []

while len(pr) > 1:

    for _ in range(k-1):
        pr.append(pr.pop(0))
    pr.pop(0)
print(pr[0])
'''


import json
import shutil
from argparse import ArgumentParser
from os import listdir, makedirs, getcwd, chdir
from os.path import join, dirname, isfile, exists, realpath
from subprocess import call, check_output, CalledProcessError

from proyo.misc import root_dir, generate_alternate_help, arrange_tree, map_tree, collect_leaves
from proyo.proyo import Proyo

def load_macros(folder):
    macros = {}
    for i in listdir(folder):
        path = join(folder, i)
        if isfile(path):
            with open(path) as f:
                macros[i] = f.read()
    return macros

def main():
    templates = join(root_dir, 'templates')
    macros = load_macros(join(root_dir, 'macros'))
    cur_dir = getcwd()

    parser = ArgumentParser()

    proyo = Proyo(templates, dict(parser=parser), macros)
    proyo.parse()

    for p in proyo.get_leaf_vars('parser'):
        p.add_argument('project_folder')

    for p in proyo.get_all_children():
        if 'parser' in p:
            p['parser'].usage = generate_alternate_help(p)

    args = parser.parse_args()

    chdir(cur_dir)
    out_folder = realpath(args.project_folder)
    if exists(out_folder):
        print('Destination must not exists!')
        exit(1)

    proyo.set_target(out_folder)
    proyo.update_global(args=args)
    proyo.run()

    chdir(cur_dir)
    for rel, text in proyo.files.items():
        path = join(out_folder, rel)
        makedirs(dirname(path), exist_ok=True)
        fmt = 'wb' if isinstance(text, bytes) else 'w'
        with open(path, fmt) as f:
            f.write(text)

    try:
        tree_output = '\n' + check_output(['tree', '-C', out_folder]).decode().split('\n', 1)[-1]
    except (CalledProcessError, FileNotFoundError):
        tree_output = json.dumps(list(proyo.files), indent=2)

    proyo.post_run_all()

    print('Generated to {}: {}'.format(args.project_folder, tree_output))

if __name__ == '__main__':
    main()


import importlib
import sys
import subprocess

from parser import Parser
import utils
import repository
from module import Module

class Mnos:
    def __init__(self):
        self.module_queue = ModuleQueue()
        self.template_engine = utils.TemplateEngine()
        self.sysinfo = utils.SystemInfo()
        self.pkg_mgr = utils.PackageManager()

        self.config = None
        self._modules = {}

    def run(self):
        self._parse_config()
        self._load_modules()
        self._initialize_modules()
        self._run_modules()

    def _load_modules(self):
        for i in self.config["modules"]:
            print("debug:", "loading module {}…".format(i))
            self._modules[i] = importlib.import_module(
                "modules.{}".format(i)
            )

    def _initialize_modules(self):
        for i in self._modules:
            print("debug:", "initializing module {}…".format(i))
            self._modules[i] = self._modules[i].Module()

    def _run_modules(self):
        for module in self._modules.values():
            self.module_queue.enqueue(
                action=module.run,
                after=module.module_info["run_after"]
            )

    def _parse_config(self):
        parser = Parser()
        print("debug:", "parsing config…")
        with open("sample-config.txt") as f:
            self.config = parser.parse(f.read())

# TODO: use real logger instead of print()
class ModuleQueue:
    def __init__(self):
        self._events_seen = set()
        self._queue = dict()

    def enqueue(self, action, after=[]):
        """
        Schedule *action* to be called after all events in *after* have
        occurred.

        *action* is expected to be a callable object.  *after* is an
        iterable of events.  An empty list of events will cause *action*
        to be run immediately.
        """

        print("debug:", "ModuleQueue:",
              "enqueueing {} to run after {}…".format(action, after))
        # filter out events already seen
        events = set(after) - self._events_seen

        print(
            "debug:", "ModuleQueue:",
            "{} is waiting for these unseen events: {}, ".format(
                action,
                events
            ),
            end=""
        )

        if not events:
            print("running now.")
            action()
        else:
            print("enqueueing.")
            if action in self._queue:
                self._queue[action].update(events)
            else:
                self._queue[action] = events

    def fire_event(self, event):
        print("debug:", "ModuleQueue:",
              "event {} is fired.".format(event))
        self._events_seen.add(event)
        for action in self._queue.keys():
            self._queue[action].discard(event)

            print("debug:", "ModuleQueue:",
                  "checking {}… ".format(action), end="")
            if not self._queue[action]:
                print("is ready, running now.")
                action()
            else:
                print("still waiting for {}.".format(self._queue[action]))

def install_file(content, target):
    print(content)
    print("installing config file to {}…".format(target))

def execute(cmd):
    #return subprocess.check_call(cmd)
    print(" ".join(cmd))

mnos = Mnos()


from functools import reduce

FSM = {'CLOSED': {'APP_PASSIVE_OPEN': 'LISTEN',
                  'APP_ACTIVE_OPEN': 'SYN_SENT'},
       'LISTEN': {'RCV_SYN': 'SYN_RCVD',
                  'APP_SEND': 'SYN_SENT',
                  'APP_CLOSE': 'CLOSED'},
       'SYN_RCVD': {'APP_CLOSE': 'FIN_WAIT_1',
                    'RCV_ACK': 'ESTABLISHED'},
       'SYN_SENT': {'RCV_SYN': 'SYN_RCVD',
                    'RCV_SYN_ACK': 'ESTABLISHED',
                    'APP_CLOSE': 'CLOSED'},
       'ESTABLISHED': {'APP_CLOSE': 'FIN_WAIT_1',
                       'RCV_FIN': 'CLOSE_WAIT'},
       'FIN_WAIT_1': {'RCV_FIN': 'CLOSING',
                      'RCV_FIN_ACK': 'TIME_WAIT',
                      'RCV_ACK': 'FIN_WAIT_2'},
       'CLOSING': {'RCV_ACK': 'TIME_WAIT'},
       'FIN_WAIT_2': {'RCV_FIN': 'TIME_WAIT'},
       'TIME_WAIT': {'APP_TIMEOUT': 'CLOSED'},
       'CLOSE_WAIT': {'APP_CLOSE': 'LAST_ACK'},
       'LAST_ACK': {'RCV_ACK': 'CLOSED'}}

def change_state(current_state, event):
    return FSM[current_state][event]

def traverse_TCP_states(events):
    try:
        return reduce(change_state, events, 'CLOSED')
    except KeyError:
        return 'ERROR'

4. A Simplistic TCP Finite State Machine (FSM)/solution.py


# -*- coding: utf-8 -*-
# 使用OpenCV播放视频
import cv2

wnd = 'OpenCV Video'

# 获得视频的格式
videoCapture = cv2.VideoCapture('C:\\Users\\Administrator\\Desktop\\video.mp4')

# 获得码率及尺寸
fps = videoCapture.get(cv2.CAP_PROP_FPS)
size = (int(videoCapture.get(cv2.CAP_PROP_FRAME_WIDTH)),
        int(videoCapture.get(cv2.CAP_PROP_FRAME_HEIGHT)))

cv2.namedWindow(wnd, flags=0)
cv2.resizeWindow(wnd, size[0] / 2, size[1] / 2)

# 读帧
success, frame = videoCapture.read()

while success:
	cv2.imshow(wnd, frame)  # 显示
	cv2.waitKey(1000 / int(fps))  # 延迟
	success, frame = videoCapture.read()  # 获取下一帧


# from django.shortcuts import render
# from django.shortcuts import redirect
from django.shortcuts import render_to_response
from .models import Post, Gallery
from comment.models import Comment
from .forms import *
from .utils import *
from comment.views import CommentView
from comment.forms import CommentForm
from natlet.custom_project_utils import GetRandomSidebarPost

from django.views.generic import View
from django.shortcuts import get_object_or_404
from django.core.files.base import ContentFile
from django.template import RequestContext
from django.db.models import Q
from django.contrib.auth.mixins import LoginRequiredMixin

import os.path

# Create your views here.

class Variables:
    def post_filter_utils(self):
        post_filter = Post.objects.filter(post_date__lte=timezone.now()).order_by('-post_date')
        return post_filter

class Index(DisplayObjectMixin, View):
    model = Post
    template = 'novatlet_temp/index.html'

class CustomSearch(DisplayObjectMixin, View):
    model = Post
    template = 'novatlet_temp/index.html'

class PostDetail(View, GetRandomSidebarPost):
    model = Post
    model_form = CommentForm

    def get(self, request, slug):
        alert_show = request.GET.get('alert_mess')

        get_object = get_object_or_404(self.model, slug__iexact=slug)
        p = get_object.related_gallery
        gallery = Photos.objects.filter(location=p)
        admin_obj = True

        comment = Comment.objects.filter(comment_for_post=get_object)
        comment_form = self.model_form(data={'hidden_slug': slug})

        get_sidebar_posts = GetRandomSidebarPost(model=self.model).get_newest_post()

        return render(request, 'novatlet_temp/post_detail.html', context={
                        self.model.__name__.lower(): get_object,
                        'gallery': gallery,
                        'admin_obj': admin_obj,
                        'comment': comment,
                        'comment_form': comment_form,
                        'returned_slug': slug,
                        'get_sidebar_posts': get_sidebar_posts,
                        })

class TagList(LoginRequiredMixin, DisplayObjectMixin, View):
    model = Tag
    template = 'novatlet_temp/tag_list.html'
    raise_exception = True

class TagDetail(View):
    def get(self, request, slug):
        get_tag = get_object_or_404(Tag, slug__iexact=slug)
        admin_obj = True

        return render(request, 'novatlet_temp/tag_detail.html', context={'tag_detail': get_tag, 'admin_obj': admin_obj})

class GallaryCreate(LoginRequiredMixin, View):
    model = [LocationForm, PhotoForm]
    raise_exception = True

    def get(self, request):
        gallery_form = self.model[0]
        gallery_form_0 = self.model[1]

        location_field = gallery_form_0.GetPhotoField()

        return render(request, 'novatlet_temp/gallery_create.html', context={
                        'gallery': gallery_form,
                        'gallery_0': gallery_form_0,
                        'location_field': location_field,
                        })

    def post(self, request):
        if request.POST:
            bound_form_0 = self.model[0](request.POST) #LocationForm: dir_object
            bound_form_1 = self.model[1](request.POST, request.FILES) #PhotoForm: location, img_object

            if bound_form_0.is_valid() or bound_form_1.is_valid():
                def save_after_validate(self, request, directory_object):

                    for f in request.FILES.getlist('img_object'):
                        data = f.read()
                        photo = Photos(location = Location.objects.order_by('id').last()) # it works
                        photo.img_object.save(save_path(directory_object) + str(f.name), ContentFile(data))
                        photo.save()
                        print("success 2 !", photo)

                def save_path(directory_object):
                    save_path = directory_object + "/photos/"
                    return save_path

                if not bound_form_0.cleaned_data.get('boolean'):
                    print("we are here")
                    if request.POST['dir_object'] == '':
                        return render(request, 'novatlet_temp/gallery_create.html', context={
                                                'gallery': bound_form_0,
                                                'gallery_0': bound_form_1,
                                                'location_field': bound_form_1.GetPhotoField(),
                                                    })
                    bound_form_0.save()
                    directory_object = request.POST['dir_object']

                    save_after_validate(self, request, directory_object)

                if bound_form_0.cleaned_data.get('boolean'):
                    if request.POST['dir_object'] != '':
                        return render(request, 'novatlet_temp/gallery_create.html', context={
                                                'gallery': bound_form_0,
                                                'gallery_0': bound_form_1,
                                                'location_field': bound_form_1.GetPhotoField(),
                                                    })
                    directory_object = Location.objects.get(id=request.POST['location']).__str__()

                    save_after_validate(self, request, directory_object)

            return render(request, 'novatlet_temp/gallery_create.html', context={
                                       'gallery': bound_form_0,
                                       'gallery_0': bound_form_1,
                                       'location_field': bound_form_1.GetPhotoField(),
                                         })

# handlers zone

def custom_handler404(request, exception):
    context = RequestContext(request)
    response = render_to_response('404.html', context={'context': context})
    response.status_code = 404
    return response

#Мед в голове


# -*- coding: utf-8 -*-

"""
彩色图像的灰度化、二值化
"""
from skimage.color import rgb2gray
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
import cv2
import time

def rgb2gray():
    # 灰度化
    img = cv2.imread("../images/lenna.png")
    h,w = img.shape[:2]
    img_gray =cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    img_gray_1 = np.zeros([h,w],img.dtype)
    t1 = time.time()
    #逐像素处理
    for i in range(h):
        for j in range(w):
            m = img[i,j]
            img_gray_1[i,j] = int(m[0]*0.11 + m[1]*0.59 + m[2]*0.3)
    t2=time.time()

    #numpy运算
    factors = np.array([0.11,0.59,0.3],dtype=np.float32)
    img_gray_2= np.clip(np.sum(img * factors,axis=2).astype(np.uint8),0,255)
    t3 = time.time()

    plt.subplot(221)
    plt.imshow(img[:,:,::-1])
    plt.subplot(222)
    plt.imshow(img_gray,cmap='gray')
    plt.subplot(223)
    plt.imshow(img_gray_1,cmap='gray')
    plt.subplot(224)
    plt.imshow(img_gray_2, cmap='gray')
    info = "逐像素循环计算耗时%.3fs，numpy api运算耗时%.3fs"%(t2-t1,t3-t2)
    print(info)
    plt.show()

if __name__=="__main__":
    rgb2gray()