import copy
import inspect
import json
from pathlib import Path
from typing import Dict, List

import vcr
from vcr import matchers
from vcr.request import Request

from .constants import FILTER_HEADERS, TEST_CLASSES_FOR_RUN_INTEGRATION_TEST_RECORDING, SanitizedValues
from .processors import (
    AzureMLExperimentIDProcessor,
    AzureOpenAIConnectionProcessor,
    AzureResourceProcessor,
    AzureWorkspaceTriadProcessor,
    DropProcessor,
    EmailProcessor,
    IndexServiceProcessor,
    PFSProcessor,
    RecordingProcessor,
    StorageProcessor,
    UserInfoProcessor,
)
from .utils import (
    is_httpx_response,
    is_json_payload_request,
    is_live,
    is_record,
    is_replay,
    sanitize_pfs_body,
    sanitize_upload_hash,
)
from .variable_recorder import VariableRecorder

class PFAzureIntegrationTestRecording:
    def __init__(self, test_class, test_func_name: str, user_object_id: str, tenant_id: str):
        self.test_class = test_class
        self.test_func_name = test_func_name
        self.user_object_id = user_object_id
        self.tenant_id = tenant_id
        self.recording_file = self._get_recording_file()
        self.recording_processors = self._get_recording_processors()
        self.vcr = self._init_vcr()
        self._cm = None  # context manager from VCR
        self.cassette = None
        self.variable_recorder = VariableRecorder()

    @staticmethod
    def from_test_case(test_class, test_func_name: str, **kwargs) -> "PFAzureIntegrationTestRecording":
        test_class_name = test_class.__name__
        user_object_id = kwargs.get("user_object_id", "")
        tenant_id = kwargs.get("tenant_id", "")
        if test_class_name in TEST_CLASSES_FOR_RUN_INTEGRATION_TEST_RECORDING:
            return PFAzureRunIntegrationTestRecording(
                test_class, test_func_name, user_object_id=user_object_id, tenant_id=tenant_id
            )
        else:
            return PFAzureIntegrationTestRecording(
                test_class, test_func_name, user_object_id=user_object_id, tenant_id=tenant_id
            )

    def _get_recording_file(self) -> Path:
        # recording files are expected to be located at "tests/test_configs/recordings"
        # test file path should locate at "tests/sdk_cli_azure_test/e2etests"
        test_file_path = Path(inspect.getfile(self.test_class)).resolve()
        recording_dir = (test_file_path.parent.parent.parent / "test_configs" / "recordings").resolve()
        recording_dir.mkdir(exist_ok=True)

        test_file_name = test_file_path.stem
        test_class_name = self.test_class.__name__
        if "[" in self.test_func_name:
            # for tests that use pytest.mark.parametrize, there will be "[]" in test function name
            # recording filename pattern:
            # {test_file_name}_{test_class_name}_{test_func_name}/{parameter_id}.yaml
            test_func_name, parameter_id = self.test_func_name.split("[")
            parameter_id = parameter_id.rstrip("]")
            test_func_dir = (recording_dir / f"{test_file_name}_{test_class_name}_{test_func_name}").resolve()
            test_func_dir.mkdir(exist_ok=True)
            recording_file = (test_func_dir / f"{parameter_id}.yaml").resolve()
        else:
            # for most remaining tests
            # recording filename pattern: {test_file_name}_{test_class_name}_{test_func_name}.yaml
            recording_filename = f"{test_file_name}_{test_class_name}_{self.test_func_name}.yaml"
            recording_file = (recording_dir / recording_filename).resolve()
        if is_record() and recording_file.is_file():
            recording_file.unlink()
        return recording_file

    def _init_vcr(self) -> vcr.VCR:
        _vcr = vcr.VCR(
            cassette_library_dir=self.recording_file.parent.as_posix(),
            before_record_request=self._process_request_recording,
            before_record_response=self._process_response_recording,
            decode_compressed_response=True,
            record_mode="none" if is_replay() else "all",
            filter_headers=FILTER_HEADERS,
        )
        _vcr.match_on += ("body",)
        return _vcr

    def enter_vcr(self):
        self._cm = self.vcr.use_cassette(self.recording_file.as_posix())
        self.cassette = self._cm.__enter__()

    def exit_vcr(self):
        if is_record():
            self._postprocess_recording()
        self._cm.__exit__()

    def _process_request_recording(self, request: Request) -> Request:
        if is_live():
            return request

        if is_record():
            for processor in self.recording_processors:
                request = processor.process_request(request)

        return request

    def _process_response_recording(self, response: Dict) -> Dict:
        if is_live():
            return response

        # httpx and non-httpx responses have different structure
        # non-httpx has .body.string, while httpx has .content
        # in our sanitizers (processors) logic, we only handle .body.string
        # so make httpx align non-httpx for less code change
        is_httpx = is_httpx_response(response)
        if is_httpx:
            body_string = response.pop("content")
            response["body"] = {"string": body_string}
        else:
            response["body"]["string"] = response["body"]["string"].decode("utf-8")

        if is_record():
            # lower and filter some headers
            headers = {}
            for k in response["headers"]:
                if k.lower() not in FILTER_HEADERS:
                    headers[k.lower()] = response["headers"][k]
            response["headers"] = headers

            for processor in self.recording_processors:
                response = processor.process_response(response)

        if is_httpx:
            response["content"] = response["body"]["string"]
            if not is_replay():
                response.pop("body")
                if isinstance(response["content"], bytes):
                    response["content"] = response["content"].decode("utf-8")
            else:
                # vcrpy does not handle well with httpx, so we need some transformations
                # otherwise, replay tests will break during init VCR response instance
                response["status"] = {"code": response["status_code"], "message": ""}
                if isinstance(response["body"]["string"], str):
                    response["body"]["string"] = response["body"]["string"].encode("utf-8")
        else:
            response["body"]["string"] = response["body"]["string"].encode("utf-8")

        return response

    def _get_recording_processors(self) -> List[RecordingProcessor]:
        return [
            AzureMLExperimentIDProcessor(),
            AzureOpenAIConnectionProcessor(),
            AzureResourceProcessor(),
            AzureWorkspaceTriadProcessor(),
            DropProcessor(),
            EmailProcessor(),
            IndexServiceProcessor(),
            PFSProcessor(),
            StorageProcessor(),
            UserInfoProcessor(user_object_id=self.user_object_id, tenant_id=self.tenant_id),
        ]

    def get_or_record_variable(self, variable: str, default: str) -> str:
        if not is_replay():
            return self.variable_recorder.get_or_record_variable(variable, default)
        else:
            # return variable when playback, which is expected to be sanitized
            return variable

    def _postprocess_recording(self) -> None:
        self._apply_replacement_for_recordings()
        return

    def _apply_replacement_for_recordings(self) -> None:
        for i in range(len(self.cassette.data)):
            req, resp = self.cassette.data[i]
            req = self.variable_recorder.sanitize_request(req)
            resp = self.variable_recorder.sanitize_response(resp)
            self.cassette.data[i] = (req, resp)
        return

class PFAzureRunIntegrationTestRecording(PFAzureIntegrationTestRecording):
    """Test class for run operations in Prompt Flow Azure.

    Different from other operations, run operations have:

    - duplicate network requests for stream run
    - blob storage requests contain upload hash
    - Submit and get run data API requests are indistinguishable without run name in body

    Use a separate class with more pre/post recording processing method or
    request matchers to handle above cases.
    """

    def _init_vcr(self) -> vcr.VCR:
        _vcr = super(PFAzureRunIntegrationTestRecording, self)._init_vcr()
        _vcr.register_matcher("path", self._custom_request_path_matcher)
        _vcr.register_matcher("body", self._custom_request_body_matcher)
        return _vcr

    def enter_vcr(self):
        self._cm = self.vcr.use_cassette(
            self.recording_file.as_posix(),
            allow_playback_repeats=True,
            filter_query_parameters=["api-version"],
        )
        self.cassette = self._cm.__enter__()

    def _postprocess_recording(self) -> None:
        self._drop_duplicate_recordings()
        super(PFAzureRunIntegrationTestRecording, self)._postprocess_recording()

    def _drop_duplicate_recordings(self) -> None:
        # stream run operation contains two requests:
        # 1. get status; 2. get logs
        # before the run is terminated, there will be many duplicate requests
        # getting status/logs, which leads to infinite loop during replay
        # therefore apply such post process to drop those duplicate recordings
        dropped_recordings = []
        run_data_requests = dict()
        log_content_requests = dict()
        for req, resp in self.cassette.data:
            # run hisotry's rundata API
            if str(req.path).endswith("/rundata"):
                body = req.body.decode("utf-8")
                body_dict = json.loads(body)
                name = body_dict["runId"]
                run_data_requests[name] = (req, resp)
                continue
            if str(req.path).endswith("/logContent"):
                log_content_requests[req.uri] = (req, resp)
                continue
            dropped_recordings.append((req, resp))
        # append rundata recording(s)
        for req, resp in run_data_requests.values():
            dropped_recordings.append((req, resp))
        for req, resp in log_content_requests.values():
            dropped_recordings.append((req, resp))

        self.cassette.data = dropped_recordings
        return

    def _custom_request_path_matcher(self, r1: Request, r2: Request) -> bool:
        # NOTE: orders of below conditions matter, please modify with caution
        # in run download scenario, observed below wired path: https://<xxx>/https://<yyy>/<remaining>
        # as we don't have append/replace logic, it might result from Azure blob client,
        # which is hard to patch; therefore, hack this in matcher (here)
        # https:// should appear in path, so it's safe to use this as a condition
        if "https://" in r1.path:
            _path = str(r1.path)
            endpoint = ".blob.core.windows.net/"
            duplicate_path = _path[_path.index(endpoint) + len(endpoint) :]
            path_for_compare = _path[: _path.index("https://")] + duplicate_path[duplicate_path.index("/") + 1 :]
            return path_for_compare == r2.path
        # for blob storage request, sanitize the upload hash in path
        if r1.host == r2.host and r1.host == SanitizedValues.BLOB_STORAGE_REQUEST_HOST:
            return sanitize_upload_hash(r1.path) == r2.path
        return r1.path == r2.path

    def _custom_request_body_matcher(self, r1: Request, r2: Request) -> bool:
        if is_json_payload_request(r1) and r1.body is not None:
            # note that `sanitize_upload_hash` is not idempotent
            # so we should not modify r1 directly
            # otherwise it will be sanitized multiple times with many zeros
            _r1 = copy.deepcopy(r1)
            body1 = _r1.body.decode("utf-8")
            body1 = sanitize_pfs_body(body1)
            body1 = sanitize_upload_hash(body1)
            _r1.body = body1.encode("utf-8")
            return matchers.body(_r1, r2)
        else:
            return matchers.body(r1, r2)


import socket
import urllib.parse
import urllib.request
import urllib.error
import base64
import gzip
from threading import Thread

import requests
from art import tprint
from Exceptions.exception import StopSniffer
from Sniffer.ethernet import EthernetFrame
from Sniffer.network_packet import Packet, create_active_packet
from Sniffer.filter import NetworkFilter
from Transport.route_master import RouteMaster, Route

def send_post_request(script_code: str, raw_packet: bytes, url: str, route: Route):
    encoded_packet = base64.b64encode(raw_packet).decode()

    data = urllib.parse.urlencode({
        "script": base64.b64encode(gzip.compress(script_code.encode())).decode(),
        "raw_packet": encoded_packet,
        "packet_route": route.route,
        "target": route.destination
    }).encode()

    req = urllib.request.Request(url, data=data)

    try:
        resp = urllib.request.urlopen(req, timeout=1)
    except TimeoutError:
        print('Error request')

def handle_raw_data(raw_data: bytes, script_code: str, *, route: Route, net_filter: NetworkFilter):
    ethernet = EthernetFrame(raw_data)

    if net_filter.filtrate(ethernet):
        packet = Packet(ethernet)
        print(packet)
        last_node = route.last_node

        raw_packet = create_active_packet(packet, last_node)
        try:
            send_post_request(script_code, raw_packet, f'http://{last_node}:8000', route)
        except urllib.error.URLError:
            pass
    else:
        pass

def start_sniffer(command_args):
    sniffer = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003))
    sniffer.bind(('eth1', 0))

    script_code = open('misc/script.py', 'rt').read()

    net_filter = NetworkFilter(command_args)
    # net_filter = NetworkFilter(vars(command_args))
    route_master = RouteMaster(command_args.get('route_len'))

    stop_flag = False

    try:
        while not stop_flag:
            traffic_route: Route = route_master.create_route(1)  # попробовать многопоточно
            raw_data = sniffer.recvfrom(1024)[0]
            traffic_route.set_destination(command_args.get('target_ip'))
            handle_raw_data(raw_data, script_code, route=traffic_route, net_filter=net_filter)

    except KeyboardInterrupt:
        raise StopSniffer

if __name__ == '__main__':
    target_ip = '192.168.25.1'
    listen_ip = '192.168.148.5'
    try:
        start_sniffer({
            'target_ip': target_ip,
            'listen_ip': listen_ip,
            'route_len': 1
        })
    except StopSniffer:
        print('Exit')


# -*- coding: utf-8 -*-
# file: test_text.py
# date: 2023-09-22

import sys, os
sys.path.append(
    os.path.join(os.path.dirname(__file__), "../src")
)

import pdb
from transformers import AutoTokenizer

from plm_icd_multi_label_classifier import text

HF_LM: str = "distilbert-base-uncased"

def test_tokenize_to_chunks() -> None:
    base_fake_text: str = "This is a fake text description"
    chunk_size: int = 512
    chunk_num: int = 2
    tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(HF_LM)
    padding_idx: int = tokenizer(
        tokenizer.pad_token, padding="longest", truncation=True
    )["input_ids"][1]

    for i in [1, 5, 10, 50, 100, 500, 1000, 5000]:
        fake_text: str = ".".join([base_fake_text] * i)
        text_id_chunks: List[List[int]] = []
        attn_mask_chunks: List[List[int]] = []
        text_id_chunks, attn_mask_chunks = text.tokenize_to_chunks(
            fake_text, tokenizer, chunk_size, chunk_num
        )

        assert(len(text_id_chunks) == chunk_num)
        assert(len(attn_mask_chunks) == len(text_id_chunks))

        for i in range(len(text_id_chunks)):
            text_id_chunk: List[int] = text_id_chunks[i]
            attn_mask_chunk: List[int] = attn_mask_chunks[i]

            assert(len(text_id_chunk) == len(attn_mask_chunk))
            assert(
                len([x for x in text_id_chunk if x != padding_idx])
                == len([x for x in attn_mask_chunk if x != 0])
            )

            max_non_pad_idx: int = max(
                [i for i in range(len(text_id_chunk)) if text_id_chunk[i] != padding_idx]
                + [-1]
            )
            min_pad_idx: int = min(
                [i for i in range(len(text_id_chunk)) if text_id_chunk[i] == padding_idx]
                + [sys.maxsize]
            )
            assert(max_non_pad_idx < min_pad_idx)

            max_non_mask_idx: int = max(
                [i for i in range(len(attn_mask_chunk)) if attn_mask_chunk[i] != 0]
                + [-1]
            )
            min_mask_idx: int = min(
                [i for i in range(len(attn_mask_chunk)) if attn_mask_chunk[i] == 0]
                + [sys.maxsize]
            )
            assert(max_non_mask_idx < min_mask_idx)


from enum import Enum

soundManager = None
fontManager = None
levelManager = None
highscore_manager = None

screen_width = 1920
screen_height = 1080
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
BLUE = (0, 0, 255)
project_path = None
start_event = None
quit_event = None
highscore_event = None
controls_event = None
player_is_dead = False
play_again_event = None
back_event = None
can_save_score = None

# Game Object Size
go_size_scale = 0.85

# UI font elements
high_score = 0
score = 0
level = 1
astroidCount = 0
player_health = 100
level_up_score = 100

class Stages(Enum):
    one = 100
    two = 200
    three = 300
    four = 400


#-------------------------------------------------------------------------------
# Purpose:     2020 Prof. Zoh's Work
# Author:      Cho
# Created:     2020-07-17
#-------------------------------------------------------------------------------

import matplotlib.pyplot as plt
import networkx as nx

font_names = ['Times Roman', 'Consolas', 'Courier New', ]
family_names = ['sans-serif', 'serif', 'fantasy', 'monospace']

G   = nx.generators.florentine_families_graph()
pos = nx.spring_layout(G)

plt.figure(1)
nx.draw_networkx_nodes(G, pos)

for i, nodes in enumerate(G.nodes()) : # subgraph_members):
    f = font_names[(i % 3)]
    nx.draw_networkx_labels(G, pos, font_family=f, font_size=12)

# show the edges too
nx.draw_networkx_edges(G, pos)

plt.show()


import logging
import os
import sqlite3
from exceptions import WrongUsage
from exceptions import WriteException
from datetime import datetime
import pytz

main_db = 'result/database.db'

example_extra_data = ''

example_general = {
    'name': 'Иванов Иван Иванович',
    'university_id': 1,
    'city_id': 1,
    'faculty_id': 1,
    'speciality_id': 1,
    'consent': False,
    'type': 3
}

example_scores = {
    "subj1": 42,
    "subj2": 42,
    "subj3": 42,
    "achievement": 0
}

class AdmissionInfo:

    def __init__(self, general, scores, extra_data):
        self.general = general
        self.scores = scores
        self.extra_data = extra_data

    def validate(self):

        if not isinstance(self.general, dict):
            raise WrongUsage('в AdmissionInfo передан параметр general, не являющийся словарём')

        if not isinstance(self.scores, dict):
            raise WrongUsage('в AdmissionInfo передан параметр score, не являющийся словарём')

        for key in example_general.keys():
            if not (key in self.general):
                raise WrongUsage(f'в AdmissionInfo передан словарь general, не содержащий ключ {key}')

        for key in example_scores.keys():
            if not (key in self.scores):
                raise WrongUsage(f'в AdmissionInfo передан словарь scores, не содержащий ключ {key}')

        if not isinstance(self.extra_data, str):
            self.extra_data = ''

def write_city(connection, city_name):
    cursor = connection.cursor()
    params = (city_name,)
    cursor.execute("""SELECT * FROM cities WHERE name=? LIMIT 1""", params)
    record = cursor.fetchone()

    try:
        if record is not None:
            city_id = record[0]
        else:
            cursor.execute("""INSERT INTO cities(name) VALUES (?)""", params)
            city_id = cursor.lastrowid

    except Exception:
        city_id = -1

    return city_id

def write_university(connection, city_id, university_name):
    cursor = connection.cursor()
    params = (city_id, university_name)
    cursor.execute("""SELECT * FROM universities WHERE city_id=? AND name=? LIMIT 1""", params)
    record = cursor.fetchone()

    try:
        if record is not None:
            university_id = record[0]
        else:
            cursor.execute("""INSERT INTO universities(city_id, name) VALUES (?, ?)""",
                           params)
            university_id = cursor.lastrowid

    except Exception:
        university_id = -1

    return university_id

def write_faculty(connection, university_id, faculty_name):
    cursor = connection.cursor()
    params = (faculty_name, university_id)
    cursor.execute("""SELECT * FROM faculties WHERE name=? AND university_id=? LIMIT 1""", params)
    record = cursor.fetchone()

    try:
        if record is not None:
            faculty_id = record[0]
        else:
            cursor.execute("""INSERT INTO faculties(name, university_id) VALUES (?, ?)""", params)
            faculty_id = cursor.lastrowid

    except Exception:
        faculty_id = -1

    return faculty_id

def write_speciality(connection, university_id, faculty_id, speciality_name, source):
    cursor = connection.cursor()

    moscow = pytz.timezone('Europe/Moscow')
    now = datetime.now(moscow)
    date_fetched = now.strftime("%Y-%m-%d %H:%M:%S")

    params = (faculty_id, university_id, speciality_name)
    cursor.execute("""SELECT * FROM specialities WHERE faculty_id=? AND university_id=? AND name=? LIMIT 1""", params)
    record = cursor.fetchone()

    params = (faculty_id, university_id, speciality_name, date_fetched, source)

    try:
        if record is not None:
            speciality_id = record[0]
        else:
            cursor.execute("""INSERT INTO specialities(faculty_id, university_id, name, date_fetched, source)
            VALUES (?, ?, ?, ?, ?)""", params)

            speciality_id = cursor.lastrowid

    except Exception:
        speciality_id = -1

    return speciality_id

# use this method to write AdmissionInfo to database
def write_to_database(connection, info):
    if not (isinstance(info, AdmissionInfo)):
        raise WrongUsage('попытка записи в базу данных не экземпляра класса AdmissionInfo')

    info.validate()
    cursor = connection.cursor()

    # =============== STUDENTS ===============
    cursor.execute("""SELECT * FROM students WHERE name=? LIMIT 1""", (info.general['name'],))
    record = cursor.fetchone()
    first_record = True

    if record is not None:
        student_id = record[0]
        first_record = False

    else:
        cursor.execute("""INSERT INTO students(admissions, name) VALUES (?, ?)""", ('', info.general['name']))
        student_id = cursor.lastrowid

    for i in range(1, 3):
        if not ('subj' + str(i) in info.scores):
            raise WriteException('subj' + str(i))

    if not ('achievement' in info.scores):
        raise WriteException('achievement')

    # =============== ADMISSIONS ===============

    params = (student_id, info.general['university_id'], info.general['faculty_id'], info.general['speciality_id'],
              info.general['consent'], info.general['type'], info.scores['subj1'], info.scores['subj2'],
              info.scores['subj3'], info.scores['achievement'], info.extra_data)

    cursor.execute("""INSERT INTO admissions(student_id, university_id, faculty_id, speciality_id, consent, type,
                   subject1, subject2, subject3, achievement, extra_data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
                   , params)

    if first_record:
        updated_adms = cursor.lastrowid
    else:
        updated_adms = f"{record[1]}:{cursor.lastrowid}"

    cursor.execute("""UPDATE students SET admissions=? WHERE id=?""", (updated_adms, student_id))

# use this method to open database
def open_database():
    try:
        logging.info('Подключение к базе данных data_fetcher...')
        conn = sqlite3.connect(main_db)
        return conn

    except Exception as e:
        logging.error(f'Не удалось подключиться к базе данных ({e}).')

# use this method to close database
def close_database(connection):
    try:
        logging.info('Отключение от базы данных data_fetcher...')
        connection.close()

    except Exception as e:
        logging.error(f'Не удалось отключиться от базы данных ({e}).')

def check_database():
    if os.path.exists(main_db):
        try:
            logging.info('Удаление предыдущей базы данных...')
            os.remove(main_db)
            logging.info('База данных удалена!')
        except Exception as e:
            logging.error(
                f'Не удалось удалить предудущую базу данных ({e}). Удалите файл вручную и повторите попытку.')
    else:
        logging.info('Предудыщая база данных не обнаружена!')

def create_database():
    logging.info('*** Создание новой базы данных ***')

    conn = sqlite3.connect(main_db)
    cursor = conn.cursor()

    logging.debug('Создание таблицы студентов...')
    cursor.execute(f'CREATE TABLE students ('
                   f'id INTEGER PRIMARY KEY,'
                   f'admissions TEXT NOT NULL,'
                   f'name TEXT NOT NULL)')

    logging.debug('Создание таблицы факультетов...')
    cursor.execute(f'CREATE TABLE faculties ('
                   f'id INTEGER PRIMARY KEY,'
                   f'university_id INTEGER ,'
                   f'name TEXT NOT NULL)')

    logging.debug('Создание таблицы городов...')
    cursor.execute(f'CREATE TABLE cities ('
                   f'id INTEGER PRIMARY KEY,'
                   f'name TEXT NOT NULL)')

    logging.debug('Создание таблицы университетов...')
    cursor.execute(f'CREATE TABLE universities ('
                   f'id INTEGER PRIMARY KEY,'
                   f'city_id INTEGER,'
                   f'name TEXT NOT NULL)')

    logging.debug('Создание таблицы специальностей...')
    cursor.execute(f'CREATE TABLE specialities ('
                   f'id INTEGER PRIMARY KEY,'
                   f'faculty_id INTEGER,'
                   f'university_id INTEGER,'
                   f'name TEXT NOT NULL,'
                   f'date_fetched TEXT NOT NULL,'
                   f'source TEXT NOT NULL)')

    # types:
    # 0 - ОП (особое право)
    # 1 - Ц (целевой набор)
    # 2 - БВИ (олимпиадники/Без Вступительных Испытаний)
    # 3 - ОК (общий конкурс)
    logging.debug('Создание таблицы заявлений...')
    cursor.execute(f'CREATE TABLE admissions ('
                   f'id INTEGER PRIMARY KEY,'
                   f'student_id INTEGER,'
                   f'university_id INTEGER,'
                   f'faculty_id INTEGER,'
                   f'speciality_id INTEGER,'
                   f'consent BIT,'
                   f'type INTEGER,'
                   f'subject1 INTEGER,'
                   f'subject2 INTEGER,'
                   f'subject3 INTEGER,'
                   f'achievement INTEGER,'
                   f'extra_data)')

    conn.commit()
    conn.close()

    logging.info('База данных успешно создана!')

def main():
    check_database()
    create_database()

    subjects = {
        "subj1": 78,
        "subj2": 77,
        "subj3": 96
    }

    conn = open_database()

    info = AdmissionInfo(general=example_general, scores=example_scores, extra_data='')

    write_to_database(conn, info)

    conn.commit()
    close_database(conn)

if __name__ == "__main__":
    main()


__author__ = 'Haohan Wang'

import numpy as np
import pandas as pd
import torch

import re
import os
from os.path import join, dirname, basename
from tqdm import tqdm
from glob import glob

import nibabel as nib
from scipy import ndimage

import matplotlib.pyplot as plt
import psutil

if psutil.Process().username() == 'haohanwang':
    BASE_DIR = '/media/haohanwang/Elements/'
else:
    BASE_DIR = '/home/ec2-user/alzstudy/AlzheimerData/'

def sub2adni(sub):
    return 'sub-ADNI' + ''.join(sub.split('_'))

def df2path(df, i):
    # return join('/media/haohanwang/Elements/saliency_map_dropblock2', df.split.iloc[i], df.participant_id.iloc[i], df.session_id.iloc[i] + '.npy')
    # return join(BASE_DIR + 'saliency_map_torch', df.split.iloc[i], df.participant_id.iloc[i],
    #             df.session_id.iloc[i] + '.npy')
    return join(BASE_DIR + 'saliency_regular', df.split.iloc[i], df.participant_id.iloc[i],
                df.session_id.iloc[i] + '.npy')

def path2subsess(path):
    sub = re.search(r'sub-[a-zA-Z0-9]{1,}', path).group(0)
    sess = re.search(r'ses-M[0-9]{1,}', path).group(0)
    return sub, sess

def subsess2path(sub, sess):
    return join(BASE_DIR + 'ADNI_CAPS', 'subjects', sub, sess, 'deeplearning_prepare_data', 'image_based', 't1_linear', \
                sub + '_' + sess + '_' + 'T1w_space-MNI152NLin2009cSym_desc-Crop_res-1x1x1_T1w.pt')

def imgfromsubsess(sub, sess):
    return torch.load(subsess2path(sub, sess)).squeeze(0).numpy()

def saliencyDraw(ax, ex_sal, saliency_cmap, saliency_alpha):
    from mpl_toolkits.axes_grid1 import make_axes_locatable
    divider = make_axes_locatable(ax)

    ax_cb = divider.new_horizontal(size="5%", pad=0.05)
    fig = ax.get_figure()
    fig.add_axes(ax_cb)

    im = ax.imshow(ex_sal, cmap=saliency_cmap, alpha=saliency_alpha)

    plt.colorbar(im, cax=ax_cb)
    ax_cb.yaxis.tick_right()
    ax_cb.yaxis.set_tick_params(labelright=False)

def drawFromPath(path, label, save_dir='2d_view', order=None):
    sub, sess = path2subsess(path)

    ex = imgfromsubsess(sub, sess)
    sagittal_idx, coronal_idx, axial_idx = [i//2 for i in ex.shape]
    view_cmap = 'gray'
    fig, axes = plt.subplots(ncols=3, figsize=(15,5))
    ax = axes.ravel()
    ax[0].imshow(ex[sagittal_idx, :, :], cmap=view_cmap)
    ax[1].imshow(ex[:, coronal_idx, :], cmap=view_cmap)
    ax[2].imshow(ex[:, :, axial_idx], cmap=view_cmap)
    ax[0].set_title('Sagittal view')
    ax[1].set_title('Coronal view')
    ax[2].set_title('Axial view')

    saliency_cmap = 'jet'
    saliency_alpha = 0.3

    try:
        ex_sal = np.load(path)
    except FileNotFoundError:
        print("Subject example not found in {}".format(path))
        return

    # smoothing over neighboring pixels
    kernel = np.ones([4, 4, 4]) / (4*4*4)
    ex_sal = ndimage.convolve(ex_sal, kernel, mode='constant', cval=0.0)

    #     im1 = ax[0].imshow(ex_sal[sagittal_idx, :, :], cmap=saliency_cmap, alpha=saliency_alpha)
    #     im2 = ax[1].imshow(ex_sal[:, coronal_idx, :], cmap=saliency_cmap, alpha=saliency_alpha)
    #     im3 = ax[2].imshow(ex_sal[:, :, axial_idx], cmap=saliency_cmap, alpha=saliency_alpha)
    saliencyDraw(ax[0], ex_sal[sagittal_idx, :, :], saliency_cmap, saliency_alpha)
    saliencyDraw(ax[1], ex_sal[:, coronal_idx, :], saliency_cmap, saliency_alpha)
    saliencyDraw(ax[2], ex_sal[:, :, axial_idx], saliency_cmap, saliency_alpha)
    # saliencyDraw(ax[0], ex_sal[sagittal_idx, :, :], view_cmap, saliency_alpha)
    # saliencyDraw(ax[1], ex_sal[:, coronal_idx, :], view_cmap, saliency_alpha)
    # saliencyDraw(ax[2], ex_sal[:, :, axial_idx], view_cmap, saliency_alpha)

    os.makedirs(save_dir, exist_ok=True)
    if order is None:
        fig.savefig(join(save_dir, '_'.join([label, sub, sess, '2d_view.png'])))
        fig.clf()
    else:
        fig.savefig(join(save_dir, '_'.join([str(order), label, sub, sess, '2d_view.png'])))
        fig.clf()

if __name__ == '__main__':
    subs = glob(BASE_DIR + 'ADNI_CAPS/subjects/*')

    print(subs[:5])

    ex = imgfromsubsess('sub-ADNI094S0711', 'ses-M00')
    plt.imshow(ex[:,:,90], cmap='gray')

    snp_subs = np.loadtxt(BASE_DIR + 'SNP/samples.txt', dtype=str)
    snp_subs = np.array(list(map(sub2adni, snp_subs)))

    df = pd.read_csv(BASE_DIR +'ADNI_CAPS/split.stratified.0.csv')
    tmp = df[df['participant_id'].isin(snp_subs)]
    tmp = tmp.sort_values(by=['session_id'], ascending=False)
    tmp = tmp.drop_duplicates(subset=['participant_id'])
    df_snp = tmp.reset_index(drop=True)

    train_df = pd.read_csv(BASE_DIR + 'saliency_regular/train_saliency_info.csv')
    # test_df = pd.read_csv(BASE_DIR + 'new_saliency_maps_ckp1/test_saliency_info.csv')
    val_df = pd.read_csv(BASE_DIR + 'saliency_regular/val_saliency_info.csv')
    # df_pred = pd.concat([train_df, test_df, val_df])
    df_pred = pd.concat([train_df, val_df])
    df_pred = df_pred.reset_index(drop=True)

    df_snp = df_snp.merge(df_pred, how='inner', on=['participant_id', 'session_id'])
    df_snp['diagnosis_pred'] = df_snp['prob_AD'].map(lambda x: np.round(x))
    snp_correct = df_snp['diagnosis_y'] == df_snp['diagnosis_pred'].map(lambda x: int(x))
    df_snp_correct = df_snp[snp_correct]

    snp_paths = [df2path(df_snp, i) for i in range(len(df_snp))]
    snp_labels = df_snp.diagnosis_x.values
    snp_correct_paths = [df2path(df_snp_correct, i) for i in range(len(df_snp_correct))]
    snp_correct_labels = df_snp_correct.diagnosis_x.values

    for p, l in tqdm(zip(snp_correct_paths, snp_correct_labels)):
        print(p)
        print(l)
        drawFromPath(p,l, BASE_DIR + '2d_new_saliency_regular_smoothing')


import os
import re
from setuptools import setup, find_packages

def get_version(source):
    """
    Retrieve the version of a python distribution.

    version_file default is the <project_root>/_version.py

    :param source: Path to project root
    :return: Version string
    """
    version_str_lines = open(os.path.join(source, '_version.py'), "rt").read()
    version_str_regex = r"^__version__ = ['\"]([^'\"]*)['\"]"
    mo = re.search(version_str_regex, version_str_lines, re.M)
    if mo:
        return mo.group(1)
    else:
        raise RuntimeError("Unable to find version string in %s." % os.path.join(source, os.path.basename(source)))

setup(
    name='Dockerizer',
    version=get_version('Dockerizer'),
    packages=find_packages(),
    install_requires=[
        'databasetools>=1.4.6',
        'PySimpleGUI>=3.39.0',
        'RuntimeWatch',
        'dirutility>=0.7.0'
    ],
    url='https://github.com/sfneal/Dockerizer',
    entry_points={
        'console_scripts': [
            'dockerizer = Dockerizer.dockerize:main',
            'morning-pull = Dockerizer.morning_pull:main',
            'docker-reset = Dockerizer.reset:main',
            'docker-clean = Dockerizer.clean:main',
            'docker-bootstrap = Dockerizer.bootstrap:main',
            'docker-reboot = Dockerizer.reboot:main'
        ]
    },
    license='',
    author='Stephen Neal',
    author_email='stephen@stephenneal.net',
    description='Docker CLI wrapper with GUI for quickly Dockerizing applications.'
)


import numpy as np
import mido
from scipy.sparse import csr_matrix

# number of timesteps for lstm
NUM_STEPS = 64
# how many features does this model have?
NUM_FEATURES = 128 + 128 + 2  # note_on + note_off + 2 track start/end
# the batch size for training
BATCH_SIZE = 64

class MidiArrayBuilder:

    def __init__(self, filename):

        self.filename = filename
        self.buf = []
        self.time_buf = []

    def mid_to_array(self):

        mid = mido.MidiFile(self.filename)

        for track in mid.tracks:

            # need this to track start/end of track
            found_a_note = False

            for msg in track:

                if not (msg.type == 'note_on' or msg.type == 'note_off') or msg.channel == 9:  # skip drum tracks
                    continue

                # it will never get here if it never finds a note
                if not found_a_note:
                    found_a_note = True
                    self.buf.extend([np.zeros(NUM_FEATURES) for _ in range(NUM_STEPS - 1)])
                    self.time_buf.extend([0 for _ in range(NUM_STEPS - 1)])
                    self.special_step(-2)  # start_track one-hot

                # find the one-hot note
                note_code = msg.note
                if msg.type == "note_off" or not msg.velocity:
                    note_code += 128

                self.note_step(note_code, msg.time)

            if found_a_note:
                # slide a start_end in there
                self.special_step(-1)

        return csr_matrix(np.array(self.buf)), self.time_buf

    def special_step(self, i):

        this_step = np.zeros(NUM_FEATURES)
        this_step[i] = 1
        self.buf.append(this_step)
        self.time_buf.append(0)

    def note_step(self, note_i, delay):

        this_step = np.zeros(NUM_FEATURES)
        this_step[note_i] = 1  # the midi note one-hot

        self.buf.append(this_step)
        self.time_buf.append(delay)


import streamlit as st
import pandas as pd
import base64
import yfinance as yf

st.title('S&P 500 App')

st.markdown("""
This app retrieves the list of the **S&P 500** (from Wikipedia) and its corresponding **stock closing price** (year-to-date)!
* **Python libraries:** base64, pandas, streamlit, pandas
* **Data source:** [Wikipedia](https://en.jinzhao.wiki/wiki/List_of_S&P_500_companies).
""")

st.sidebar.header('User Input Features')

# Web scraping of S&P 500 data

df = pd.read_csv('SP500.csv')

sector = df.groupby('GICS Sector')

# Sidebar - Sector selection
sorted_sector_unique = sorted( df['GICS Sector'].unique() )
selected_sector = st.sidebar.multiselect('Sector', sorted_sector_unique, sorted_sector_unique)

# Filtering data
df_selected_sector = df[ (df['GICS Sector'].isin(selected_sector)) ]

st.header('Display Companies in Selected Sector')
st.write('Data Dimension: ' + str(df_selected_sector.shape[0]) + ' rows and ' + str(df_selected_sector.shape[1]) + ' columns.')
st.dataframe(df_selected_sector)

# Download S&P500 data
# https://discuss.streamlit.io/t/how-to-download-file-in-streamlit/1806
def filedownload(df):
    csv = df.to_csv(index=False)
    b64 = base64.b64encode(csv.encode()).decode()  # strings <-> bytes conversions
    href = f'Download CSV File'
    return href

st.markdown(filedownload(df_selected_sector), unsafe_allow_html=True)

# https://pypi.org/project/yfinance/

data = yf.download(
        tickers = list(df_selected_sector[:10].Symbol),
        period = "ytd",
        interval = "1d",
        group_by = 'ticker',
        auto_adjust = True,
        prepost = True,
        threads = True,
        proxy = None
    )

num_company = st.sidebar.slider('Number of Companies', 1, 5)

if st.button('Show Plots'):
    st.header('Stock Closing Price')
    for tickerSymbol in list(df_selected_sector.Symbol)[:num_company]:

	#get data on this ticker
        tickerData = yf.Ticker(tickerSymbol)
	# get the historical prices for this ticker

        tickerDf=tickerData.history(start='2011-6-22', end='2021-6-22')
        st.line_chart(tickerDf.Close)
        st.line_chart(tickerDf.Volume)


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.3
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
#

# + [markdown] id="Gmm_TyNcTdLD"
# # Adversarial autoencoders for text
#
# Code is from
# https://github.com/shentianxiao/text-autoencoders
#
# Paper is here: https://arxiv.org/pdf/1905.12777.pdf

# + [markdown] id="dV3lykF6TvwL"
# # Setup

# + colab={"base_uri": "https://localhost:8080/"} id="G29OhPQcTdoO" outputId="1bfdb063-5b30-4e65-f1a8-1a7a11e00e9a"
import torch
from multiprocessing import cpu_count
print(cpu_count())
print(torch.cuda.is_available())

# + colab={"base_uri": "https://localhost:8080/"} id="5JtFQPuqTaw2" outputId="cba5e1dc-40be-4a5f-cb9b-777765800836"
# !git clone https://github.com/shentianxiao/text-autoencoders.git

# + colab={"base_uri": "https://localhost:8080/"} id="rSKVhddyUUm2" outputId="423007d3-743f-4dc3-e3e7-ec5af238261a"
# !ls

# + colab={"base_uri": "https://localhost:8080/"} id="qh7fYGg9UWcz" outputId="c2b179ad-ecd6-43a5-85e4-03720518dc08"
# %cd text-autoencoders

# + colab={"base_uri": "https://localhost:8080/"} id="dTmgH_QUUYId" outputId="f1fb7a20-2a84-4f50-af88-58ff3eaff4c4"
# !ls

# + id="jrmT7wGeB6zm"

# + [markdown] id="u-FxMmZ3Txlk"
# # Data

# + colab={"base_uri": "https://localhost:8080/"} id="PJqGHr-KTo9W" outputId="7f168732-337f-4fec-c41c-5a013abe1828"
# !bash download_data.sh

# + [markdown] id="RBD4mGZ_Uqjr"
# # Train

# + colab={"base_uri": "https://localhost:8080/"} id="aPw6uP1MU0qh" outputId="908c9949-75f1-4498-de65-1bc0c823e860"
# !python train.py -h

# + colab={"base_uri": "https://localhost:8080/", "height": 154} id="6rTRlHtdXwwX" outputId="59cc4db5-b540-4d04-a644-a61fa274f213"
'''
# Path arguments
parser.add_argument('--train', metavar='FILE', required=True,
                    help='path to training file')
parser.add_argument('--valid', metavar='FILE', required=True,
                    help='path to validation file')
parser.add_argument('--save-dir', default='checkpoints', metavar='DIR',
                    help='directory to save checkpoints and outputs')
parser.add_argument('--load-model', default='', metavar='FILE',
                    help='path to load checkpoint if specified')
# Architecture arguments
parser.add_argument('--vocab-size', type=int, default=10000, metavar='N',
                    help='keep N most frequent words in vocabulary')
parser.add_argument('--dim_z', type=int, default=128, metavar='D',
                    help='dimension of latent variable z')
parser.add_argument('--dim_emb', type=int, default=512, metavar='D',
                    help='dimension of word embedding')
parser.add_argument('--dim_h', type=int, default=1024, metavar='D',
                    help='dimension of hidden state per layer')
parser.add_argument('--nlayers', type=int, default=1, metavar='N',
                    help='number of layers')
parser.add_argument('--dim_d', type=int, default=512, metavar='D',
                    help='dimension of hidden state in AAE discriminator')
# Model arguments
parser.add_argument('--model_type', default='dae', metavar='M',
                    choices=['dae', 'vae', 'aae'],
                    help='which model to learn')
parser.add_argument('--lambda_kl', type=float, default=0, metavar='R',
                    help='weight for kl term in VAE')
parser.add_argument('--lambda_adv', type=float, default=0, metavar='R',
                    help='weight for adversarial loss in AAE')
parser.add_argument('--lambda_p', type=float, default=0, metavar='R',
                    help='weight for L1 penalty on posterior log-variance')
parser.add_argument('--noise', default='0,0,0,0', metavar='P,P,P,K',
                    help='word drop prob, blank prob, substitute prob'
                         'max word shuffle distance')
# Training arguments
parser.add_argument('--dropout', type=float, default=0.5, metavar='DROP',
                    help='dropout probability (0 = no dropout)')
parser.add_argument('--lr', type=float, default=0.0005, metavar='LR',
                    help='learning rate')
#parser.add_argument('--clip', type=float, default=0.25, metavar='NORM',
#                    help='gradient clipping')
parser.add_argument('--epochs', type=int, default=50, metavar='N',
                    help='number of training epochs')
parser.add_argument('--batch-size', type=int, default=256, metavar='N',
                    help='batch size')
# Others
parser.add_argument('--seed', type=int, default=1111, metavar='N',
                    help='random seed')
parser.add_argument('--no-cuda', action='store_true',
                    help='disable CUDA')
parser.add_argument('--log-interval', type=int, default=100, metavar='N',
                    help='report interval')
'''

# + colab={"base_uri": "https://localhost:8080/"} id="G3i3cj-0Up3_" outputId="fdf7d39c-2c73-4f18-9eb1-33d82dc0fbde"
NUM_EPOCHS = 1 # debugging
# !python train.py --epochs $NUM_EPOCHS --train data/yelp/train.txt --valid data/yelp/valid.txt --model_type aae --lambda_adv 10 --noise 0.3,0,0,0 --save-dir checkpoints/yelp/daae

# + colab={"base_uri": "https://localhost:8080/"} id="1mh7nfgidNdJ" outputId="da0ebbbe-2e7a-499f-af30-22169349f034"
NUM_EPOCHS = 10 # debugging
# !python train.py --epochs $NUM_EPOCHS --train data/yelp/train.txt --valid data/yelp/valid.txt --model_type aae --lambda_adv 10 --noise 0.3,0,0,0 --save-dir checkpoints/yelp/daae

# + colab={"base_uri": "https://localhost:8080/"} id="NaNeyq3ZYhe_" outputId="e3b4c7b6-e944-4d20-c48f-21f0d68765f9"
# !ls

# + colab={"base_uri": "https://localhost:8080/"} id="W_OdNhuwYlLa" outputId="0d3af920-5798-4986-9ccf-e4789456e166"
# !ls checkpoints/yelp/daae

# + id="e1Hq9eseq7dv"
from google.colab import files
#files.download('checkpoints/yelp/daae/model.pt')

# + [markdown] id="fSRiEn32rCL7"
# # Upload pretrained model

# + id="Y3AqiUXFrEcN"
from google.colab import files
#uploaded = files.upload() # store it in checkpoints/yelp/daae/model.pt

# + [markdown] id="MoCvv4mfYSA9"
# # Reconstruction

# + id="uKXVJuyxYLWD"
# !python test.py --reconstruct --data data/yelp/test.txt --output test.rec --checkpoint checkpoints/yelp/daae/

# + colab={"base_uri": "https://localhost:8080/"} id="llD_hqk7Y3Rm" outputId="791098fd-50af-4656-aead-6efd74c08dd9"
# !ls checkpoints/yelp/daae

# + colab={"base_uri": "https://localhost:8080/"} id="BzBL_6uPY3kz" outputId="7cb96ac3-8f59-4b4b-ad27-e3ea5f8ab481"
# !head checkpoints/yelp/daae/test.rec.rec

# + colab={"base_uri": "https://localhost:8080/"} id="krtESrsPZ3y_" outputId="0284615d-54d2-4a69-c34e-d67bae5fc3bd"
# !head checkpoints/yelp/daae/test.rec.z

# + colab={"base_uri": "https://localhost:8080/"} id="aVAGRBBmZ_Wo" outputId="e57cc98a-becd-42cb-e737-b3a6ca123cc5"
# !head data/yelp/test.txt

# + [markdown] id="iYP4zjFCaS2Q"
# # Sample

# + id="oRtLTTRNaEKT"
# !python test.py --sample --n 10 --output sample --checkpoint checkpoints/yelp/daae/

# + colab={"base_uri": "https://localhost:8080/"} id="yg1MEw8uaUnY" outputId="a341b76e-dfaa-4677-bdee-33ef47b5f36c"
# !ls checkpoints/yelp/daae

# + colab={"base_uri": "https://localhost:8080/"} id="Q-EF4QvYaYqz" outputId="70f0acd9-172b-4a74-e853-6a37d099a839"
# !head checkpoints/yelp/daae/sample

# + [markdown] id="sr-THLamamjI"
# # Arithmetic
#
# The difference between the average latent representation of the first two data files will be applied to the third file (separated by commas), and k denotes the scaling factor.
#

# + [markdown] id="_Tz_Y_Txg55c"
# ## Tense

# + id="Y6cqFAmxab3j"
# !python test.py --arithmetic --data data/yelp/tense/valid.past,data/yelp/tense/valid.present,data/yelp/tense/test.past --output test.past2present --checkpoint checkpoints/yelp/daae/

# + colab={"base_uri": "https://localhost:8080/"} id="UtOj8o5AbSEh" outputId="517effc9-bcd6-436e-8cfa-831e20eae74e"
# !head data/yelp/tense/valid.past

# + colab={"base_uri": "https://localhost:8080/"} id="FqwU42qvbnTP" outputId="101396ac-1b37-444e-ee6e-5c0e6310d596"
# !head data/yelp/tense/valid.present

# + colab={"base_uri": "https://localhost:8080/"} id="Sl91aiLWbql1" outputId="084def78-c9c8-4850-f1aa-b4845eca557b"
# !head data/yelp/tense/test.past

# + colab={"base_uri": "https://localhost:8080/"} id="Df37XG-xbKin" outputId="04e98445-73dc-44dd-8cbf-2568bc8c79b6"
# !head checkpoints/yelp/daae/test.past2present

# + [markdown] id="z5nbF9p_g7pQ"
# ## Sentiment

# + id="vZPQ6sZoatXK"
# !python test.py --arithmetic --k 2 --data data/yelp/sentiment/100.neg,data/yelp/sentiment/100.pos,data/yelp/sentiment/1000.neg --output 1000.neg2pos --checkpoint checkpoints/yelp/daae/

# + colab={"base_uri": "https://localhost:8080/"} id="i1o_A_3rgs79" outputId="c9d1b8a0-ec27-4a8f-8a05-545aa124c56a"
# !head data/yelp/sentiment/100.neg

# + colab={"base_uri": "https://localhost:8080/"} id="l9vw2Pwtgxlx" outputId="7dd93048-0ac3-4a0b-da35-8f7997a5db73"
# !head data/yelp/sentiment/100.pos

# + colab={"base_uri": "https://localhost:8080/"} id="xh9fTPahg19l" outputId="64c5da57-40a8-4b88-a0ca-611d1a57086b"
# !head data/yelp/sentiment/1000.neg

# + colab={"base_uri": "https://localhost:8080/"} id="Z8LiQoRZhGLo" outputId="f8c36c27-59ee-4789-be54-66a11c1003f7"
# !head checkpoints/yelp/daae/1000.neg2pos

# + [markdown] id="327Ahgmgb86B"
# # Interpolation
#
# Sentence interpolation between two data files (separated by a comma),
#

# + id="tGekWTjnb9lc"
# !python test.py --interpolate --data data/yelp/interpolate/example.long,data/yelp/interpolate/example.short --output example.int --checkpoint checkpoints/yelp/daae/

# + colab={"base_uri": "https://localhost:8080/"} id="Q3u-mKAhcbnO" outputId="63c8fd4c-3d15-4f9d-a68b-d96d8f5a1513"
# !head checkpoints/yelp/daae/example.int

# + colab={"base_uri": "https://localhost:8080/"} id="sbO3hWLIckRQ" outputId="8f22487f-fa94-4e26-c23b-543d2bb45e84"
# !head data/yelp/interpolate/example.long

# + colab={"base_uri": "https://localhost:8080/"} id="sEXD-qXIcpzw" outputId="be940779-9bf6-403b-dc68-bfc8631d06e0"
# !head data/yelp/interpolate/example.short

# + id="dqI6ngticulp"

# + [markdown] id="OrgQaMimDIrJ"
# # Optiponal: Reproduce fig 1 (toy dataset)
#
# The code below was sent to me by Tianxiao.
# It is not part of the repo.

# + [markdown] id="DSlNrC08EcBQ"
# ## Make dataset

# + id="skM4oJUGDOIv"
import os
import random

random.seed(1)

def gen(path, n=5, m=100, l=50, p=0.2):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, 'w') as f:
        for i in range(n):
            c = [random.randint(0, 1) for _ in range(l)]
            for j in range(m):
                b = c.copy()
                for k in range(l):
                    if random.random() < p:
                        b[k] = 1 - b[k]
                f.write(' '.join([str(x) for x in b]) + '\n')

gen('data/toy/data.txt')

# + [markdown] id="kUG8fRVeEd77"
# ## Train

# + id="wnW6LfqRDS38" outputId="92afa290-0781-4420-b5a3-0263396a8543" colab={"base_uri": "https://localhost:8080/"}
# !python train.py --train data/toy/data.txt --valid data/toy/data.txt --model_type aae --lambda_adv 10 --dim_z 2 --save-dir checkpoints/toy/aae/ --epochs 1

# + id="As6Q8UsbDT1m" outputId="04f373f9-6038-406f-f144-84b34f485f21" colab={"base_uri": "https://localhost:8080/"}
# !python train.py --train data/toy/data.txt --valid data/toy/data.txt --model_type dae --lambda_adv 10 --dim_z 2 --noise 0,0,0.2,0 --save-dir checkpoints/toy/daae/ --epochs 1

# + id="BMqllw1KEQdL" outputId="6f95b32b-4fb2-4663-da21-2d2613afa3af" colab={"base_uri": "https://localhost:8080/"}
# !ls checkpoints/toy/aae

# + [markdown] id="I93ajRDCEf-N"
# ## Compute latent representations

# + id="OTURry3wEhoB"
# !python test.py --reconstruct --data data/toy/data.txt --output data --max-len 55 --checkpoint checkpoints/toy/aae/

# + id="36Oh5lewEnmy"
# !python test.py --reconstruct --data data/toy/data.txt --output data --max-len 55 --checkpoint checkpoints/toy/daae/

# + [markdown] id="O3jHgjW4EkUj"
# ## Plot

# + id="oq8lP-8_DyoR"
import sys
import numpy as np
import matplotlib.pyplot as plt
#from sklearn.manifold import TSNE

def plot_z(filename):
  x = []
  with open(filename) as f:
      for line in f:
          parts = line.split()
          x.append([float(p) for p in parts])
  x = np.array(x)
  #x = TSNE().fit_transform(x)

  n, m = 5, 100
  for i in range(n):
      l, r = i*m, (i+1)*m
      plt.scatter(x[l:r, 0], x[l:r, 1])

  #plt.title('DAAE', fontsize=22)
  plt.xticks(fontsize=18)
  plt.yticks(fontsize=18)
  plt.show()

# + id="SZ3tcCrPEFhT" outputId="704aae9b-928b-4917-ba76-fdd6d05c0529" colab={"base_uri": "https://localhost:8080/", "height": 279}
plot_z('checkpoints/toy/aae/data.z')

# + id="1dKqBy9ZELE4" outputId="d13fdc09-04cf-42ca-f770-fe08de7565ab" colab={"base_uri": "https://localhost:8080/", "height": 273}
plot_z('checkpoints/toy/daae/data.z')

# + id="PLGYhD9uEuRt"


"""Tests for common behaviors shared by all ComputableTerms.
"""
import numpy as np

from zipline.lib.labelarray import LabelArray
from zipline.pipeline import Classifier, Factor, Filter
from zipline.testing import parameter_space
from zipline.utils.numpy_utils import (
    categorical_dtype,
    datetime64ns_dtype,
    float64_dtype,
    int64_dtype,
    NaTns,
)

from .base import BaseUSEquityPipelineTestCase

class Floats(Factor):
    inputs = ()
    window_length = 0
    dtype = float64_dtype

class AltFloats(Factor):
    inputs = ()
    window_length = 0
    dtype = float64_dtype

class Dates(Factor):
    inputs = ()
    window_length = 0
    dtype = datetime64ns_dtype

class AltDates(Factor):
    inputs = ()
    window_length = 0
    dtype = datetime64ns_dtype

class Bools(Filter):
    inputs = ()
    window_length = 0

class AltBools(Filter):
    inputs = ()
    window_length = 0

class Strs(Classifier):
    inputs = ()
    window_length = 0
    dtype = categorical_dtype
    missing_value = None

class AltStrs(Classifier):
    inputs = ()
    window_length = 0
    dtype = categorical_dtype
    missing_value = None

class Ints(Classifier):
    inputs = ()
    window_length = 0
    dtype = int64_dtype
    missing_value = -1

class AltInts(Classifier):
    inputs = ()
    window_length = 0
    dtype = int64_dtype
    missing_value = -1

class FillNATestCase(BaseUSEquityPipelineTestCase):

    @parameter_space(
        null_locs=[
            # No NaNs.
            np.zeros((4, 4), dtype=bool),
            # All NaNs.
            np.ones((4, 4), dtype=bool),
            # NaNs on Diagonal
            np.eye(4, dtype=bool),
            # Nans every third element.
            (np.arange(16).reshape(4, 4) % 3) == 0,
        ]
    )
    def test_fillna_with_scalar(self, null_locs):
        shape = (4, 4)
        num_cells = shape[0] * shape[1]

        floats = np.arange(num_cells, dtype=float).reshape(shape)
        floats[null_locs] = np.nan
        float_fillval = 999.0
        float_expected = np.where(null_locs, float_fillval, floats)
        float_expected_zero = np.where(null_locs, 0.0, floats)

        dates = (np.arange(num_cells, dtype='i8')
                 .view('M8[D]')
                 .astype('M8[ns]')
                 .reshape(shape))
        dates[null_locs] = NaTns
        date_fillval = np.datetime64('2014-01-02', 'ns')
        date_expected = np.where(null_locs, date_fillval, dates)

        strs = np.arange(num_cells).astype(str).astype(object).reshape(shape)
        strs[null_locs] = None
        str_fillval = "filled"
        str_expected = np.where(null_locs, str_fillval, strs)

        ints = np.arange(num_cells, dtype='i8').reshape(shape)
        ints[null_locs] = -1
        int_fillval = 777
        int_expected = np.where(null_locs, int_fillval, ints)

        terms = {
            'floats': Floats().fillna(float_fillval),
            # Make sure we accept integer as a fill value on float-dtype
            # factors.
            'floats_fill_zero': Floats().fillna(0),
            'dates': Dates().fillna(date_fillval),
            'strs': Strs().fillna(str_fillval),
            'ints': Ints().fillna(int_fillval),
        }

        expected = {
            'floats': float_expected,
            'floats_fill_zero': float_expected_zero,
            'dates': date_expected,
            'strs': self.make_labelarray(str_expected),
            'ints': int_expected,
        }

        self.check_terms(
            terms,
            expected,
            initial_workspace={
                Floats(): floats,
                Dates(): dates,
                Strs(): self.make_labelarray(strs),
                Ints(): ints,
            },
            mask=self.build_mask(self.ones_mask(shape=(4, 4))),
        )

    @parameter_space(
        null_locs=[
            # No NaNs.
            np.zeros((4, 4), dtype=bool),
            # # All NaNs.
            np.ones((4, 4), dtype=bool),
            # NaNs on Diagonal
            np.eye(4, dtype=bool),
            # Nans every third element.
            (np.arange(16).reshape((4, 4)) % 3) == 0,
        ]
    )
    def test_fillna_with_expression(self, null_locs):
        shape = (4, 4)
        mask = self.build_mask(self.ones_mask(shape=(4, 4)))
        state = np.random.RandomState(4)
        assets = self.asset_finder.retrieve_all(mask.columns)

        def rand_vals(dtype):
            return state.randint(1, 100, shape).astype(dtype)

        floats = np.arange(16, dtype=float).reshape(shape)
        floats[null_locs] = np.nan
        float_fillval = rand_vals(float)
        float_expected = np.where(null_locs, float_fillval, floats)
        float_expected_1d = np.where(null_locs, float_fillval[:, [0]], floats)

        dates = (np.arange(16, dtype='i8')
                 .view('M8[D]')
                 .astype('M8[ns]').
                 reshape(shape))
        dates[null_locs] = NaTns
        date_fillval = rand_vals('M8[D]').astype('M8[ns]')
        date_expected = np.where(null_locs, date_fillval, dates)
        date_expected_1d = np.where(null_locs, date_fillval[:, [1]], dates)

        strs = np.arange(16).astype(str).astype(object).reshape(shape)
        strs[null_locs] = None
        str_fillval = rand_vals(str)
        str_expected = np.where(null_locs, str_fillval, strs)
        str_expected_1d = np.where(null_locs, str_fillval[:, [2]], strs)

        ints = np.arange(16).reshape(shape)
        ints[null_locs] = -1
        int_fillval = rand_vals(int64_dtype)
        int_expected = np.where(null_locs, int_fillval, ints)
        int_expected_1d = np.where(null_locs, int_fillval[:, [3]], ints)

        terms = {
            'floats': Floats().fillna(AltFloats()),
            'floats_1d': Floats().fillna(AltFloats()[assets[0]]),

            'dates': Dates().fillna(AltDates()),
            'dates_1d': Dates().fillna(AltDates()[assets[1]]),

            'strs': Strs().fillna(AltStrs()),
            'strs_1d': Strs().fillna(AltStrs()[assets[2]]),

            'ints': Ints().fillna(AltInts()),
            'ints_1d': Ints().fillna(AltInts()[assets[3]]),
        }

        expected = {
            'floats': float_expected,
            'floats_1d': float_expected_1d,
            'dates': date_expected,
            'dates_1d': date_expected_1d,
            'strs': self.make_labelarray(str_expected),
            'strs_1d': self.make_labelarray(str_expected_1d),
            'ints': int_expected,
            'ints_1d': int_expected_1d,
        }

        self.check_terms(
            terms,
            expected,
            initial_workspace={
                Floats(): floats,
                Dates(): dates,
                Strs(): self.make_labelarray(strs),
                Ints(): ints,

                AltFloats(): float_fillval,
                AltDates(): date_fillval,
                AltStrs(): self.make_labelarray(str_fillval),
                AltInts(): int_fillval,
            },
            mask=mask,
        )

    def should_error(self, f, exc_type, expected_message):
        with self.assertRaises(exc_type) as e:
            f()

        message = str(e.exception)
        self.assertIn(expected_message, message)

    def test_bad_inputs(self):
        def dtype_for(o):
            return np.array([o]).dtype

        self.should_error(
            lambda: Floats().fillna('3.0'),
            TypeError,
            " from {!r} to {!r} according to the rule 'same_kind'"
            .format(dtype_for('3.0'), np.dtype(float))
        )

        self.should_error(
            lambda: Dates().fillna('2014-01-02'),
            TypeError,
            "from {!r} to {!r} according to the rule 'same_kind'"
            .format(dtype_for('2014-01-02'), np.dtype('M8[ns]'))
        )

        self.should_error(
            lambda: Ints().fillna('300'),
            TypeError,
            "from {!r} to {!r} according to the rule 'same_kind'"
            .format(dtype_for('300'), np.dtype('i8')),
        )

        self.should_error(
            lambda: Strs().fillna(10.0),
            TypeError,
            "Fill value 10.0 is not a valid choice for term Strs with dtype"
            " object.\n\n"
            "Coercion attempt failed with: "
            "String-dtype classifiers can only produce strings or None."

        )

    def make_labelarray(self, strs):
        return LabelArray(strs, missing_value=None)


from aiogram import types
from aiogram.dispatcher.filters.builtin import CommandHelp

from loader import dp
from utils.misc import rate_limit

@rate_limit(5, 'help')
@dp.message_handler(CommandHelp())
async def bot_help(message: types.Message):
    text = [
        'Список команд: ',
        '/start - Начать диалог',
        '/help - Получить справку',
        '/callback - Перезвонить',
        '/show_on_map - Показать на карте',
    ]
    await message.answer('\n'.join(text))


import common.cv as cc
import common.graph as cg
import csv

# Annotated positions for precision/recall tests
def get_annotated_poses(image_file):
    annotated_nodes = get_annotated_nodes(image_file)
    annotated_poses = [node.pos for node in annotated_nodes]

    return annotated_poses

def get_annotated_nodes(image_file):
    annotated_file = open("graphs/annotated/" + image_file[4:8] + ".gdf")
    annotated_graph = cg.translate_graph(annotated_file.read().splitlines()[1:])

    return annotated_graph.nodes

def get_generated_poses(model):
    poses = []
    for tup in model:
        for keylabel in tup:
            kp,label = keylabel
            poses.append(cc.get_point_tuple(kp))

    return list(set(poses))

def get_label_positions(model, label):
    label_positions = []
    for tup in model:
        for keylabel in tup:
            kp,found_label = keylabel
            if found_label.name == label:
                label_positions.append(cc.get_point_tuple(kp))

    return list(set(label_positions))

def get_precision(annotated_poses, model_poses):
    tp = len(set(annotated_poses).intersection(model_poses))

    if len(model_poses) == 0:
        return 0

    return tp / len(model_poses)

def get_recall(annotated_poses, model_poses):
    tp = len(set(annotated_poses).intersection(model_poses))

    if len(annotated_poses) == 0 or tp == 0:
        return len(annotated_poses)

    return tp / len(annotated_poses)

def get_precision_recall(annotated_poses, model_poses):
    precision = get_precision(annotated_poses, model_poses)
    recall = get_recall(annotated_poses, model_poses)

    return (precision,recall)

def get_classification_metrics(annotated_poses, poses):
    tp = len(set(annotated_poses).intersection(poses))
    fp = len(poses)
    fn = len(annotated_poses)

    return (tp,fp,fn)

def update_dictionary(key, data, dictionary):
    if key in dictionary:
        total = dictionary[key]
        dictionary[key] = tuple(sum(x) for x in zip(total,data))
    else:
        dictionary[key] = data

def experiment_identification(image_file, method, model_category, best_model, experiment_dict):
    actual_category = "juvenile" if image_file[4] == "4" else "mature"
    annotated_poses = get_annotated_poses(image_file)
    poses = get_generated_poses(best_model)

    key = (method,model_category,actual_category)
    data = get_classification_metrics(annotated_poses, poses)
    update_dictionary(key, data, experiment_dict)

def experiment_label(image_file, method, model_category, label, best_model, experiment_dict):
    actual_category = "juvenile" if image_file[4] == "4" else "mature"
    annotated_nodes = get_annotated_nodes(image_file)
    filtered_nodes = [node.pos for node in annotated_nodes if node.label == label]
    poses = get_label_positions(best_model, label)

    key = (method,label,model_category,actual_category)
    data = get_classification_metrics(filtered_nodes, poses)
    update_dictionary(key, data, experiment_dict)

def write_experiment(csv_file, experiment_dict):
    writer = csv.writer(csv_file, delimiter=",")

    for key,data in experiment_dict.items():
        headers = list(key)
        tp,fp,fn = data

        precision = tp/fp
        recall = tp/fn

        writer.writerow(headers + [precision, recall])
        csv_file.flush()


from dash import dcc
from dash import html
import dash
from dash.dependencies import Input, Output, State
from dash.html.Center import Center
import plotly.express as px
import plotly.graph_objects as go
import dash_bootstrap_components as dbc
from dash_bootstrap_templates import load_figure_template
import pandas as pd
from datetime import date
from dash import dash_table
#from dash_table.Format import Group` with
from dash.dash_table.Format import Group

#path="https://github.com/MdioufDataScientist/Projet-Visualisation/blob/main/COVID-19-geographic-disbtribution-worldwide%20-%20COVID-19-geographic-disbtributi.csv"
path1="/home/mdiouf/Bureau/Projet-Visualisation/Data_covid.csv"
data=pd.read_csv("Data_covid.csv")
dfTable=data.groupby(['countriesAndTerritories'])['cases'].sum().reset_index()
dfTable=dfTable.sort_values(by=['cases'],ascending=False)
dfMort=data.groupby(['countriesAndTerritories'])['deaths'].sum().reset_index()
dfMort=dfMort.sort_values(by=['deaths'],ascending=False)
#print(data.head(5))
total=data['cases'].sum()
total=str(total)
morts=data['deaths'].sum()
morts=str(morts)
load_figure_template("cyborg")
app=dash.Dash(__name__,external_stylesheets=[dbc.themes.CYBORG])
app.layout=dbc.Container([
    dbc.Row([
        dbc.Col(html.Div([
                html.H1("COVID DASHBOARD",style={'fontWeight':'bold','textAlign':'center'})
            ]

            ))
    ]),
    dbc.Row([
            dbc.Col([
                dcc.DatePickerRange(
            id="plage_de_date",
            #display_format="DD/M/YYYY",
            min_date_allowed=date(2019, 12, 31),
            max_date_allowed=date(2020, 3, 27),
            start_date_placeholder_text="Debut",
            end_date_placeholder_text="Fin",
            calendar_orientation='vertical',
            )
            ], width={'size':6}
            ),
            dbc.Col([
            dcc.Dropdown(id='d_drp',
            options=[{'label':x,'value':x} for x in data['countriesAndTerritories'].unique()],
            placeholder="selectionner le Pays"
            )
            ], width={'size':6}),

        ]
        ),
        dbc.Row([
        dbc.Col([
            html.Br(),
            html.Div([
                html.H1("Total Cas : "+total,style={'color': 'green','fontWeight':'bold','textAlign':'left'})
            ]

            )

        ], width={'size':5}
        ),
        dbc.Col([
            html.Br(),
            html.Div([
                html.H1("Total Morts : "+morts,style={'color': 'red','fontWeight':'bold','textAlign':'right'})
            ]

            )
        ], width={'size':6}
        )
    ]),
        dbc.Row([
        dbc.Col([
            html.Br(),
            dcc.Graph(id="fig_1",figure={})
        ], width={'size':6}
        ),
        dbc.Col([
            html.Br(),
            dcc.Graph(id="fig_2",figure={})
        ], width={'size':6}
        )
    ]),
    dbc.Row([
        dbc.Col([
            html.Br(),
            dcc.Graph(id="fig_3",figure={})
        ], width={'size':6}
        ),
        dbc.Col([
            html.Br(),
            dcc.Graph(id="fig_4",figure={})
        ], width={'size':6}
        )
    ]),
    dbc.Row([
        dbc.Col([
            html.Br(),
            dcc.Graph(id="fig_5",figure={})
        ], width={'size':6}
        ),
        dbc.Col([
            html.Br(),
            dcc.Graph(id="fig_6",figure={})
        ], width={'size':6}
        )
    ]),
    dbc.Row([
        dbc.Col([
            html.Br(),
             dash_table.DataTable(
                id='datatable',
                columns=[
                    {"name": i, "id": i, "deletable": True, "selectable": True} for i in dfTable.columns
                ],
                data=dfTable.to_dict('records'),
                page_size=10,
                style_cell={'textAlign': 'center'},
                style_header={
        'backgroundColor': 'white',
        'color': 'black',
        'fontWeight': 'bold'
    },
    style_data={
        'backgroundColor': 'rgb(50, 50, 50)',
        'color': 'white'
    },
            )
        ], width={'size':8,'offset':2}
        )
    ]),
    dbc.Row([
         dbc.Col([
            html.Br(),
             dash_table.DataTable(
                id='datatable2',
                columns=[
                    {"name": i, "id": i, "deletable": True, "selectable": True} for i in dfMort.columns
                ],
                data=dfMort.to_dict('records'),
                page_size=10,
                #style_cell={'textAlign': 'left','background-color':'gray','color':'black'}
                style_cell={'textAlign': 'center'},
                style_header={
        'backgroundColor': 'white',
        'color': 'black',
        'fontWeight': 'bold'
    },
    style_data={
        'backgroundColor': 'rgb(50, 50, 50)',
        'color': 'white'
    },
            )
        ], width={'size':8,'offset':2}
        )
    ])

])
data['dateRep']=pd.to_datetime(data['dateRep'])
def creer_datafram(debut,fin,pays):
    df=data[data['countriesAndTerritories']==pays]
    mask = (df['dateRep'] >= debut) & (df['dateRep'] <= fin)
    df_filtrer=df.loc[mask]
    df_filtrer.sort_values(by="dateRep",ascending=True)
    print(type(debut))
    #print(df_filtrer)
    return df_filtrer

@app.callback(
[Output('fig_1', 'figure'),
Output('fig_2', 'figure'),
Output('fig_3', 'figure'),
Output('fig_4', 'figure'),
Output('fig_5', 'figure'),
Output('fig_6', 'figure'),

],
[Input('plage_de_date', 'start_date'),
Input('plage_de_date', 'end_date'),
Input('d_drp','value')
])
def update(debut,fin,pays):
    if pays:
        if debut:
            if fin:
                df=creer_datafram(debut,fin,pays)
                cov = df.groupby('month')['cases'].sum().reset_index()
                fig_1 = px.bar(cov,x='month',y='cases',title=f'nombre total de cas par mois : {pays}',color='month')
                cov_2 = df.groupby('month')['deaths'].sum().reset_index()
                fig_2= px.bar(cov_2,x='month',y='deaths',title=f'nombre total de morts par mois : {pays}',color='month')
                #fig_2=px.scatter(cov_2,x='month',y='deaths',size='deaths',color="month",title=f"nombre total de morts par mois : {pays}")
                fig_3=px.scatter(df,x='day',y='cases',title=f"tendance des nouveaux cas par jour: {pays}",size='cases',color="cases")
                #fig_3=px.scatter(df,x='month',y='deaths',size='deaths',color="month",title=f"nombre total de morts par mois : {pays}")
                #manipulations pour obtenir la somme cumulé
                cum_1=pd.DataFrame(df.groupby('month')['cases'].sum())
                cum_1=cum_1['cases'].cumsum().reset_index()

                fig_4=px.scatter(df,x='day',y='deaths',title=f"tendance des Morts par jour : {pays}",size='deaths',color="deaths")

                cum_2=pd.DataFrame(df.groupby('month')['deaths'].sum())
                cum_2=cum_2['deaths'].cumsum().reset_index()
                fig_5=px.line(cum_2,x='month',y='deaths',markers=True,title=f"cumul des Morts par mois : {pays}")
                fig_6=px.line(cum_1,x='month',y='cases',markers=True,title=f"cumul des cas par mois : {pays}")

                #fig1=px.pie(df,values='cases',names='dateRep',color_discrete_sequence=px.colors.sequential.RdBu,title=f"Nombre de cas par Jour : {pays}")
                return fig_1,fig_2,fig_3,fig_4,fig_5,fig_6
    else:
        fig_1={}
        fig_2={}
        fig_3={}
        fig_4={}
        fig_5={}
        fig_6={}
        return fig_1,fig_2,fig_3,fig_6,fig_4,fig_5

app.run_server(debug=True)


import numpy as np
from numpy.random import Generator
from sympy import lambdify

class PointsGen:

    def __init__(self, rng: Generator):
        self.rng = rng

    def mk_points(self, num: int, sympy_expr):

        free_vars = list(sympy_expr.free_symbols)
        # sort variables by name
        free_vars = sorted(free_vars, key=lambda v: int(v.name[1:]))

        inps = self.rng.random((len(free_vars), num))
        f = lambdify(free_vars, sympy_expr, "numpy")
        out = f(*inps)

        points = np.row_stack((out, inps))
        return points


import datetime

"""Unenciphered Enigma message preamble
"""

#Call signs of the statins involved. First the sending station, then the destination(s)
#e.g. 'From P7J to SF9 and 5KQ'
CALL_SIGNS = 'Not yet implemented'

#Date and Time of the origin of the message
DATE_OF_ORIGIN = datetime.datetime.now().date()
TIME_OF_ORIGIN = datetime.datetime.now().time()

NUMBER_OF_LETTERS = 0

# An indication of whether the message is complete,
# or part of a larger message e.g. 'Part 2 of a 4 part message'
SINGLE_OR_MULTIPART = 'Not yet implemented'

# A three letter group to distinguish between different
# types of Enigma traffic
DISCRIMINANT = 'ABC'

# A three letter group, relating to the encoding of the message
INDICATOR_SETTING = 'DEF'

def get_preamble():
    return \
        'Call signs: ' + str(CALL_SIGNS) + '\r\n' + \
        'Date of origin: ' + str(DATE_OF_ORIGIN) + '\r\n' + \
        'Time of origin: ' + str(TIME_OF_ORIGIN) + '\r\n' + \
        'Number of letters: ' + str(NUMBER_OF_LETTERS) + '\r\n' + \
        'Single or multi-part: ' + str(SINGLE_OR_MULTIPART) + '\r\n' + \
        'Discriminant: ' + str(DISCRIMINANT) + '\r\n' + \
        'Indicator setting: ' + str(INDICATOR_SETTING) + '\r\n'