#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" A command line utility for The Cannon. """

import argparse
import logging
import os
from collections import OrderedDict
from numpy import ceil, loadtxt, zeros, nan, diag, ones
from subprocess import check_output
from six.moves import cPickle as pickle
from tempfile import mkstemp
from time import sleep

def fit(model_filename, spectrum_filenames, threads, clobber, from_filename,
    **kwargs):
    """
    Fit a series of spectra.
    """

    import AnniesLasso as tc

    model = tc.load_model(model_filename, threads=threads)
    logger = logging.getLogger("AnniesLasso")
    assert model.is_trained

    chunk_size = kwargs.pop("parallel_chunks", 1000) if threads > 1 else 1
    fluxes = []
    ivars = []
    output_filenames = []
    failures = 0

    fit_velocity = kwargs.pop("fit_velocity", False)

    # MAGIC HACK
    delete_meta_keys = ("fjac", ) # To save space...
    initial_labels = loadtxt("initial_labels.txt")

    if from_filename:
        with open(spectrum_filenames[0], "r") as fp:
            _ = list(map(str.strip, fp.readlines()))
        spectrum_filenames = _

    output_suffix = kwargs.get("output_suffix", None)
    output_suffix = "result" if output_suffix is None else str(output_suffix)
    N = len(spectrum_filenames)
    for i, filename in enumerate(spectrum_filenames):
        logger.info("At spectrum {0}/{1}: {2}".format(i + 1, N, filename))

        basename, _ = os.path.splitext(filename)
        output_filename = "-".join([basename, output_suffix]) + ".pkl"

        if os.path.exists(output_filename) and not clobber:
            logger.info("Output filename {} already exists and not clobbering."\
                .format(output_filename))
            continue

        try:
            with open(filename, "rb") as fp:
                flux, ivar = pickle.load(fp)
                fluxes.append(flux)
                ivars.append(ivar)

            output_filenames.append(output_filename)

        except:
            logger.exception("Error occurred loading {}".format(filename))
            failures += 1

        else:
            if len(output_filenames) >= chunk_size:

                results, covs, metas = model.fit(fluxes, ivars,
                    initial_labels=initial_labels, model_redshift=fit_velocity,
                    full_output=True)

                for result, cov, meta, output_filename \
                in zip(results, covs, metas, output_filenames):

                    for key in delete_meta_keys:
                        if key in meta:
                            del meta[key]

                    with open(output_filename, "wb") as fp:
                        pickle.dump((result, cov, meta), fp, 2) # For legacy.
                    logger.info("Saved output to {}".format(output_filename))

                del output_filenames[0:], fluxes[0:], ivars[0:]

    if len(output_filenames) > 0:

        results, covs, metas = model.fit(fluxes, ivars,
            initial_labels=initial_labels, model_redshift=fit_velocity,
            full_output=True)

        for result, cov, meta, output_filename \
        in zip(results, covs, metas, output_filenames):

            for key in delete_meta_keys:
                if key in meta:
                    del meta[key]

            with open(output_filename, "wb") as fp:
                pickle.dump((result, cov, meta), fp, 2) # For legacy.
            logger.info("Saved output to {}".format(output_filename))

        del output_filenames[0:], fluxes[0:], ivars[0:]

    logger.info("Number of failures: {}".format(failures))
    logger.info("Number of successes: {}".format(N - failures))

    return None

def join_results(output_filename, result_filenames, model_filename=None,
    from_filename=False, clobber=False, errors=False, cov=False, **kwargs):
    """
    Join the test results from multiple files into a single table file.
    """

    import AnniesLasso as tc
    from astropy.table import Table, TableColumns

    meta_keys = kwargs.pop("meta_keys", {})
    meta_keys.update({
        "chi_sq": nan,
        "r_chi_sq": nan,
        "snr": nan,
    #    "redshift": nan,
    })

    logger = logging.getLogger("AnniesLasso")

    # Does the output filename already exist?
    if os.path.exists(output_filename) and not clobber:
        logger.info("Output filename {} already exists and not clobbering."\
            .format(output_filename))
        return None

    if from_filename:
        with open(result_filenames[0], "r") as fp:
            _ = list(map(str.strip, fp.readlines()))
        result_filenames = _

    # We might need the label names from the model.
    if model_filename is not None:
        model = tc.load_model(model_filename)
        assert model.is_trained
        label_names = model.vectorizer.label_names
        logger.warn(
            "Results produced from newer models do not need a model_filename "\
            "to be specified when joining results.")

    else:
        with open(result_filenames[0], "rb") as fp:
            contents = pickle.load(fp)
            if "label_names" not in contents[-1]:
                raise ValueError(
                    "cannot find label names; please provide the model used "\
                    "to produce these results")
            label_names = contents[-1]["label_names"]

    # Load results from each file.
    failed = []
    N = len(result_filenames)

    # Create an ordered dictionary of lists for all the data.
    data_dict = OrderedDict([("FILENAME", [])])
    for label_name in label_names:
        data_dict[label_name] = []

    if errors:
        for label_name in label_names:
            data_dict["E_{}".format(label_name)] = []

    if cov:
        data_dict["COV"] = []

    for key in meta_keys:
        data_dict[key] = []

    # Iterate over all the result filenames
    for i, filename in enumerate(result_filenames):
        logger.info("{}/{}: {}".format(i + 1, N, filename))

        if not os.path.exists(filename):
            logger.warn("Path {} does not exist. Continuing..".format(filename))
            failed.append(filename)
            continue

        with open(filename, "rb") as fp:
            contents = pickle.load(fp)

        assert len(contents) == 3, "You are using some old school version!"

        labels, Sigma, meta = contents

        if Sigma is None:
            Sigma = nan * ones((labels.size, labels.size))

        result = [filename] + list(labels)
        if errors:
            result.extend(diag(Sigma)**0.5)
        if cov:
            result.append(Sigma)
        result += [meta.get(k, v) for k, v in meta_keys.items()]

        for key, value in zip(data_dict.keys(), result):
            data_dict[key].append(value)

    # Warn of any failures.
    if failed:
        logger.warn(
            "The following {} result file(s) could not be found: \n{}".format(
                len(failed), "\n".join(failed)))

    # Construct the table.
    table = Table(TableColumns(data_dict))
    table.write(output_filename, overwrite=clobber)
    logger.info("Written to {}".format(output_filename))

def main():
    """
    The main command line interpreter. This is the console script entry point.
    """

    # Create the main parser.
    parser = argparse.ArgumentParser(
        description="The Cannon", epilog="http://TheCannon.io")

    # Create parent parser.
    parent_parser = argparse.ArgumentParser(add_help=False)
    parent_parser.add_argument("-v", "--verbose",
        dest="verbose", action="store_true", default=False,
        help="Verbose logging mode.")
    parent_parser.add_argument("-t", "--threads",
        dest="threads", type=int, default=1,
        help="The number of threads to use.")

    # Allow for multiple actions.
    subparsers = parser.add_subparsers(title="action", dest="action",
        description="Specify the action to perform.")

    # Fitting parser.
    fit_parser = subparsers.add_parser("fit", parents=[parent_parser],
        help="Fit stacked spectra using a trained model.")
    fit_parser.add_argument("model_filename", type=str,
        help="The path of a trained Cannon model.")
    fit_parser.add_argument("spectrum_filenames", nargs="+", type=str,
        help="Paths of spectra to fit.")
    fit_parser.add_argument("--parallel-chunks", dest="parallel_chunks",
        type=int, default=1000, help="The number of spectra to fit in a chunk.")
    fit_parser.add_argument("--clobber", dest="clobber", default=False,
        action="store_true", help="Overwrite existing output files.")
    fit_parser.add_argument(
        "--output-suffix", dest="output_suffix", type=str,
        help="A string suffix that will be added to the spectrum filenames "\
             "when creating the result filename")
    fit_parser.add_argument("--from-filename", dest="from_filename",
        action="store_true", default=False, help="Read spectrum filenames from file")
    fit_parser.set_defaults(func=fit)

    # Join results parser.
    join_parser = subparsers.add_parser("join", parents=[parent_parser],
        help="Join results from individual stars into a single table.")
    join_parser.add_argument("output_filename", type=str,
        help="The path to write the output filename.")
    join_parser.add_argument("result_filenames", nargs="+", type=str,
        help="Paths of result files to include.")
    join_parser.add_argument("--from-filename",
        dest="from_filename", action="store_true", default=False,
        help="Read result filenames from a file.")
    join_parser.add_argument(
        "--errors", dest="errors", default=False, action="store_true",
        help="Include formal errors in destination table.")
    join_parser.add_argument(
        "--cov", dest="cov", default=False, action="store_true",
        help="Include covariance matrix in destination table.")
    join_parser.add_argument(
        "--clobber", dest="clobber", default=False, action="store_true",
        help="Ovewrite an existing table file.")

    join_parser.set_defaults(func=join_results)

    # Parse the arguments and take care of any top-level arguments.
    args = parser.parse_args()
    if args.action is None: return

    logger = logging.getLogger("AnniesLasso")
    if args.verbose:
        logger.setLevel(logging.DEBUG)

    # Do things.
    return args.func(**args.__dict__)

if __name__ == "__main__":

    """
    Usage examples:
    # tc train model.pickle --condor --chunks 100
    # tc train model.pickle --threads 8
    # tc join model.pickle --from-filename files

    """
    _ = main()


from dacite import from_dict
from elasticsearch import Elasticsearch, NotFoundError

from service.models import FinancialBusinessOffer

class ElasticSearchService:
    def __init__(self, host, port, index, secrets):
        self.es_client = Elasticsearch(
            [{'host': host, 'port': int(port), 'scheme': 'https'}],
            basic_auth=(secrets.ES_USER, secrets.ES_PASSWORD),
        )
        self.index = index

    def get_document_by_id(self, document_id) -> FinancialBusinessOffer:
        try:
            document = dict(
                self.es_client.get(index=self.index, id=document_id)['_source']
            )
            return from_dict(data=document, data_class=FinancialBusinessOffer)
        except NotFoundError:
            print(f'Elastic document Not found for {document_id}')

    def get_batch(self, batch_size):
        return self.es_client.search(index=self.index, size=batch_size)

    def get_all_documents(self):
        return self.es_client.search(index=self.index, size=10000)


import time
from collections import defaultdict

start = None

def print_with_time(string, restart=False):
    global start
    if not start or restart:
        start = time.time()
    elapsed = time.time() - start
    print('{} (+{:.2f}s) {}'.format(time.strftime('%a %H:%M:%S'), elapsed, string))

def read_dfs(file_name):
    dfs = defaultdict(int)
    with open(file_name, 'r') as file:
        num_docs, vocab_size = map(int, file.readline().strip().split())
        for line in file:
            token, count = line.strip().split()
            dfs[token] = int(count)
    assert vocab_size == len(dfs)
    return dfs, num_docs


# Databricks notebook source
# MAGIC %md
# MAGIC
# MAGIC Let us go through ranking using Spark SQL and Pyspark.
# MAGIC * Create Data Frame for Daily Product Revenue
# MAGIC * Create Temporary View for Daily Product Revenue
# MAGIC * Ranking using Spark SQL (using `OVER (PARTITION BY grouping_column ORDER BY sorting_column)`)
# MAGIC * Ranking using Pyspark Data Frame APIs (using functions from `pyspark.sql.window`)

# COMMAND ----------

daily_product_revenue = spark. \
    read. \
    format('delta'). \
    load('dbfs:/public/retail_db/daily_product_revenue')

# COMMAND ----------

daily_product_revenue.createOrReplaceTempView('daily_product_revenue')

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC SHOW tables

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC SELECT count(*) FROM daily_product_revenue

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC SELECT dpr.*,
# MAGIC   dense_rank() OVER (PARTITION BY order_date ORDER BY revenue DESC) AS drnk
# MAGIC FROM daily_product_revenue AS dpr
# MAGIC ORDER BY dpr.order_date, dpr.revenue DESC

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC WITH dpr_ranked AS (
# MAGIC   SELECT dpr.*,
# MAGIC     dense_rank() OVER (PARTITION BY order_date ORDER BY revenue DESC) AS drnk
# MAGIC   FROM daily_product_revenue AS dpr
# MAGIC ) SELECT * FROM dpr_ranked
# MAGIC WHERE drnk <= 5
# MAGIC ORDER BY order_date, revenue DESC

# COMMAND ----------

from pyspark.sql.functions import col

# COMMAND ----------

from pyspark.sql.window import Window

# COMMAND ----------

spec = Window. \
    partitionBy('order_date'). \
    orderBy(col('revenue').desc())

# COMMAND ----------

from pyspark.sql.functions import dense_rank

# COMMAND ----------

daily_product_revenue. \
    withColumn('drnk', dense_rank().over(spec)). \
    filter('drnk <= 5'). \
    orderBy('order_date', col('revenue').desc()). \
    show()

# COMMAND ----------

045 Data Processing using Spark SQL/115 Ranking using Spark SQL Windowing Functions/11 Ranking using Spark SQL and Pyspark.py


import pytest
from flask.testing import FlaskClient
from flask_sqlalchemy import SQLAlchemy

from tests.functional.settings import get_settings
from tests.functional.testdata.users_registration import (test_data_for_registration_successfull,
                              test_data_for_registration_fail)

conf = get_settings()
pytestmark = pytest.mark.asyncio

@pytest.mark.parametrize(
    'request_body, expected_body, expected_answer', test_data_for_registration_successfull
)
async def test_registration_successfull(
        auth_api_client: FlaskClient, sqlalchemy_postgres: SQLAlchemy,
        request_body: dict, expected_body: dict, expected_answer: dict
):
    """
    Тест для проверки успешных запросов на регистрацию.

    Проверяет:
    - наличие пользователя после регистрации в бд
    - успешный ответ от серивиса
    """
    from models.db_models import User

    response = auth_api_client.post(f'/registration', json=request_body)
    response_body = response.json
    user = User.query.filter_by(username=request_body['username']).first()

    assert user.username == request_body['username']
    assert response.status_code == expected_answer['status']
    assert response_body == expected_body

@pytest.mark.parametrize(
    'request_body, expected_body, expected_answer', test_data_for_registration_fail
)
async def test_registration_fail(
        auth_api_client: FlaskClient, sqlalchemy_postgres: SQLAlchemy,
        request_body: dict, expected_body: dict, expected_answer: dict
):
    """
    Тест для проверки ошибки регистрации при существовании пользователя с таким логином.

    Проверяет:
    - неудачный ответ от серивиса
    """
    auth_api_client.post(f'/registration', json=request_body)

    response = auth_api_client.post(f'/registration', json=request_body)
    response_body = response.json

    assert response.status_code == expected_answer['status']
    assert response_body == expected_body


from wx import PySimpleApp, DirDialog, DD_DEFAULT_STYLE, DD_NEW_DIR_BUTTON, ID_OK

def choose_dir():
    """图形方式选择路径"""
    app = PySimpleApp()
    dialog = DirDialog(None, "选择安装目录", style=DD_DEFAULT_STYLE | DD_NEW_DIR_BUTTON)
    if dialog.ShowModal() == ID_OK:
        return dialog.GetPath()

if __name__ == '__main__':
    choose_dir()


from django.shortcuts import render, redirect
from .credentials import REDIRECT_URI, CLIENT_SECRET, CLIENT_ID
from rest_framework.views import APIView
from requests import Request, post
from rest_framework import status
from rest_framework.response import Response
from .util import *

class AuthURL(APIView):
    def get(self, request, fornat=None):
        scopes = 'user-read-playback-state user-modify-playback-state user-read-currently-playing playlist-modify-public playlist-modify-private'

        url = Request('GET', 'https://accounts.spotify.com/authorize', params={
            'scope': scopes,
            'response_type': 'code',
            'redirect_uri': REDIRECT_URI,
            'client_id': CLIENT_ID
        }).prepare().url

        return Response({'url': url}, status=status.HTTP_200_OK)

def spotify_callback(request, format=None):
    code = request.GET.get('code')
    error = request.GET.get('error')

    response = post('https://accounts.spotify.com/api/token', data={
        'grant_type': 'authorization_code',
        'code': code,
        'redirect_uri': REDIRECT_URI,
        'client_id': CLIENT_ID,
        'client_secret': CLIENT_SECRET
    }).json()

    access_token = response.get('access_token')
    token_type = response.get('token_type')
    refresh_token = response.get('refresh_token')
    expires_in = response.get('expires_in')
    error = response.get('error')

    if not request.session.exists(request.session.session_key):
        request.session.create()

    update_or_create_user_tokens(
        request.session.session_key, access_token, token_type, expires_in, refresh_token)

    return redirect('/user')

class IsAuthenticated(APIView):
    def get(self, request, format=None):
        is_authenticated = is_spotify_authenticated(
            self.request.session.session_key)
        return Response({'status': is_authenticated}, status=status.HTTP_200_OK)

class GetPlaylists(APIView):
    def get(self, request, format=None):
        # Send request to Spotify for user's playlists.
        host = self.request.session.session_key
        endpoint = "playlists"
        response = execute_user_spotify_api_request(host, endpoint)

        # Return 204 if error.
        if 'error' in response or 'items' not in response:
            return Response({}, status=status.HTTP_204_NO_CONTENT)

        # Get list of playlists.
        items = response.get('items')

        # Return 204 if user has no playlists.
        if not items:
            return Response({}, status=status.HTTP_204_NO_CONTENT)

        # Collect playlist information into a list.
        playlists = []
        for item in items:
            playlist = {
                'playlist_id': item.get('id'),
                'name': item.get('name'),
                'tracks_total': item.get('tracks').get('total'),
            }
            playlists.append(playlist)

        return Response(playlists, status=status.HTTP_200_OK)

class GetPlaylistInfo(APIView):
    def get(self, request, playlist_id, format=None):
        playlist_info = get_playlist_info(self, playlist_id)

        if playlist_info is None:
            return Response({}, status=status.HTTP_204_NO_CONTENT)
        else:
            return Response(playlist_info, status=status.HTTP_200_OK)

class PutPlaylistInfo(APIView):
    def put(self, request, playlist_id, insert_before, range_length, range_start, format=None):
        print(playlist_id, insert_before, range_length, range_start)
        print("------------------------------------------------------------")
        response = put_playlist_info(self, playlist_id, insert_before,
                                     range_length, range_start)
        return Response({}, status=response.status_code)


'''
방문체크는 왔던 곳 한 번 더 안 오려고!!
벽을 한 번도 안 부셨을 때, 벽을 이미 부수고 지나왔을 때
나눠서 방문 체크 두 번 하기
벽을 한 번 부수고 왔을 때는 갈 수 없는 길이어도(-1 return)
한 번도 부수지 않았을 때는 가능한 길이 될 수 있음
'''
import sys
input = sys.stdin.readline
from collections import deque

N, M = map(int, input().split())
graph = [list(map(int, input().rstrip())) for _ in range(N)]

def bfs(N, M):
    visited = [[[0] * M for _ in range(N)] for _ in range(2)]       # 3차원 배열 (벽을 부시지 않았을 때, 부셨을 때 방문 체크)
    deq = deque([(0, 0, 1, 0)])
    visited[0][0][0] = 1

    di = [-1, 1, 0, 0]
    dj = [0, 0, -1, 1]

    while deq:
        i, j, cnt, broken = deq.popleft()

        if i == N-1 and j == M-1:
            return cnt

        for k in range(4):
            ni = i + di[k]
            nj = j + dj[k]
            if ni in range(N) and nj in range(M):
                if graph[ni][nj] == 0 and not visited[broken][ni][nj]:
                    deq.append((ni, nj, cnt+1, broken))
                    visited[broken][ni][nj] = 1

                elif graph[ni][nj] == 1 and not visited[1][ni][nj]:    # 벽이면 한 번 부수기
                    if broken == 0:
                        deq.append((ni, nj, cnt+1, 1))
                        visited[1][ni][nj] = 1

    return -1

print(bfs(N, M))


import time
import numpy as np
import matplotlib.pyplot as plt
import cv2
import h5py
import os

from tqdm import tqdm
from PIL import Image
from sklearn.preprocessing import StandardScaler

import webdataset as wds
import sys

from nsd_access import NSDAccess
nsda = NSDAccess('/fsx/proj-medarc/fmri/natural-scenes-dataset/s3')

import nibabel as nib

import torch
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("device:",device)

sub=int(sys.argv[1])
preset_cnt=int(sys.argv[2])

subject=f'subj0{sub+1}'
subj=subject
print("subject",subject)

print("preset_cnt",preset_cnt)

# CLIP
import clip
from torchvision import transforms
from models import *
clip_extractor = Clipper("ViT-L/14", device=device, hidden_state=True, norm_embs=False)
clip_extractor_last = Clipper("ViT-L/14", device=device, hidden_state=False, norm_embs=False)

# openclip_extractor = OpenClipper('ViT-H-14', device=device, hidden_state=False, norm_embs=False)
# openclip_extractor_last = OpenClipper('ViT-H-14', device=device, hidden_state=True, norm_embs=False)

# ImageBind
import sys
sys.path.insert(0, "/fsx/proj-medarc/fmri/ImageBind")
sys.path.insert(0, "/fsx/proj-medarc/fmri/ImageBind/models")
import data
import imagebind_model
from imagebind_model import ModalityType

imagebind = imagebind_model.imagebind_huge(pretrained=True)
imagebind.eval().requires_grad_(False)
imagebind.to(device)

imagebind_transform = transforms.Compose(
    [transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC),
    transforms.CenterCrop(224),
    transforms.Normalize(
        mean=(0.48145466, 0.4578275, 0.40821073),
        std=(0.26862954, 0.26130258, 0.27577711))])

import copy
imagebind_hidden = copy.deepcopy(imagebind)
imagebind_hidden.modality_heads.vision = nn.Identity()
imagebind_hidden.modality_postprocessors.vision = nn.Identity()
imagebind_hidden.eval().requires_grad_(False)
imagebind_hidden.to(device)

# FCN segmentation model
from torchvision.models.feature_extraction import create_feature_extractor, get_graph_node_names
from torchvision.models.segmentation import fcn_resnet50, FCN_ResNet50_Weights
fcn_weights = FCN_ResNet50_Weights.COCO_WITH_VOC_LABELS_V1
fcn_model = create_feature_extractor(fcn_resnet50(weights=fcn_weights), return_nodes=["backbone.maxpool"]).to(device)
fcn_model.eval().requires_grad_(False)
pass

# NOTE: you will want to parallelize this
print(time.strftime("\nCurrent time: %H:%M:%S", time.localtime()))
total_cnt = 100

lh_file = f"/fsx/proj-medarc/fmri/natural-scenes-dataset/algonauts_data/dataset/{subject}/roi_masks/lh.all-vertices_fsaverage_space.npy"
lh_mask = np.load(lh_file)
rh_file = f"/fsx/proj-medarc/fmri/natural-scenes-dataset/algonauts_data/dataset/{subject}/roi_masks/rh.all-vertices_fsaverage_space.npy"
rh_mask = np.load(rh_file)
mask = np.hstack((lh_mask,rh_mask))
mask = np.asarray(mask,bool)

# load coco 73k indices
indices_path = "/fsx/proj-medarc/fmri/natural-scenes-dataset/COCO_73k_subj_indices.hdf5"
hdf5_file = h5py.File(indices_path, "r")
indices = hdf5_file[f"{subj}"][:]

# load orig images
f = h5py.File('/fsx/proj-medarc/fmri/natural-scenes-dataset/s3/nsddata_stimuli/stimuli/nsd/nsd_stimuli.hdf5', 'r')
images = f['imgBrick']

# load shared 73k indices
import scipy.io as sio
nsd_design = sio.loadmat('/fsx/proj-medarc/fmri/natural-scenes-dataset/s3/nsddata/experiments/nsd/nsd_expdesign.mat')
shared_idx = nsd_design['sharedix'][0]-1
order = nsd_design['subjectim'][sub]-1

# for each unique image, get the betas and average
all_betas = None
cnt = 0; abs_cnt = 0;
for uniq_idx in tqdm(np.unique(indices)):
    if abs_cnt==preset_cnt:
        trial_numbers = np.where(indices==uniq_idx)[0]
        for itrial, trial in enumerate(trial_numbers):
            sess = int(np.floor(trial / 750))
            sess_trial = trial - (sess * 750)
            sess += 1

            betas = nsda.read_betas(subject=subject,
                    session_index=int(sess),
                    trial_index=[int(sess_trial)], # empty list as index means get all for this session
                    data_type='betas_fithrf_GLMdenoise_RR',
                    data_format='fsaverage')

            # load mean + scale to get zero mean and unit variance depending on the given session
            betas_nsd_mean = np.load(f"/fsx/proj-medarc/fmri/natural-scenes-dataset/challenge_scalars/challenge_mean_sess{sess}_{subj}.npy")
            betas_nsd_std = np.load(f"/fsx/proj-medarc/fmri/natural-scenes-dataset/challenge_scalars/challenge_std_sess{sess}_{subj}.npy")

            betas_nsd = betas[mask]
            shape = betas_nsd.shape
            betas_nsd = betas_nsd.reshape(-1,betas_nsd.shape[-2]*betas_nsd.shape[-1])
            betas_nsd = (betas_nsd - betas_nsd_mean) / betas_nsd_std
            betas_nsd = betas_nsd.astype('float16') # (1, 15724)

            if itrial==0:
                same_betas_nsd = betas_nsd
            else:
                same_betas_nsd = np.concatenate((same_betas_nsd,betas_nsd),axis=0)

        if same_betas_nsd.shape[0] == 1:
            num_unique = 1
            same_betas_nsd = np.concatenate((same_betas_nsd, same_betas_nsd, same_betas_nsd), axis=0)
        elif same_betas_nsd.shape[0] == 2:
            num_unique = 2
            same_betas_nsd = np.concatenate((same_betas_nsd, np.mean(same_betas_nsd,axis=0)[None]), axis=0)
        else:
            num_unique = 3

        if all_betas is None:
            all_betas = same_betas_nsd[None]
            ordering = uniq_idx
            ordered_imgs = trial_numbers[0]
            num_uniques = num_unique
        else:
            all_betas = np.concatenate((all_betas,same_betas_nsd[None]),axis=0)
            ordering = np.append(ordering,uniq_idx)
            ordered_imgs = np.append(ordered_imgs, trial_numbers[0])
            num_uniques = np.append(num_uniques, num_unique)

    cnt += 1
    if cnt==total_cnt:
        if abs_cnt==preset_cnt:
            sink = wds.TarWriter(f"/fsx/proj-medarc/fmri/natural-scenes-dataset/algonauts_data/wds/{subj}_{abs_cnt}.tar")
            for i,idx in enumerate(range(total_cnt*preset_cnt,total_cnt*preset_cnt+total_cnt)):
                image = Image.fromarray(images[ordering[i]].astype('uint8').clip(0,255))
                with torch.no_grad():
                    cur_imgs_tensor = torch.Tensor(images[ordering[i]]).permute(2,0,1).unsqueeze(0).to(device) / 255
                    ib_inputs = {ModalityType.VISION: imagebind_transform(cur_imgs_tensor)}
                    sink.write({
                        "__key__": "sample%09d" % idx,
                        "num_uniques.npy": np.array([num_uniques[i]]),
                        "coco73k.npy": np.array([ordering[i]]),
                        "vert.npy": all_betas[i],
                        "trial.npy": np.array([ordered_imgs[i]]),
                        "jpg": image,
                        "clip_emb_hidden.npy": clip_extractor.embed_image(cur_imgs_tensor).detach().cpu().numpy(),
                        "clip_emb_final.npy": clip_extractor_last.embed_image(cur_imgs_tensor).detach().cpu().numpy(),
                        "imagebind_hidden.npy": imagebind_hidden(ib_inputs)['vision'].detach().cpu().numpy(),
                        "imagebind_final.npy": imagebind(ib_inputs)['vision'].detach().cpu().numpy(),
                        "fcn_maxpool.npy": fcn_model(cur_imgs_tensor)['backbone.maxpool'].detach().cpu().numpy(),
                    })
            sink.close()

        all_betas = None
        cnt = 0
        abs_cnt += 1

if abs_cnt==preset_cnt: # for the last batch, won't be equal to exactly total_cnt probably
    sink = wds.TarWriter(f"/fsx/proj-medarc/fmri/natural-scenes-dataset/algonauts_data/wds/{subj}_{abs_cnt}.tar")
    for i,idx in enumerate(range(total_cnt*preset_cnt,total_cnt*preset_cnt+total_cnt)):
        image = Image.fromarray(images[ordering[i]].astype('uint8').clip(0,255))
        with torch.no_grad():
            cur_imgs_tensor = torch.Tensor(images[ordering[i]]).permute(2,0,1).unsqueeze(0).to(device) / 255
            ib_inputs = {ModalityType.VISION: imagebind_transform(cur_imgs_tensor)}
            sink.write({
                "__key__": "sample%09d" % idx,
                "num_uniques.npy": np.array([num_uniques[i]]),
                "coco73k.npy": np.array([ordering[i]]),
                "vert.npy": all_betas[i],
                "trial.npy": np.array([ordered_imgs[i]]),
                "jpg": image,
                "clip_emb_hidden.npy": clip_extractor.embed_image(cur_imgs_tensor).detach().cpu().numpy(),
                "clip_emb_final.npy": clip_extractor_last.embed_image(cur_imgs_tensor).detach().cpu().numpy(),
                "imagebind_hidden.npy": imagebind_hidden(ib_inputs)['vision'].detach().cpu().numpy(),
                "imagebind_final.npy": imagebind(ib_inputs)['vision'].detach().cpu().numpy(),
                "fcn_maxpool.npy": fcn_model(cur_imgs_tensor)['backbone.maxpool'].detach().cpu().numpy(),
            })
    sink.close()

print(time.strftime("\nCurrent time: %H:%M:%S", time.localtime()))


# -*- coding: utf-8 -*-
"""
Created on Wed Dec 14 22:09:47 2022

@author: Shahir
"""

import os
import pickle
from typing import Any, Optional

import torch
import torch.nn as nn
import torch.optim as optim

from cloudsift.utils import get_timestamp_str

class ModelTracker:
    def __init__(
            self,
            root_dir: str) -> None:

        experiment_dir = "Experiment {}".format(get_timestamp_str())
        self.save_dir = os.path.join(root_dir, experiment_dir)
        self.best_model_metric = float('-inf')
        self.record_per_epoch = {}

    def update_info_history(
            self,
            epoch: int,
            info: Any) -> None:

        os.makedirs(self.save_dir, exist_ok=True)
        self.record_per_epoch[epoch] = info
        fname = "Experiment Epoch Info History.pckl"
        with open(os.path.join(self.save_dir, fname), 'wb') as f:
            pickle.dump(self.record_per_epoch, f)

    def update_model_weights(
            self,
            epoch: int,
            model_state_dict: dict,
            metric: Optional[float] = None,
            save_best: bool = True,
            save_latest: bool = True,
            save_current: bool = False) -> None:

        os.makedirs(self.save_dir, exist_ok=True)
        update_best = metric is None or metric > self.best_model_metric
        if update_best and metric is not None:
            self.best_model_metric = metric

        if save_best and update_best:
            torch.save(model_state_dict, os.path.join(self.save_dir,
                                                      "Weights Best.pckl"))
        if save_latest:
            torch.save(model_state_dict, os.path.join(self.save_dir,
                                                      "Weights Latest.pckl"))
        if save_current:
            torch.save(model_state_dict, os.path.join(self.save_dir,
                                                      "Weights Epoch {} {}.pckl".format(epoch, get_timestamp_str())))

def make_optimizer(
        params_to_update: list[torch.Tensor],
        lr: float = 0.001,
        weight_decay: float = 1e-9,
        clip_grad_norm: bool = False) -> optim.Optimizer:

    optimizer = optim.AdamW(
        params_to_update,
        lr=lr,
        betas=(0.9, 0.999),
        eps=1e-08,
        weight_decay=weight_decay,
        amsgrad=True)

    if clip_grad_norm:
        nn.utils.clip_grad_norm_(params_to_update, 3.0)

    return optimizer

def get_lr(
        optimizer: optim.Optimizer) -> None:

    for param_group in optimizer.param_groups:
        return param_group['lr']

def set_optimizer_lr(
        optimizer: optim.Optimizer,
        lr: float) -> None:

    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def make_scheduler(
        optimizer: optim.Optimizer) -> optim.lr_scheduler.ReduceLROnPlateau:

    scheduler = optim.lr_scheduler.ReduceLROnPlateau(
        optimizer, factor=0.5, threshold=0.025, patience=10, cooldown=5, min_lr=1e-6, verbose=True)

    return scheduler

def shrink_and_preturb(
        base_net: nn.Module,
        new_net: nn.Module,
        shrink: float = 0.5,
        perturb: float = 0.1) -> nn.Module:

    params1 = base_net.parameters()
    params2 = new_net.parameters()
    with torch.set_grad_enabled(False):
        for p1, p2 in zip(params1, params2):
            p1.mul_(shrink).add_(p2, alpha=perturb)

    return base_net

def load_weights(
        model: nn.Module,
        weights_fname,
        map_location=None) -> nn.Module:

    model.load_state_dict(torch.load(weights_fname, map_location=map_location))

    return model

def save_training_session(
        model: nn.Module,
        optimizer: optim.Optimizer,
        sessions_save_dir: str) -> str:

    sub_dir = "Session {}".format(get_timestamp_str())
    sessions_save_dir = os.path.join(sessions_save_dir, sub_dir)
    os.makedirs(sessions_save_dir, exist_ok=True)

    torch.save(model.state_dict(), os.path.join(
        sessions_save_dir, "Model State.pckl"))
    torch.save(optimizer.state_dict(), os.path.join(
        sessions_save_dir, "Optimizer State.pckl"))
    print("Saved session to", sessions_save_dir)

    return sessions_save_dir

def load_training_session(
        model: nn.Module,
        optimizer: optim.Optimizer,
        session_dir: str,
        update_models: bool = True,
        map_location: Optional[torch.device] = None) -> dict[str, Any]:

    if update_models:
        model.load_state_dict(torch.load(os.path.join(
            session_dir, "Model State.pckl"), map_location=map_location))
        optimizer.load_state_dict(
            torch.load(os.path.join(session_dir, "Optimizer State.pckl"), map_location=map_location))

    print("Loaded session from", session_dir)

    out_data = {
        'model': model,
        'optimizer': optimizer
    }

    return out_data


from typing import Tuple, Optional

import aiohttp
import orjson

from polyschedule.exceptions import GettingScheduleError

async def _request_schedule_json(group_id, date) -> dict:
    async with aiohttp.ClientSession() as session:
        async with session.get(f'https://ruz.spbstu.ru/api/v1/ruz/scheduler/{group_id}?date={date}') as response:
            return orjson.loads(await response.text())

async def get_week(group_id, date) -> Tuple[dict, Optional[list]]:
    week_json = await _request_schedule_json(group_id, date)

    if week_json.get('error'):
        raise GettingScheduleError(week_json['text'])

    week_info = week_json['week']
    week_schedule = week_json['days']

    return week_info, week_schedule or None

async def get_day(group_id, date) -> Optional[dict]:
    _, week_schedule = await get_week(group_id, date)

    if not week_schedule:
        return None

    matched_day = [i for i in week_schedule if i['date'] == date]

    return matched_day[0] if matched_day else None


#!/usr/bin/env python3
"""
analyze and plot statistics from load test
"""

# standard library modules
import argparse
import logging
import math
import sys
# third-party modules
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import json
import os
import platform
import numpy as np
import math

logger = logging.getLogger(__name__)

def getColumn(inputList,column):
    return [inputList[i][column] for i in range(0,len(inputList))]

def makeTimelyXTicks():
    # x-axis tick marks at multiples of 60 and 10
    ax = plt.gca()
    ax.xaxis.set_major_locator( mpl.ticker.MultipleLocator(60) )
    ax.xaxis.set_minor_locator( mpl.ticker.MultipleLocator(10) )

#mpl.rcParams.update({'font.size': 28})
#mpl.rcParams['axes.linewidth'] = 2 #set the value globally

if __name__ == "__main__":
    # configure logger formatting
    logFmt = '%(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s'
    logDateFmt = '%Y/%m/%d %H:%M:%S'
    formatter = logging.Formatter(fmt=logFmt, datefmt=logDateFmt )
    logging.basicConfig(format=logFmt, datefmt=logDateFmt)
    logger.setLevel(logging.INFO)
    logger.debug('the logger is configured')

    ap = argparse.ArgumentParser( description=__doc__, fromfile_prefix_chars='@', formatter_class=argparse.ArgumentDefaultsHelpFormatter )
    ap.add_argument( '--statsFilePath', default='G:/Share/Neo/loadtest/data/2019-08-13T14/locustStats.csv', help='the path to locustStats csv file' )
    ap.add_argument( '--launchedFilePath', default='G:/Share/Neo/loadtest/data/2019-08-13T14/launched.json', help='the path to launched json file' )
    ap.add_argument( '--mapFilePath', default='WorldCountryBoundaries.csv', help='the path to launched json file' )
    ap.add_argument( '--outDirPath', default='data', help='the path to to dir for output' )
    args = ap.parse_args()

    statsFilePath = args.statsFilePath
    launchedFilePath = args.launchedFilePath
    outDirPath = args.outDirPath

    os.makedirs( outDirPath, exist_ok=True )

    figSize1 = (10,6)
    #figSize1 = None
    fontFactor = .5

    logger.info("Reading World Map data")
    #mapFileName = "C:\\neocortix\\CloudServices\\2019-08-10_LoadTestData\\WorldCountryBoundaries.csv"
    mapFileName = args.mapFilePath
    mapFile = open(mapFileName, "r")
    mapLines = mapFile.readlines()
    mapFile.close()
    mapNumLines = len(mapLines)

    CountryData = []
    CountrySphericalData = []

    # for i in range(1,8) :
    for i in range(1,mapNumLines) :
        firstSplitString = mapLines[i].split("\"")
        nonCoordinateString = firstSplitString[2]
        noncoordinates = nonCoordinateString.split(",")
        countryString = noncoordinates[6]

        if firstSplitString[1].startswith('<Polygon><outerBoundaryIs><LinearRing><coordinates>') and firstSplitString[1].endswith('</coordinates></LinearRing></outerBoundaryIs></Polygon>'):
            coordinateString = firstSplitString[1].replace('<Polygon><outerBoundaryIs><LinearRing><coordinates>','').replace('</coordinates></LinearRing></outerBoundaryIs></Polygon>','').replace(',0 ',',0,')
            # print("coordinateString = %s" % coordinateString)
            # print("nonCoordinateString = %s" % nonCoordinateString)
            coordinates = [float(j) for j in coordinateString.split(",")]
            coordinateList = np.zeros([int(len(coordinates)/3),2])
            for j in range(0,len(coordinateList)) :
                coordinateList[j,:] = coordinates[j*3:j*3+2]
            coordinateSphericalList = np.zeros([int(len(coordinates)/3),3])
            for j in range(0,len(coordinateSphericalList)) :
                r = 1
                phi = 2*math.pi*coordinates[j*3]/360
                theta = 2*math.pi*(90-coordinates[j*3+1])/360
                coordinateSphericalList[j,0] = r * np.sin(theta) * np.cos(phi)
                coordinateSphericalList[j,1] = r * np.sin(theta) * np.sin(phi)
                coordinateSphericalList[j,2] = r * np.cos(theta)

            # print("noncoordinates = %s" % str(noncoordinates))
            # print("countryString = %s" % countryString)
            # print("coordinateList = %s" % str(coordinateList))
            CountryData.append([countryString,coordinateList])
            CountrySphericalData.append([countryString,coordinateSphericalList])
        else :
            # print("Exception Line %i  %s" % (i,countryString))
            # if firstSplitString[1].startswith("<MultiGeometry>") :
            #     print("MultiGeometry  Line %i  %s" % (i,countryString))
            # else :
            #     print("Inner Boundary Line %i  %s" % (i,countryString))
            reducedCoordinateString = firstSplitString[1].replace('<MultiGeometry>','').replace('</MultiGeometry>','').replace('<Polygon>','').replace('</Polygon>','').replace('<outerBoundaryIs>','').replace('</outerBoundaryIs>','').replace('<innerBoundaryIs>','').replace('</innerBoundaryIs>','').replace('<LinearRing>','').replace('</LinearRing>','').replace('</coordinates>','').replace(',0 ',',0,')
            # print("reducedCoordinateString = %s" % reducedCoordinateString)
            coordinateStringSets = reducedCoordinateString.split("<coordinates>")
            # print("coordinateStringSets = %s" % str(coordinateStringSets))
            coordinateSets= []
            for j in range(1,len(coordinateStringSets)) :
                coordinateSets.append([float(k) for k in coordinateStringSets[j].split(",")])
            # print("coordinateSets = %s" % str(coordinateSets))
            coordinateList = []
            coordinateSphericalList = []
            for j in range(0,len(coordinateSets)) :
                # print("\ncoordinateSets[%i] = %s" % (j,str(coordinateSets[j])))
                coordinateList.append(np.zeros([int(len(coordinateSets[j])/3),2]))
                for k in range(0,len(coordinateList[j])) :
                    coordinateList[j][k,:] = coordinateSets[j][k*3:k*3+2]
                # print("\ncoordinateList[%i] = %s" % (j,str(coordinateList[j])))
                coordinateSphericalList.append(np.zeros([int(len(coordinateSets[j])/3),3]))
                for k in range(0,len(coordinateSphericalList[j])) :
                    r = 1
                    phi = 2*math.pi*coordinateSets[j][k*3]/360
                    theta = 2*math.pi*(90-coordinateSets[j][k*3+1])/360
                    coordinateSphericalList[j][k,0] = r * np.sin(theta) * np.cos(phi)
                    coordinateSphericalList[j][k,1] = r * np.sin(theta) * np.sin(phi)
                    coordinateSphericalList[j][k,2] = r * np.cos(theta)

            CountryData.append([countryString,coordinateList])
            CountrySphericalData.append([countryString,coordinateSphericalList])

    logger.info("Reading launched json file")
    #launchedFilePath = 'C:\\neocortix\\CloudServices\\2019-08-10_LoadTestData\\2019-08-11_launched.json'
    #launchedFilePath = 'data/2019-08-13T14/launched.json'
    with open( launchedFilePath, 'r') as jsonInFile:
        launched = json.load(jsonInFile)
    # print(len(launched))
    # print(launched[0])
    # print(launched[0]['instanceId'])
    # print(launched[0]['device-location']['country'])

    instanceIdsAndCountries = []
    for i in range(0,len(launched)):
        instanceIdsAndCountries.append([launched[i]['instanceId'],launched[i]['device-location']['country'],launched[i]['device-location']['latitude'],launched[i]['device-location']['longitude']])

    # for i in range(0,len(launched)):
    #     print(instanceIdsAndCountries[i])

    logger.info("Reading locustStats csv file")

    #rawStats = pd.read_csv( 'G:/Share/Neo/loadtest/data/locustStats_examples/locustStats_small.csv' )
    #rawStats = pd.read_csv( 'G:/Share/Neo/loadtest/data/locustStats_examples/locustStats_big.csv' )
    # rawStats = pd.read_csv( 'G:/Share/Neo/loadtest/data/locustStats_examples/locustStats_08-06.csv' )
    # rawStats = pd.read_csv( 'C:\\neocortix\\CloudServices\\2019-08-10_LoadTestData\\locustStats_08-06.csv' )
    #rawStats = pd.read_csv( 'C:\\neocortix\\CloudServices\\2019-08-10_LoadTestData\\2019-08-11_locustStats.csv' )
    rawStats = pd.read_csv( statsFilePath )
    rawStats = rawStats[rawStats.nFails == 0].reset_index()

    # parse calculable time values from strings
    rawStats['startPdts'] = pd.to_datetime( rawStats.dateTime )
    unixTimestamps = rawStats.startPdts.map( lambda x: x.to_pydatetime().timestamp())
    rawStats['startRelTime'] = unixTimestamps - unixTimestamps.min()
    rawStats['endRelTime'] = rawStats['startRelTime']+3

#     print(len(rawStats))
#     print(len(rawStats.mspr))
#     print(rawStats.mspr[0:10])
#     print(rawStats['startPdts'][0:10])

    logger.info("Analyzing location data")

    startRelTimesAndPhoneIDsAndMSPRsAndCountries = []
    startRelTimesAndMSPRsUnitedStates = []
    startRelTimesAndMSPRsRussia = []
    startRelTimesAndMSPRsOther = []
    startRelTimesAndMSPRsUnitedStatesLoaded = []
    startRelTimesAndMSPRsRussiaLoaded = []
    startRelTimesAndMSPRsOtherLoaded = []
    clipTimeInMs = 3000
    startRelTimeMin = 150
    startRelTimeMax = 250

    for i in range(0,len(rawStats)):
        worker = rawStats.worker[i][7:]
        if worker in getColumn(instanceIdsAndCountries,0):
            index = getColumn(instanceIdsAndCountries,0).index(worker)
            country = getColumn(instanceIdsAndCountries,1)[index]
            if rawStats['startRelTime'][i] <= clipTimeInMs:
                startRelTimesAndPhoneIDsAndMSPRsAndCountries.append([rawStats['startRelTime'][i],rawStats.worker[i],rawStats.mspr[i],country])
                if country == 'United States':
                    startRelTimesAndMSPRsUnitedStates.append([rawStats['startRelTime'][i],rawStats.mspr[i]])
                    if rawStats['startRelTime'][i]>=startRelTimeMin and rawStats['startRelTime'][i]<=startRelTimeMax:
                        startRelTimesAndMSPRsUnitedStatesLoaded.append([rawStats['startRelTime'][i],rawStats.mspr[i]])
                elif country == 'Russia':
                    startRelTimesAndMSPRsRussia.append([rawStats['startRelTime'][i],rawStats.mspr[i]])
                    if rawStats['startRelTime'][i]>=startRelTimeMin and rawStats['startRelTime'][i]<=startRelTimeMax:
                        startRelTimesAndMSPRsRussiaLoaded.append([rawStats['startRelTime'][i],rawStats.mspr[i]])
                else:
                    startRelTimesAndMSPRsOther.append([rawStats['startRelTime'][i],rawStats.mspr[i]])
                    if rawStats['startRelTime'][i]>=startRelTimeMin and rawStats['startRelTime'][i]<=startRelTimeMax:
                        startRelTimesAndMSPRsOtherLoaded.append([rawStats['startRelTime'][i],rawStats.mspr[i]])

#     print(len(startRelTimesAndPhoneIDsAndMSPRsAndCountries))
#     for i in range(0,100):
#         print(startRelTimesAndPhoneIDsAndMSPRsAndCountries[i])

#     print(len(startRelTimesAndMSPRsUnitedStates))
#     print(len(startRelTimesAndMSPRsRussia))
#     print(len(startRelTimesAndMSPRsOther))

    # index the data by start timne, for efficient selection
    istats = rawStats.set_index( 'startRelTime', drop=False )
    istats = istats.sort_index()

    nrThresh = 0*10000 # threshold below which frames have too few requests
    windowLen = 6
    stepSize = 1
    endTime = math.floor( istats.startRelTime.max() )

    logger.info("Doing Temporal Integration")

    # temporal integration loop
    dicts=[]  # list of integrated data records
    for xx in range( windowLen, endTime, stepSize ):
        startRelTime = xx-windowLen
        subset = istats.loc[ xx-windowLen : xx ]
        nr = subset.nr.sum()
        rpsMean = nr / windowLen if nr else float('nan')
        if nr <= nrThresh:
            msprMed = float('nan')
        else:
            msprMed = subset.msprMed.median()
        if nr <= nrThresh:
            msprMean = float('nan')
        else:
            msprMean = (subset.mspr * subset.nr).sum() / subset.nr.sum()

        nFails = subset.nFails.sum()
        if nr <= nrThresh:
            failRate = float('nan')
        else:
            failRate = nFails / nr if nr else 0

        nw = len(subset.worker.unique() )
        upwMean = subset.nUsers.mean()
        nUsersMean = upwMean * nw

        dicts.append( {'startRelTime': xx-windowLen, 'endRelTime': xx, 'nr': nr,
            'rps': rpsMean, 'msprMed': msprMed, 'msprMean': msprMean,
            'nWorkers': nw, 'nUsersMean': nUsersMean,
            'failRate': failRate, 'upwMean': upwMean } )
    # convert to dataframe
    outDf = pd.DataFrame( dicts )
    outDf.to_csv( outDirPath+'/integratedStats.csv' )

    logger.info("Plotting")

    plt.figure()
    plt.plot( outDf.startRelTime, outDf.nWorkers )
    plt.title("# workers present" )
    plt.savefig( outDirPath+'/nWorkers.png' )

    plt.figure()
    plt.plot( outDf.startRelTime, outDf.nUsersMean )
    plt.title("# simulated users" )
    plt.savefig( outDirPath+'/simulatedUsers.png' )

    if False:
        # pandas-style plotting
        outDf.plot( x='startRelTime', y=['rps','msprMed', 'msprMean', 'failRate'], subplots=True )
        # x-axis tick marks
        makeTimelyXTicks()

        # pyplot-style plotting
        # plt.figure()
        plt.figure(3,figsize=(25,20))
        plt.plot( outDf.startRelTime, outDf.msprMean )
        plt.plot( outDf.startRelTime, outDf.msprMed )
        makeTimelyXTicks()

    plt.figure(figsize=figSize1)  # was figure 2
    plt.plot( outDf.startRelTime, outDf.rps, linewidth=5, color=(0.0, 0.6, 1.0) )
    makeTimelyXTicks()
    plt.xlim([0, outDf.startRelTime.max()+10])
    plt.title("Delivered Load During Test\n", fontsize=42*fontFactor)
    plt.xlabel("Time during Test (s)", fontsize=32*fontFactor)
    plt.ylabel("Requests per second", fontsize=32*fontFactor)
    #plt.show()
    plt.savefig( outDirPath+'/rps.png' )

    plt.figure(figsize=figSize1)  # was figure 10
    plt.plot(getColumn(startRelTimesAndMSPRsUnitedStates,0),getColumn(startRelTimesAndMSPRsUnitedStates,1), linestyle='', color=(0.0, 0.6, 1.0),marker='o',markersize=2)
    plt.plot(getColumn(startRelTimesAndMSPRsRussia,0),getColumn(startRelTimesAndMSPRsRussia,1), linestyle='', color=(1.0, 0.0, 0.0),marker='o',markersize=2)
    plt.plot(getColumn(startRelTimesAndMSPRsOther,0),getColumn(startRelTimesAndMSPRsOther,1), linestyle='', color=(0.0, 1.0, 0.0),marker='o',markersize=2)
    plt.ylim([0,clipTimeInMs])
    plt.title("Response Times (ms)\n", fontsize=42*fontFactor)
    plt.xlabel("Time during Test (s)", fontsize=32*fontFactor)
    plt.ylabel("Response Times (ms)", fontsize=32*fontFactor)
    # plt.savefig(outputPath + "\\plot_benchmarkVsDPR.png",bbox_inches='tight')
    #plt.show()
    # plt.clf()
    # plt.close()
    plt.savefig( outDirPath+'/msprScatter1.png' )

    if True:
        plt.figure(figsize=figSize1)  # was figure 4
        plt.hist(getColumn(startRelTimesAndMSPRsUnitedStates,1),bins=4000, density=False, facecolor=(0,0.2,1), alpha=0.75)
        plt.hist(getColumn(startRelTimesAndMSPRsRussia,1),bins=4000, density=False, facecolor=(1.0,0.0,0), alpha=0.75)
        plt.hist(getColumn(startRelTimesAndMSPRsOther,1),bins=4000, density=False, facecolor=(0,1.0,0), alpha=0.75)
        plt.xlim([0,clipTimeInMs])
        plt.title("Duration Histograms (Overall)\n", fontsize=32*fontFactor)
        plt.xlabel("Response Time (ms)", fontsize=32*fontFactor)
        plt.ylabel("Number of Occurrences", fontsize=32*fontFactor)
        props={'size': 32*fontFactor}
        plt.gca().legend(["United States","Russia","Other"],loc='center right', prop=props, bbox_to_anchor=(1, 0.50))
        # plt.savefig(outputPath + "\\plot_durationHistogram.png",bbox_inches='tight')
        #plt.show()
        # plt.clf()
        # plt.close()
        plt.savefig( outDirPath+'/durationHistogram.png' )

    plt.figure(figsize=figSize1)  # was figure 5
    plt.hist(getColumn(startRelTimesAndMSPRsUnitedStatesLoaded,1),bins=4000, density=False, facecolor=(0,0.2,1), alpha=0.75)
    plt.hist(getColumn(startRelTimesAndMSPRsRussiaLoaded,1),bins=4000, density=False, facecolor=(1.0,0.0,0), alpha=0.75)
    plt.hist(getColumn(startRelTimesAndMSPRsOtherLoaded,1),bins=4000, density=False, facecolor=(0,1.0,0), alpha=0.75)
    plt.xlim([0,clipTimeInMs])
    plt.title("Duration Histograms (Loaded)\n", fontsize=32*fontFactor)
    plt.xlabel("Response Time (ms)", fontsize=32*fontFactor)
    plt.ylabel("Number of Occurrences", fontsize=32*fontFactor)
    props={'size': 32*fontFactor}
    plt.gca().legend(["United States","Russia","Other"],loc='center right', prop=props, bbox_to_anchor=(1, 0.50))
    # plt.savefig(outputPath + "\\plot_durationHistogram.png",bbox_inches='tight')
    #plt.show()
    # plt.clf()
    # plt.close()
    plt.savefig( outDirPath+'/durationHistogramLoaded.png' )

    if True :

        # plot filled boundaries
        fig = plt.figure(figsize=figSize1)  # was figure 3
        ax = fig.gca()
        # Turn off tick labels
        ax.set_yticklabels([])
        ax.set_xticklabels([])
        # ax.set_aspect('equal')
        # for i in range(0,20) :
        colorValue = 0.85
        for i in range(0,len(CountryData)) :
            if len(np.shape(CountryData[i][1]))==2 :
                # plt.plot(np.transpose(CountryData[i][1])[0],np.transpose(CountryData[i][1])[1])
                ax.add_artist(plt.Polygon(CountryData[i][1],edgecolor='None', facecolor=(colorValue,colorValue,colorValue),aa=True))

            else :
                # print("%s       %s" % (CountryData[i][0],np.shape(CountryData[i][1])[0]))
                for j in range(0,np.shape(CountryData[i][1])[0]) :
                    # print("%s" % CountryData[i][1][j])
                    # plt.plot(np.transpose(CountryData[i][1][j])[0],np.transpose(CountryData[i][1][j])[1])
                    ax.add_artist(plt.Polygon(CountryData[i][1][j],edgecolor='None', facecolor=(colorValue,colorValue,colorValue),aa=True))
        plt.plot(getColumn(instanceIdsAndCountries,3),getColumn(instanceIdsAndCountries,2),linestyle='', color=(0.0, 0.5, 1.0),marker='o',markersize=15*fontFactor)
        plt.xlim([-180,180])
        plt.ylim([-60,90])
        #plt.show()
        plt.savefig( outDirPath+'/countryData.png' )

    logger.info("finished")


from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.shortcuts import render, redirect, get_object_or_404
from question_box.models import Question, Answer, Favorite, Category
from users.models import User
from question_box.forms import QuestionForm, AnswerForm
from django.utils.text import slugify
from django.http import JsonResponse
import json
from django.contrib import messages

@login_required
def question_list(request):
    questions = Question.objects.all()
    # favorite_questions = get_user_favorite(request)
    return render(request, 'core/question_list.html', {'questions': questions})

def question_details(request, pk):
    question = get_object_or_404(Question, pk=pk)
    answers = Answer.objects.filter(response=question.pk)
    if request.method == "POST":
        form = AnswerForm(request.POST)
        if form.is_valid():
            answer = form.save(commit=False)
            answer.response = question
            answer.save()
            return redirect('question-details', pk=pk)
    else:
        form = AnswerForm()
        return render(request, 'core/question_details.html', {'question': question, 'answers':answers, 'form': form, 'user':request.user})

def question_add(request):
    if request.method == "POST":
        form = QuestionForm(request.POST)
        if form.is_valid():
            question = form.save(commit=False)
            question.save()
            return redirect('question-list')
    else:
        form = QuestionForm()
        return render(request, 'core/question_add.html', {'form': form})

# @csrf_exempt
# def question_add(request):
#     data = json.loads(request.body.decode('utf-8'))
#     question_body = data.get("questionBody")
#     question_title = data.get("questionTitle")
#     new_question = Question.objects.create(title=question_title, body=question_body,)
#     if question_body and question_title:
#         return JsonResponse(
#             {
#         "status":"ok",
#         "data": {
#             "pk": new_question.pk,
#             "title": new_question.title,
#             "body": new_question.body,
#         },
#     })

def question_delete(request, pk):
    question = get_object_or_404(Question, pk=pk)
    if question.creator.pk == request.user.pk:
        question.delete()
        return redirect('question-list')
    else:
        messages.add_message(request, messages.INFO, 'You do not have the authority to delete this question.')

# def answer_add(request, pk):
#     question = get_object_or_404(Question, pk=pk)
#     answers = Answer.objects.filter(response=question.pk)
#     if request.method == "POST":
#         form = AnswerForm(request.POST)
#         if form.is_valid():
#             answer = form.save(commit=False)
#             answer.save()
#             return redirect('question-details')
#     else:
#         form = AnswerForm()
#     return render(request, 'core/answer_add.html', {'form': form})

def tagged(request, slug):
    # Filter books by tag name
    tag = Category.objects.get(slug=slug)
    questions = Question.objects.filter(tag=tag)
    return render(request, 'core/question_list.html', {'tag': tag, 'questions': questions})

# ********trying to make favorites work************

def favorite(request, question_pk):
    # for favorite in books:
    question = get_object_or_404(Question, pk=question_pk)
    favorite = Favorite.objects.create(owner=request.user, question=question)
    print(favorite)
    return render(request, 'core/question_details.html', {'question': question})

# def get_user_favorite(request):
#     user = User.objects.get(username=request.user.username)
#     favorite_questions = [favorite.questions for favorite in user.favorites.all()]
#     return favorite_questions


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib.request as urllib
import json
from datetime import datetime,timedelta
import re

output="/tmp/muell.ics"
idx="termins"
city_id="71186"
area_id="71186"
#alert_lead_time_sec=21600
alert_length_time_sec=3600
url = "https://mymuell.jumomind.com/webservice.php?idx="+idx+"&city_id="+city_id+"&area_id="+area_id
#import_garbage_type=["EICH_BIO", "LK_EICH_PAP", "LK_EICH_REST", "LK_EICH_GELB3","LK_EICH_GELB2","LK_EICH_GELB1"]
import_garbage_type=["EICH_BIO", "LK_EICH_PAP", "LK_EICH_REST", "LK_EICH_GELB2"]
import_translation = {
'EICH_BIO': 'Grünzeug',
'LK_EICH_PAP': 'Pappzeug',
'LK_EICH_REST': 'Restzeug',
'LK_EICH_GELB3': 'Plastikzeug'}

def create_ical(cal_id, cal_garbage_type, cal_date_start, cal_date_end, cal_date_created):
    return  "PRODID:peppisLittleHelper\r\n" \
            "METHOD:PUBLISH\r\n" \
            "BEGIN:VEVENT\r\n" \
            "UID:"+cal_id + "\r\n"\
            "LOCATION:Strasse\r\n" \
            "SUMMARY:"+cal_garbage_type + "\r\n" \
            "DESCRIPTION:"+cal_garbage_type + "\r\n" \
            "CLASS:PUBLIC\r\n" \
            "DTSTART;VALUE=DATE:" + datetime.strftime(cal_date_start,"%Y%m%d") + "\r\n" \
            "DTEND;VALUE=DATE:" + datetime.strftime(cal_date_end,"%Y%m%d") + "\r\n" \
            "DTSTAMP:" + datetime.strftime(cal_date_created,"%Y%m%dT%H%M%SZ") + "\r\n" \
            "END:VEVENT\r\n\r\n"  \

def multipleReplace(text, wordDict):
    """
    take a text and replace words that match the key in a dictionary
    with the associated value, return the changed text
    """
    for key in wordDict:
        text = text.replace(key, wordDict[key])
    return text

json_object = urllib.urlopen(url)
data = json.loads(json_object.read().decode())
f = open(output, "a")
f.write("BEGIN:VCALENDAR\r\n" \
"VERSION:2.0\r\n" )

for item in data:
    _data=(item["_data"])
    for _item in _data:
        if _item["cal_garbage_type"] in import_garbage_type:
            cal_id=_item["cal_id"]
            cal_date=_item["cal_date"]
            cal_date_normal=_item["cal_date_normal"]
            cal_garbage_type=_item["cal_garbage_type"]
            cal_comment=_item["cal_comment"]

            #convert string to datetime
            cal_date = datetime.strptime(cal_date, "%Y-%m-%d")

            #calc the cal_date_start & cal_date_end
            #cal_date_start=cal_date - timedelta(seconds=alert_lead_time_sec)
            #cal_date_end=cal_date - timedelta(seconds=alert_lead_time_sec) + timedelta(seconds=alert_length_time_sec)
            cal_date_start=cal_date
            cal_date_end=cal_date + timedelta(seconds=alert_length_time_sec)

            cal_date_created = datetime.now()
            #replace the strings
            cal_garbage_type = multipleReplace(cal_garbage_type,import_translation)

            ical= create_ical(_item["cal_id"], cal_garbage_type, cal_date_start, cal_date_end, cal_date_created)

            f.write(ical)
f.write("END:VCALENDAR")
f.close()


from django.conf import settings
ugettext = lambda s: s

Jenis_Material = getattr(settings, 'Jenis_Material',((1, ugettext('Major')),
														(2, ugettext('Minor')),
														(3, ugettext('Colour Material')),
														(4, ugettext('Uncolour')),
														(5, ugettext('Stabilitator')),
														(6, ugettext('Cat Label')),
														(7, ugettext('Case'))))

Warna_Produk = getattr(settings, 'Warna_Produk',((1, ugettext('Flint')),
													(2, ugettext('Amber')),
													(3, ugettext('Green'))))

Label_Produk = getattr(settings, 'Label_Produk',((1, ugettext('Polos')),
													(2, ugettext('Dekorasi'))))

Dapur = getattr(settings, 'Dapur',(('G.1.1 D.G', ugettext('G.1.1 D.G')),
									('G.1.2 D.G', ugettext('G.1.2 D.G')),
									('G.1.3 D.G', ugettext('G.1.3 D.G')),
									('G.2.1 D.G', ugettext('G.2.1 D.G')),
									('G.2.2 D.G', ugettext('G.2.2 D.G')),
									('G.2.3 D.G', ugettext('G.2.3 D.G'))))

Status_Dapur = getattr(settings, 'Status_Dapur',((1, ugettext('PRODUKSI')),
												(2, ugettext('STOP PRODUKSI'))))

ACL = getattr(settings, 'ACL',(('ACL-1', ugettext('ACL-1')),
									('ACL-2', ugettext('ACL-2')),
									('ACL-3', ugettext('ACL-3')),
									('ACL-4', ugettext('ACL-4'))))

Status_ACL = getattr(settings, 'Status_ACL',((1, ugettext('PRODUKSI')),
												(2, ugettext('STOP PRODUKSI'))))

TIM = getattr(settings, 'TIM',(('PRODUKSI', ugettext('PRODUKSI')),
											('ACL', ugettext('ACL')),
											('PENGEMASAN', ugettext('PENGEMASAN'))))

Initial = getattr(settings, 'Initial',(('A', ugettext('A')),
										('B', ugettext('B')),
										('C', ugettext('C')),
										('D', ugettext('D')),
										('E', ugettext('E')),
										('F', ugettext('F'))))

Inspect = getattr(settings, 'Inspect',((1, ugettext('Target')),
												(2, ugettext('Pencapaian'))))

LEVEL_AKSES_CHOICES =  getattr(settings, 'LEVEL_AKSES_CHOICES', (('Seksi', ugettext('Seksi')),
																	('Kadep', ugettext('Kadep')),
																	('Direktur', ugettext('Direktur')))) #nanti Direktur rubah ke kasi

STACHOLDER =  getattr(settings, 'STACHOLDER', (('Departemen Inventori', ugettext('Departemen Inventori')),
																	('Departemen Sales', ugettext('Departemen Sales')),
																	('Departemen Assest', ugettext('Departemen Assest'))))

Kategori_Produk = getattr(settings, 'Kategori_Produk', ((1, ugettext('Generic')),
										(2, ugettext('Patent'))))


#!/usr/bin/env python

import json
from os import environ

import boto3
boto3.setup_default_session( region_name='eu-west-1', profile_name='connectedboiler')

AWS_S3_BUCKET= "cbprod-cloudtrail"
AWS_SQS_URL = "https://sqs.eu-west-1.amazonaws.com/116645666073/traildash"

bucket = boto3.resource('s3').Bucket(AWS_S3_BUCKET)
queue = boto3.resource('sqs').Queue(AWS_SQS_URL)

items_queued = 0
for item in bucket.objects.all():
    if not item.key.endswith('.json.gz'):
        continue

    queue.send_message(
        MessageBody=json.dumps({
            'Message': json.dumps({
                's3Bucket': AWS_S3_BUCKET,
                's3ObjectKey': [item.key]
            })
        })
    )
    items_queued += 1

print('Done! {} items were backfilled'.format(items_queued))


import typing as T
import traceback
import asyncio
import signal

from .pickler import Pickler
from .stream import Streamer
from ..utils import parse_address, get_event_loop
from .transport import Transport

def get_handler(calls, streamer: Streamer):

    async def handler(transport: Transport):
        try:
            command = await streamer.load(transport)
        except Exception:
            transport.close_connection()
            return

        if command == "call":
            func = await streamer.load(transport)
            args = await streamer.load(transport)
            kwargs = await streamer.load(transport)
            try:
                output = calls[func](*args, **kwargs)
            except Exception:
                await streamer.dump(True, transport)  # flag
                await streamer.dump(traceback.format_exc(), transport)
            else:
                try:
                    streamer.pre_serialize(output)
                except Exception:
                    await streamer.dump(True, transport)  # flag
                    await streamer.dump(traceback.format_exc(), transport)
                else:
                    await streamer.dump(False, transport)  # flag
                    await streamer.dump(output, transport)
        else:
            print(f"unsupported command: {command}, close the connection.")
        transport.close_connection()

    return handler

class Server():
    def __init__(
            self, address: str = "127.0.0.1:8686",
            pickler_cls: type = Pickler,
            transport_cls: type = Transport,
            streamer: T.Optional[Streamer] = None,
            funcs: T.Optional[T.Dict[str, T.Callable]] = None) -> None:
        if funcs is None:
            funcs = {}
        self.funcs = funcs
        self.address = parse_address(address)
        if streamer is None:
            self.streamer = Streamer(pickler_cls())
        else:
            self.streamer = streamer
        self.transport_cls = transport_cls

    def register_func(self, func: T.Callable, key=None):
        if key is None:
            key = func.__name__
        self.funcs[key] = func

    def unregister_func(self, key: str):
        self.funcs.pop(key)

    def run(self):
        transport: Transport = self.transport_cls(self.address, True)
        handler = get_handler(self.funcs, self.streamer)

        async def server_coro():
            print(f"Start server at {self.address}", flush=True)
            loop = asyncio.get_event_loop()

            while True:
                await transport.init_connection()
                loop.create_task(handler(transport))

        with get_event_loop() as loop:
            signal.signal(signal.SIGINT, lambda s, f: loop.stop())
            loop.run_until_complete(server_coro())

class Client():
    def __init__(
            self, address: str,
            pickler_cls: type = Pickler,
            transport_cls: type = Transport,
            streamer: T.Optional[Streamer] = None,
            ) -> None:
        if streamer is not None:
            self.streamer = streamer
        else:
            self.streamer = Streamer(pickler_cls())
        self.server_addr = parse_address(address)
        self.transport: Transport = transport_cls(self.server_addr, False)

    async def call(self, func_name: str, *args, **kwargs):
        await self.transport.init_connection()
        await self.streamer.dump("call", self.transport)
        await self.streamer.dump(func_name, self.transport)
        await self.streamer.dump(args, self.transport)
        await self.streamer.dump(kwargs, self.transport)
        flag = await self.streamer.load(self.transport)
        if flag:
            error_msg = await self.streamer.load(self.transport)
            raise RuntimeError(error_msg)
        output = await self.streamer.load(self.transport)
        self.transport.close_connection()
        return output


"""
=====
Shape
=====

``Shape`` is used to calculate shape properties using 3D capsule geometry.
Outputs `length and `surface_area` are determined from inputs `volume` and `width`.
These variables are required to plug into a `Lattice Environment`
"""

import math

from scipy.constants import N_A
from vivarium.core.process import Step
from vivarium.library.units import units, Quantity

PI = math.pi
AVOGADRO = N_A / units.mol

def length_from_volume(volume, width):
    """
    get cell length from volume, using the following equation for capsule volume, with V=volume, r=radius,
    a=length of cylinder without rounded caps, l=total length:

    :math:`V = (4/3)*PI*r^3 + PI*r^2*a`
    :math:`l = a + 2*r`
    """
    radius = width / 2
    cylinder_length = (volume - (4/3) * PI * radius**3) / (PI * radius**2)
    total_length = cylinder_length + 2 * radius
    return total_length

def volume_from_length(length, width):
    """
    get volume from length and width, using 3D capsule geometry
    """
    radius = width / 2
    cylinder_length = length - width
    volume = cylinder_length * (PI * radius**2) + (4 / 3) * PI * radius**3
    return volume

def surface_area_from_length(length, width):
    """
    get surface area from length and width, using 3D capsule geometry

    :math:`SA = 4*PI*r^2 + 2*PI*r*a`
    """
    radius = width / 2
    cylinder_length = length - width
    surface_area = 4 * PI * radius**2 + 2 * PI * radius * cylinder_length
    return surface_area

def mmol_to_counts_from_volume(volume):
    '''mmol_to_counts has units L/mmol'''
    return (volume * AVOGADRO).to(units.L/units.mmol)

class Shape(Step):
    """Shape Step

    Derives cell length and surface area from width and volume.

    Ports:

    * **cell_global**: Should be given the agent's boundary store.
      Contains variables: **volume**, **width**, **length**, and
      **surface_area**.
    * **periplasm_global**: Contains the **volume** variable for the
      volume of the periplasm.

    Arguments:
        parameters (dict): A dictionary that can contain the
            following configuration options:

            * **width** (:py:class:`float`): Initial width of the cell in
              microns
    """

    name = 'ecoli-shape'
    defaults = {
        'width': 1.0 * units.um,
        'periplasm_fraction': 0.2,
        'cytoplasm_fraction': 0.8,
        'initial_cell_volume': 1.2 * units.fL,
        'initial_mass': 1339 * units.fg,
    }

    def __init__(self, parameters=None):
        super().__init__(parameters)
        self.outer_to_inner_area = math.pow(self.parameters[
            'cytoplasm_fraction'], 1/3)**2

    def ports_schema(self):

        schema = {
            'cell_global': {
                'volume': {
                    '_default': 0 * units.fL,
                    '_updater': 'set',
                    '_emit': True,
                    '_divider': 'split',
                },
                'width': {
                    '_default': 0 * units.um,
                    '_updater': 'set',
                    '_emit': True,
                    '_divider': 'set',
                },
                'length': {
                    '_default': 0 * units.um,
                    '_updater': 'set',
                    '_emit': True,
                    '_divider': 'split',
                },
                'outer_surface_area': {
                    '_default': 0 * units.um**2,
                    '_updater': 'set',
                    '_emit': True,
                    '_divider': 'split',
                },
                'inner_surface_area': {
                    '_default': 0 * units.um**2,
                    '_updater': 'set',
                    '_emit': True,
                    '_divider': 'split',
                },
                'mmol_to_counts': {
                    '_default': 0 / units.millimolar,
                    '_emit': True,
                    '_divider': 'split',
                    '_updater': 'set',
                },
                'mass': {
                    '_default': 0 * units.fg,
                    '_updater': 'set',
                    '_emit': True,
                    '_divider': 'split',
                },
            },
            'listener_cell_mass': {
                '_default': self.parameters['initial_mass'].magnitude,  # fg
            },
            'listener_cell_volume': {
                '_default': self.parameters['initial_cell_volume'].magnitude,  # fL
            },
            'periplasm_global': {
                'volume': {
                    '_default': self.parameters['initial_cell_volume'] * self.parameters['periplasm_fraction'],  # fL
                    '_emit': True,
                    '_divider': 'split',
                    '_updater': 'set',
                },
                'mmol_to_counts': {
                    '_default': 0 / units.millimolar,
                    '_emit': True,
                    '_divider': 'split',
                    '_updater': 'set',
                },
            },
            'cytoplasm_global': {
                'volume': {
                    '_default': self.parameters['initial_cell_volume'] * self.parameters['cytoplasm_fraction'],  # fL
                    '_emit': True,
                    '_divider': 'split',
                    '_updater': 'set',
                },
                'mmol_to_counts': {
                    '_default': 0 / units.millimolar,
                    '_emit': True,
                    '_divider': 'split',
                    '_updater': 'set',
                },
            },
        }
        return schema

    def initial_state(self, config=None):
        cell_volume = self.parameters['initial_cell_volume']
        assert isinstance(cell_volume, Quantity)
        width = self.parameters['width']
        assert isinstance(width, Quantity)
        length = length_from_volume(cell_volume, width)
        outer_surface_area = surface_area_from_length(length, width)
        inner_surface_area = self.outer_to_inner_area * outer_surface_area

        assert self.parameters['periplasm_fraction'] + self.parameters['cytoplasm_fraction'] == 1
        periplasm_volume = cell_volume * self.parameters['periplasm_fraction']
        cytoplasm_volume = cell_volume * self.parameters['cytoplasm_fraction']

        mass = self.parameters['initial_mass']
        assert isinstance(mass, Quantity)
        return {
            'cell_global': {
                'volume': cell_volume,
                'width': width,
                'length': length,
                'outer_surface_area': outer_surface_area,
                'inner_surface_area': inner_surface_area,
                'mmol_to_counts': mmol_to_counts_from_volume(
                    cell_volume),
                'mass': mass,
            },
            'listener_cell_mass': mass.magnitude,
            'listener_cell_volume': cell_volume.magnitude,
            'periplasm_global': {
                'volume': periplasm_volume,
                'mmol_to_counts': mmol_to_counts_from_volume(
                    periplasm_volume),
            },
            'cytoplasm_global': {
                'volume': cytoplasm_volume,
                'mmol_to_counts': mmol_to_counts_from_volume(
                    cytoplasm_volume),
            },
        }

    def next_update(self, timestep, states):
        for port in (
                'cell_global', 'periplasm_global', 'cytoplasm_global'):
            for variable, value in states[port].items():
                assert isinstance(value, Quantity), f"{variable}={value} is not a Quantity"

        width = states['cell_global']['width']
        cell_volume = states['listener_cell_volume'] * units.fL

        assert self.parameters['periplasm_fraction'] + self.parameters['cytoplasm_fraction'] == 1
        periplasm_volume = cell_volume * self.parameters['periplasm_fraction']
        cytoplasm_volume = cell_volume * self.parameters['cytoplasm_fraction']

        # calculate length and surface area
        length = length_from_volume(cell_volume, width)
        outer_surface_area = surface_area_from_length(length, width)
        inner_surface_area = self.outer_to_inner_area * outer_surface_area

        update = {
            'cell_global': {
                'length': length,
                'outer_surface_area': outer_surface_area,
                'inner_surface_area': inner_surface_area,
                'mmol_to_counts': mmol_to_counts_from_volume(
                    cell_volume),
                'mass': states['listener_cell_mass'] * units.fg,
                'volume': cell_volume,
            },
            'periplasm_global': {
                'volume': periplasm_volume,
                'mmol_to_counts': mmol_to_counts_from_volume(
                    periplasm_volume),
            },
            'cytoplasm_global': {
                'volume': cytoplasm_volume,
                'mmol_to_counts': mmol_to_counts_from_volume(
                    cytoplasm_volume),
            }
        }
        return update


# -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod
import time
import datetime

class PrettendToBeEvent(object):
    #общий класс Event`ов
    __metaclass__ = ABCMeta
    @abstractmethod
    def message(self):
        pass

    def time_created(self):
        t = time.clock()
        return str(round(t, 3))

class GoodEvent(PrettendToBeEvent):
    #хороший Event

    def __init__(self):
        PrettendToBeEvent.__init__(self)

    def message(self):
        hello_message = 'Я хороший, найди меня'
        date = datetime.datetime.now().replace(microsecond=0).isoformat()
        name = type(self).__name__
        code = 200
        return name, date, id(self), code, hello_message

class BadEvent(PrettendToBeEvent):
    #плохой Event

    def __init__(self):
        PrettendToBeEvent.__init__(self)

    def message(self):
        hello_message = 'Я плохой, лучше не находи меня'
        date = datetime.datetime.now().replace(microsecond=0).isoformat()
        name = type(self).__name__
        code = 404
        return name, date, id(self), code, hello_message

class UselessEvent(PrettendToBeEvent):
    #Бесполезный Event для общего шума

    def __init__(self):
        PrettendToBeEvent.__init__(self)

    def message(self):
        hello_message = 'А я просто бесполезный'
        date = datetime.datetime.now().replace(microsecond=0).isoformat()
        name = type(self).__name__
        code = 100
        return name, date, id(self), code, hello_message

if __name__ == '__main__':
    pass


import torch
import torch.nn as nn
from torch.nn import functional as F

class ResNet_Block(nn.Module):
    def __init__(self, inchannel, outchannel, pre=False):
        super(ResNet_Block, self).__init__()
        self.pre = pre
        self.right = nn.Sequential(
            nn.Conv2d(inchannel, outchannel, (1, 1)),
        )
        self.left = nn.Sequential(
            nn.Conv2d(inchannel, outchannel, (1, 1)),
            nn.InstanceNorm2d(outchannel),
            nn.BatchNorm2d(outchannel),
            nn.ReLU(),
            nn.Conv2d(outchannel, outchannel, (1, 1)),
            nn.InstanceNorm2d(outchannel),
            nn.BatchNorm2d(outchannel),
        )
    def forward(self, x):
        x1 = self.right(x) if self.pre is True else x
        out = self.left(x)
        out = out + x1
        return F.relu(out)

class LGC_net(nn.Module):
    def __init__(self):
        super(LGC_net, self).__init__()
        self.conv1 = nn.Sequential(nn.Conv2d(5, 128, (1, 1)),
                                   nn.BatchNorm2d(128),
                                   nn.ReLU(inplace=True),
                                   nn.Conv2d(128, 128, (1, 1)))
        self.conv2 = nn.Sequential(nn.Conv2d(128, 128, (1, 1)),
                                   nn.BatchNorm2d(128))
        self.res_blocks = nn.ModuleList()
        for _ in range(12):
            self.res_blocks.append(ResNet_Block(128,128))
        self.linear = nn.Conv2d(128, 1, (1, 1))
        self.initialize_weights()

    def initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                torch.nn.init.xavier_normal_(m.weight.data)
                if m.bias is not None:
                    torch.nn.init.constant_(m.bias.data, 0)

    def forward(self, x):
        out = self.conv1(x)
        for res_blk in self.res_blocks:
            out = res_blk(out)
        out = self.conv2(out)
        logit = self.linear(out)
        # probs = torch.sigmoid(logit.squeeze(-1).squeeze(1))

        # log_probs = F.logsigmoid(logit.squeeze(-1).squeeze(1))
        # normalizer = torch.logsumexp(log_probs, dim=1,keepdim=True)
        # log_probs = log_probs - normalizer
        return logit

# model = LGC_net()
# model.cuda()
# x = torch.rand(32, 5, 200).cuda()  # Normalized matches: Batch_size * N *4
# weights, Es = model(x)       # Lists of predicted weights and Es
# mask = weights > 0


import PySimpleGUI as sg

layout = [[sg.Text('Enter your name and age')],
          [sg.Text('Name', size=(15, 1)), sg.InputText()],
          [sg.Text('Age', size=(15, 1)), sg.InputText()],
          [sg.Submit(), sg.Cancel()]]

window = sg.Window('Mini Form', layout)

while True:
    event, values = window.read()
    if event == sg.WIN_CLOSED or event == 'Cancel':
        break
    if event == 'Submit':
        name = values[0]
        age = values[1]
        sg.popup_yes_no()

window.close()


#!/usr/bin/env python
import argparse
import pickle
import numpy as np

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('activations', help='pkl file with the activations')
    args = ap.parse_args()

    pkl = pickle.load(open(args.activations, 'rb'))

    log_probs = pkl['log_probabilities']

    for sent_log_probs in log_probs:
        print(np.exp(-sent_log_probs.mean()))

main()


#!/usr/bin/env python
# coding: utf-8

"""
In this script, the variable dimensions of our models are determined.
In the first part we calculate the piston sealing grooves.
Secondly, the dimensions of the non-circular shapes are determined.
"""

__author__ = "Eva Zillen"
__copyright__ = "Copyright 2021, TU Delft Biomechanical Design"
__credits__ = ["Eva Zillen, Heike Vallery, Gerwin Smit"]
__license__ = "CC0-1.0 License"
__version__ = "1.0.0"
__maintainer__ = "Eva Zillen"
__email__ = "e.zillen@student.tudelft.nl"

# # Definitions

# ## Imports
import numpy as np
import math
from sympy import symbols, Eq, solve, pi

# ## Functions

def calculate_groove(ID,S,r_sq,C):
    # Determine the outer diameter of the O-ring
    OD = ID+2*S
    # Determine the squeeze dimension of the O-ring
    S_sq = S * (1 - r_sq)
    # Determine the pison groove diameter
    D_PG = OD - 2 * S_sq
    # Set the width of the piston groove
    W_PG = S + 1
    # Determine the piston diameter
    P_D = OD - 2 * C
    return f'\nOuter diameter O-ring: {OD} mm \nPiston diameter: {P_D} mm \nPiston groove diameter: {D_PG} mm \nWidth piston groove: {round(W_PG,2)} mm\n'

# # Determining the dimensions of the O-ring groove

# #### O-ring Lidl, SLA print

# The O-ring used had the dimensions (18x3.5)
ID = 18
S = 3.5
# A clearance of 0.5 mm was used between the piston and the cylinder
C = 0.5
# 10% squeeze ration was used in this study
r_sq = 0.1

print ("O-ring Lidl, 18*3.5")
print(calculate_groove(ID,S,r_sq,C))

print('------------------')

# #### O-ring en X-ring Eriks

# The O-ring used had the dimensions (18.64x3.53)
ID = 18.64
S = 3.53
# A clearance of 0.5 mm was used between the piston and the cylinder
C = 0.5
# 10% squeeze ration was used in this study
r_sq = 0.1

print ("O-ring and X-ring Eriks, 18.64*3.53")
print(calculate_groove(ID,S,r_sq,C))

print('------------------')
# # Determining the dimensions of the non-conventional cylinders

# ### Stadium shape

def stadium(ID,S):
    # Variables defining the stadium shape (see Figure 2 in report)
    L = symbols('L')
    D = 0

    # Target area - the area of the circular shape with diameter = 25 mm
    A_target = math.pi * (25/2)**2
    # Determine the outer diameter of the O-ring
    OD = ID + 2 * S
    # Determine the perimeter of the O-ring
    P_c = math.pi * OD

    # Variable necessary to perform the optimization
    A_s = 0

    # To find the variables D & L we use for the stadium surface area (A_s) and the perimeter (P_s) equations
    # Optimization is done by varying D and finding corresponding L
    for D_optimize in np.arange(5,20,0.1):
        # Variables necessary for optimization, to remember the previous values
        previous_A_s = A_s
        previous_L = L
        previous_D = D

        # L is reset each round, define it again
        L = symbols('L')

        # Rounding necessary for limitations on floating point numbers in Python
        D = round(D_optimize,2)

        # Solve for the perimeter
        P_s_eq = (math.pi * D) + 2 * L - P_c
        result = solve((P_s_eq), (L))

        # Solving returns a list, L is the only value in this list
        L = result[0]

        # Applying the equation for the surface area
        A_s = math.pi * (D/2)**2 + L * D

        # Finding the right value: if the surface area is larger than the target area, take the previous and continue
        if previous_A_s <= A_target and A_s >= A_target:
            print(f"Optimizing found: \narea = {previous_A_s} with D = {previous_D} and L = {previous_L} is closest to the target surface area of {A_target}")
            break

    return previous_D,previous_L

# Example calculations for an O-ring of 22x3.5mm
ID = 22
S = 3.5
D, L = stadium(ID,S)
print(f"\n--------------------------------------")
print(f"Resulting values for an stadium shape with O-ring of {ID} x {S} mm")
print(f"D = {D} mm")
print(f"L = {L} mm")
print(f"--------------------------------------")

# ### Kidney shape

def optimize_range(ID, S, gamma):
    # Variables defining the kidney shape (see Figure 2 in report)
    r = symbols('r')
    a = 0

    # Target area - the area of the circular shape with diameter = 25 mm
    A_target = math.pi*(25/2)**2
    # Determine the outer diameter of the O-ring
    OD = ID+2*S
    # Determine the perimeter of the O-ring
    P_c = math.pi*OD

    # Variable necessary to perform the optimization
    A_k = 0

    # To find the variables r & a we use the stadium surface area (A_k) and the perimeter (P_k) equations
    # Optimization is done by varying a and finding corresponding r
    for a_optimize in np.arange(5,20,0.1):
        # Variables necessary for optimization, to remember the previous values
        previous_A_k = A_k
        previous_r = r
        previous_a = a

        # L is reset each round, define it again
        r = symbols('r')

        # Rounding necessary for limitations on floating point numbers in Python
        a = round(a_optimize,2)

        # Solve for the perimeter
        P_k_eq = math.pi* a +(gamma*(2 * r + a)) - P_c
        result = solve((P_k_eq), (r))

        # Solving returns a list, L is the only value in this list
        r = result[0]

        # Applying the equation for the surface area
        A_k = math.pi * (a / 2)**2 + (gamma * (r * a + (a**2 / 2)))

        # Finding the right value: if the surface area is larger than the target area, take the previous and continue
        if previous_A_k <= A_target and A_k >= A_target:
            print(f"Optimizing found:\narea = {previous_A_k} with a = {previous_a} and r = {previous_r} is closest to the target surface area of {A_target}")
            break

    return previous_a,previous_r

# Example calculations for an O-ring of 22x3.5mm
ID = 22
S = 3.5
gamma = 100/180*math.pi

a, r = optimize_range(ID, S, gamma)
print(f"\n--------------------------------------")
print(f"Resulting values for a kidney shape with O-ring of {ID} x {S} mm")
print(f"a = {a} mm")
print(f"r = {r} mm")
print(f"--------------------------------------")


from bs4 import BeautifulSoup
import glob, re
import subprocess
import os
import json
import pprint as pp
import re
import nltk
from nltk.tokenize import word_tokenize
import string
from string import punctuation
from gensim.models import Word2Vec
from nltk.corpus import stopwords
import csv
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import altair as alt
import pandas as pd
stop_words = set(stopwords.words('french'))

to_add = ['où',"cette","si","moins","cet","enfin","quand","mesme","comme","encore","tant","quelque", "tout", "sous", 'luy',"estre", "sans","plus","faire","fait","a","toute","moy","vostre", "leurs",'.', ',', ';', ':', '?', '!', '(', ')', '[', ']', '{', '}', '-', '_', '+', '=', '"', "'", '/', '\\', '@', '#', '$', '%', '^', '&', '*', '`', '~', '<', '>', '|', '，', '。', '；', '：', '？', '！', '（', '）', '【', '】', '『', '』', '—', '–', '－', '_', '+', '=' ,'“', '”', '‘', '’', '·', '…', '‖', '”', '’', '〝', '〞', '〈', '〉', '《', '》', '「', '」', '『', '』', '【', '】', '〔', '〕', '｛', '｝', '〖', '〗', '〘', '〙', '〚', '〛', '⸨', '⸩']

for w in to_add:
	stop_words.add(w)

'''
CLEANR = re.compile('<.*?>')
with open('termes.json') as outfile:
    TERMES = json.load(outfile)
'''

def cleanhtml(raw_html):
  cleantext = raw_html.replace(")",' ')
  cleantext = cleantext.replace("(",' ')
  cleantext = cleantext.replace(",",' ')
  cleantext = cleantext.replace(".",' ')
  cleantext = cleantext.replace("\\n",' ')
  cleantext = cleantext.replace("\\",' ')
  cleantext = cleantext.replace("\xa0",' ')
  cleantext = cleantext.replace("xa0",' ')
  cleantext = cleantext.replace("/",' ')

  return cleantext

def removeStopWords(tokens):
	filtered_tokens = [token for token in tokens if token.lower() not in stop_words]
	return filtered_tokens

all_sentences = []

csv_file = open('douleur_douloureux.csv', 'r')
reader = csv.reader(csv_file)

i = 0
for row in reader:

	if '-' in row[5] : continue
	if int(row[5]) < 1913: continue

	print(row[5])

	#print(row[3] + ' ' + row[2]+ ' '+ row[5])
	phrase = str(row[11]) + ' '+ str(row[12]) + ' ' + str(row[13])
	phrase = cleanhtml(phrase)
	tokens = nltk.word_tokenize(phrase, language='french')
	""" TRAITEMENT DES TERMES POLYLEXICAUX
	for terme in TERMES:
			if terme in passage and len(terme.split(" ")) > 1:
				condensed_terme = "_".join(terme.split(' '))
				new_passage = passage.replace(terme,condensed_terme).lower()
	"""

	tokens = removeStopWords(tokens)
	all_sentences.append(tokens)

	#if i == 1000: break
	i+=1

print(i)
print(all_sentences)

model = Word2Vec(all_sentences,
                 min_count=3,   # Ignore words that appear less than this # Dimensionality of word embeddings
                 workers=2,     # Number of processors (parallelisation)
                 window=5      # Context window for words during training
                 )       # Number of epochs training over corpus

# Get the top N most similar words to "death"
similar_words = model.wv.most_similar("douleur",topn=15)
# Extract the similar word from the list
similar_words.append(("douleur",1))
similar_words = [word[0] for word in similar_words]

# Get the embeddings of the similar words
X = model.wv[similar_words]

# Perform PCA on the word embeddings
pca = PCA(n_components=3)
result = pca.fit_transform(X)

data = pd.DataFrame(result, columns=['x', 'y', 'z'], index=similar_words)

data["word"] = data.index

chart = alt.Chart(data,width = 800, height = 600).mark_circle().encode(
    x='x',
    y='y',
    color=alt.Color('z', scale=alt.Scale(scheme='greenblue')),
    tooltip=['word'],
    size=alt.Size('z', scale=alt.Scale(range=[10, 500]), legend=None)
).interactive()

chart.save("pca_word_embeddings.html")

"""
# Create a 3D scatter plot of the PCA result
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(result[:, 0], result[:, 1], result[:, 2])

for i, word in enumerate(similar_words):
    ax.text(result[i, 0], result[i, 1], result[i, 2], word)

plt.show()

"""

'''
print(dir(model.wv))
print('expérience_intérieure')
pp.pprint(model.wv.most_similar("expérience_intérieure",topn=50))
'''


from collections import Counter
polymer, insert_data = open('./input.txt').read().split('\n\n')

insert = [l.split(' -> ') for l in insert_data.split('\n') if l]
insert_dict = {i: Counter([i[0]+m, m+i[1]]) for i, m in insert}

def main(steps):
    pol_counts = Counter([c + polymer[i+1] for i, c in enumerate(polymer[:-1])])

    for s in range(steps):
        n_pol_counts = Counter()
        for pair, count in pol_counts.items():
            n_pol_counts += {k: count*v for k,v in insert_dict[pair].items()}
        pol_counts = n_pol_counts

    singular_counter = Counter()
    for pair, count in pol_counts.items():
        for c in pair:
            singular_counter += {c: count}

    singular_counter += Counter([polymer[0], polymer[-1]])
    freq = singular_counter.most_common()
    return (freq[0][1] - freq[-1][1])//2

print(main(10))
print(main(40))


import cv2
import numpy as np
from core import Consumer

class VideoView():
    def __init__(self):
        self.consumer = Consumer("frame")
        self.consumer.start(on_message=self.on_message)

    @staticmethod
    def on_message(chan, method_frame, header_frame, body, userdata=None):
        """Called when a message is received. Log message and ack it."""
       # print(chan)
       # print(method_frame)

        body = np.frombuffer(body, np.uint8)
        img = cv2.imdecode(body,cv2.IMREAD_COLOR)
        cv2.imshow(method_frame.routing_key,img)
        cv2.waitKey(1)

        chan.basic_ack(delivery_tag=method_frame.delivery_tag)

if __name__ == "__main__":
    view = VideoView()


import xml.etree.ElementTree as ET

# read all the xml files and store record elements in a list
def get_docs():
    files = ["./cfc-xml/cf74.xml", "./cfc-xml/cf75.xml", "./cfc-xml/cf76.xml", "./cfc-xml/cf77.xml",
             "./cfc-xml/cf78.xml", "./cfc-xml/cf79.xml"]
    res = []
    for file in files:
        with open(file) as f:
            tree = ET.parse(f)
            records = tree.getroot()
            for record in records:
                res.append(record)
    return res

# parse a tree element to json with fields: ["RECORDNUM", 'AUTHORS', 'MAJORSUBJ', "MINORSUBJ", 'ABSTRACT', 'TITLE', 'EXTRACT', 'SOURCE']
def parse_json(elem):
    # get id field information
    record_id = "00000"
    if elem.find('RECORDNUM') != -1 and elem.find('RECORDNUM') is not None and elem.find('RECORDNUM').text is not None:
        record_id = elem.find('RECORDNUM').text

    # get title field information
    title = ""
    if elem.find('TITLE') != -1 and elem.find('TITLE') is not None and elem.find('TITLE').text is not None:
        title = elem.find('TITLE').text

    # get abstract field information
    abstract = ""
    if elem.find('ABSTRACT') != -1 and elem.find('ABSTRACT') is not None and elem.find('ABSTRACT').text is not None:
        abstract = elem.find('ABSTRACT').text

    # get extract field information
    extract = ""
    if elem.find('EXTRACT') != -1 and elem.find('EXTRACT') is not None and elem.find('EXTRACT').text is not None:
        extract = elem.find('EXTRACT').text

    # get source field information
    source = ""
    if elem.find('SOURCE') != -1 and elem.find('SOURCE') is not None and elem.find('SOURCE').text is not None:
        source = elem.find('SOURCE').text

    # get authors field information
    authors = ""
    authors_elem = elem.find('AUTHORS')
    if authors_elem != -1 and authors_elem is not None:
        for author in authors_elem:
            if author is not None and author.text is not None:
                authors = authors + " " + author.text

    # get major_subjs field information
    major_subjs = ""
    major_subj_elem = elem.find('MAJORSUBJ')
    if major_subj_elem != -1 and major_subj_elem is not None:
        for major_subj in major_subj_elem:
            if major_subj is not None and major_subj.text is not None:
                major_subjs = major_subjs + " " + major_subj.text

    # get minor_subjs field information
    minor_subjs = ""
    minor_subj_elem = elem.find('MINORSUBJ')
    if minor_subj_elem != -1 and minor_subj_elem is not None:
        for minor_subj in minor_subj_elem:
            if minor_subj is not None and minor_subj.text is not None:
                minor_subjs = minor_subjs + " " + minor_subj.text

    res = {
        "RECORDNUM": record_id,
        "AUTHORS": authors,
        "TITLE": title,
        "MAJORSUBJ": major_subjs,
        "MINORSUBJ": minor_subjs,
        "ABSTRACT": abstract,
        "EXTRACT": extract,
        "SOURCE": source
    }
    return res