from typing import List, Optional, Type, Union

from opyoid.exceptions import BindingError
from opyoid.provider import Provider
from opyoid.scopes import Scope, SingletonScope
from opyoid.utils import EMPTY, InjectedT
from .binding import Binding
from .binding_registry import BindingRegistry
from .class_binding import ClassBinding
from .instance_binding import InstanceBinding
from .multi_binding import ItemBinding, MultiBinding
from .provider_binding import ProviderBinding
from .registered_binding import RegisteredBinding
from .registered_multi_binding import RegisteredMultiBinding
from .self_binding import SelfBinding

class AbstractModule:
    """Base class for Modules, should not be used outside of the library."""

    def __init__(self, log_bindings: bool = False):
        self._is_configured = False
        self._binding_registry = BindingRegistry(log_bindings)

    @property
    def binding_registry(self) -> BindingRegistry:
        return self._binding_registry

    def __repr__(self) -> str:
        return ".".join([self.__class__.__module__, self.__class__.__qualname__])

    def configure(self) -> None:
        """Contains all bindings, called at injector initialization.

        Should not be called directly, but through configure_once.
        Only public to have a simpler API.
        """
        raise NotImplementedError

    def install(self, module: "AbstractModule") -> None:
        """Adds bindings from another Module to this one."""
        # pylint: disable=import-outside-toplevel
        from .private_module import PrivateModule

        module.configure_once()
        for binding in module.binding_registry.get_bindings_by_target().values():
            if isinstance(module, PrivateModule):
                if not module.is_exposed(binding.target):
                    continue
                if isinstance(binding, RegisteredMultiBinding):
                    binding = RegisteredMultiBinding(
                        binding.raw_binding,
                        item_bindings=[
                            RegisteredBinding(
                                registered_item_binding.raw_binding,
                                (module,) + binding.source_path,
                            )
                            for registered_item_binding in binding.item_bindings
                        ],
                    )
                else:
                    binding = RegisteredBinding(
                        binding.raw_binding,
                        (module,) + binding.source_path,
                    )
            self._binding_registry.register(binding, add_self_binding=False)

    # pylint: disable=too-many-arguments
    def bind(self,
             target_type: Type[InjectedT],
             *,
             to_class: Type[InjectedT] = EMPTY,
             to_instance: InjectedT = EMPTY,
             to_provider: Union[Provider, Type[Provider]] = EMPTY,
             scope: Type[Scope] = SingletonScope,
             named: Optional[str] = None) -> RegisteredBinding:
        try:
            binding = self._create_binding(
                target_type,
                to_class,
                to_instance,
                to_provider,
                scope,
                named,
            )
        except BindingError as error:
            raise BindingError(f"Error in {self!r} when binding to {target_type!r}: {error}") from None
        registered_binding = self._register(binding)
        return registered_binding

    def configure_once(self):
        """Calls configure if it has not already been called."""

        if not self._is_configured:
            self._is_configured = True
            self.configure()

    def multi_bind(self,
                   item_target_type: Type[InjectedT],
                   item_bindings: List[ItemBinding[InjectedT]],
                   *,
                   scope: Type[Scope] = SingletonScope,
                   named: Optional[str] = None,
                   override_bindings: bool = True) -> RegisteredBinding:

        return self._register_multi_binding(
            MultiBinding(item_target_type, item_bindings, scope=scope, named=named, override_bindings=override_bindings)
        )

    @staticmethod
    def bind_item(*,
                  to_class: Type[InjectedT] = EMPTY,
                  to_instance: InjectedT = EMPTY,
                  to_provider: Union[Provider, Type[Provider]] = EMPTY) -> ItemBinding[InjectedT]:
        return ItemBinding(bound_class=to_class, bound_instance=to_instance, bound_provider=to_provider)

    @staticmethod
    def _create_binding(target_type: Type[InjectedT],
                        bound_class: Type[InjectedT],
                        bound_instance: InjectedT,
                        bound_provider: Union[Provider, Type[Provider]],
                        scope: Type[Scope],
                        named: Optional[str]) -> Binding:
        if bound_instance is not EMPTY:
            return InstanceBinding(target_type, bound_instance, named=named)
        if bound_provider is not EMPTY:
            return ProviderBinding(target_type, bound_provider, scope=scope, named=named)
        if bound_class is not EMPTY and bound_class != target_type:
            return ClassBinding(target_type, bound_class, scope=scope, named=named)
        return SelfBinding(target_type, scope=scope, named=named)

    def _register(self, binding: Binding[InjectedT]) -> RegisteredBinding:
        if isinstance(binding, MultiBinding):
            registered_binding = self._register_multi_binding(binding)
        else:
            registered_binding = RegisteredBinding(binding)
            self._binding_registry.register(registered_binding)
        return registered_binding

    def _register_multi_binding(self, binding: MultiBinding[InjectedT]) -> RegisteredMultiBinding:
        registered_binding = RegisteredMultiBinding(binding)
        for item_binding in binding.item_bindings:
            if item_binding.bound_class is not EMPTY:
                item_binding = SelfBinding(
                    item_binding.bound_class,
                    scope=binding.scope,
                    named=binding.named,
                )
            elif item_binding.bound_instance is not EMPTY:
                item_binding = InstanceBinding(
                    binding.item_target_type,
                    item_binding.bound_instance,
                    named=binding.named,
                )
            elif item_binding.bound_provider is not EMPTY:
                item_binding = ProviderBinding(
                    binding.item_target_type,
                    item_binding.bound_provider,
                    scope=binding.scope,
                    named=binding.named,
                )
            else:
                raise BindingError(
                    f"ItemBinding in {binding!r} has no instance, class or provider, one should be set")

            # pylint: disable=no-member
            registered_binding.item_bindings.append(RegisteredBinding(item_binding))
        self._binding_registry.register(registered_binding)
        return registered_binding


# import
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State

# app
app = dash.Dash()

# layout
app.layout = html.Div([
    dcc.Input(id='text-input', value='placeholder input'),
    html.Button(id='submit-button', children='Submit'),
    html.Div(id='text-output')
])

# callback & functions
@app.callback(
    Output('text-output', 'children'),
    [Input('submit-button', 'n_clicks')],
    [State('text-input', 'value')]
)
def return_output(n_clicks,text_input):
    return "{} and clicked {}".format(text_input,n_clicks)

# run server
if __name__ == '__main__':
    app.run_server()


#!/usr/bin/env python

# Copyright 2017 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""DialogFlow API Context Python sample showing how to manage session
contexts.

Examples:
  python context_management.py -h
  python context_management.py --project-id PROJECT_ID \
  list --session-id SESSION_ID
  python context_management.py --project-id PROJECT_ID \
  create --session-id SESSION_ID --context-id CONTEXT_ID
  python context_management.py --project-id PROJECT_ID \
  delete --session-id SESSION_ID --context-id CONTEXT_ID
"""

import argparse

# [START dialogflow_list_contexts]
def list_contexts(project_id, session_id):
    import dialogflow_v2 as dialogflow
    contexts_client = dialogflow.ContextsClient()

    session_path = contexts_client.session_path(project_id, session_id)

    contexts = contexts_client.list_contexts(session_path)

    print('Contexts for session {}:\n'.format(session_path))
    for context in contexts:
        print('Context name: {}'.format(context.name))
        print('Lifespan count: {}'.format(context.lifespan_count))
        print('Fields:')
        for field, value in context.parameters.fields.items():
            if value.string_value:
                print('\t{}: {}'.format(field, value))
# [END dialogflow_list_contexts]

# [START dialogflow_create_context]
def create_context(project_id, session_id, context_id, lifespan_count):
    import dialogflow_v2 as dialogflow
    contexts_client = dialogflow.ContextsClient()

    session_path = contexts_client.session_path(project_id, session_id)
    context_name = contexts_client.context_path(
        project_id, session_id, context_id)

    context = dialogflow.types.Context(
        name=context_name, lifespan_count=lifespan_count)

    response = contexts_client.create_context(session_path, context)

    print('Context created: \n{}'.format(response))
# [END dialogflow_create_context]

# [START dialogflow_delete_context]
def delete_context(project_id, session_id, context_id):
    import dialogflow_v2 as dialogflow
    contexts_client = dialogflow.ContextsClient()

    context_name = contexts_client.context_path(
        project_id, session_id, context_id)

    contexts_client.delete_context(context_name)
# [END dialogflow_delete_context]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '--project-id',
        help='Project/agent id.  Required.',
        required=True)

    subparsers = parser.add_subparsers(dest='command')

    list_parser = subparsers.add_parser(
        'list', help=list_contexts.__doc__)
    list_parser.add_argument(
        '--session-id',
        required=True)

    create_parser = subparsers.add_parser(
        'create', help=create_context.__doc__)
    create_parser.add_argument(
        '--session-id',
        required=True)
    create_parser.add_argument(
        '--context-id',
        help='The id of the context.',
        required=True)
    create_parser.add_argument(
        '--lifespan-count',
        help='The lifespan_count of the context.  Defaults to 1.',
        default=1)

    delete_parser = subparsers.add_parser(
        'delete', help=delete_context.__doc__)
    delete_parser.add_argument(
        '--session-id',
        required=True)
    delete_parser.add_argument(
        '--context-id',
        help='The id of the context.',
        required=True)

    args = parser.parse_args()

    if args.command == 'list':
        list_contexts(args.project_id, args.session_id, )
    elif args.command == 'create':
        create_context(
            args.project_id, args.session_id, args.context_id,
            args.lifespan_count)
    elif args.command == 'delete':
        delete_context(args.project_id, args.session_id, args.context_id)


from enum import Enum
from queue import Queue
from .event import Event
from .repository import Repository
from .test_manager import TestManager

class Bisector:
    class State(Enum):
        INITIAL = 1
        CREATING_REPO = 2
        BISECTING = 3

    def __init__(self, conf, batch):
        self._state = self.State.INITIAL
        self._event_queue = Queue()
        self._conf = conf
        self._batch = batch
        self._next_batch = self.next_batch(0, self._batch.len())
        self._last_test_result = None
        self._first_run = True

        self._event_processor = {

            Event.Type.TEST_BATCH_FINISHED: self.test_finished_handler,

        }

    def run(self):

        """ Initialize repository """
        repo = Repository(self._conf)
        repo.initialize(self._batch)

        start_batch = (0, self._batch.len())
        tm = TestManager(self._conf, self._batch, start_batch, self._event_queue)
        tm.start()

        while True:
            event = self._event_queue.get()
            res = self._event_processor[event.type](event)
            if (res == -1):
                break;
            else:
                pass

    def next_batch(self, a, b):
        start, end = a, b
        if (start == end):
            return
        mid = (end - start) // 2
        res_left = yield (start, start + mid)
        res_right = yield (start + mid + 1, end)

        if not res_left:

            yield from self.next_batch(start, start + mid)
        if not res_right:

            yield from self.next_batch(start + mid + 1, end)

    def test_finished_handler(self, event):
        bisection_finished = False
        if self._first_run:
            self._first_run = False
        else:
            self._last_test_result = event.data['result']

        print("Test Finished with result = %s" % (event.data['result']))
        try:
            new_change_indexes = self._next_batch.send(self._last_test_result)
        except:
            bisection_finished = True

        if bisection_finished:
            return -1
        else:
            tm = TestManager(self._conf, self._batch, new_change_indexes, self._event_queue)
            tm.start()
            return 1

    def get_event_queue(self):
        return self._event_queue


#!/usr/bin/env python3
import sys
import tempfile

from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont

if __name__ == '__main__':
    WHITE = "#ffffff"
    RED = "#ff0000"

    width, height = 600, 450
    im = Image.new(mode="RGBA", size=(width, height), color=WHITE)
    draw = ImageDraw.Draw(im)

    if sys.platform == "darwin":
        filename = "/Library/Fonts/Microsoft/Times New Roman Bold.ttf"
    elif sys.platform == "win32":
        filename = "timesbd.ttf"
    elif sys.platform in ["linux", "linux2"]:
        # this script required wqy truetype font,
        # install it on Debian/Ubuntu: apt-get install ttf-wqy-microhei
        filename = "/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc"
    else:
        raise NotImplementedError

    font_size = 16
    font = ImageFont.truetype(font=filename, size=font_size, encoding="unic")

    text = "hello world 中文"
    draw.text(xy=(10, 10), text=text, fill=RED, font=font)

    fd, fullpath = tempfile.mkstemp()
    save_to = fullpath + ".png"
    im.save(save_to)
    msg = "save %s" % save_to
    print(msg)
    im.show()


#!/usr/bin/python3

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
import numpy as np
import copy
from model_Daikon import RNN_model, CNN_model
import config
from itertools import groupby
import math
from tqdm import tqdm

def load_data(mode):
  path = ''
  if mode == 'train':
    path = './data/trainframes.npy'
    file = open('./data/labels.npy', 'rb')
    labels = np.load(file)
    file.close()
  elif mode == 'test':
    path = './data/testframes.npy'
    labels = None

  file = open(path, 'rb')
  frames = np.load(file)
  file.close()

  print('Data loaded.')
  return frames, labels

def write_result(frame_scores, path):
  def get_intchar_map(path):
    phone2int = {}
    phone2char = {}
    with open(path) as file:
      for line in file:
        sep = line[:-1].split('\t')
        phone2int[sep[0]] = int(sep[1])
        phone2char[sep[0]] = str(sep[2])
    return phone2int, phone2char

  def get_phone_map(path):
    phone_map = {}
    with open(path) as file:
      for line in file:
        sep = line[:-1].split('\t')
        phone_map[sep[0]] = str(sep[1])
    return phone_map
  phone2int, phone2char = get_intchar_map('./data/48phone_char.map')
  int2phone = {v: k for k, v in phone2int.items()}
  phone_map = get_phone_map('./data/48_39.map')

  split_videos=[]
  ids = []
  with open('./data/mfcc/test.ark') as file:
    pre_id = ''
    video = []
    for line_id, line in enumerate(file):
      line = line.strip('\n').split(' ')
      id = '_'.join(line[0].split('_')[:2])
      if pre_id != id and len(video) != 0:
        split_videos.append(video)
        ids.append(pre_id)
        video = []
      pre_id = id
      video.append(frame_scores[line_id].argmax())
    split_videos.append(video)
    ids.append(pre_id)

  with open(path, 'w+') as file:
    file.write('id,phone_sequence\n')
    for video_id, sequence_48 in enumerate(split_videos):
      sequence_39 = [phone_map[int2phone[integer]] for integer in sequence_48]
      def comb(a, thres=3):
        assert (len(a) > 0)

        pre_char = a[0]
        cnt = 0
        filtered_a = []
        for entry in a:
          cur_char = entry
          if pre_char == cur_char:
            cnt += 1
          else:
            if cnt >= thres:
              filtered_a.append(pre_char)
            pre_char = cur_char
            cnt = 1
        if cnt >= thres:
          filtered_a.append(a[-1])
        return filtered_a
      comb_phone_sequence = comb(comb(sequence_39), 1)
      char_sequence = [phone2char[phone] for phone in comb_phone_sequence]

      string = ''.join(char_sequence).strip('L')
      line = ids[video_id]+","+string
      print(line)
      # input()
      file.write(line+'\n')

args = config.parse_arguments()
args.fac = int(args.use_bidirection) + 1

with tf.Graph().as_default():
  with tf.name_scope('test'):
    test_args = copy.deepcopy(args)
    test_args.mode = 'test'
  with tf.variable_scope('model', reuse=None):
    test_model = CNN_model(args=test_args)

  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True
  config.graph_options.optimizer_options.global_jit_level =\
    tf.OptimizerOptions.ON_1
  sv = tf.train.Supervisor(logdir=args.log_dir)

  test_frames, _ = load_data('test')

  with sv.managed_session() as sess:
    global_step = sess.run(test_model.step)
    print('global step = {}'.format(global_step))
    frame_scores = np.zeros([test_frames.shape[0], args.n_class], dtype=float)
    n_frames = test_frames.shape[0]
    n_batch = math.ceil(n_frames/args.batch_size)
    for batch_id in tqdm(range(n_batch)):
      def get_batch(frames, start_id):
        batch_frames = np.zeros([args.batch_size, args.window_size, args.dim])
        for idx in range(args.batch_size):
          batch_frames[idx, :, :] = frames.take(
          range(start_id+idx, start_id+idx + args.window_size),
          mode='wrap', axis=0).copy()
        return batch_frames
      start_id = batch_id*args.batch_size
      batch_frames = get_batch(test_frames, start_id)
      feed_dict = {test_model.frames_holder: batch_frames}
      prediction = sess.run(test_model.pred, feed_dict=feed_dict)
      for pred_batch_idx in range(args.batch_size):
        batch_start_id = pred_batch_idx*args.window_size
        batch_end_id = batch_start_id + args.window_size
        batch_pred = prediction[batch_start_id:batch_end_id]

        def filter_batch_pred(batch_pred, mode):
          filter_pred = np.zeros([args.window_size, args.n_class], dtype=float)
          if mode == 'filter':
            for idx, frame_pred in enumerate(batch_pred):
              def softmax(x):
                """Compute softmax values for each sets of scores in x."""
                e_x = np.exp(x - np.max(x))
                return e_x / e_x.sum(axis=0)
              softmax_frame_pred = softmax(frame_pred)
              n_over_threshold = np.sum(softmax_frame_pred > 0.5)
              # print(n_over_threshold)
              if n_over_threshold > 0 and n_over_threshold < 3:
                filter_pred[idx] = frame_pred.copy()
            return filter_pred
          elif mode == 'strip':
            filter_pred[args.vote_num: -args.vote_num] = batch_pred[args.vote_num: -args.vote_num]
            return filter_pred
        batch_pred = filter_batch_pred(batch_pred, 'strip')

        # input()
        frame_start_id = start_id+pred_batch_idx
        frame_end_id = frame_start_id + args.window_size
        if frame_end_id <= n_frames:
          frame_scores[frame_start_id:frame_end_id]\
            += batch_pred
        elif frame_end_id > n_frames and frame_start_id < n_frames:
          frame_scores[frame_start_id:] += batch_pred[:n_frames-frame_start_id]
          frame_scores[:frame_end_id-n_frames] += batch_pred[n_frames-frame_start_id:]

    write_result(frame_scores, args.pred_file)


from gpiozero import LED
import time

#%%   GPIO setup
enable = LED(17)
direc = LED(23)
step = LED(24)
MS1 = LED(5)
MS2 = LED(6)

#%%
class motor:
    def __init__(self):
        self.enable = enable
        self.direc = direc
        self.step = step
        self.MS1 = MS1
        self.MS2 = MS2

        self.stepdelay = 0.001
        self.MS1.off()
        self.MS2.on()

    def move(self,steps,direc):
        if direc == 0:
            self.direc.on()
        else:
            self.direc.off()
        self.enable.off()

        for i in range(steps):
            self.step.on()
            time.sleep(self.stepdelay)
            step.off()
            time.sleep(self.stepdelay)

        self.enable.on()

#%%  test settings


from django.conf.urls import url
from . import views
# from django.contrib import admin

urlpatterns = [
    url(r'^$', views.index, name='homepage'),
    url(r'^process$', views.process),
    url(r'^login$', views.login),
    url(r'^logout$', views.logout, name='logout'),
]


#! /usr/bin/python
# -*- coding: utf-8 -*-
import os
from configparser import ConfigParser

import yaml

def get_data_from_ini(node,key):
    """
    封装：获取element.ini文件方法
    :return:
    """
    from app_frame.my_configs import config
    path = os.path.join(config.BASE_DIR,'data/element.ini')
    target = ConfigParser()
    target.read(path,encoding='utf-8')
    result = target.get(node,key)
    result = result.split('|')
    return result

def get_data_from_yaml(path):
    """
    获取指定yaml文件的内容
    :param path:
    :return:
    """

    with open(path,'r',encoding='utf-8') as f:
        result = yaml.safe_load(f)
        return result

if __name__ == "__main__":
    # print(get_data_from_ini(node='main', key='goto_address'))
    from app_frame.my_configs import config
    s = get_data_from_yaml(path=os.path.join(config.BASE_DIR,'data/test_markpage_search.yaml'))
    print(s)


import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import numpy as np
from PIL import Image
from resnet import resnet18
import logging
from utils import ims_to_comet

logger = logging.getLogger("iCaRL")

class iCaRL(nn.Module):
    def __init__(self, feature_size, n_classes, learning_rate):
        # Network architecture
        super(iCaRL, self).__init__()
        self.feature_extractor = resnet18()
        self.feature_extractor.fc = nn.Linear(self.feature_extractor.fc.in_features,
                                              feature_size)
        self.fc = nn.Linear(feature_size, n_classes, bias=False)
        self.n_classes = n_classes
        self.n_known = 0

        # List containing exemplar_sets
        # Each exemplar_set is a np.array of N images
        # with shape (N, C, H, W)
        self.exemplar_sets = []

        self.crossentropy = nn.CrossEntropyLoss()
        self.binary_entropy = nn.BCEWithLogitsLoss()
        # self.optimizer = optim.Adam(self.parameters(), lr=learning_rate,
        #                             weight_decay=0.00001)
        # Means of exemplars
        self.exemplar_means = []

    def forward(self, x):
        x = self.feature_extractor(x)
        # x = self.bn(x)
        # x = self.ReLU(x)
        x = self.fc(x)
        return x

    def increment_classes(self, n):
        """Add n classes in the final fc layer"""
        in_features = self.fc.in_features
        out_features = self.fc.out_features
        weight = self.fc.weight.data

        self.fc = nn.Linear(in_features, out_features + n, bias=False)
        self.fc.weight.data[:out_features] = weight
        self.n_classes += n

    def recalculate_means(self, transform):
        with torch.no_grad():
            exemplar_means = []
            for P_y in self.exemplar_sets:
                features = []
                # Extract feature for each exemplar in P_y
                for ex in P_y:
                    ex = Variable(transform(Image.fromarray(ex))).cuda()
                    feature = self.feature_extractor(ex.unsqueeze(0))
                    feature = feature.squeeze()
                    feature.data = feature.data / feature.data.norm()  # Normalize
                    features.append(feature)
                features = torch.stack(features)
                mu_y = features.mean(0).squeeze()
                mu_y.data = mu_y.data / mu_y.data.norm()  # Normalize
                exemplar_means.append(mu_y)
            self.exemplar_means = exemplar_means

    def classify(self, x):
        """Classify images by neares-means-of-exemplars

        Args:
            x: input image batch
        Returns:
            preds: Tensor of size (batch_size,)
        """
        batch_size = x.size(0)
        exemplar_means = self.exemplar_means
        means = torch.stack(exemplar_means)  # (n_classes, feature_size)
        means = torch.stack([means] * batch_size)  # (batch_size, n_classes, feature_size)
        means = means.transpose(1, 2)  # (batch_size, feature_size, n_classes)

        feature = self.feature_extractor(x)  # (batch_size, feature_size)
        for i in np.arange(feature.size(0)):  # Normalize
            feature.data[i] = feature.data[i] / feature.data[i].norm()
        feature = feature.unsqueeze(2)  # (batch_size, feature_size, 1)
        feature = feature.expand_as(means)  # (batch_size, feature_size, n_classes)

        dists = (feature - means).pow(2).sum(1).squeeze()  # (batch_size, n_classes)
        _, preds = dists.min(1)

        return preds

    def construct_exemplar_set(self, images, m, transform):
        """Construct an exemplar set for image set

        Args:
            images: np.array containing images of a class
        """
        # Compute and cache features for each example
        features = []
        for img in images:
            x = Variable(transform(Image.fromarray(img)), requires_grad=False).cuda()
            feature = self.feature_extractor(x.unsqueeze(0)).data.cpu().numpy()
            feature = feature / np.linalg.norm(feature)  # Normalize
            features.append(feature[0])

        features = np.array(features)
        class_mean = np.mean(features, axis=0)
        class_mean = class_mean / np.linalg.norm(class_mean)  # Normalize

        exemplar_set = []
        exemplar_features = []  # list of Variables of shape (feature_size,)
        for k in np.arange(m):
            S = np.sum(exemplar_features, axis=0)
            phi = features
            mu = class_mean
            mu_p = 1.0 / (k + 1) * (phi + S)
            mu_p = mu_p / np.linalg.norm(mu_p)
            i = np.argmin(np.sqrt(np.sum((mu - mu_p) ** 2, axis=1)))

            exemplar_set.append(images[i])
            exemplar_features.append(features[i])
            """
            print "Selected example", i
            print "|exemplar_mean - class_mean|:",
            print np.linalg.norm((np.mean(exemplar_features, axis=0) - class_mean))
            #features = np.delete(features, i, axis=0)
            """
        self.exemplar_sets.append(np.array(exemplar_set))

    def reduce_exemplar_sets(self, exp_per_class):
        for y, P_y in enumerate(self.exemplar_sets):
            self.exemplar_sets[y] = P_y[:exp_per_class]

    def combine_dataset_with_exemplars(self, dataset):
        for y, P_y in enumerate(self.exemplar_sets):
            exemplar_images = P_y
            exemplar_labels = [y] * len(P_y)
            dataset.append(exemplar_images, exemplar_labels)

    def update_representation(self, experiment, dataset, batch_size, epochs, inc_phase, learning_rate):
        # Increment number of weights in final fc layer
        optimizer = optim.SGD(self.parameters(), lr=learning_rate, weight_decay=0.00001, momentum=0.9)
        classes = list(set(dataset.labels))
        new_classes = [cls for cls in classes if cls > self.n_classes - 1]
        self.increment_classes(len(new_classes))
        self.cuda()
        print("%d new classes" % (len(new_classes)))

        # Form combined training set
        self.combine_dataset_with_exemplars(dataset)

        loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
                                             shuffle=True, num_workers=2)

        # Store network outputs with pre-update parameters
        previous_logits = torch.zeros(len(dataset), self.n_classes).cuda()
        for indices, images, labels in loader:
            images = Variable(images).cuda()
            indices = indices.cuda()
            net_output = torch.sigmoid(self.forward(images))
            previous_logits[indices] = net_output.data
        previous_logits = Variable(previous_logits).cuda()

        # lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, params.LR_EPOCHS,
        #                                                     params.LR_DECAY)
        step = 0
        self.feature_extractor.train()
        for epoch in np.arange(epochs):
            # lr_scheduler.step()
            for i, (indices, images, labels) in enumerate(loader):
                images = Variable(images).cuda()
                labels = Variable(labels).cuda()
                one_hot_labels = torch.nn.functional.one_hot(labels, num_classes=self.n_classes)
                one_hot_labels = one_hot_labels.float()
                indices = indices.cuda()

                optimizer.zero_grad()
                net_output = self.forward(images)

                # simple loss
                # if self.n_known > 0:
                #      Classification loss for new classes
                #     loss = self.crossentropy(net_output, labels)
                #
                #     # Distilation loss for old classes
                #     q_i = previous_logits[indices]
                #     dist_loss = self.binary_entropy(net_output, q_i)
                #     loss += dist_loss
                # else:
                #     loss = self.crossentropy(net_output, labels)

                # more complex loss
                if self.n_known > 0:
                    # Classification loss for new classes

                    # valid_output = net_output[:, torch.tensor(new_classes)]
                    # valid_targets = one_hot_labels[:, torch.tensor(new_classes)]
                    # loss = self.binary_entropy(valid_output, valid_targets)

                    loss = self.crossentropy(net_output, labels)

                    # Distilation loss for old classes
                    q_i = previous_logits[indices].detach()
                    dist_output = net_output[:torch.tensor(self.n_known)]
                    dist_targets = q_i[:torch.tensor(self.n_known)]
                    # dist_loss = [self.dist_loss(net_output[:, y], q_i[:, y])
                    #              for y in range(self.n_known)]
                    # dist_loss = sum(dist_loss) #/ self.n_known
                    dist_loss = self.binary_entropy(dist_output, dist_targets)
                    loss += dist_loss
                else:
                    # loss = self.binary_entropy(net_output, one_hot_labels)
                    loss = self.crossentropy(net_output, labels)

                loss.backward()
                optimizer.step()

                experiment.log_metric("loss_inc_phase_{}".format(inc_phase), loss.data.item(),
                                      step=step, epoch=epoch)
                step += 1

                if (i + 1) % 10 == 0:
                    logger.info('Epoch {}, Iter {} Loss: {}'.format(epoch, i, loss.data.item()))
        self.feature_extractor.eval()


import chainer
from chainer import Variable, cuda
import numpy as np
import os
from common.utils.save_images import save_images_grid

def _sample_generation(gen_g, gen_f, dataset, eval_folder, batch_size, random, indices, trigger, count):
    if not os.path.exists(eval_folder):
        os.makedirs(eval_folder)

    xp = gen_f.xp

    if random:
        datasize_A = dataset.len_A
        datasize_B = dataset.len_B
        indices_A = np.random.choice(datasize_A, batch_size, replace=False)
        indices_B = np.random.choice(datasize_B, batch_size, replace=False)
    else:
        indices_A = indices[0]
        indices_B = indices[1]
    batches_A = [xp.asarray(dataset.get_example_raw_A(i)) for i in indices_A]
    batches_B = [xp.asarray(dataset.get_example_raw_B(i)) for i in indices_B]

    x = xp.stack(batches_A)
    y = xp.stack(batches_B)

    x = Variable(x)
    y = Variable(y)

    with chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
        x_y = gen_g(x)
        y_x = gen_f(y)
        x_y_x = gen_f(x_y)
        y_x_y = gen_g(y_x)

    imgs = xp.concatenate((x.data, x_y.data, x_y_x.data, y.data, y_x.data, y_x_y.data), axis=0)

    if trigger == 'iteration':
        prefix = 'iter_'
    elif trigger == 'epoch':
        prefix = 'epoch_'
    filename = prefix + str(count) + ".jpg"

    save_images_grid(imgs, path=os.path.join(eval_folder, filename))

def visualization(gen_g, gen_f, dataset, eval_folder=".", batch_size=1, random = True, indices = None):
    @chainer.training.make_extension()
    def samples_generation(trainer):
        trigger = 'iteration'
        count = getattr(trainer.updater, trigger)
        _sample_generation(gen_g, gen_f, dataset, eval_folder, batch_size, random, indices, trigger, count)

    return samples_generation


'''
1.6.8
25
Australian Voting
PC/UVa IDs: 110108/10142, Popularity: B, Success rate: low Level: 1
'''
import collections

def getWinner(voters, n):
    for aRound in range(n):
        currentVote = []
        for voter in voters:
            currentVote.append(voter[0])

        result = collections.Counter(currentVote)

        # winner
        for i in result:
            if result[i] > len(voters) / 2:
                return i

        # loser
        losers = set()
        minVotes = len(voters)
        for i in result:
            if result[i] < minVotes:
                minVotes = result[i]

        for i in result:
            if result[i] == minVotes:
                losers.add(i)

        for voter in voters:
            if voter[0] in losers:
                del voter[0]

if __name__ == '__main__':

    f = open('input.txt')

    numOfTests = int(f.readline())
    f.readline()
    for i in range(numOfTests):
        n = int(f.readline())
        names = []
        for j in range(n):
            names.append(f.readline().strip())

        voters = []
        while True:
            line = f.readline()
            if line =='':
                break
            voters.append([int(x) for x in line.split()])

        winner = getWinner(voters, n)
        print(names[winner - 1])

10142 - Australian Voting/austrilianBallot.py


#!/usr/bin/python3

import datetime
import os
import re
import subprocess
import sys

import mechanicalsoup
from urllib import parse

import test_common

def print_error(alias, link_text, link_url):
    base_path = alias.split("/")[0]

    # print to the main file
    fh = open("bot_output/all_data.csv", "a")
    print("'{0}','{1}','{2}'".format(alias, link_text, link_url), file=fh)
    fh.close()

    # print to a path-specific file
    if not os.path.isfile("bot_output/{0}.csv".format(base_path)):
        fh = open("bot_output/{0}.csv".format(base_path), "w")
        print("'Alias', 'Link text', 'Link URL'", file=fh)
        print("'{0}','{1}','{2}'".format(alias, link_text, link_url), file=fh)
        fh.close()
    else:
        fh = open("bot_output/{0}.csv".format(base_path), "a")
        print("'{0}','{1}','{2}'".format(alias, link_text, link_url), file=fh)
        fh.close()

def main(aliases, recover=False):
    browser = mechanicalsoup.Browser()
    browser.session.headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0"

    now = datetime.datetime.now()

    base_url = "https://www.une.edu"
    tested = {}

    counter = 0
    count = len(aliases)

    # setup output directory / big file
    if not os.path.isdir("bot_output"):
        os.mkdir("bot_output")

    if not recover: # remove old CSV files and open fresh all_data.csv
        for filename in os.listdir("bot_output/"):
            if os.path.isfile(filename) and filename.endswith(".csv"):
                os.remove("bot_output/{0}".format(filename))

        fh = open("bot_output/all_data.csv", "w")
        print("'Alias', 'Link text', 'Link URL'", file=fh)
        fh.close()

    une_re = re.compile("^https?:\/\/(www\.)?une\.edu")
    i_une_edu = re.compile("^https?:\/\/i\.une\.edu") # skip i.une.edu TODO: fix underlying SSL verification error
    doc_re = re.compile("(xls|doc|ppt|pdf)x?$", re.I) # pdfs and MS Office files

    while aliases:
        counter += 1

        alias = aliases.pop()

        # ignore news stories two years old or older
        components = alias.split("/")
        if components[0] == "news" and len(components) > 1:
            try:
                year = int(components[1])
                if year < now.year - 1:
                    continue
            except ValueError:
                pass

        uri = os.path.join(base_url, alias)

        print("testing {0} ({1}/{2})".format(uri, counter, count))
        try:
            page = browser.get(uri)
        except:
            print("skipping {0}: failed to GET".format(uri))
            tested[uri] = "???"
            continue

        if page.url != uri and (doc_re.search(page.url) or not une_re.match(page.url)):  # skip redirects off site or to PDFs
            print("skipping {0}: redirects to {1}".format(uri, page.url))
            tested[uri] = 300
            continue
        elif not page.ok: # >= 400
            print("skipping {0}: {1}".format(uri, page.status_code))
            tested[uri] = page.status_code
            continue

        links = page.soup.find_all("a")

        for link in links:
            link_text = re.sub(r"\s+", " ", link.text) # eliminate line breaks and tabstops

            try:
                link_url = link.attrs["href"]
            except KeyError: # probably javascript
                continue

            link_text = link_text.strip()
            link_url = link_url.strip()

            if i_une_edu.match(link_url):
                continue

            scheme = parse.urlparse(link_url).scheme
            if not scheme:
                link_url = base_url + link_url
            if scheme in ("mailto", "javascript"):
                continue

            if link_url not in tested.keys():
                try:
                    res = browser.get(link_url)
                except:
                    tested[link_url] = "???"
                    print("\ttesting {0} ...???".format(link_url))
                    print_error(alias, link_text, link_url)
                    continue

                if res.ok:
                    print("\ttesting {0} ...success!".format(link_url))
                else:
                    print("\ttesting {0} ...failure!".format(link_url))
                    print_error(alias, link_text, link_url)

                tested[link_url] = res.status_code

    with open("bot_output/raw_data.txt", "w") as fh:
        for link_url in tested.keys():
            print("{0}: {1}".format(link_url, tested[link_url]), file=fh)

if __name__ == "__main__":
    recover = False

    if os.path.isfile(".bot-recovery.txt"):
        with open(".bot-recovery.txt", "r") as fh:
            aliases = [l.strip() for l in fh.readlines()]
        recover = True
    else:
        aliases = test_common.get_all_aliases()

    try:
        main(aliases, recover)

        if os.path.isfile(".bot-recovery.txt"):
            os.remove(".bot-recovery.txt") # cleanup recovery file
    except:
        with open(".bot-recovery.txt", "w") as fh:
            for alias in aliases:
                print(alias, file=fh)
        raise


from PIL import Image
from winsound import Beep
import numpy as np
import sounddevice as sd

def compile_sounds(width:int, h_curve:list, img):
    """
        Given the length of one of the sides in a square image
        assign a frequency based on RGBA value.

        Then it adds them all together.

        Intuitively, I think it makes sense that the brighter
        colors have a higher freqency and so forth.

        The frequency is louder the higher the transparency.
    """

    """
        First we need a mapping of RGB values to frequencies
        RGB values go from 0x000000 (cooler)
        to 0xFFFFFF (warmer).

        winsound's Beep takes frequencys 37 hertz through 32,767 hertz
        This is only 0x7fda values
    """
    #define the range that winsound takes:
    min_hertz = 37
    max_hertz = 32767

    #try to evenly map out the smaller frequency lane to the color space of RBG
    colors = np.zeros((256,256,256), dtype = np.int32)
    for red in range(256):
        for green in range(256):
            for blue in range(256):
                #log scale of mapping frequency
                freq = np.exp( (np.log(max_hertz) - np.log(min_hertz))
                                    / 16777215 * (red * 65536 + green * 256 + blue)
                                    + np.log(min_hertz) )
                colors[red, green, blue] = int(freq)

    sounds = []
    area = width * width
    for pixel_index in range(area):
        pixel_coords = h_curve[pixel_index]
        rgba_val = img.getpixel(pixel_coords)
        red_ = rgba_val[0]
        green_ = rgba_val[1]
        blue_ = rgba_val[2]
        pixel_freq = colors[red_, green_, blue_]
        sounds.append(pixel_freq)

    play_multiple_freq_at_once(freqs=sounds,duration=1.0)

def play_multiple_freq_at_once(freqs:list, duration=1.0):
    """
        Given a list of freqencies to play and a duration,
        play the frequencies at once using math
        to calculate and summate their sinwaves in the air.
    """
    #The sound duration will be one second by default

    #This creates a time array which corresponds to the duration of the sinwave
    time = np.arange(int(44100 * duration)) / 44100

    #We need to add together the sinwaves of each frequency to play them at once.
    #This will hold our sinwaves:
    sounds = []
    for freq in freqs:
        #calculate the sinwave:
        wave = np.sin(2*np.pi * freq * time)
        sounds.append(wave)

    #add together the sinwaves now:
    comosite_sinwave = np.sum(sounds, axis = 0)
    #This normalizes the composite_sinwave as a range in [-1, 1]
    comosite_sinwave /= np.max(np.abs(comosite_sinwave))

    #Finally, play the composite wave:
    sd.play(comosite_sinwave, samplerate=44100, blocking=True)


import logging
import os
import sys
import time

from typing import Union
from io import IOBase

from .base import Client
from tftpy.shared import TIMEOUT_RETRIES
from tftpy.packet import types
from tftpy.exceptions import TftpException,TftpTimeout,TftpFileNotFoundError
from tftpy.states import SentReadRQ,SentWriteRQ

logger = logging.getLogger('tftpy.context.client')

class Upload(Client):
    """The upload context for the client during an upload.
    Note: If input is a hyphen, then we will use stdin."""

    def __init__(self, host: str, port: int, timeout: int,
                 input: Union[IOBase,str], **kwargs) -> None:
        """Upload context for uploading data to a server.

        Args:
            host (str): Server Address
            port (int): Server Port
            timeout (int): socket timeout
            input ([IOBase,str]): Input data, can be one of
                - An open file object
                - A path to a file
                - a '-' indicating read from STDIN
        """

        super().__init__(host, port, timeout, **kwargs)

        # If the input object has a read() function, assume it is file-like.
        if hasattr(input, 'read'):
            self.fileobj = input
        elif input == '-':
            self.fileobj = sys.stdin
        else:
            self.fileobj = open(input, "rb")

        logger.debug("tftpy.context.client.upload.__init__()")
        logger.debug(f" file_to_transfer = {self.file_to_transfer}, options = {self.options}")

    def start(self) -> None:
        """Main loop to read data in and send file to the server."""

        logger.info(f"Sending tftp upload request to {self.host}")
        logger.info(f"    filename -> {self.file_to_transfer}")
        logger.info(f"    options -> {self.options}")

        self.metrics.start_time = time.time()
        logger.debug(f"Set metrics.start_time to {self.metrics.start_time}")

        pkt = types.WriteRQ()
        pkt.filename = self.file_to_transfer
        pkt.mode = self.mode
        pkt.options = self.options

        self.send(pkt)
        self.state = SentWriteRQ(self)

        while self.state:
            try:
                logger.debug(f"State is {self.state}")
                self.cycle()
            except TftpTimeout as err:
                logger.error(str(err))
                self.retry_count += 1

                if self.retry_count >= TIMEOUT_RETRIES:
                    logger.debug("hit max retries, giving up")
                    raise
                else:
                    logger.warning("resending last packet")
                    self.state.resend_last()

    def end(self, *args):
        """Finish up the context."""

        super().end()

        self.metrics.end_time = time.time()
        logger.debug(f"Set metrics.end_time to {self.metrics.end_time}")
        self.metrics.compute()

class Download(Client):
    """The download context for the client during a download.
    Note: If output is a hyphen, then the output will be sent to stdout."""

    def __init__(self, host: str, port: int, timeout: int,
                 output: Union[IOBase,str], **kwargs) -> None:
        """Initalize the Download context with the server and
           where to save the data

        Args:
            host (str): Server Address
            port (int): Server port
            timeout (int): Socket Timeout
            output (Union[IOBase,str]): Output data, can be one of
                - An open file object
                - A path to a file
                - '-' indicating write to STDOUT

        Raises:
            TftpException: unable to open the destiation file for writing
        """

        super().__init__(host, port, timeout, **kwargs)

        self.filelike_fileobj = False

        # If the output object has a write() function, assume it is file-like.
        if hasattr(output, 'write'):
            self.fileobj = output
            self.filelike_fileobj = True
        # If the output filename is -, then use stdout
        elif output == '-':
            self.fileobj = sys.stdout
            self.filelike_fileobj = True
        else:
            try:
                self.fileobj = open(output, "wb")
            except OSError as err:
                raise TftpException("Could not open output file", err)

        logger.debug("tftpy.context.client.Download.__init__()")
        logger.debug(f" file_to_transfer = {self.file_to_transfer}, options = {self.options}")

    def start(self) -> None:
        """Initiate the download.

        Raises:
            TftpTimeout: Failed to connect to the server
            TftpFileNotFoundError: Recieved a File not fount error
        """

        logger.info(f"Sending tftp download request to {self.host}")
        logger.info(f"    filename -> {self.file_to_transfer}")
        logger.info(f"    options -> {self.options}")

        self.metrics.start_time = time.time()
        logger.debug(f"Set metrics.start_time to {self.metrics.start_time}")

        pkt = types.ReadRQ()
        pkt.filename = self.file_to_transfer
        pkt.mode = self.mode
        pkt.options = self.options

        self.send(pkt)
        self.state = SentReadRQ(self)

        while self.state:
            try:
                logger.debug(f"State is {self.state}")
                self.cycle()

            except TftpTimeout as err:
                logger.error(str(err))
                self.retry_count += 1
                if self.retry_count >= TIMEOUT_RETRIES:
                    logger.debug("hit max retries, giving up")
                    raise TftpTimeout("Max retries reached")
                else:
                    logger.warning("resending last packet")
                    self.state.resend_last()

            except TftpFileNotFoundError as err:
                # If we received file not found, then we should not save the open
                # output file or we'll be left with a size zero file. Delete it,
                # if it exists.

                logger.error("Received File not found error")
                if self.fileobj is not None and not self.filelike_fileobj and os.path.exists(self.fileobj.name):
                    logger.debug(f"unlinking output file of {self.fileobj.name}")
                    os.unlink(self.fileobj.name)

                raise TftpFileNotFoundError(err)

    def end(self) -> None:
        """Finish up the context."""

        super().end(not self.filelike_fileobj)
        self.metrics.end_time = time.time()
        logger.debug(f"Set metrics.end_time to {self.metrics.end_time}")
        self.metrics.compute()


import numpy as np
from copy import deepcopy
from numbers import Number
from itertools import islice

from sklearn.exceptions import NotFittedError
from sklearn.utils.validation import check_is_fitted
from sklearn.base import clone

def is_multi_response(y):
    """
    Checks whether or not y is a multiple output response.

    Parameters
    ----------
    y: array-like

    Output
    ------
    is_mat: bool
        Whether or not y is a multi-output response
    """

    y = np.array(y)
    if y.ndim == 2 and y.shape[1] >= 2:
        return True
    else:
        return False

def lasso_and_ridge_from_enet(pen_val, l1_ratio):
    """
    Returns the lasso and L2 penalties from the elastic net parameterization

    Parameters
    ----------
    pen_val: float

    l1_ratio: float, None

    Output
    ------
    lasso_pen, ridge_pen
    """

    if l1_ratio is None or l1_ratio == 0:
        lasso_pen = None
        ridge_pen = pen_val

    elif l1_ratio == 1:
        lasso_pen = pen_val
        ridge_pen = None

    else:
        lasso_pen = pen_val * l1_ratio
        ridge_pen = pen_val * (1 - l1_ratio)

    return lasso_pen, ridge_pen

def get_coef_and_intercept(est, copy=False, error=False):
    """
    Extracts the coefficient and intercept from an estimatator.

    Parameters
    ----------
    est:
        The estimator.

    copy: bool
        Whether or not to copy the underlying data.

    error: bool
        If no coefficient is found, whether or not to throw an exception.

    Output
    ------
    coef, intercept

    coef: array-like
        The coefficient.

    intercept: float, array-like, None
        The intercept. Returns None if not found.
    """

    if hasattr(est, 'coef_'):
        coef = est.coef_
        if hasattr(est, 'intercept_'):
            intercept = est.intercept_
        else:
            intercept = None

    elif hasattr(est, 'best_estimator_'):
        # try getting the data from the selected estimator
        return get_coef_and_intercept(est=est.best_estimator_,
                                      copy=copy,
                                      error=error)

    else:
        if error:
            raise ValueError("Unable to detect coefficient")

    if copy:
        return deepcopy(coef), deepcopy(intercept)

    else:
        return coef, intercept

def maybe_add(d, exclude_false=False, **kws):
    """
    Adds keywork argumnts to a dict if their values are not None.

    Parameters
    ----------
    d: dict
        The dictionary to add to.

    exclude_false: bool
        Exclue keys whose values are false.

    kws: dict
        The keys to maybe add

    """
    for k, v in kws.items():
        if v is not None:
            d[k] = v

    return d

def clip_zero(x, zero_tol=1e-8):
    """
    Sets x or the elements of x to zero when they are very small.

    Parameters
    ----------
    x: Number, array-like
        The value or values to clip.

    zero_tol: float
        The tolerance below which we declare a number to be zero.

    Output
    ------
    x_clipped: Number or np.array

    """
    if isinstance(x, Number):
        if abs(x) <= zero_tol:
            return 0
        else:
            return x

    x_ = np.zeros_like(x)
    non_zero_mask = abs(np.array(x)) > zero_tol
    x_[non_zero_mask] = x[non_zero_mask]
    return x_

def at_most_one_none(*args):
    """
    Returns True if at most one of the args is not None.
    """
    n_none = sum([a is None for a in args])
    return n_none <= 1

def is_fitted(estimator):
    """
    Checks if an estimator has been fitted.

    Parameters
    ----------
    estimator: sklearn estimator
        The estimator to check.

    Output
    ------
    bool:
        Returns True iff the estimator has been fitted.
    """

    try:
        check_is_fitted(estimator)
        is_fitted = True
    except NotFittedError:
        is_fitted = False
    return is_fitted

def fit_if_unfitted(estimator, X, y=None, **fit_params):
    """
    Fits an estimator if it has not yet been fitted. If it has been fit then will just return the input estimator.

    Parameters
    -----------
    estimator: sklearn estimator
        The estimator.

    X, y:
        The input to estimator.fit(X, y, **fit_params)

    **fit_params:
        Keyword args to be  passed to fit

    Output
    ------
    estimator:
        The fitted estimator.
    """
    if not is_fitted(estimator):
        return clone(estimator).fit(X, y, **fit_params)
    return estimator

def delete_fit_attrs(est):
    """
    Removes any fit attribute from an estimator.
    """
    fit_keys = [k for k in est.__dict__.keys() if k.endswith('_')]
    for k in fit_keys:
        del est.__dict__[k]
    return est

def is_str_and_matches(check, value, lower=True):
    """
    Checks if an input is a string and if it matches a given value.

    Parameters
    ----------
    check:
        The object to check.

    value: str
        The value we want to match.

    lower: bool
        Lower case both check and value before matching.
    """

    if not isinstance(check, str):
        return False

    else:
        if lower:
            return check.lower() == value.lower()
        else:
            return check == value

def get_shapes(n_features, n_responses):
    """
    Gets the shapes of the coefficeint and intercept.

    Parameters
    ----------
    n_features: int
        Number of features.

    n_responses: int
        Number of responses

    Output
    ------
    coef_shape, intercept_shape

    coef_shape: tuple
        Shape of the coefficient.

    intercept_shape: tuple
        Shape of the intercept.
    """
    if n_responses == 1:
        coef_shape = (n_features, )
        intercept_shape = ()
    else:
        coef_shape = (n_features, n_responses)
        intercept_shape = (n_responses, )

    return coef_shape, intercept_shape

def get_shapes_from(X, y):
    """
    Gets the shapes of the coefficeint and intercept.

    Parameters
    ----------
    X: array-like, shape (n_samples, n_features)
        The covariate matrix.

    y: array-like, shape (n_samples, ) or (n_samples, n_responses)
        The response data.

    Output
    ------
    coef_shape, intercept_shape

    coef_shape: tuple
        Shape of the coefficient.

    intercept_shape: tuple
        Shape of the intercept.
    """
    n_features = X.shape[1]

    if y.ndim == 1:
        n_respones = 1
    else:
        n_respones = y.shape[1]

    return get_shapes(n_features=n_features, n_responses=n_respones)

def get_from(iterable, idx):
    """
    Selects an index from an iterable that may be a generator without going through the entire generator.
    Acts like list(iterable)[idx]
    """
    try:
        return next(islice(iterable, idx, idx + 1))
    except StopIteration:
        raise ValueError("idx={} was out of range for iterable".format(idx))

def count_support(x, zero_tol='machine'):
    """
    Counts the number of non-zero elements in an array.
    Parameters
    ----------
    x: array-like
        The array whose support we want to count.

    zero_tol: float, str
        Tolerance for declaring a small value equal to zero. If zero_tol=='machine' then we use the machine precision i.e. np.finfo(x.dtype).eps.

    Output
    ------
    n_nonzero: int
        Number of non zero elements of x.
    """

    _x = np.array(x)
    if zero_tol == 'machine':
        zero_tol = np.finfo(_x.dtype).eps

    return (abs(_x) > zero_tol).sum()

def lb_transform_to_indices(lb, y):
    """
    Transforms the dummay variable encoding output by a LabelBinarizer to integers. Safely returns a numpy vector even if the binerizer returns a sparse output.

    Essentially just

    lb.fit_transform(y).argmax(axis=1),

    but with additional numpy converstion.

    Parameters
    ----------
    lb: sklearn.preprocessing.LabelBinarizer
        The fit label binarizer.

    y: array-like
        The target values to transform

    Output
    ------
    y_idxs: array-like of ints
        The class label indices.
    """
    return np.array(lb.fit_transform(y).argmax(axis=1)).reshape(-1)

def enet_params_from_sum(pen_val_1, pen_val_2):
    """
    Computes the elastic net pen_val and mix_val from the two penalty values.

    pen_val_1 = pen_val * mix_val
    pen_val_2 = pen_val * (1 - mix_val )

    Parameters
    ----------
    pen_val_1, pen_val_2: float
        The two penalty values in the sum.

    Output
    ------
    pen_val, mix_val: float
        The elastic net parameters.
    """
    pen_val = pen_val_1 + pen_val_2
    mix_val = pen_val_1 / (pen_val_1 + pen_val_2)
    return pen_val, mix_val


"""Steidel+2010 Table 4: Absorption Line Strengths at z_fg in Galaxy Pairs.

DOI: 10.1088/0004-637X/717/1/289

Only corrected values are kept for fg samples.
"""
import re

from astropy.table import Table
import matplotlib.pyplot as plt
import numpy as np
import requests

from toolbox.env import root

url = 'http://iopscience.iop.org/0004-637X/717/1/289/suppdata/apj348432t4_ascii.txt'
path = root / 'data/ref/steidel_2010.ecsv'

def _rm_fn_mark(s):
    return re.sub('\^[a-z]', '', s)

def _parse_num(s):
    if s.startswith('<'):
        val = float(s[1:])
        err = np.nan
        uplim = True
    else:
        s = _rm_fn_mark(s)
        s = s.strip('()')
        val = float(s.split()[0])
        err = float(s.split()[-1])
        uplim = False
    return val, err, uplim

def make():
    """Make an easily loadable local copy from the online ASCII table."""
    r = requests.get(url)
    for line in r.text.split('\n'):
        tokens = line.strip().split('\t')
        parse_row = False
        # Initialize table with columns
        if tokens[0] == 'Sample':
            # Read column names
            names = line.strip().split('\t')
            names[1] = 'b'
            names = [_rm_fn_mark(name) for name in names]
            n_lines = len(names) - 2
            # Expand line names into three parts
            line_names = names[2:]
            names = names[:2]
            for name in line_names:
                names += [name, name + ' err', name + ' uplim']
            dtype = [object, float] + [float, float, bool] * n_lines
            table = Table(names=names, dtype=dtype)
            # Assign column units
            for cn in table.colnames:
                if cn == 'Sample':
                    continue
                if cn == 'b':
                    table[cn].unit = 'kpc'
                elif 'uplim' not in cn:
                    table[cn].unit = 'AA'
        # Mark rows to parse
        if 'fg' in tokens[0]:
            # Skip the rows with uncorrescted values
            sample = tokens[0]
        if tokens[0] == '0^c':
            tokens = [sample] + tokens
            parse_row = True
        if 'bg' in tokens[0]:
            parse_row = True
        # Parse row
        if parse_row:
            row = [tokens[0], float(_rm_fn_mark(tokens[1]))]
            for ew in tokens[2:]:
                row += list(_parse_num(ew))
            table.add_row(row)
    table.meta['code'] = 'toolbox.data.steidel_2010'
    table.write(path, overwrite=True)

def load():
    """Load the table from local path."""
    return Table.read(path)

def plot(line, x0=2, uplim_err=0.02, ax=None, **kwargs):
    if ax is None:
        ax = plt.gca()
    table = load()
    x = table['b']
    order = x.argsort()
    x = x[order]
    x[:3] = x0
    y = table[line][order]
    yerr = table[line + ' err'][order]
    uplims = table[line + ' uplim'][order]
    yerr[uplims] = uplim_err
    return ax.errorbar(x, y, yerr=yerr, uplims=uplims, **kwargs)


#!/usr/bin/python
import xml.sax
from itertools import product
import argparse
import sys
parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-x", "--orthoXML", type=str, required=True,
                    help="Flat non-hierarchical orthoXML file")
args = parser.parse_args()

class OrthoXMLHandler(xml.sax.ContentHandler):
    def __init__(self):
        xml.sax.ContentHandler.__init__(self)
        self.genes = {}
        self.ingroup = -1
        self.groups = {}
        self.species = None

    def startElement(self, name, attrs):
        if name == 'species':
            self.species = attrs['name']

        if name == 'gene' and self.species:
            self.genes[attrs['id']] = '{}_{}'.format(
                attrs['protId'], self.species)

        if name == 'orthologGroup':
            self.ingroup = attrs['id']

        if name == 'geneRef' and self.ingroup != -1:
            self.groups.setdefault(self.ingroup, []).append(
                self.genes[attrs['id']])

    def endElement(self, name):
        if name == 'orthologGroup':
            self.ingroup = -1
        if name == 'species':
            self.species = None

    def getGroups(self):
        return(self.groups)

with open(args.orthoXML, "r") as oxmlfile:
    handler = OrthoXMLHandler()
    parser = xml.sax.make_parser()
    parser.setContentHandler(handler)
    parser.parse(oxmlfile)
    g = handler.getGroups()
    # Parse groups
    for gid, gval in g.items():
        paralogs = {}
        for p in gval:
            prot, sp = p.split('_', 1)
            paralogs.setdefault(sp, []).append(prot)
        # Output inparalogs
        xsp, ysp = paralogs.values()
        for x, y in product(xsp, ysp):
            # Some error in OrthoXML file one identifier assigned to two species
            # <gene id="11143" protId="P63058"/>
            # <gene id="11144" protId="P63058"/>
            if x != y:
                print('{}\t{}'.format(x, y))


from enum import Enum
from tkinter import VERTICAL

from model.display import Dimens, Colors
from widget.inflater import Inflater
from widget.layout import LinearLayout

class Tab(Enum):

    @property
    def image(self):
        return self.value.get('image')

    @property
    def label(self):
        return self.value.get('label')

    @property
    def view(self):
        return self.value.get('view')

class TabWidget(LinearLayout):
    def __init__(self, parent, orientation=VERTICAL):
        LinearLayout.__init__(self, parent, orientation)
        self._tabs = {}
        self._active_tab = None
        self._content = None
        self._btn_config = {
            'width': 10,
            'height': 2,
            'anchor': 'w' if orientation == VERTICAL else 'center',
            'padx': Dimens.SPACE_BIG,
            'bd': 0
        }
        self._separator = None
        if orientation == VERTICAL:
            self.columnconfigure(2, weight=1)
        else:
            self.rowconfigure(2, weight=1)

    def add(self, tab: Tab):
        self._tabs[tab] = Inflater.button(self, tab.label, lambda t=tab: self.switch(t), **self._btn_config)
        LinearLayout.add(self, self._tabs[tab])
        size = len(self._tabs)
        if size and self._orientation == VERTICAL:
            self.rowconfigure(size - 1, weight=0)
            self.rowconfigure(size, weight=1)
        elif size:
            self.columnconfigure(size - 1, weight=0)
            self.columnconfigure(size, weight=1)
        self._update_separator()

    def switch(self, tab):
        if self._active_tab:
            self._tabs[self._active_tab]['bg'] = Colors.BG_PRIMARY
        if self._content:
            self._content.destroy()
        self._active_tab = tab
        self._tabs[tab]['bg'] = Colors.BG_DARK
        self._content = tab.view(self)
        if self._orientation == VERTICAL:
            self._content.grid(row=0, column=2, rowspan=len(self._tabs) + 1, sticky='nswe')
        else:
            self._content.grid(row=2, column=0, columnspan=len(self._tabs) + 1, sticky='nswe')

    def _update_separator(self):
        if self._separator:
            self._separator.destroy()
        self._separator = Inflater.separator(self, orientation=self._orientation)
        if self._orientation == VERTICAL:
            self._separator.grid(row=0, column=1, rowspan=len(self._tabs) + 1, sticky='ns')
        else:
            self._separator.grid(row=1, column=0, columnspan=len(self._tabs) + 1, sticky='we')


#!/usr/bin/env python3.6

# renames image files in given folder according to their creation date
# taken from
# https://orthallelous.wordpress.com/2015/04/19/extracting-date-and-time-from-images-with-python/

import os
from datetime import datetime as dt
from datetime import timedelta
from PIL import Image
import time

folder_path = 'C:/Users/karel.gavenciak/Desktop/Gruzie fotky Eva'

def imgDate(fn):
    "returns the image date from image (if available)\nfrom Orthallelous"
    std_fmt = '%Y:%m:%d %H:%M:%S.%f'
    # for subsecond prec, see doi.org/10.3189/2013JoG12J126 , sect. 2.2, 2.3
    tags = [(36867, 37521),  # (DateTimeOriginal, SubsecTimeOriginal)
            (36868, 37522),  # (DateTimeDigitized, SubsecTimeOriginal)
            (306, 37520), ]  # (DateTime, SubsecTime)
    exif = Image.open(fn)._getexif()

    for t in tags:
        dat_stmp = exif.get(t[0])
        sub_stmp = exif.get(t[1], 0)

        # PIL.PILLOW_VERSION >= 3.0 returns a tuple
        dat_stmp = dat_stmp[0] if type(dat_stmp) == tuple else dat_stmp
        sub_stmp = sub_stmp[0] if type(sub_stmp) == tuple else sub_stmp
        if dat_stmp != None: break

    if dat_stmp == None: return None
    full = '{}.{}'.format(dat_stmp, sub_stmp)
    # print(full)
    T = dt.strptime(full, std_fmt)
    # T = time.mktime(time.strptime(T, '%Y:%m:%d %H:%M:%S')) + float('0.%s' % sub)
    return T

for f in os.listdir(folder_path):
    date = imgDate(os.path.join(folder_path,f))

    os.rename(os.path.join(folder_path,f),os.path.join(folder_path,date.strftime('IMG_%Y%m%d_%H%M%S.JPG')))


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
print(os.listdir("../input"))
import sys
import matplotlib.pyplot as plt
sys.path.append('rxrx1-utils')
from tqdm import tqdm
from PIL import Image
import rxrx.io as rio
#!git clone https://github.com/recursionpharma/rxrx1-utils
import zipfile

# Load the Pandas libraries with alias 'pd'
import pandas as pd
# Read data from file 'filename.csv'
data = pd.read_csv("../input/balanced_sample_train.csv")
# Preview the first 5 lines of the loaded data
print(data.head())
#
# # pos_control = data[data['well_type']=='positive_control']
# # print(pos_control.head())
# #
# # well_type = 'positive_control'
#
# filename = "../input/train.csv"
# data = pd.read_csv(filename)

# data = data[data['well_type'] == well_type]
print((data.shape))

storage_path = '../input/train'

def convert_to_rgb(df, split, resize=False, new_size=224, extension='png'):
    RGB_MAP = {
        1: {
            'rgb': np.array([19, 0, 249]),
            'range': [0, 51]
        },
        2: {
            'rgb': np.array([42, 255, 31]),
            'range': [0, 107]
        },
        3: {
            'rgb': np.array([255, 0, 25]),
            'range': [0, 64]
        },
        4: {
            'rgb': np.array([45, 255, 252]),
            'range': [0, 191]
        },
        5: {
            'rgb': np.array([250, 0, 253]),
            'range': [0, 89]
        },
        6: {
            'rgb': np.array([254, 255, 40]),
            'range': [0, 191]
        }
    }

    N = df.shape[0]

    print(df.head())
    print(df['id_code'].iloc[1])
    for i in tqdm(range(N)):

        code = df['id_code'].iloc[i]
        experiment = df['experiment'].iloc[i]
        plate = df['plate'].iloc[i]
        well = df['well'].iloc[i]
        sirna = df['sirna'].iloc[i]

        for site in [1, 2]:

            folder = '../input/'+str(split)+'/'+str(sirna)+'/'

            if not os.path.exists(folder):
                os.makedirs(folder)

            save_path = '../input/'+str(split)+'/'+str(sirna)+'/'+str(code)+'_s'+str(site)+'.'+str(extension)

            im = rio.load_site_as_rgb(
                split, experiment, plate, well, site, channels=(1, 2, 3, 4, 5, 6),
                base_path='../input/', rgb_map=RGB_MAP
            )
            im = im.astype(np.uint8)
            im = Image.fromarray(im)

            if resize:
                im = im.resize((new_size, new_size), resample=Image.BILINEAR)

            im.save(save_path)

def build_new_df(df, extension='.png'):
    new_df = pd.concat([df, df])
    new_df['filename'] = pd.concat([
        df['id_code'].apply(lambda string: string + '_s1' ),
        df['id_code'].apply(lambda string: string + '_s2' )

    ])

    new_df['filename'] = new_df['filename'] + "_" + new_df["sirna"].map(str) + str(extension)

    return new_df

def zip_and_remove(path):
    ziph = zipfile.ZipFile(str(path)+'.zip', 'w', zipfile.ZIP_DEFLATED)

    for root, dirs, files in os.walk(path):
        for file in tqdm(files):
            file_path = os.path.join(root, file)
            ziph.write(file_path)
            os.remove(file_path)

    ziph.close()

convert_to_rgb(data, 'train')
zip_and_remove('train')

new_train = build_new_df(data)
new_train.to_csv('../input/balanced_sample_new_train.csv', index=False)


import re
from typing import List

import nltk as nltk
from pyaspeller import YandexSpeller

def text_correction(speller: YandexSpeller, text: str) -> str:
    changes = {change['word']: change['s'][0] for change in speller.spell(text) if len(change['s']) > 0}
    for word, suggestion in changes.items():
        if word[0].isupper():
            continue
        text = text.replace(word, suggestion)
    return text

def to_ascii(sentence):
    """
    Encode sentence in ASCII
    :type sentence: str
    """
    try:
        return sentence.decode('utf-8').encode('ascii', 'ignore').decode('ascii')
    except:
        return sentence.encode('ascii', 'ignore').decode('ascii')

def split_by_houses(rows: List[str]) -> List[List[str]]:
    objectives = []
    indexes = [i for i, row in enumerate(rows) if re.search(r'\s*[А-Я,A-Z]+\d+\n$', row)]
    indexes.append(len(rows) - 1)
    start_objective_index = indexes[0]
    for i in indexes[1:-1]:
        objectives.append(rows[start_objective_index:i])
        start_objective_index = i
    return objectives

def text_correct(speller: YandexSpeller, rows: List[str]) -> list:
    results = []

    # for line in lines:
    #     if line.endswith('-\n'):
    # lines = filter(lambda line: not re.search('^\n', line), lines)
    for line in rows:
        if line == '\n':
            continue
        line = re.sub('\n', ' ', line)
        results.append(line)
    text = ' '.join(results)
    text = re.sub('-[\n|\s]+', '', text)
    text = text_correction(speller, text)
    tokens = nltk.sent_tokenize(text)
    return tokens

def run(rows: List[str]) -> str:
    speller = YandexSpeller()
    address_description = split_by_houses(rows)
    result = []
    for block in address_description:
        result.append(text_correct(speller, block))
    return result

if __name__ == '__main___':
    text_photos = '/Users/rinatahmetov/code/sandbox/python/secrets_of_arkham/text_photo2.txt'
    with open(text_photos, 'r') as f:
        lines = f.readlines()
    results = run(lines)
    print('\n'.join(results))


#!/usr/bin/env python
# -*- coding:utf-8 -*-

import urllib2
from saml2 import (
    BINDING_HTTP_POST,
    BINDING_HTTP_REDIRECT,
    entity,
)
from saml2.client import Saml2Client
from saml2.config import Config as Saml2Config

from django.conf import settings
from django.core.urlresolvers import reverse
from django.contrib.auth.models import (User, Group)
from django.contrib.auth.decorators import login_required
from django.contrib.auth import login, logout
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt
from django.template import TemplateDoesNotExist
from django.http import HttpResponseRedirect
from django.utils.module_loading import import_string

def get_current_domain(r):
    return '{scheme}://{host}'.format(
        scheme=r.scheme,
        host=r.get_host(),
    )

def _get_saml_client(domain):
    acs_url = domain + reverse('acs')
    import tempfile
    tmp = tempfile.NamedTemporaryFile()
    f = open(tmp.name, 'w')
    f.write(urllib2.urlopen(settings.SAML2_AUTH['METADATA_AUTO_CONF_URL']).read())
    f.close()
    saml_settings = {
        'metadata': {
            "local": [tmp.name],
        },
        'service': {
            'sp': {
                'endpoints': {
                    'assertion_consumer_service': [
                        (acs_url, BINDING_HTTP_REDIRECT),
                        (acs_url, BINDING_HTTP_POST)
                    ],
                },
                'allow_unsolicited': True,
                'authn_requests_signed': False,
                'logout_requests_signed': True,
                'want_assertions_signed': True,
                'want_response_signed': False,
            },
        },
    }

    spConfig = Saml2Config()
    spConfig.load(saml_settings)
    spConfig.allow_unknown_attributes = True
    saml_client = Saml2Client(config=spConfig)
    tmp.close()
    return saml_client

@login_required
def welcome(r):
    try:
        return render(r, 'django_saml2_auth/welcome.html', context={'user': r.user})
    except TemplateDoesNotExist:
        return HttpResponseRedirect(reverse('admin:index'))

def denied(r):
    return render(r, 'django_saml2_auth/denied.html')

def _create_new_user(username, email, firstname, lastname):
    user = User.objects.create_user(username, email)
    user.first_name = firstname
    user.last_name = lastname
    user.groups = [Group.objects.get(name=x) for x in settings.SAML2_AUTH.get('NEW_USER_PROFILE', {}).get('USER_GROUPS', [])]
    user.is_active = settings.SAML2_AUTH.get('NEW_USER_PROFILE', {}).get('ACTIVE_STATUS', True)
    user.is_staff = settings.SAML2_AUTH.get('NEW_USER_PROFILE', {}).get('STAFF_STATUS', True)
    user.is_superuser = settings.SAML2_AUTH.get('NEW_USER_PROFILE', {}).get('SUPERUSER_STATUS', False)
    user.save()
    return user

@csrf_exempt
def acs(r):
    saml_client = _get_saml_client(get_current_domain(r))
    resp = r.POST.get('SAMLResponse', None)
    next_url = r.session.get('login_next_url', reverse('admin:index'))

    if not resp:
        return HttpResponseRedirect(reverse('denied'))

    authn_response = saml_client.parse_authn_request_response(
        resp, entity.BINDING_HTTP_POST)
    if authn_response is None:
        return HttpResponseRedirect(reverse('denied'))

    user_identity = authn_response.get_identity()
    if user_identity is None:
        return HttpResponseRedirect(reverse('denied'))

    user_email = user_identity[settings.SAML2_AUTH.get('ATTRIBUTES_MAP', {}).get('email', 'Email')][0]
    user_name = user_identity[settings.SAML2_AUTH.get('ATTRIBUTES_MAP', {}).get('username', 'UserName')][0]
    user_first_name = user_identity[settings.SAML2_AUTH.get('ATTRIBUTES_MAP', {}).get('first_name', 'FirstName')][0]
    user_last_name = user_identity[settings.SAML2_AUTH.get('ATTRIBUTES_MAP', {}).get('last_name', 'LastName')][0]

    target_user = None
    is_new_user = False

    try:
        target_user = User.objects.get(username=user_name)
        if settings.SAML2_AUTH.get('TRIGGER', {}).get('BEFORE_LOGIN', None):
            import_string(settings.SAML2_AUTH['TRIGGER']['BEFORE_LOGIN'])(user_identity)
    except User.DoesNotExist:
        target_user = _create_new_user(user_name, user_email, user_first_name, user_last_name)
        if settings.SAML2_AUTH.get('TRIGGER', {}).get('CREATE_USER', None):
            import_string(settings.SAML2_AUTH['TRIGGER']['CREATE_USER'])(user_identity)
        is_new_user = True

    r.session.flush()

    if target_user.is_active:
        target_user.backend = 'django.contrib.auth.backends.ModelBackend'
        login(r, target_user)
    else:
        return HttpResponseRedirect(reverse('denied'))

    if is_new_user:
        try:
            return render(r, 'django_saml2_auth/welcome.html', context={'user': r.user})
        except TemplateDoesNotExist:
            return HttpResponseRedirect(next_url)
    else:
        return HttpResponseRedirect(next_url)

def signin(r):
    import urlparse
    from urllib import unquote
    next_url = r.GET.get('next', reverse('admin:index'))

    try:
        if "next=" in unquote(next_url):
            next_url = urlparse.parse_qs(urlparse.urlparse(unquote(next_url)).query)['next'][0]
    except:
        next_url = r.GET.get('next', reverse('admin:index'))

    r.session['login_next_url'] = next_url

    saml_client = _get_saml_client(get_current_domain(r))
    _, info = saml_client.prepare_for_authenticate()

    redirect_url = None

    for key, value in info['headers']:
        if key == 'Location':
            redirect_url = value
            break

    return HttpResponseRedirect(redirect_url)

def signout(r):
    logout(r)
    return render(r, 'django_saml2_auth/signout.html')


'''
Created on Jun 7, 2016

@author: leegc
'''
import sidekick.checks as checks
from sidekick.checks import error_messages
from sidekick.metadata import ActiveContainerMetadata
from sidekick.metadata import MessageGenerator
import shutil
import os
import logging
import json
import random
from tempfile import mkdtemp

DIV = "__***__"
PULL_TAG_KEY = "PULL_TAG"
UNTRUSTED_IMAGE="deleidos/de-schema-wizard-untrusted"
UNTRUSTED_IMAGE_KEY = "U_IMAGE"
UNTRUSTED_USER = 1111
UNTRUSTED_GROUP = 999
UNTRUSTED_PROFILE_KEY = "U_PROFILE"

class ContainerTracker(object):

    def __init__(self):
        self.tag = os.environ[PULL_TAG_KEY]
        if UNTRUSTED_IMAGE_KEY in os.environ:
            self.untrusted_image_name = os.environ[UNTRUSTED_IMAGE_KEY]
        else:
            self.untrusted_image_name = UNTRUSTED_IMAGE
        if UNTRUSTED_PROFILE_KEY not in os.environ:
            raise OSError("You must specify the AppArmor profile that untrusted code will use \
                with the \'U_PROFILE\' environment variable.  To use the default Docker profile,\
                 set this variable to \'docker-default\'.")
        self.untrusted_profile = os.environ[UNTRUSTED_PROFILE_KEY]
        logging.info("Using untrusted profile - " + self.untrusted_profile)

    def start_untrusted(self, docker_cli, label_list=['untrusted'], image_str='hello-world', volumes_from=[], environment={}, user=UNTRUSTED_USER, entrypoint=None, mem_limit="100m", security_opt=None):
        host_config = docker_cli.create_host_config(volumes_from=volumes_from, network_mode = 'none',
                                                     mem_limit=mem_limit, read_only=True, security_opt=security_opt)

        logging.info("Host config: " + str(host_config))
        logging.info("Starting " + image_str+ " as user " + str(user) + ".")
        user_with_group = str(user) + ":" + str(UNTRUSTED_GROUP)
        container = docker_cli.create_container(image=image_str, labels=label_list, host_config=host_config,
                                                 environment=environment, network_disabled=True, user=user_with_group)
        c_id = container.get('Id')
        logging.info("Starting "+image_str+" with id " + c_id + ".")
        docker_cli.start(container=c_id)
        return c_id

    def send_result_to_ie(self, redis, container_metadata, data):
        tmp_dir = container_metadata.tmp_dir
        response_channel = container_metadata.response_channel
        redis.publish(response_channel, json.dumps(data))
        logging.info("Sent " + str(data) + " to " + response_channel + ".")
        logging.info("Removing tree: " + tmp_dir + ".")
        shutil.rmtree(tmp_dir)

    def send_error_result_to_ie(self, redis, response_channel):
        data = {'result':False,'error':True,'message':checks.error_messages[checks.INTERNAL_ERROR]}
        redis.publish(response_channel, json.dumps(data))
        logging.info("Sent " + str(data) + " to " + response_channel + ".")

    def handle_results(self, docker_cli, redis_conn, container_id, shutdown_metadata_code):

        std_out = docker_cli.logs(container=container_id, stdout=True, tail=10).decode("utf-8")
        logging.info("Raw script logs: " + std_out)

        message_gen = MessageGenerator()
        message_gen.report_metadata_shutdown_code(shutdown_metadata_code)

        data = {'result':False,'error':False,'message':None }
        string_result = None
        lines = std_out.split('\n')
        is_next = False
        for line in reversed(lines):
            if line == DIV:
                is_next = True
            elif is_next:
                string_result = line
                break
        if string_result is None:
            if shutdown_metadata_code == checks.OK:
                message_gen.reset_with_new_message(checks.error_messages[checks.SUSPICIOUS])
            else:
                message_gen.add_to_message("No result detected.  Defaulted to false.")
            logging.warning(message_gen.get_message())
        elif len(string_result) > 1000:
            message_gen.add_to_message("Length of result is suspiciously long.  Defaulted to false.")
            logging.warning(message_gen.get_message())
            # mark as suspicious
        else:
            logging.info("Raw script results: " + string_result)
            try:
                raw_data = json.loads(string_result)

                message_gen.report_raw_result(raw_data['exception'], raw_data['line'])

                if raw_data['result'] == "True":
                    data['result'] = True
                elif not raw_data['result'] == "False":
                    message_gen.reset_with_new_message("Unexpected result - must return boolean (True or False).")

                if raw_data['error'] == "True":
                    if data['result']:
                        message_gen.add_to_message("Unexpected result - 'true' result with a reported error.  Defaulted to false.")
                        data['result'] = False

            except Exception as exc:
                logging.warning(message_gen.get_message())
                logging.exception(exc)
                message_gen.reset_with_new_message("Result could not be determined.  Defaulted to false.")
        data['message'] = message_gen.get_message()
        logging.info("Processed script results: " + str(data) + ".")
        return data

    def handle_requests(self, docker_cli, redis, pubsub, data_volume):

            message = pubsub.get_message()
            if message is not None:
                if message['type'] == 'message':
                    # receive request from interpretation engine via Redis
                    data = message['data'].decode("utf-8")
                    j = json.loads(data)
                    response_channel = j['response-channel-id']
                    try:
                        j.pop('response-channel-id')

                        user_id = random.randint(1001, 5000)
                        tmp_dir = mkdtemp(dir="/usr/local/shared/untrusted/")
                        tmp = open(os.path.join(tmp_dir, "metrics-dictionary.json"), 'w')
                        tmp.write(json.dumps(j))
                        tmp.close()
                        logging.info("Updating " + tmp_dir + " and " + tmp.name + ".")
                        os.chown(tmp_dir, user_id, UNTRUSTED_GROUP)
                        os.chown(tmp.name, user_id, UNTRUSTED_GROUP)
                        #out = check_output(["usermod", "-g", str(UNTRUSTED_GROUP), str(user_id)])
                        #logging.info(out)
                        os.chmod(tmp_dir, 0o700)
                        os.chmod(tmp.name, 0o700)
                        env = {'RESOURCE_PATH':tmp.name}

                        image_str=self.untrusted_image_name+":"+self.tag
                        if self.untrusted_profile is not None:
                            sec_opt = ["apparmor="+self.untrusted_profile]
                        else:
                            sec_opt = None
                        c_id = self.start_untrusted(docker_cli, volumes_from=[data_volume['Id']], image_str=image_str, environment=env, user=user_id, security_opt=sec_opt)

                        container_metadata = ActiveContainerMetadata(response_channel, tmp_dir, checks.get_checks())
                        logging.info("Response channel for " + c_id + " is " + response_channel + ".")
                        return c_id, container_metadata
                    except Exception as err:
                        logging.exception(err)
                        logging.error("Container did not successfully start.")
                        self.send_error_result_to_ie(redis, response_channel)
                        return None


import json
from requests import get, post

def getBlockchain(URL, PORT):
    op = "blocks"
    target = URL + ":" + str(PORT) + '/' + op

    res = get(target)
    return res

def addNewBlock(URL, PORT, req=""):
    op = "mineBlock"
    target = URL + ":" + str(PORT) + '/' + op

    headers = {'Content-type': 'application/json'}
    data = {"data": req}
    res = post(target, data=json.dumps(data), headers=headers)

    return res

def getPeers(URL, PORT):
    op = "peers"
    target = URL + ":" + str(PORT) + '/' + op

    res = get(target)
    return res

def addPeer(URL, PORT, req=None):
    op = "addPeer"
    target = URL + ":" + str(PORT) + '/' + op

    headers = {'Content-type': 'application/json'}
    data = {"peer": req}
    res = post(target, data=json.dumps(data), headers=headers)
    return res

def stopNode(URL, PORT):
    op = "stop"
    target = URL + ":" + str(PORT) + '/' + op

    res = post(target)
    return res


import numpy as np
import scipy
import ot

def filter_for_common_genes(slices):
    """
    param: slices - list of slices (AnnData objects)
    """
    assert len(slices) > 0, "Cannot have empty list."

    common_genes = slices[0].var.index
    for s in slices:
        common_genes = intersect(common_genes, s.var.index)
    for i in range(len(slices)):
        slices[i] = slices[i][:, common_genes]
    print('Filtered all slices for common genes. There are ' + str(len(common_genes)) + ' common genes.')

def kl_divergence(X, Y):
    """
    Returns pairwise KL divergence (over all pairs of samples) of two matrices X and Y.

    param: X - np array with dim (n_samples by n_features)
    param: Y - np array with dim (m_samples by n_features)

    return: D - np array with dim (n_samples by m_samples). Pairwise KL divergence matrix.
    """
    assert X.shape[1] == Y.shape[1], "X and Y do not have the same number of features."

    X = X/X.sum(axis=1, keepdims=True)
    Y = Y/Y.sum(axis=1, keepdims=True)
    log_X = np.log(X)
    log_Y = np.log(Y)
    X_log_X = np.matrix([np.dot(X[i],log_X[i].T) for i in range(X.shape[0])])
    D = X_log_X.T - np.dot(X,log_Y.T)
    return np.asarray(D)

def intersect(lst1, lst2):
    """
    param: lst1 - list
    param: lst2 - list

    return: list of common elements
    """

    temp = set(lst2)
    lst3 = [value for value in lst1 if value in temp]
    return lst3

def norm_and_center_coordinates(X):
    """
    param: X - numpy array

    return:
    """
    return (X-X.mean(axis=0))/min(scipy.spatial.distance.pdist(X))

def match_spots_using_spatial_heuristic(X,Y,use_ot=True):
    """
    param: X - numpy array
    param: Y - numpy array

    return: pi- mapping of spots using spatial heuristic
    """
    n1,n2=len(X),len(Y)
    X,Y = norm_and_center_coordinates(X),norm_and_center_coordinates(Y)
    dist = scipy.spatial.distance_matrix(X,Y)
    if use_ot:
        pi = ot.emd(np.ones(n1)/n1, np.ones(n2)/n2, dist)
    else:
        row_ind, col_ind = scipy.sparse.csgraph.min_weight_full_bipartite_matching(scipy.sparse.csr_matrix(dist))
        pi = np.zeros((n1,n2))
        pi[row_ind, col_ind] = 1/max(n1,n2)
        if n1<n2: pi[:, [(j not in col_ind) for j in range(n2)]] = 1/(n1*n2)
        elif n2<n1: pi[[(i not in row_ind) for i in range(n1)], :] = 1/(n1*n2)
    return pi

## Covert a sparse matrix into a dense matrix
to_dense_array = lambda X: np.array(X.todense()) if isinstance(X,scipy.sparse.csr.spmatrix) else X

## Returns the data matrix or representation
extract_data_matrix = lambda adata,rep: adata.X if rep is None else adata.obsm[rep]


"""add show

Revision ID: 36f659f65c7
Revises: None
Create Date: 2013-08-25 21:26:14.813041

"""

# revision identifiers, used by Alembic.
revision = '36f659f65c7'
down_revision = '212bfc51b4ed'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('like',
                  sa.Column('username', sa.VARCHAR(255),
                  server_default='', nullable=False, index=True))
    op.add_column('follow',
                  sa.Column('username', sa.VARCHAR(255),
                  server_default='', nullable=False, index=True))

def downgrade():
    op.drop_column('like', 'username')
    op.drop_column('follow', 'username')


#!/usr/bin/env python3
"""
Author : oom
Date   : 2020-06-07
Purpose: Emulate wc (word count)
"""

import argparse
import sys

# -----------------------------------------------------------------------------
def get_args():
    """Get command-line arguments"""

    parser = argparse.ArgumentParser(
        description='Emulate wc (word count)',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('file',
                        help='Input file(s)',
                        metavar='FILE',
                        nargs="*",
                        type=argparse.FileType('r'),
                        default=[sys.stdin])

    return parser.parse_args()

# -----------------------------------------------------------------------------
def main():
    """Make a jazz noise here"""

    args = get_args()
    files_arg = args.file

    total_lines = 0
    total_words = 0
    total_bytes = 0
    for file_arg in files_arg:
        num_lines = 0
        num_words = 0
        num_bytes = 0
        for line in file_arg:
            num_lines += 1
            num_words += len(line.split())
            num_bytes += len(line)

        total_lines += num_lines
        total_words += num_words
        total_bytes += num_bytes

        print("{:8}{:8}{:8} {}".format(
            num_lines, num_words, num_bytes, file_arg.name))

    if len(files_arg) > 1:
        print("{:8}{:8}{:8} {}".format(
            total_lines, total_words, total_bytes, "total"))

# -----------------------------------------------------------------------------
if __name__ == '__main__':
    main()