"""This file includes a collection of utility functions that are useful for
implementing DQN."""
import gym
import numpy as np
import random

def sample_n_unique(sampling_f, n):
    res = []
    while len(res) < n:
        candidate = sampling_f()
        if candidate not in res:
            res.append(candidate)
    return res

class Schedule(object):
    def value(self, t):
        raise NotImplementedError()

class ConstantSchedule(object):
    def __init__(self, value):
        self._v = value

    def value(self, t):
        """See Schedule.value"""
        return self._v

def linear_interpolation(l, r, alpha):
    return l + alpha * (r - l)

class PiecewiseSchedule(object):
    def __init__(self, endpoints, interpolation=linear_interpolation, outside_value=None):
        idxes = [e[0] for e in endpoints]
        assert idxes == sorted(idxes)
        self._interpolation = interpolation
        self._outside_value = outside_value
        self._endpoints      = endpoints

    def value(self, t):
        """See Schedule.value"""
        for (l_t, l), (r_t, r) in zip(self._endpoints[:-1], self._endpoints[1:]):
            if l_t <= t and t < r_t:
                alpha = float(t - l_t) / (r_t - l_t)
                return self._interpolation(l, r, alpha)

        # t does not belong to any of the pieces, so doom.
        assert self._outside_value is not None
        return self._outside_value

class LinearSchedule(object):
    def __init__(self, schedule_timesteps, final_p, initial_p=1.0):
        self.schedule_timesteps = schedule_timesteps
        self.final_p            = final_p
        self.initial_p          = initial_p

    def value(self, t):
        fraction  = min(float(t) / self.schedule_timesteps, 1.0)
        return self.initial_p + fraction * (self.final_p - self.initial_p)

def get_wrapper_by_name(env, classname):
    currentenv = env
    while True:
        if classname in currentenv.__class__.__name__:
            return currentenv
        elif isinstance(env, gym.Wrapper):
            currentenv = currentenv.env
        else:
            raise ValueError("Couldn't find wrapper named %s"%classname)

class ReplayBuffer(object):
    def __init__(self, size, frame_history_len):
        self.size = size
        self.frame_history_len = frame_history_len

        self.next_idx      = 0
        self.num_in_buffer = 0

        self.obs      = None
        self.action   = None
        self.reward   = None
        self.done     = None

    def can_sample(self, batch_size):
        return batch_size + 1 <= self.num_in_buffer

    def _encode_sample(self, idxes):
        obs_batch      = np.concatenate([self._encode_observation(idx)[np.newaxis, :] for idx in idxes], 0)
        act_batch      = self.action[idxes]
        rew_batch      = self.reward[idxes]
        next_obs_batch = np.concatenate([self._encode_observation(idx + 1)[np.newaxis, :] for idx in idxes], 0)
        done_mask      = np.array([1.0 if self.done[idx] else 0.0 for idx in idxes], dtype=np.float32)

        return obs_batch, act_batch, rew_batch, next_obs_batch, done_mask

    def sample(self, batch_size):
        assert self.can_sample(batch_size)
        idxes = sample_n_unique(lambda: random.randint(0, self.num_in_buffer - 2), batch_size)
        return self._encode_sample(idxes)

    def encode_recent_observation(self):
        assert self.num_in_buffer > 0
        return self._encode_observation((self.next_idx - 1) % self.size)

    def _encode_observation(self, idx):
        end_idx   = idx + 1 # make noninclusive
        start_idx = end_idx - self.frame_history_len
        # this checks if we are using low-dimensional observations, such as RAM
        # state, in which case we just directly return the latest RAM.
        if len(self.obs.shape) == 2:
            return self.obs[end_idx-1]
        # if there weren't enough frames ever in the buffer for context
        if start_idx < 0 and self.num_in_buffer != self.size:
            start_idx = 0
        for idx in range(start_idx, end_idx - 1):
            if self.done[idx % self.size]:
                start_idx = idx + 1
        missing_context = self.frame_history_len - (end_idx - start_idx)
        # if zero padding is needed for missing context
        # or we are on the boundry of the buffer
        if start_idx < 0 or missing_context > 0:
            frames = [np.zeros_like(self.obs[0]) for _ in range(missing_context)]
            for idx in range(start_idx, end_idx):
                frames.append(self.obs[idx % self.size])
            return np.concatenate(frames, 0)
        else:
            # this optimization has potential to saves about 30% compute time \o/
            img_h, img_w = self.obs.shape[2], self.obs.shape[3]
            return self.obs[start_idx:end_idx].reshape(-1, img_h, img_w)

    def store_frame(self, frame):
        if len(frame.shape) > 1:
            # transpose image frame into (img_c, img_h, img_w)
            frame = frame.transpose(2, 0, 1)

        if self.obs is None:
            self.obs      = np.empty([self.size] + list(frame.shape), dtype=np.uint8)
            self.action   = np.empty([self.size],                     dtype=np.int32)
            self.reward   = np.empty([self.size],                     dtype=np.float32)
            self.done     = np.empty([self.size],                     dtype=np.bool)
        self.obs[self.next_idx] = frame

        ret = self.next_idx
        self.next_idx = (self.next_idx + 1) % self.size
        self.num_in_buffer = min(self.size, self.num_in_buffer + 1)

        return ret

    def store_effect(self, idx, action, reward, done):
        self.action[idx] = action
        self.reward[idx] = reward
        self.done[idx]   = done


import json
import socket
from uuid import getnode as get_mac
import os
import sys

class Settings:

  def __init__(self, etcf, log):
    self.etcfname = etcf
    self.log = log
    self.mqtt_server = "192.168.1.7"   # From json
    self.mqtt_port = 1883              # From json
    self.mqtt_client_name = "detection_1"   # From json
    self.homie_device = None            # From json
    self.homie_name = None              # From json
    # IP and MacAddr are not important (should not be important).
    if sys.platform.startswith('linux'):
      s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
      s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
      s.connect(('<broadcast>', 0))
      self.our_IP =  s.getsockname()[0]
      # from stackoverflow (of course):
      self.macAddr = ':'.join(("%012x" % get_mac())[i:i+2] for i in range(0, 12, 2))
    elif sys.platform.startswith('darwin'):
      host_name = socket.gethostname()
      self.our_IP = socket.gethostbyname(host_name)
      self.macAddr = ':'.join(("%012x" % get_mac())[i:i+2] for i in range(0, 12, 2))
    else:
      self.our_IP = "192.168.1.255"
      self.macAddr = "de:ad:be:ef"
    self.load_settings(self.etcfname)
    print("Settings from", self.etcfname)

  def load_settings(self, fn):
    conf = json.load(open(fn))
    self.mqtt_server = conf.get("mqtt_server_ip", "192.168.1.255")
    self.mqtt_port = conf.get("mqtt_port", 1883)
    self.mqtt_client_name = conf.get("mqtt_client_name", "Bad Client")
    #  Homie,
    self.mqtt_root = conf.get("mqtt_root", "homie/network/status")
    self.mqtt_basic = conf.get("mqtt_basic", "basic")     # is a property, root+basic+/set
    self.mqtt_tb = conf.get("mqtt_tb" "trumpybear")
    self.check_every = conf.get("check_every", 120)
    self.ping_list = conf.get("ping_list", None)
    self.hubitat = conf.get("hubitat", "192.168.1.5")

  def print(self):
    print("==== Settings ====")
    print(self.settings_serialize())

  def settings_serialize(self):
    st = {}
    st['mqtt_server_ip'] = self.mqtt_server
    st['mqtt_port'] = self.mqtt_port
    st['mqtt_client_name'] = self.mqtt_client_name
    st["mqtt_root"] = self.mqtt_root
    st["mqtt_basic"] = self.mqtt_basic
    st["mqtt_tb"] = self.mqtt_tb
    st["check_every"] = self.check_every
    st["ping_list"] = self.ping_list
    st["hubitat"] = self.hubitat
    str = json.dumps(st)
    return str

  def settings_deserialize(self, jsonstr):
    st = json.loads(jsonstr)


import os
import sys
import setuptools
from setuptools import setup, find_packages

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.md')).read()
CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()

requires = [
    'simplejson',
    'psycopg2',
    'pyparsing',
]

#
# eggs that you need if you're running a version of python lower than 2.7
#
if sys.version_info[:2] < (2, 7):
    requires.extend(['argparse>=1.2.1', 'unittest2>=0.5.1'])

#
# eggs you need for development, but not production
#
dev_extras = (
    'zc.buildout',
    'coverage>=3.5.2',
    'fabric>=1.4.3',
    'zest.releaser>=3.37',
    'nose'
)

setup(
    name='ott.osm',
    version='0.1.0',
    description='Open Transit Tools - OSM Tools (Python)',
    long_description=README + '\n\n' + CHANGES,
    classifiers=[
        "Programming Language :: Python",
    ],
    author="Open Transit Tools",
    author_email="info@opentransittools.com",
    dependency_links=('http://opentransittools.com',),
    license="Mozilla-derived (http://opentransittools.com)",
    url='http://opentransittools.com',
    keywords='ott, otp, view, transit, osm',
    packages=find_packages(),
    include_package_data=True,
    zip_safe=False,
    install_requires=requires,
    extras_require=dict(dev=dev_extras),
    tests_require=requires,
    test_suite="ott.osm",
    entry_points="""\
        [paste.app_factory]
        main = ott.view.pyramid.config:main
    """,
)


#!/usr/bin/python3
__author__ = 'Philip'
from app.keys import FileNameKeys, JsonKeys, GeoFileNameKeys, TweetKeywordKeys
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
# from twitter_api_filterer import lupus_api_filterer
# from twitter_api_util import save_new_geo_data, GEO_STATES_UPDATE_STATUS
import json
# from twitter_api_states import api_states
# from twitter_api_additional_map_filters import update_geo_option_data

class StdOutListener(StreamListener):
    def on_data(self, data):
        with open(FileNameKeys.FETCHED_TWEETS_KEY, 'a') as tf:
            # Check to make sure data is not null and can be loaded from json to text
            if data:
                text = json.loads(data)
                # filters tweets before storing them to see if they are related to lupus -WIP
                if lupus_api_filterer(text):
                    # checks if the tweet has any valid geo data otherwise skips updating the json file
                    state = api_states(text)
                    if state is False:
                        with open(GeoFileNameKeys.GEO_STATES_UPDATE_STATUS_KEY, 'r') as update_status_infile:
                            status_check = json.load(update_status_infile)
                        if status_check[JsonKeys.STATUS_KEY]:
                            update_status = {"status": False}
                            with open(GeoFileNameKeys.GEO_STATES_UPDATE_STATUS_KEY, 'w') as update_status_outfile:
                                json.dump(update_status, update_status_outfile)
                    else:
                        # Updates geo data if there was a update to the geo data counter
                        save_new_geo_data(state)
                        update_geo_option_data(text, state)

                    # Appends the new tweet to the list of previous tweets
                    tf.write(data)
                    # Prints the tweet into the console for visual observance of tweet coming in
                    # print('@%s : %s\n' % (text[JsonKeys.USER_KEY][JsonKeys.SCREEN_NAME_KEY], text[JsonKeys.TEXT_KEY]))
                    return True

    def on_error(self, status):
        print(status)

def lupus_tweet_tracker_setup():
    # This handles Twitter authentication and the connection to Twitter Streaming API
    l = StdOutListener()
    auth = OAuthHandler(TwitterApiAuthKeys.CONSUMER_KEY, TwitterApiAuthKeys.CONSUMER_SECRET_KEY)
    auth.set_access_token(TwitterApiAuthKeys.ACCESS_TOKEN_KEY, TwitterApiAuthKeys.ACCESS_TOKEN_SECRET_KEY)
    stream = Stream(auth, l)

    stream.filter(track=TweetKeywordKeys.LUPUS_KEYWORDS_KEY)

if __name__ == '__main__':
    lupus_tweet_tracker_setup()


import sys
import nltk
from nltk.stem.porter import PorterStemmer
from glob import glob
from infi.clickhouse_orm import Database
from models import Fragment

def trim_punctuation(word):
    '''
    Trim punctuation characters from the beginning and end of the word
    '''
    start = end = len(word)
    for i in range(len(word)):
        if word[i].isalnum():
            start = min(start, i)
            end = i + 1
    return word[start : end]

def parse_file(filename):
    '''
    Parses a text file at the give path.
    Returns a generator of tuples (original_word, stemmed_word)
    The original_word may include punctuation characters.
    '''
    stemmer = PorterStemmer()
    with open(filename, 'r', encoding='utf-8') as f:
        for line in f:
            for word in line.split():
                yield (word, stemmer.stem(trim_punctuation(word)))

def get_fragments(filename):
    '''
    Converts a text file at the given path to a generator
    of Fragment instances.
    '''
    from os import path
    document = path.splitext(path.basename(filename))[0]
    idx = 0
    for word, stem in parse_file(filename):
        idx += 1
        yield Fragment(document=document, idx=idx, word=word, stem=stem)
    print('{} - {} words'.format(filename, idx))

if __name__ == '__main__':

    # Load NLTK data if necessary
    nltk.download('punkt')
    nltk.download('wordnet')

    # Initialize database
    db = Database('default')
    db.create_table(Fragment)

    # Load files from the command line or everything under ebooks/
    filenames = sys.argv[1:] or glob('ebooks/*.txt')
    for filename in filenames:
        db.insert(get_fragments(filename), batch_size=100000)


import io
import os
import time
from collections import defaultdict, deque
import datetime
import pandas as pd

import torch
import torch.distributed as dist

from pathlib import Path
import logging
log = logging.getLogger()
console = logging.StreamHandler()
log.addHandler(console)
log.setLevel(logging.INFO)

CSV_COL_ORDER = ['init_timestamp', 'train_timestamp', 'train_dur', 'train_speed',
                 'train_eval_start_timestamp', 'train_eval_end_timestamp', 'train_eval_dur', 'train_eval_speed',
                 'val_eval_start_timestamp', 'val_eval_end_timestamp', 'val_eval_dur', 'val_eval_speed',
                 'f_timestamp', 'epoch_dur',
                 'train_loss', 'train_acc1', 'train_acc5', 'train_best_acc1_batch', 'train_best_acc5_batch',
                 'val_loss', 'val_acc1', 'val_acc5', 'val_best_acc1_batch', 'val_best_acc5_batch'
                 ]

class SmoothedValue(object):
    """Track a series of values and provide access to smoothed values over a
    window or the global series average.
    """

    def __init__(self, window_size=20, fmt=None):
        if fmt is None:
            fmt = "{median:.4f} ({global_avg:.4f})"
        self.deque = deque(maxlen=window_size)
        self.total = 0.0
        self.count = 0
        self.fmt = fmt

    def update(self, value, n=1):
        self.deque.append(value)
        self.count += n
        self.total += value * n

    def synchronize_between_processes(self):
        """
        Warning: does not synchronize the deque!
        """
        if not is_dist_avail_and_initialized():
            return
        t = torch.tensor([self.count, self.total],
                         dtype=torch.float64, device='cuda')
        dist.barrier()
        dist.all_reduce(t)
        t = t.tolist()
        self.count = int(t[0])
        self.total = t[1]

    @property
    def median(self):
        d = torch.tensor(list(self.deque))
        return d.median().item()

    @property
    def avg(self):
        d = torch.tensor(list(self.deque), dtype=torch.float32)
        return d.mean().item()

    @property
    def global_avg(self):
        return self.total / self.count

    @property
    def max(self):
        return max(self.deque)

    @property
    def value(self):
        return self.deque[-1]

    def __str__(self):
        return self.fmt.format(
            median=self.median,
            avg=self.avg,
            global_avg=self.global_avg,
            max=self.max,
            value=self.value)

class MetricLogger(object):
    def __init__(self, delimiter="\t"):
        self.meters = defaultdict(SmoothedValue)
        self.delimiter = delimiter

    def update(self, **kwargs):
        for k, v in kwargs.items():
            if isinstance(v, torch.Tensor):
                v = v.item()
            assert isinstance(v, (float, int))
            self.meters[k].update(v)

    def __getattr__(self, attr):
        if attr in self.meters:
            return self.meters[attr]
        if attr in self.__dict__:
            return self.__dict__[attr]
        raise AttributeError("'{}' object has no attribute '{}'".format(
            type(self).__name__, attr))

    def __str__(self):
        loss_str = []
        for name, meter in self.meters.items():
            loss_str.append(
                "{}: {}".format(name, str(meter))
            )
        return self.delimiter.join(loss_str)

    def synchronize_between_processes(self):
        for meter in self.meters.values():
            meter.synchronize_between_processes()

    def add_meter(self, name, meter):
        self.meters[name] = meter

    def log_every(self, iterable, print_freq, header=None):
        i = 0
        if not header:
            header = ''
        start_time = time.time()
        end = time.time()
        iter_time = SmoothedValue(fmt='{avg:.4f}')
        data_time = SmoothedValue(fmt='{avg:.4f}')
        space_fmt = ':' + str(len(str(len(iterable)))) + 'd'
        log_msg = [
            header,
            '[{0' + space_fmt + '}/{1}]',
            'eta: {eta}',
            '{meters}',
            'time: {time}',
            'data: {data}'
        ]
        if torch.cuda.is_available():
            log_msg.append('max mem: {memory:.0f}')
        log_msg = self.delimiter.join(log_msg)
        MB = 1024.0 * 1024.0
        for obj in iterable:
            data_time.update(time.time() - end)
            yield obj
            iter_time.update(time.time() - end)
            if i % print_freq == 0 or i == len(iterable) - 1:
                eta_seconds = iter_time.global_avg * (len(iterable) - i)
                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
                if torch.cuda.is_available():
                    print(log_msg.format(
                        i, len(iterable), eta=eta_string,
                        meters=str(self),
                        time=str(iter_time), data=str(data_time),
                        memory=torch.cuda.max_memory_allocated() / MB))
                else:
                    print(log_msg.format(
                        i, len(iterable), eta=eta_string,
                        meters=str(self),
                        time=str(iter_time), data=str(data_time)))
            i += 1
            end = time.time()
        total_time = time.time() - start_time
        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
        print('{} Total time: {} ({:.4f} s / it)'.format(
            header, total_time_str, total_time / len(iterable)))

class CustomLogger:
    def __init__(self, args):
        self.logs = {}
        self.msg_list = []
        self.epochs = args.epochs
        self.output_dir = Path(args.output_dir)
        self.temp_epoch = {'train_loss': [], 'train_acc1': [], 'train_acc5': [],
                           'val_loss': [], 'val_acc1': [], 'val_acc5': []
                           }
        self.best_acc_batch = {'train_acc1': 0,
                               'train_acc5': 0,
                               'val_acc1': 0,
                               'val_acc5': 0
                               }
        self.list_args(args)
        self.n_batches = {}

    @staticmethod
    def get_ep_name(epoch):
        ep_str = str(epoch + 1)
        for i in [2, 3]:
            if len(ep_str) < i:
                ep_str = '0' + ep_str
        return f'ep_{ep_str}'

    @staticmethod
    def now():
        return datetime.datetime.now().replace(microsecond=0)

    def save_num_batches(self, train_dataloader, val_dataloader, batch_size):
        for loader, split in zip([train_dataloader, val_dataloader], ['train', 'val']):
            num_batches, total_count = 0, 0
            for _, targets in loader:
                num_batches += 1
                total_count += targets.shape[0]
            msg = f'{split} data volume: {total_count} images, separated to {num_batches} batches ' \
                  f'(batch size {batch_size})'
            self.log(msg)
            self.n_batches[split] = num_batches

    def list_args(self, args):
        start_msg = "LeViT + FFCV" if args.ffcv_load == 'Y' else "LeViT ONLY"
        self.log(start_msg)
        self.log("Listing arguments:")
        self.log(args)
        # for k, v in vars(args).items():
        #     self.log(f'{k}={v}')

    def log(self, msg):
        log.info(msg)
        self.msg_list.append(str(msg))

    def start_epoch(self, epoch):
        ep_name = self.get_ep_name(epoch)
        self.logs[ep_name] = {}
        self.logs[ep_name]['init_timestamp'] = self.now()

    def start_eval(self, epoch, split):
        ep_name = self.get_ep_name(epoch)
        self.logs[ep_name][f'{split}_eval_start_timestamp'] = self.now()

    def update(self, **kwargs):
        for k, v in kwargs.items():
            self.temp_epoch[k].append(v)
            if 'acc' in k:
                self.best_acc_batch[k] = max(self.best_acc_batch[k], v)

    def sum_train(self, epoch):
        ep_name = self.get_ep_name(epoch)
        self.logs[ep_name]['train_timestamp'] = self.now()
        self.logs[ep_name]['train_dur'] = self.logs[ep_name]['train_timestamp'] - self.logs[ep_name]['init_timestamp']
        train_dur = self.logs[ep_name]['train_dur'].seconds
        train_speed = train_dur / self.n_batches['train']
        self.logs[ep_name]['train_speed'] = train_speed
        msg = f'{self.now()} - EPOCH [{ep_name.split("_")[-1]}] TRAIN_SPEED {round(train_speed, 5)} sec/batch'
        self.log(msg)

    def sum_eval(self, epoch, split):
        ep_name = self.get_ep_name(epoch)
        now = self.now()
        self.logs[ep_name][f'{split}_eval_end_timestamp'] = now
        self.logs[ep_name][f'{split}_eval_dur'] = now - self.logs[ep_name][f'{split}_eval_start_timestamp']
        eval_dur = self.logs[ep_name][f'{split}_eval_dur'].seconds
        eval_speed = eval_dur / self.n_batches[split]
        self.logs[ep_name][f'{split}_eval_speed'] = eval_speed
        loss = round(sum(self.temp_epoch[f'{split}_loss']) / len(self.temp_epoch[f'{split}_loss']), 4)
        acc1 = round(sum(self.temp_epoch[f'{split}_acc1']) / len(self.temp_epoch[f'{split}_acc1']), 2)
        acc5 = round(sum(self.temp_epoch[f'{split}_acc5']) / len(self.temp_epoch[f'{split}_acc5']), 2)
        self.logs[ep_name][f'{split}_best_acc1_batch'] = self.best_acc_batch[f'{split}_acc1']
        self.logs[ep_name][f'{split}_best_acc5_batch'] = self.best_acc_batch[f'{split}_acc5']
        self.best_acc_batch[f'{split}_acc1'] = 0
        self.best_acc_batch[f'{split}_acc5'] = 0
        best_batch_acc1 = round(self.logs[ep_name][f'{split}_best_acc1_batch'], 2)
        best_batch_acc5 = round(self.logs[ep_name][f'{split}_best_acc5_batch'], 2)
        msg = f'{self.now()} - EPOCH [{ep_name.split("_")[-1]}] ' \
              f'EVAL_{split}_SPEED {round(eval_speed, 5)} sec/batch | ' \
              f'LOSS {loss}, ACC1 {acc1}% (BEST {best_batch_acc1}%), ACC5 {acc5}% (BEST {best_batch_acc5}%)'
        self.log(msg)

    def sum_epoch(self, epoch):
        ep_name = self.get_ep_name(epoch)
        now = datetime.datetime.now().replace(microsecond=0)
        self.logs[ep_name]['f_timestamp'] = now
        self.logs[ep_name]['epoch_dur'] = now - self.logs[ep_name]['init_timestamp']
        ep = ep_name.split('_')[-1]
        all_duration = sum([self.logs[x]['train_dur'].seconds for x in self.logs if 'ep' in x])
        avg_time_diff = all_duration / (epoch+1)
        eta = (self.epochs - (epoch+1)) * avg_time_diff
        eta = datetime.timedelta(seconds=round(eta))
        train_speed = self.logs[ep_name]['train_speed']
        msg = f'{now} - EPOCH [{ep}]/{self.epochs}, TRAIN_SPEED {round(train_speed, 5)} sec/batch  ETA {eta}| '
        for k in self.temp_epoch:
            if len(self.temp_epoch[k]) == 0:
                # self.logs[ep_name][k] = None
                msg += f'{k}: NO CALC, '
            else:
                avg = sum(self.temp_epoch[k]) / len(self.temp_epoch[k])
                self.logs[ep_name][k] = avg
                if 'acc' in k:
                    val = str(round(avg, 2)) + '%'
                else:
                    val = str(round(avg, 4))
                if len(val) < 7:
                    msg += ' '*(7-len(val))
                msg += f'{k}: {val}, '
                self.temp_epoch[k] = []
        msg = msg[:-2]
        self.log(msg)

    def save_logged_data(self):
        df = pd.DataFrame.from_dict(self.logs, orient='index')
        cols = [col for col in CSV_COL_ORDER if col in df.columns]
        df = df[cols]
        if self.output_dir:
            df.to_csv(f'{self.output_dir}/logged_data.csv')
        with open(f'{self.output_dir}/raw_logs.txt', 'w') as f:
            for msg in self.msg_list:
                f.write(msg+'\n')

def is_photo(x: str):
    for f in ['jpg', 'jpeg', 'JPG', 'JPEG']:
        if f in x:
            return True
    return False

def _load_checkpoint_for_ema(model_ema, checkpoint):
    """
    Workaround for ModelEma._load_checkpoint to accept an already-loaded object
    """
    mem_file = io.BytesIO()
    torch.save(checkpoint, mem_file)
    mem_file.seek(0)
    model_ema._load_checkpoint(mem_file)

def setup_for_distributed(is_master):
    """
    This function disables printing when not in master process
    """
    import builtins as __builtin__
    builtin_print = __builtin__.print

    def print(*args, **kwargs):
        force = kwargs.pop('force', False)
        if is_master or force:
            builtin_print(*args, **kwargs)

    __builtin__.print = print

def is_dist_avail_and_initialized():
    if not dist.is_available():
        return False
    if not dist.is_initialized():
        return False
    return True

def get_world_size():
    if not is_dist_avail_and_initialized():
        return 1
    return dist.get_world_size()

def get_rank():
    if not is_dist_avail_and_initialized():
        return 0
    return dist.get_rank()

def is_main_process():
    return get_rank() == 0

def save_on_master(*args, **kwargs):
    if is_main_process():
        torch.save(*args, **kwargs)

def init_distributed_mode(args):
    args.distributed = False
    return
    # if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
    #     args.rank = int(os.environ["RANK"])
    #     args.world_size = int(os.environ['WORLD_SIZE'])
    #     args.gpu = int(os.environ['LOCAL_RANK'])
    # elif 'SLURM_PROCID' in os.environ:
    #     args.rank = int(os.environ['SLURM_PROCID'])
    #     args.gpu = args.rank % torch.cuda.device_count()
    # else:
    #     print('Not using distributed mode')
    #     args.distributed = False
    #     return
    #
    # args.distributed = True
    #
    # torch.cuda.set_device(args.gpu)
    # args.dist_backend = 'nccl'
    # print('| distributed init (rank {}): {}'.format(
    #     args.rank, args.dist_url), flush=True)
    # torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
    #                                      world_size=args.world_size, rank=args.rank)
    # torch.distributed.barrier()
    # setup_for_distributed(args.rank == 0)

def replace_batchnorm(net):
    for child_name, child in net.named_children():
        if hasattr(child, 'fuse'):
            setattr(net, child_name, child.fuse())
        elif isinstance(child, torch.nn.Conv2d):
            child.bias = torch.nn.Parameter(torch.zeros(child.weight.size(0)))
        elif isinstance(child, torch.nn.BatchNorm2d):
            setattr(net, child_name, torch.nn.Identity())
        else:
            replace_batchnorm(child)

def replace_layernorm(net):
    import apex
    for child_name, child in net.named_children():
        if isinstance(child, torch.nn.LayerNorm):
            setattr(net, child_name, apex.normalization.FusedLayerNorm(
                child.weight.size(0)))
        else:
            replace_layernorm(child)


import torch
from torch import nn
from torch import functional as F

class EncoderModel(nn.Module):
    def __init__(self, params):
        super(EncoderModel, self).__init__()
        self.name = "Parent encoder model"
        self.layers = None
        self.input_size = 300
        self.dropout = 0.5
        self.embedding = nn.Embedding(params["vocab_size"], params["vector_size"], padding_idx= params["pad_idx"]).from_pretrained(params["pretrained"])

    def processPadded(self, textTuple):
        text, text_lens = textTuple
        self.lens = text_lens
        embedded = self.embedding(text)
        return torch.nn.utils.rnn.pack_padded_sequence(embedded, text_lens, batch_first=True, enforce_sorted=False)

    def forward(self, x):
        x = self.processPadded(x)
        out, _ = self.layers(x)
        unpacked, unpacked_len = torch.nn.utils.rnn.pad_packed_sequence(out, batch_first = True)
        ## get the last hidden
        idx = (self.lens - 1).view(-1, 1).expand(len(self.lens), unpacked.size(2))
        idx = idx.unsqueeze(1)
        last_hidden = torch.gather(unpacked, dim = 1, index = idx)
        return torch.squeeze(last_hidden)

class MeanEncoder(EncoderModel):
    def __init__(self, params, hidden_size, layer_num):
        super(MeanEncoder, self).__init__(params)
        self.name = "Vector mean"
        self.layer_size = self.input_size

    def forward(self, x):
        text, text_lens = x
        text_lens = text_lens.float()
        embedded = self.embedding(text)
        averaged = torch.sum(embedded, dim = 1)
        averaged = torch.div(averaged, text_lens.view(-1,1))
        return averaged

class LSTMEncoder(EncoderModel):
    def __init__(self, params, hidden_size, layer_num):
        super(LSTMEncoder, self).__init__(params)
        self.name = "LSTM"
        self.layer_size = hidden_size
        self.layers = nn.LSTM(self.input_size, hidden_size, num_layers=layer_num, dropout=self.dropout, bidirectional=False, batch_first=True)

class BiLSTMEncoder(EncoderModel):
    def __init__(self, params, hidden_size, layer_num):
        super(BiLSTMEncoder, self).__init__(params)
        self.name = "BiLSTM"
        self.layer_size = hidden_size *2
        self.layers = nn.LSTM(self.input_size, hidden_size, num_layers=layer_num, dropout=self.dropout, bidirectional=True, batch_first=True)

class MaxBiLSTMEncoder(EncoderModel):
    def __init__(self, params, hidden_size, layer_num):
        super(MaxBiLSTMEncoder, self).__init__(params)
        self.name = "Pooled BiLSTM"
        self.layer_size = hidden_size *2
        self.layers = nn.LSTM(self.input_size, hidden_size, num_layers=layer_num, dropout=self.dropout, bidirectional=True, batch_first=True)

    def forward(self, x):
        x = self.processPadded(x)
        out, _ = self.layers(x)
        unpacked, unpacked_len = torch.nn.utils.rnn.pad_packed_sequence(out, batch_first = True)
        max_pooled = torch.squeeze(torch.max(unpacked, dim = 1).values)

        return max_pooled


from unittest import mock

import pytest

from game import Game, GameDoesNotExist
from parser import MemoryGameRepository

class TestMemoryGameRepository:

    @mock.patch.object(MemoryGameRepository, 'store', {})
    def test_should_add_new_game(self):
        game = Game('abc')
        memory_repo = MemoryGameRepository()
        memory_repo.add(game)
        assert memory_repo.get_active_game() is game
        assert memory_repo.active_game_uid == 'abc'

    @mock.patch.object(MemoryGameRepository, 'store', {})
    def test_should_active_game_uid_be_the_last_added(self):
        memory_repo = MemoryGameRepository()
        memory_repo.add(Game('xyz'))
        memory_repo.add(Game('abc'))
        assert memory_repo.active_game_uid == 'abc'

    @mock.patch.object(MemoryGameRepository, 'store', {})
    def test_should_get_games(self):
        memory_repo = MemoryGameRepository()
        memory_repo.add(Game('abc'))
        assert 'abc' in memory_repo.get_games()

    @mock.patch.object(MemoryGameRepository, 'store', {})
    def test_should_get_games_by_id(self):
        memory_repo = MemoryGameRepository()
        memory_repo.add(Game('abc'))
        assert memory_repo.get_game_by_uid('abc')

        with pytest.raises(GameDoesNotExist):
            memory_repo.get_game_by_uid('asdasdasd')


#!/usr/bin/env python3

import os
import re
import glob
from izip import Izip
import argparse
import xml.etree.ElementTree as ET
from tempfile import mkdtemp
from shutil import copy,move,rmtree
from utils import get_settings

xmlsettings = get_settings()

class Ixml():
    """处理xml"""

    def __init__(self, file):
        self.file = file
        (filepath, fullname) = os.path.split(file)
        filename, fileextension = os.path.splitext(fullname)
        self.path = filepath
        self.name = filename
        self.ext = fileextension

    def build_idic(self, xmli, mtype, idic):
        tree = ET.parse(xmli)
        root = tree.getroot()
        for child in root:
            id = child.attrib.get('Id')
            fp = child.attrib.get('Target')
            if child.attrib.get('Type') == "http://schemas.openxmlformats.org/officeDocument/2006/relationships/image":
                (p, f) = os.path.split(fp)
                idic[id] = xmlsettings.mediatype[mtype]['path'] + f

    #This is for Excel document only
    def build_xl_xdic(self, xmlx, xdic):
        tree = ET.parse(xmlx)
        root = tree.getroot()
        namespaces = dict([node for _, node in ET.iterparse(xmlx, events=['start-ns'])])
        for pic in root.findall('.//xdr:twoCellAnchor', namespaces):
            row  = pic.find("xdr:to/xdr:row", namespaces)
            col  = pic.find("xdr:to/xdr:col", namespaces)
            id_elem = pic.find("xdr:pic/xdr:blipFill/a:blip", namespaces)
            try:
                id = id_elem.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed")
                xdic[id] = [row.text, col.text]
            except AttributeError:
                if xmlsettings.debug == True:
                    print("Ignoring empty elem at row: " + row.text + " col: " + col.text)

    def extract_xl_image(self, xz, idic, xdic, usenamefile, sheet):
        for id in xdic.keys():
            (imagepath, imagefullname) = os.path.split(idic[id])
            imagename, imageextension = os.path.splitext(imagefullname)
            if imageextension == ".jpeg":
                imageextension = ".jpg"
            fixed_row = '{:04}'.format(int(xdic[id][0]))
            fixed_col = '{:04}'.format(int(xdic[id][1]))
            name = ""
            if usenamefile == True:
                textnamefile = self.path + self.name + ".txt"
                with open(textnamefile, mode='r', encoding='UTF-8') as nf:
                    names = nf.read().splitlines()
                    name = names[int(xdic[id][0])]
                    if name != "":
                        name = "_" + name
                    else:
                        name = ""
            sheet = '{:02}'.format(int(sheet))
            targetpath = self.path + self.name + "/"
            try:
                os.stat(targetpath)
            except:
                os.mkdir(targetpath)
            targetfile = targetpath + sheet + "_" + fixed_row + "_" + fixed_col + name + imageextension
            sourcefile = idic[id]

            if not os.path.exists(targetfile):
                xz.writefp(sourcefile, targetfile)

    def extract_image(self, xz, idic, page):
        for id in sorted(idic.keys()):
            (imagepath, imagefullname) = os.path.split(idic[id])
            imagename, imageextension = os.path.splitext(imagefullname)
            name = re.search(r'[0-9]', id).group()
            name = '{:03}'.format(int(name))
            if imageextension == ".jpeg":
                imageextension = ".jpg"
            page = '{:03}'.format(int(page))
            targetpath = self.path + self.name + "/"
            try:
                os.stat(targetpath)
            except:
                os.mkdir(targetpath)
            targetfile = targetpath + page + "_" + name + imageextension
            sourcefile = idic[id]

            if not os.path.exists(targetfile):
                xz.writefp(sourcefile, targetfile)


import requests
from lib import config

def request_html(url, params=None):
    params = params or {}
    res = requests.get(url, headers={'User-Agent': config.USER_AGENT}, params=params)
    if res.status_code != 200:
        raise Exception('Non-200 status code')
    body = res.content
    return body


#!/usr/bin/env python2
import socket
import struct
import time
import dns
import dns.message
import libnacl
import libnacl.utils
import binascii
from builtins import bytes

class DNSCryptResolverCertificate(object):
    DNSCRYPT_CERT_MAGIC = b'\x44\x4e\x53\x43'
    DNSCRYPT_ES_VERSION = b'\x00\x01'
    DNSCRYPT_PROTOCOL_MIN_VERSION = b'\x00\x00'

    def __init__(self, serial, validFrom, validUntil, publicKey, clientMagic):
        self.serial = serial
        self.validFrom = validFrom
        self.validUntil = validUntil
        self.publicKey = publicKey
        self.clientMagic = clientMagic

    def isValid(self):
        now = time.time()
        return self.validFrom <= now and self.validUntil >= now

    @staticmethod
    def fromBinary(binary, providerFP):
        if len(binary) != 124:
            raise Exception("Invalid binary certificate")

        certMagic = binary[0:4]
        esVersion = binary[4:6]
        protocolMinVersion = binary[6:8]

        if certMagic != DNSCryptResolverCertificate.DNSCRYPT_CERT_MAGIC or esVersion != DNSCryptResolverCertificate.DNSCRYPT_ES_VERSION or protocolMinVersion != DNSCryptResolverCertificate.DNSCRYPT_PROTOCOL_MIN_VERSION:
            raise Exception("Invalid binary certificate")

        orig = libnacl.crypto_sign_open(binary[8:124], providerFP)

        resolverPK = orig[0:32]
        clientMagic = orig[32:40]
        serial = struct.unpack_from("!I", orig[40:44])[0]
        validFrom = struct.unpack_from("!I", orig[44:48])[0]
        validUntil = struct.unpack_from("!I", orig[48:52])[0]
        return DNSCryptResolverCertificate(serial, validFrom, validUntil, resolverPK, clientMagic)

class DNSCryptClient(object):
    DNSCRYPT_NONCE_SIZE = 24
    DNSCRYPT_MAC_SIZE = 16
    DNSCRYPT_PADDED_BLOCK_SIZE = 64
    DNSCRYPT_MIN_UDP_LENGTH = 256
    DNSCRYPT_RESOLVER_MAGIC = b'\x72\x36\x66\x6e\x76\x57\x6a\x38'

    @staticmethod
    def _addrToSocketType(addr):
        result = None
        try:
            socket.inet_pton(socket.AF_INET6, addr)
            result = socket.AF_INET6
        except socket.error:
            socket.inet_pton(socket.AF_INET, addr)
            result = socket.AF_INET

        return result

    def __init__(self, providerName, providerFingerprint, resolverAddress, resolverPort=443, timeout=2):
        self._providerName = providerName
        self._providerFingerprint = binascii.unhexlify(providerFingerprint.lower().replace(':', ''))
        self._resolverAddress = resolverAddress
        self._resolverPort = resolverPort
        self._resolverCertificates = []
        self._publicKey, self._privateKey = libnacl.crypto_box_keypair()
        self._timeout = timeout

        addrType = self._addrToSocketType(self._resolverAddress)
        self._sock = socket.socket(addrType, socket.SOCK_DGRAM)
        self._sock.settimeout(timeout)
        self._sock.connect((self._resolverAddress, self._resolverPort))

    def _sendQuery(self, queryContent, tcp=False):
        if tcp:
            addrType = self._addrToSocketType(self._resolverAddress)
            sock = socket.socket(addrType, socket.SOCK_STREAM)
            sock.settimeout(self._timeout)
            sock.connect((self._resolverAddress, self._resolverPort))
            sock.send(struct.pack("!H", len(queryContent)))
        else:
            sock = self._sock

        sock.send(queryContent)

        data = None
        if tcp:
            got = sock.recv(2)
            if got:
                (rlen,) = struct.unpack("!H", got)
                data = sock.recv(rlen)
        else:
            data = sock.recv(4096)

        return data

    def _hasValidResolverCertificate(self):

        for cert in self._resolverCertificates:
            if cert.isValid():
                return True

        return False

    def clearExpiredResolverCertificates(self):
        newCerts = []

        for cert in self._resolverCertificates:
            if cert.isValid():
                newCerts.append(cert)

        self._resolverCertificates = newCerts

    def refreshResolverCertificates(self):
        self.clearExpiredResolverCertificates()

        query = dns.message.make_query(self._providerName, dns.rdatatype.TXT, dns.rdataclass.IN)
        data = self._sendQuery(query.to_wire())

        response = dns.message.from_wire(data)
        if response.rcode() != dns.rcode.NOERROR or len(response.answer) != 1:
            raise Exception("Invalid response to public key request")

        an = response.answer[0]
        if an.rdclass != dns.rdataclass.IN or an.rdtype != dns.rdatatype.TXT or len(an.items) == 0:
            raise Exception("Invalid response to public key request")

        self._resolverCertificates = []

        for item in an.items:
            if len(item.strings) != 1:
                continue

            cert = DNSCryptResolverCertificate.fromBinary(item.strings[0], self._providerFingerprint)
            if cert.isValid():
                self._resolverCertificates.append(cert)

    def getResolverCertificate(self):
        certs = self._resolverCertificates
        result = None
        for cert in certs:
            if cert.isValid():
                if result is None or cert.serial > result.serial:
                    result = cert

        return result

    def getAllResolverCertificates(self, onlyValid=False):
        certs = self._resolverCertificates
        result = []
        for cert in certs:
            if not onlyValid or cert.isValid():
                result.append(cert)

        return result

    @staticmethod
    def _generateNonce():
        nonce = libnacl.utils.rand_nonce()
        return nonce[:int(DNSCryptClient.DNSCRYPT_NONCE_SIZE / 2)]

    def _encryptQuery(self, queryContent, resolverCert, nonce, tcp=False):
        header = resolverCert.clientMagic + self._publicKey + nonce
        requiredSize = len(header) + self.DNSCRYPT_MAC_SIZE + len(queryContent)
        paddingSize = self.DNSCRYPT_PADDED_BLOCK_SIZE - (len(queryContent) % self.DNSCRYPT_PADDED_BLOCK_SIZE)
        # padding size should be DNSCRYPT_PADDED_BLOCK_SIZE <= padding size <= 4096
        if not tcp and requiredSize < self.DNSCRYPT_MIN_UDP_LENGTH:
            paddingSize += self.DNSCRYPT_MIN_UDP_LENGTH - requiredSize
            requiredSize = self.DNSCRYPT_MIN_UDP_LENGTH

        padding = b'\x80'
        idx = 0
        while idx < (paddingSize - 1):
            padding = padding + b'\x00'
            idx += 1

        data = queryContent + padding
        nonce = nonce + (b'\x00'*int(self.DNSCRYPT_NONCE_SIZE / 2))
        box = libnacl.crypto_box(data, nonce, resolverCert.publicKey, self._privateKey)
        return header + box

    def _decryptResponse(self, encryptedResponse, resolverCert, clientNonce):
        resolverMagic = encryptedResponse[:8]
        if resolverMagic != self.DNSCRYPT_RESOLVER_MAGIC:
            raise Exception("Invalid encrypted response: bad resolver magic")

        nonce = encryptedResponse[8:32]
        if nonce[0:int(self.DNSCRYPT_NONCE_SIZE / 2)] != clientNonce:
            raise Exception("Invalid encrypted response: bad nonce")

        cleartext = libnacl.crypto_box_open(encryptedResponse[32:], nonce, resolverCert.publicKey, self._privateKey)
        cleartextBytes = bytes(cleartext)
        idx = len(cleartextBytes) - 1
        while idx > 0:
            if cleartextBytes[idx] != 0:
                break
            idx -= 1

        if idx == 0 or cleartextBytes[idx] != 128:
            raise Exception("Invalid encrypted response: invalid padding")

        idx -= 1
        paddingLen = len(cleartextBytes) - idx

        return cleartext[:idx+1]

    def query(self, queryContent, tcp=False):

        if not self._hasValidResolverCertificate():
            self.refreshResolverCertificates()

        nonce = self._generateNonce()
        resolverCert = self.getResolverCertificate()
        if resolverCert is None:
            raise Exception("No valid certificate found")
        encryptedQuery = self._encryptQuery(queryContent, resolverCert, nonce, tcp)
        encryptedResponse = self._sendQuery(encryptedQuery, tcp)
        response = self._decryptResponse(encryptedResponse, resolverCert, nonce)
        return response


from __future__ import annotations

import unittest
from pathlib import Path
from unittest import mock

from mantidimaging.core.io.filenames import FilenameGroup
from mantidimaging.core.io.instrument_log import InstrumentLog
from mantidimaging.gui.windows.image_load_dialog.field import Field
from mantidimaging.gui.windows.image_load_dialog.presenter import LoadPresenter
from mantidimaging.core.utility.data_containers import FILE_TYPES, Indices

class ImageLoadDialogPresenterTest(unittest.TestCase):

    def _files_equal(self, file1, file2):
        self.assertEqual(Path(file1).absolute(), Path(file2).absolute())

    def setUp(self):
        self.fields = {ft.fname: mock.create_autospec(Field) for ft in FILE_TYPES}
        self.v = mock.MagicMock(fields=self.fields)
        self.v.sample = self.fields["Sample"]
        self.p = LoadPresenter(self.v)

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.LoadPresenter.do_update_flat_or_dark")
    def test_do_update_field(self, mock_do_update_flat_or_dark):
        selected_file = "SelectedFile"
        self.v.select_file.return_value = selected_file
        ft = FILE_TYPES.FLAT_BEFORE
        field = mock.MagicMock(file_info=ft)

        self.p.do_update_field(field)

        self.v.select_file.assert_called_once_with(ft.fname, True)
        mock_do_update_flat_or_dark.assert_called_once_with(field, selected_file)

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.LoadPresenter.do_update_flat_or_dark")
    def test_do_update_field_when_no_file_selected(self, mock_do_update_flat_or_dark):
        self.v.select_file.return_value = None
        ft = FILE_TYPES.FLAT_BEFORE
        field = mock.MagicMock(file_info=ft)

        self.p.do_update_field(field)

        self.v.select_file.assert_called_once_with(ft.fname, True)
        mock_do_update_flat_or_dark.assert_not_called()

    def test_update_field_with_filegroup_sample(self):
        mock_file_group = mock.create_autospec(FilenameGroup)
        file_list = [mock.Mock()]
        file_log_path = mock.Mock()
        mock_file_group.all_files.return_value = file_list
        mock_file_group.log_path = file_log_path

        mock_field_path = mock.PropertyMock()
        type(self.fields["Sample Log"]).path = mock_field_path

        self.p.update_field_with_filegroup(FILE_TYPES.SAMPLE, mock_file_group)

        mock_file_group.find_all_files.assert_called_once_with()
        mock_file_group.find_log_file.assert_called_once_with()
        self.fields["Sample"].set_images.assert_called_once_with(file_list)
        mock_field_path.assert_called_once_with(file_log_path)

    def test_update_field_with_filegroup_dark_before(self):
        mock_file_group = mock.create_autospec(FilenameGroup)
        file_list = [mock.Mock()]
        mock_file_group.all_files.return_value = file_list

        self.p.update_field_with_filegroup(FILE_TYPES.DARK_BEFORE, mock_file_group)

        mock_file_group.find_all_files.assert_called_once_with()
        mock_file_group.find_log_file.assert_not_called()
        self.fields["Dark Before"].set_images.assert_called_once_with(file_list)

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.FilenameGroup")
    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.read_image_dimensions")
    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.LoadPresenter.update_field_with_filegroup")
    def test_do_update_sample_no_related(self, mock_update_field, mock_read_image_dimensions, mock_filename_group):
        selected_file = "/a/b/img_000.tif"
        mock_read_image_dimensions.return_value = [10, 11]
        mock_sample_fg = mock.create_autospec(FilenameGroup)
        mock_filename_group.from_file.return_value = mock_sample_fg
        mock_sample_fg.all_indexes = [0, 1, 2, 3]
        mock_sample_fg.find_related.return_value = None

        self.p.do_update_sample(selected_file)

        mock_update_field.assert_called_once_with(FILE_TYPES.SAMPLE, mock_sample_fg)
        self.fields["Sample"].update_indices.assert_called_once_with(4)
        self.fields["Sample"].update_shape.assert_called_once_with([10, 11])
        self.v.ok_button.setEnabled.assert_called_once_with(True)

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.FilenameGroup")
    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.read_image_dimensions")
    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.LoadPresenter.update_field_with_filegroup")
    def test_do_update_sample_related_flat_before(self, mock_update_field, mock_read_image_dimensions,
                                                  mock_filename_group):
        selected_file = "/a/b/img_000.tif"
        mock_read_image_dimensions.return_value = [10, 11]
        mock_sample_fg = mock.create_autospec(FilenameGroup)
        mock_fb_fg = mock.create_autospec(FilenameGroup)
        mock_filename_group.from_file.return_value = mock_sample_fg
        mock_sample_fg.all_indexes = [0, 1, 2, 3]
        mock_sample_fg.find_related.side_effect = lambda ft: mock_fb_fg if ft == FILE_TYPES.FLAT_BEFORE else None

        self.p.do_update_sample(selected_file)

        updated_fields = [c.args[0] for c in mock_update_field.mock_calls]
        self.assertIn(FILE_TYPES.SAMPLE, updated_fields)
        self.assertIn(FILE_TYPES.FLAT_BEFORE, updated_fields)
        self.assertNotIn(FILE_TYPES.FLAT_AFTER, updated_fields)
        self.assertNotIn(FILE_TYPES.DARK_BEFORE, updated_fields)
        self.assertNotIn(FILE_TYPES.DARK_AFTER, updated_fields)

    @mock.patch("mantidimaging.core.io.filenames.FilenameGroup.all_files")
    @mock.patch("mantidimaging.core.io.filenames.FilenameGroup.find_all_files")
    def test_do_update_flat_or_dark(self, mock_find_all_files, mock_all_files):
        file_name = "/aaa_0000.tiff"
        ft = FILE_TYPES.FLAT_BEFORE
        field = mock.MagicMock(file_info=ft)
        files_list = [file_name, "aaa_0001.tiff"]
        mock_all_files.return_value = files_list

        self.p.do_update_flat_or_dark(field, file_name)

        field.set_images.assert_called_once_with(files_list)

    def test_do_update_single_file(self):
        file_name = "file_name"
        field = mock.MagicMock(file_info=FILE_TYPES.PROJ_180)

        self.p.do_update_single_file(field, file_name)

        self.assertEqual(field.path, Path(file_name))

    def test_do_update_single_file_no_file_selected(self):
        file_name = "file_name"
        field = mock.MagicMock(file_info=FILE_TYPES.PROJ_180)

        self.p.do_update_single_file(field, None)

        self.assertNotEqual(field.path, file_name)

    def test_do_update_sample_log_no_sample_selected(self):
        field = mock.MagicMock()
        self.p.sample_fg = None
        self.assertRaises(RuntimeError, self.p.do_update_sample_log, field, "")

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.LoadPresenter.ensure_sample_log_consistency")
    def test_do_update_sample_log(self, mock_ensure):
        file_name = "file_name"

        field = mock.MagicMock(file_info=FILE_TYPES.SAMPLE_LOG)

        self.p.sample_fg = mock.create_autospec(FilenameGroup)
        self.p.do_update_sample_log(field, file_name)

        mock_ensure.assert_called_once_with(field, file_name, [])

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.load_log")
    def test_ensure_sample_log_consistency_matching(self, mock_load_log):
        """
        Test behaviour when the number of projection angles and files matches
        """
        mock_log = mock.create_autospec(InstrumentLog)
        mock_load_log.return_value = mock_log
        file_name = "file_name"
        field = mock.MagicMock()
        field.path = None
        field.use = None
        test_filenames = ["file1", "file2"]

        self.p.ensure_sample_log_consistency(field, file_name, test_filenames)

        mock_load_log.assert_called_once_with(Path(file_name))
        mock_log.raise_if_angle_missing.assert_called_once_with(test_filenames)
        self.assertIsNotNone(field.path)
        self.assertIsNotNone(field.use)

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.load_log")
    def test_ensure_sample_log_consistency_exits_when_none_or_empty_str(self, mock_load_log):
        mock_log = mock.create_autospec(InstrumentLog)
        mock_load_log.return_value = mock_log
        file_name = None
        field = mock.MagicMock()
        test_filenames = ["file1", "file2"]

        self.p.ensure_sample_log_consistency(field, file_name, test_filenames)

        mock_load_log.assert_not_called()
        mock_log.raise_if_angle_missing.assert_not_called()

        file_name = ""
        self.p.ensure_sample_log_consistency(field, file_name, test_filenames)

        mock_load_log.assert_not_called()
        mock_log.raise_if_angle_missing.assert_not_called()

    @mock.patch("mantidimaging.gui.windows.image_load_dialog.presenter.FilenameGroup.find_all_files")
    def test_get_parameters(self, _):
        sample_path = Path("/sample/tomo/tomo_0001.tiff")
        sample_log_path = Path("/sample/tomo.log")
        sample_indices = Indices(0, 10, 1)

        type(self.fields["Sample"]).path = mock.PropertyMock(return_value=sample_path)
        self.fields["Sample"].use.isChecked.return_value = True
        type(self.fields["Sample"]).indices = mock.PropertyMock(return_value=sample_indices)
        type(self.fields["Sample Log"]).path = mock.PropertyMock(return_value=sample_log_path)

        flat_before_path = Path("/sample/flat_before/flat_before_0001.tiff")
        flat_before_log = Path("/sample/flat_before.log")
        flat_after_path = Path("/sample/flat_after/flat_after_0001.tiff")
        flat_after_log = Path("/sample/flat_after.log")
        dark_before_path = Path("/sample/dark_before/dark_before_0001.tiff")
        dark_after_path = Path("/sample/dark_after/dark_after_0001.tiff")
        proj_180_path = Path("/sample/180deg/180deg.tiff")
        self.fields["Flat Before"].use.isChecked.return_value = True
        type(self.fields["Flat Before"]).path = mock.PropertyMock(return_value=flat_before_path)
        type(self.fields["Flat Before Log"]).path = mock.PropertyMock(return_value=flat_before_log)
        self.fields["Flat After"].use.isChecked.return_value = True
        type(self.fields["Flat After"]).path = mock.PropertyMock(return_value=flat_after_path)
        type(self.fields["Flat After Log"]).path = mock.PropertyMock(return_value=flat_after_log)
        type(self.fields["Dark Before"]).path = mock.PropertyMock(return_value=dark_before_path)
        type(self.fields["Dark After"]).path = mock.PropertyMock(return_value=dark_after_path)
        type(self.fields["180 degree"]).path = mock.PropertyMock(return_value=proj_180_path)

        pixel_size = 24
        dtype = "float32"
        sinograms = True
        self.v.pixelSize.value.return_value = pixel_size
        self.v.pixel_bit_depth.currentText.return_value = dtype
        self.v.images_are_sinograms.isChecked.return_value = sinograms

        lp = self.p.get_parameters()

        lp_sample = lp.image_stacks[FILE_TYPES.SAMPLE]
        lp_flat_before = lp.image_stacks[FILE_TYPES.FLAT_BEFORE]
        lp_flat_after = lp.image_stacks[FILE_TYPES.FLAT_AFTER]
        lp_dark_before = lp.image_stacks[FILE_TYPES.DARK_BEFORE]
        lp_dark_after = lp.image_stacks[FILE_TYPES.DARK_AFTER]
        lp_proj_180deg = lp.image_stacks[FILE_TYPES.PROJ_180]

        self._files_equal(next(lp_sample.file_group.all_files()), sample_path)
        self._files_equal(lp_sample.log_file, sample_log_path)
        self.assertEqual(lp_sample.indices, sample_indices)

        self._files_equal(next(lp_flat_before.file_group.all_files()), flat_before_path)
        self._files_equal(next(lp_flat_after.file_group.all_files()), flat_after_path)
        self._files_equal(next(lp_dark_before.file_group.all_files()), dark_before_path)
        self._files_equal(next(lp_dark_after.file_group.all_files()), dark_after_path)
        self._files_equal(next(lp_proj_180deg.file_group.all_files()), proj_180_path)

        self._files_equal(lp_flat_before.log_file, flat_before_log)
        self._files_equal(lp_flat_after.log_file, flat_after_log)
        self.assertIsNone(lp_dark_before.log_file)
        self.assertIsNone(lp_dark_after.log_file)
        self.assertIsNone(lp_proj_180deg.log_file)

        self.assertEqual(lp.name, sample_path.name)
        self.assertEqual(lp.dtype, dtype)
        self.assertEqual(lp.sinograms, sinograms)
        self.assertEqual(lp.pixel_size, pixel_size)

        self.assertNotIn(FILE_TYPES.SAMPLE_LOG, lp.image_stacks.keys())
        self.assertNotIn(FILE_TYPES.FLAT_BEFORE_LOG, lp.image_stacks.keys())
        self.assertNotIn(FILE_TYPES.FLAT_AFTER_LOG, lp.image_stacks.keys())


from dataclasses import dataclass, field
from typing import List, Optional
from nova.ausgabe_status_typ import AusgabeStatusTyp
from nova.freizeit_beleg_parameter import FreizeitBelegParameter
from nova.vertriebs_response import VertriebsResponse

__NAMESPACE__ = "http://nova.voev.ch/services/v14/vertrieb"

@dataclass
class LeseFreizeitBelegStatusVertriebsResponse(VertriebsResponse):
    """Die Belegantwort ist das Antwortsobjekt des 4.

    Klangs. Sie enthält für jede übermittelte LeistungId ein Objekt
    DruckDaten in dem alle zum Drucken notwendige Druckattribute
    enthalten sind.
    """
    leistung: List["LeseFreizeitBelegStatusVertriebsResponse.Leistung"] = field(
        default_factory=list,
        metadata={
            "type": "Element",
            "namespace": "http://nova.voev.ch/services/v14/vertrieb",
        }
    )

    @dataclass
    class Leistung:
        leistungs_id: Optional[int] = field(
            default=None,
            metadata={
                "name": "leistungsId",
                "type": "Element",
                "namespace": "http://nova.voev.ch/services/v14/vertrieb",
                "required": True,
                "min_inclusive": 0,
            }
        )
        traeger_medium_code: Optional[str] = field(
            default=None,
            metadata={
                "name": "traegerMediumCode",
                "type": "Element",
                "namespace": "http://nova.voev.ch/services/v14/vertrieb",
                "required": True,
                "min_length": 0,
                "max_length": 50,
            }
        )
        ausgabe_status: Optional[AusgabeStatusTyp] = field(
            default=None,
            metadata={
                "name": "ausgabeStatus",
                "type": "Element",
                "namespace": "http://nova.voev.ch/services/v14/vertrieb",
                "required": True,
            }
        )
        beleg_parameter: List[FreizeitBelegParameter] = field(
            default_factory=list,
            metadata={
                "name": "belegParameter",
                "type": "Element",
                "namespace": "http://nova.voev.ch/services/v14/vertrieb",
            }
        )


import pickle

from gensim.models.doc2vec import Doc2Vec
from sqlalchemy import update

from model.db import DB_ENGINE, rawcontents
from utils.log import getLogger

logger = getLogger('d2v_dump')

model = Doc2Vec.load('RuntimeTY/d2v_2048_5_1216')

logger.warning('Model Loaded')

with DB_ENGINE.connect() as conn:

    stmt = 'SELECT rid, content FROM rawcontents where rid > 1582112'

    pool = []

    for (rid, content) in conn.execute(stmt):
        pool.append((rid, pickle.dumps(model.infer_vector(content))))

        if rid % 0x10 == 0:
            transcation = conn.begin()
            try:
                for (rid, vector) in pool:
                    conn.execute(
                        update(rawcontents)
                        .where(rawcontents.c.rid == rid)
                        .values(vector=vector)
                    )

                transcation.commit()
            except:
                transcation.rollback()
                logger.critical('{} dump failed'.format(rid))
                raise
            else:
                logger.info('{} dump success'.format(rid))
                pool = []

        if rid % 0x1000 == 0:
            logger.warning('{} handling'.format(rid))

    if len(pool) != 0:
        transcation = conn.begin()
        try:
            for (rid, vector) in pool:
                conn.execute(
                    update(rawcontents)
                    .where(rawcontents.c.rid == rid)
                    .values(vector=vector)
                )

            transcation.commit()
        except:
            transcation.rollback()
            logger.critical('dump failed')
            raise
        else:
            logger.info('dump success')
            pool = []


import os

from flask import Flask
from flask_ckeditor import CKEditor
from flask_wtf.csrf import CSRFProtect

from . import auth, article, db

def create_app(test_config=None):
    # Create and configure the app
    app = Flask(__name__, instance_relative_config=True)
    app.config.from_mapping(
        SECRET_KEY="dev",
        DATABASE=os.path.join(app.instance_path, "flaskr.sqlite"),
        UPLOAD_FOLDER=os.path.join("/static/images/")
    )
    ckeditor = CKEditor()
    csrf = CSRFProtect()

    if test_config is None:
        app.config.from_pyfile("config.py", silent=True)
    else:
        app.config.from_mapping(test_config)

    os.makedirs(app.instance_path, exist_ok=True)

    db.init_app(app)
    ckeditor.init_app(app)
    csrf.init_app(app)
    app.register_blueprint(auth.bp)
    app.register_blueprint(article.bp)
    app.add_url_rule("/", view_func=article.view_by_id)

    @app.context_processor  # Bit of a fudge to put it here
    def get_articles():     # Unfortunately no nav at / otherwise
        data = db.get_db()
        articles = data.execute(
            "SELECT a.id, title, body, image, caption, created, author_id, username"
            " FROM article a JOIN user u ON a.author_id = u.id"
            " ORDER BY created ASC"
        ).fetchall()
        return {"articles": articles}

    return app


import pytest
from diopter.compiler import Language, ObjectCompilationOutput, SourceProgram
from program_markers.instrumenter import InstrumenterMode, instrument_program
from program_markers.iprogram import find_non_eliminated_markers_impl, rename_markers
from program_markers.markers import (
    AsmCommentDetectionStrategy,
    AsmCommentEmptyOperandsDetectionStrategy,
    AsmCommentGlobalOutOperandDetectionStrategy,
    AsmCommentLocalOutOperandDetectionStrategy,
    AsmCommentVolatileGlobalOutOperandDetectionStrategy,
    FunctionCallDetectionStrategy,
    GlobalIntDetectionStrategy,
    GlobalVolatileIntDetectionStrategy,
    LocalVolatileIntDetectionStrategy,
    VRMarker,
)

from .utils import get_system_gcc_O0, get_system_gcc_O3

def test_asm_parsing() -> None:
    asm = """
    call VRMarker0_@PLT
.L2:
    cmpl	$0, -4(%rbp)
    js	.L3
    call	VRMarker1_@PLT
    """
    markers = (
        VRMarker.from_str("VRMarker0_", "int"),
        VRMarker.from_str("VRMarker1_", "int"),
    )
    assert (
        set(
            find_non_eliminated_markers_impl(
                asm, markers, FunctionCallDetectionStrategy()
            )
        )
    ) == set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    )

    asm = """
    js	.L2
    call	VRMarker1_@PLT
"""
    assert (
        set(
            find_non_eliminated_markers_impl(
                asm, markers, FunctionCallDetectionStrategy()
            )
        )
    ) == set((VRMarker.from_str("VRMarker1_", "int"),))

def test_instrumentation() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
    int foo(int a, int b){
        if (a+b)
            return 1;
        return 0;
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )
    assert set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    ) == set(iprogram.enabled_markers())

    gcc = get_system_gcc_O0()
    assert set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    ) == set(iprogram.find_non_eliminated_markers(gcc))
    assert set() == set(iprogram.find_eliminated_markers(gcc))

def test_disable_markers() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
    int foo(int a, int b){
        if (a+b)
            return 1;
        return 0;
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )

    gcc = get_system_gcc_O0()

    with pytest.raises(AssertionError):
        iprogram.disable_markers((VRMarker.from_str("VRMarker2_", "int"),))

    iprogram0 = iprogram.disable_markers((VRMarker.from_str("VRMarker0_", "int"),))
    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram0.disabled_markers()
    )
    assert set((VRMarker.from_str("VRMarker1_", "int"),)) == set(
        iprogram0.find_non_eliminated_markers(gcc)
    )
    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram0.find_eliminated_markers(gcc, include_all_markers=True)
    )
    assert set() == set(
        iprogram0.find_eliminated_markers(gcc, include_all_markers=False)
    )
    # disabling the same marker twice shouldn't have any effect
    assert iprogram0 == iprogram0.disable_markers(
        (VRMarker.from_str("VRMarker0_", "int"),)
    )

    iprogram1 = iprogram.disable_remaining_markers()
    assert set(
        (VRMarker.from_str("VRMarker0_", "int"), VRMarker.from_str("VRMarker1_", "int"))
    ) == set(iprogram1.disabled_markers())
    assert set() == set(iprogram1.find_non_eliminated_markers(gcc))
    assert set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    ) == set(iprogram1.find_eliminated_markers(gcc, include_all_markers=True))
    assert set() == set(
        iprogram1.find_eliminated_markers(gcc, include_all_markers=False)
    )

    iprogram1_1 = iprogram1.disable_remaining_markers()
    assert set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    ) == set(iprogram1_1.disabled_markers())
    assert set() == set(iprogram1_1.find_non_eliminated_markers(gcc))
    assert set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    ) == set(iprogram1_1.find_eliminated_markers(gcc, include_all_markers=True))
    assert set() == set(
        iprogram1_1.find_eliminated_markers(gcc, include_all_markers=False)
    )

    iprogram2 = iprogram.disable_remaining_markers()
    # disabling all remaining markers twice shouldn't have any effect
    assert iprogram2 == iprogram2.disable_remaining_markers()
    assert set(
        (VRMarker.from_str("VRMarker0_", "int"), VRMarker.from_str("VRMarker1_", "int"))
    ) == set(iprogram2.disabled_markers())
    assert set(()) == set(iprogram2.find_non_eliminated_markers(gcc))
    assert set(
        (VRMarker.from_str("VRMarker0_", "int"), VRMarker.from_str("VRMarker1_", "int"))
    ) == set(iprogram2.find_eliminated_markers(gcc, include_all_markers=True))
    assert set() == set(
        iprogram2.find_eliminated_markers(gcc, include_all_markers=False)
    )

def test_unreachable() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
    int foo(int a, int b){
        if (a+b){
            return 1;
        }
        return 0;
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )
    gcc = get_system_gcc_O0()

    with pytest.raises(AssertionError):
        iprogram.make_markers_unreachable((VRMarker.from_str("VRMarker2_", "int"),))

    iprogram0 = iprogram.make_markers_unreachable(
        (VRMarker.from_str("VRMarker0_", "int"),)
    )
    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram0.unreachable_markers()
    )
    assert set((VRMarker.from_str("VRMarker1_", "int"),)) == set(
        iprogram0.find_non_eliminated_markers(gcc)
    )
    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram0.find_eliminated_markers(gcc, include_all_markers=True)
    )
    assert set() == set(
        iprogram0.find_eliminated_markers(gcc, include_all_markers=False)
    )
    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram0.find_eliminated_markers(gcc, include_all_markers=True)
    )
    assert set() == set(
        iprogram0.find_eliminated_markers(gcc, include_all_markers=False)
    )
    # Making the same marker unreachable twice shouldn't have any effect
    assert iprogram0 == iprogram0.make_markers_unreachable(
        (VRMarker.from_str("VRMarker0_", "int"),)
    )

def test_disable_and_unreachable() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
    int foo(int a, int b){
        if (a+b){
            return 1;
        }
        return 0;
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )
    gcc = get_system_gcc_O0()

    iprogram = iprogram.make_markers_unreachable(
        (VRMarker.from_str("VRMarker0_", "int"),)
    )
    with pytest.raises(AssertionError):
        # We can't disable a marker that was already make unreachable
        iprogram.disable_markers((VRMarker.from_str("VRMarker0_", "int"),))

    iprogram = iprogram.disable_markers((VRMarker.from_str("VRMarker1_", "int"),))
    with pytest.raises(AssertionError):
        # We can't make ureachable a marker that was already disabled
        iprogram = iprogram.make_markers_unreachable(
            (VRMarker.from_str("VRMarker1_", "int"),)
        )
    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram.unreachable_markers()
    )
    assert set((VRMarker.from_str("VRMarker1_", "int"),)) == set(
        iprogram.disabled_markers()
    )

    assert set(()) == set(iprogram.find_non_eliminated_markers(gcc))
    assert set(
        (
            VRMarker.from_str("VRMarker0_", "int"),
            VRMarker.from_str("VRMarker1_", "int"),
        )
    ) == set(iprogram.find_eliminated_markers(gcc, include_all_markers=True))
    assert set() == set(
        iprogram.find_eliminated_markers(gcc, include_all_markers=False)
    )

    # All markers have already been disabled or made unreachable
    assert iprogram == iprogram.disable_remaining_markers()

def test_refinement() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
    static int foo(int a, int b){
        if (a+b){
            return 1;
        }
        return 0;
    }
    int main(){
        return foo(1,1) - foo(1,2);
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )
    gcc = get_system_gcc_O0()
    rprogram, reachable_markers = iprogram.refine_markers_with_runtime_information(
        (), gcc
    )
    assert len(reachable_markers) == 2, reachable_markers
    refined_markers = list(sorted(reachable_markers, key=lambda m: m.id))
    assert isinstance(refined_markers[0], VRMarker)
    assert refined_markers[0].lower_bound == 1
    assert refined_markers[0].upper_bound == 1
    assert isinstance(refined_markers[1], VRMarker)
    assert refined_markers[1].lower_bound == 1
    assert refined_markers[1].upper_bound == 2
    rprogram = iprogram.replace_markers(tuple(refined_markers))
    gcc = get_system_gcc_O3()
    assert len(iprogram.find_eliminated_markers(gcc)) == 0
    assert len(rprogram.find_eliminated_markers(gcc)) == 2

    reachable_markers2 = rprogram.track_reachable_markers((), gcc)
    # The refined markers should be unreachable since their new bounds are tight
    assert len(reachable_markers2) == 0

def test_strategies() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
        static int foo(int a, int b){
            return a + b;
        }
        int main()
        {
            return foo(-1,1);
        }
        """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )
    marker0 = iprogram.enabled_markers()[0]
    marker0 = VRMarker(marker0.name, marker0.id, "int", -10, 10)
    marker1 = iprogram.enabled_markers()[1]
    iprogram = iprogram.replace_markers((marker0,))

    all_marker_strategies = (
        FunctionCallDetectionStrategy(),
        AsmCommentDetectionStrategy(),
        AsmCommentEmptyOperandsDetectionStrategy(),
        AsmCommentLocalOutOperandDetectionStrategy(),
        AsmCommentGlobalOutOperandDetectionStrategy(),
        AsmCommentVolatileGlobalOutOperandDetectionStrategy(),
        LocalVolatileIntDetectionStrategy(),
        GlobalVolatileIntDetectionStrategy(),
        GlobalIntDetectionStrategy(),
    )
    gcc = get_system_gcc_O3()

    for strategy in all_marker_strategies:
        ip = iprogram.with_marker_strategy(strategy)
        markers = ip.find_non_eliminated_markers(gcc)

        assert marker0 not in markers
        assert marker1 in markers

def test_variable_name_and_type() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
        static int foo(long a, int b){
            return a + b;
        }
        """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )

    markers = [marker for marker in iprogram.markers if isinstance(marker, VRMarker)]
    assert len(markers) == 2
    assert {marker.get_variable_name_and_type(iprogram.code) for marker in markers} == {
        ("a", '"long"'),
        ("b", '"int"'),
    }

def test_number_occurences() -> None:
    iprogram = instrument_program(
        SourceProgram(
            code="""
        static int foo(long a, int b){
            return a + b;
        }
        """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )

    markers = [marker for marker in iprogram.markers if isinstance(marker, VRMarker)]
    assert len(markers) == 2
    for marker in markers:
        assert marker.number_occurences_in_code(iprogram.code) == 1

    code = """
        static int foo(long a, int b){
            VRMARKERMACRO0_(a, "long")
            VRMARKERMACRO0_(a, "long")
            VRMARKERMACRO1_(b, "int")
            return a + b;
        }
        """

    markers = sorted(markers, key=lambda m: m.id)
    assert markers[0].number_occurences_in_code(code) == 2
    assert markers[1].number_occurences_in_code(code) == 1

def test_marker_renaming() -> None:
    iprogram_0 = instrument_program(
        SourceProgram(
            code="""
    int foo(int a){
        if (a)
            return 1;
        return 0;
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )
    iprogram_1 = instrument_program(
        SourceProgram(
            code="""
    int foo(int a, int b){
        if (a+b)
            return 1;
        return 0;
    }
    """,
            language=Language.C,
        ),
        mode=InstrumenterMode.VR,
    )

    iprogram0, iprogram1 = rename_markers((iprogram_0, iprogram_1))

    assert set((VRMarker.from_str("VRMarker0_", "int"),)) == set(
        iprogram0.enabled_markers()
    )
    assert set(
        (
            VRMarker.from_str("VRMarker1_", "int"),
            VRMarker.from_str("VRMarker2_", "int"),
        )
    ) == set(iprogram1.enabled_markers())

    assert (
        VRMarker.from_str("VRMarker0_", "int").macro_without_arguments()
        in iprogram0.code
    ), iprogram0.code

    assert (
        VRMarker.from_str("VRMarker1_", "int").macro_without_arguments()
        in iprogram1.code
    ), iprogram1.code
    assert (
        VRMarker.from_str("VRMarker2_", "int").macro_without_arguments()
        in iprogram1.code
    ), iprogram1.code

    gcc = get_system_gcc_O0()
    gcc.compile_program(iprogram1, ObjectCompilationOutput())


#!/usr/bin/env python3

import argparse
import os
import re

parser = argparse.ArgumentParser(description='Add a value to the file name for numbered files (renames the file with a new number).')
parser.add_argument('--add', required=True, type=int,
                    help='add this to file name number')
parser.add_argument('--write', action='store_true', help='write changes, otherwise script just displays what it would have done')
parser.add_argument('files', metavar='files', nargs='+',
                    help='list of files to modify')
args = parser.parse_args()

# scan src dir
files = []
for file in args.files:
    print(file)
    dirname = os.path.dirname(file)
    basename = os.path.basename(file)
    print("dir:", dirname)
    print("base:", basename)
    m = re.search('(\D*)(\d+)\.(.+)', basename)
    print(m.group(0), ":", m.group(1), m.group(2), m.group(3))
    new_num =  "%d" % (int(m.group(2)) + args.add)
    while len(new_num) < len(m.group(2)):
        new_num = '0' + new_num
    new_base = "%s%s.%s" % (m.group(1), new_num, m.group(3))
    print("rename:", file, os.path.join(dirname, new_base))
    if args.write:
        os.rename(file, os.path.join(dirname, new_base))


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

#Importing the libraries
import base64
import json
import numpy as np
import pandas as pd
import logging
from google.cloud import storage

class PubsubToGCS:
    def __init__(self):
        self.bucket_name = 'egen_bucket01'

    def extract_attributes(self, data):                            #data extraction
        transform_tweet = {
            "Tweet_id": data['id'],
            "Tweet_time":data['created_at'],
            "Tweet_source":data['source'],
            "User_id":data['user']['id'],
            "User_name":data['user']['name'].replace(',' , ''),
            "Location":data['user']['location'].replace(',' , '') if data['user']['location'] else None,
            "Place": data["place"]["country_code"] if data["place"] else None,
            "User_follower_count":data['user']['followers_count'],
            "User_friend_count":data['user']['friends_count']}

        if 'retweeted_status' in data:                            #tweet text extraction
            try:                                                  #retweet text extraction
                transform_tweet['text'] = data['retweeted_status']['extended_tweet']['full_text']
            except KeyError:
                transform_tweet['text'] = data['retweeted_status']['text']
        else:
            try:
                transform_tweet['text'] = data['extended_tweet']['full_text']
            except KeyError:
                transform_tweet['text'] = data['text']
        return transform_tweet

    def structure_payload(self,message) -> pd.DataFrame:
        try:                                                      #error handling for empty dataframe and creation error
            df=pd.DataFrame(message, index=[1])
            if not df.empty:
                logging.info("DataFrame created")
            else:
                logging.info("Empty DataFrame created")
            return df
        except Exception as e:
            logging.error(f"Error creating DataFrame {str(e)}")
            raise

    def upload_to_storage(self, df, filename):
        #using the API
        storage_client = storage.Client()

        #getting the bucket name
        bucket = storage_client.get_bucket(self.bucket_name)

        #defining the path for the files
        blob = bucket.blob(f'twitter_message/{filename}.csv')
        #converting and storing the dataframe as csv
        blob.upload_from_string(data=df.to_csv(index=False), content_type='text/csv')
        logging.info('Sucessfully written file to Cloud storage.')

def hello_pubsub(event, context):
    """Triggered from a message on a Cloud Pub/Sub topic.
    Args:
         event (dict): Event payload.
         context (google.cloud.functions.Context): Metadata for the event.
    """
    #decoding the messages
    pubsub_message = base64.b64decode(event['data']).decode('utf-8')

    #json string to dictionary
    message_dict = json.loads(pubsub_message)

    pubsub_to_gcs = PubsubToGCS()

    #extracting required data
    filtered_data = pubsub_to_gcs.extract_attributes(message_dict)

    #creating a dataframe for the filtered data
    data_frame = pubsub_to_gcs.structure_payload(filtered_data)

    #uploading to cloud storage
    tweet_id = filtered_data['Tweet_id']
    pubsub_to_gcs.upload_to_storage(data_frame, "Twitter_data_" + str(tweet_id))


from django.urls.base import reverse

import requests
from requests import ConnectionError

from dimagi.utils.logging import notify_exception

from corehq.apps.app_manager.dbaccessors import wrap_app
from corehq.apps.hqmedia.models import CommCareMultimedia
from corehq.apps.linked_domain.auth import ApiKeyAuth
from corehq.apps.linked_domain.exceptions import (
    ActionNotPermitted,
    RemoteAuthError,
    RemoteRequestError,
)
from corehq.util.view_utils import absolute_reverse

def get_tableau_server_and_visualizations(domain_link):
    return _do_simple_request('linked_domain:tableau_server_and_visualizations', domain_link)

def get_toggles_previews(domain_link):
    return _do_simple_request('linked_domain:toggles', domain_link)

def get_custom_data_models(domain_link, limit_types=None):
    url = reverse('linked_domain:custom_data_models', args=[domain_link.master_domain])
    params = None
    if limit_types:
        params = [('type', type_) for type_ in limit_types]
    return _do_request_to_remote_hq_json(url, domain_link.remote_details, domain_link.linked_domain, params)

def get_fixture(domain_link, tag):
    url = reverse('linked_domain:fixture', args=[domain_link.master_domain, tag])
    return _do_request_to_remote_hq_json(url, domain_link.remote_details, domain_link.linked_domain)

def get_user_roles(domain_link):
    return _do_simple_request('linked_domain:user_roles', domain_link)['user_roles']

def get_brief_apps(domain_link):
    apps = _do_simple_request('linked_domain:brief_apps', domain_link)['brief_apps']
    return [wrap_app(app) for app in apps]

def get_app_by_version(domain_link, upstream_app_id, upstream_version):
    url = reverse('linked_domain:app_by_version', args=[domain_link.master_domain,
                                                        upstream_app_id,
                                                        upstream_version])
    response = _do_request_to_remote_hq_json(url, domain_link.remote_details, domain_link.linked_domain)
    return response['app']

def get_case_search_config(domain_link):
    return _do_simple_request('linked_domain:case_search_config', domain_link)

def get_released_app(master_domain, app_id, linked_domain, remote_details):
    url = reverse('linked_domain:latest_released_app_source', args=[master_domain, app_id])
    response = _do_request_to_remote_hq_json(url, remote_details, linked_domain)
    return _convert_app_from_remote_linking_source(response)

def get_latest_released_versions_by_app_id(domain_link):
    return _do_simple_request('linked_domain:released_app_versions', domain_link)['versions']

def get_ucr_config(domain_link, report_config_id):
    from corehq.apps.userreports.util import wrap_report_config_by_type, _wrap_data_source_by_doc_type
    url = reverse('linked_domain:ucr_config', args=[domain_link.master_domain,
                                                    report_config_id])
    response = _do_request_to_remote_hq_json(url, domain_link.remote_details, domain_link.linked_domain)
    return {
        "report": wrap_report_config_by_type(response["report"]),
        "datasource": _wrap_data_source_by_doc_type(response["datasource"]),
    }

def get_data_dictionary(domain_link):
    return _do_simple_request('linked_domain:data_dictionary', domain_link)

def get_dialer_settings(domain_link):
    return _do_simple_request('linked_domain:dialer_settings', domain_link)

def get_otp_settings(domain_link):
    return _do_simple_request('linked_domain:otp_settings', domain_link)

def get_hmac_callout_settings(domain_link):
    return _do_simple_request('linked_domain:hmac_callout_settings', domain_link)

def get_auto_update_rules(domain_link):
    return _do_simple_request('linked_domain:auto_update_rules', domain_link)['rules']

def get_remote_linkable_ucr(domain_link):
    from corehq.apps.linked_domain.ucr import get_linked_reports_in_domain
    all_remote_reports = _do_simple_request('linked_domain:linkable_ucr', domain_link).get('reports', [])
    linked_report_master_ids = [r.report_meta.master_id
                                for r in get_linked_reports_in_domain(domain_link.linked_domain)]
    for report in all_remote_reports:
        report['already_linked'] = report['id'] in linked_report_master_ids
    return all_remote_reports

def _convert_app_from_remote_linking_source(app_json):
    attachments = app_json.pop('_LAZY_ATTACHMENTS', {})
    app = wrap_app(app_json)
    app._LAZY_ATTACHMENTS = attachments
    return app

def fetch_remote_media(local_domain, missing_media, remote_app_details):
    for filename, item in missing_media:
        media_class = CommCareMultimedia.get_doc_class(item['media_type'])
        content = _fetch_remote_media_content(item, remote_app_details)
        media_item = media_class.get_by_data(content)
        if getattr(media_item, '_id', None) and not getattr(item, 'upstream_media_id', None):
            # media already exists locally; update the multimedia_map
            item['multimedia_id'], item['upstream_media_id'] = media_item._id, item['multimedia_id']
        else:
            media_item._id = item['multimedia_id']
        media_item.attach_data(content, original_filename=filename)
        media_item.add_domain(local_domain, owner=True)

def _fetch_remote_media_content(media_item, remote_app_details):
    remote_media_id = media_item['upstream_media_id'] or media_item['multimedia_id']
    url = reverse('hqmedia_download', args=[media_item['media_type'], remote_media_id])
    response = _do_request_to_remote_hq(url, remote_app_details, None)
    return response.content

def _do_simple_request(url_name, domain_link):
    url = reverse(url_name, args=[domain_link.master_domain])
    return _do_request_to_remote_hq_json(url, domain_link.remote_details, domain_link.linked_domain)

def _do_request_to_remote_hq_json(relative_url, remote_details, linked_domain, params=None, method='get'):
    return _do_request_to_remote_hq(relative_url, remote_details, linked_domain, params, method).json()

def _do_request_to_remote_hq(relative_url, remote_details, linked_domain, params=None, method='get'):
    """
    :param relative_url: Relative URL on remote HQ
    :param remote_details: RemoteDetails object containing remote URL base and auth details
    :param linked_domain: Used for permission check on remote system
    :param params: GET/POST params to include
    :param method:
    :return:
    """
    url_base = remote_details.url_base
    username = remote_details.username
    api_key = remote_details.api_key
    full_url = '%s%s' % (url_base, relative_url)
    headers = {
        'HQ-REMOTE-REQUESTER': absolute_reverse('domain_homepage', args=[linked_domain])
    }
    try:
        response = requests.request(
            method, full_url,
            params=params, auth=ApiKeyAuth(username, api_key), headers=headers
        )
    except ConnectionError:
        notify_exception(None, "Error performing remote app request", details={
            'remote_url': full_url,
            'params': params,
            'headers': headers
        })
        raise RemoteRequestError()
    if response.status_code == 401:
        raise RemoteAuthError(response.status_code)
    elif response.status_code == 403:
        raise ActionNotPermitted(response.status_code)
    elif response.status_code != 200:
        notify_exception(None, "Error performing remote app request", details={
            'remote_url': full_url,
            'response_code': response.status_code,
            'params': params
        })
        raise RemoteRequestError(response.status_code)
    return response


import numpy as np
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense

# 1. 데이터
x = np.array([1,2,3,4,5,6,7,8,9,10])
y = np.array([1,2,3,4,5,6,7,8,9,10])

# [검색] train과 test를 섞어서 7:3으로 찾을 수 있는 방법을 찾아라.
# 힌트: 사이킷런

x_train, x_test, y_train, y_test =  train_test_split(x, y,
                                                     train_size=0.7,
                                                      shuffle=True, # 명시 안할 경우 True가 default
                                                     random_state=1004)
# seed : 랜덤 난수에 따라 정해진 랜덤치로 뽑음 난수 바뀌기 전까지 고정되는 셈
# size : 퍼센트로 접근

print(x_test) # [ 1  9 10]
print(x_train) # [2 7 6 3 4 8 5]
print(y_test)
print(y_train)

# 디버그 시 줄 옆에 빨간 점 + F5
# 빨간 점 전까지 실행, 다시 F5 누르면 거기서부터 이어서 실행, 빈 곳 말고 내용 있는 곳 찍어야됨

# 2. 모델
model = Sequential()
model.add(Dense(10, input_dim=1))
model.add(Dense(1))

# 3. 컴파일, 훈련
model.compile(loss = 'mse', optimizer='adam')
model.fit(x_train, y_train, epochs=100, batch_size=1)

# 4. 평가, 예측
loss = model.evaluate(x_test, y_test)
print("loss : ", loss)
result = model.predict([11])
print('[11]의 예측값 : ', result)

# model.add(Dense(10, input_dim=1))
# model.add(Dense(1))
# epochs=100, batch_size=1
# loss :  7.579122740649855e-14
# [11]의 예측값 :  [[11.]]


#!/usr/bin/env python
# coding: utf-8

# # Exploratory Data Analysis - Terrorism
# Author - Vinay Kumar

# In[1]:

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
get_ipython().system('py -m pip install folium')

# In[2]:

import folium
from folium.plugins import MarkerCluster

# In[4]:

df = pd.read_csv(r'C:\Users\n\Desktop\Dataset\globalterrorismdb_0718dist.csv')  #loading dataset
df.head()    #display starting 3 rows

# In[5]:

df.shape

# In[6]:

df.info()

# In[7]:

df.describe()

# In[8]:

df.head()

# In[9]:

df.corr()

# In[10]:

df.columns

# # US Terror Attacks and Death Injuries

# In[11]:

df.nkillus.plot(kind = 'line', color = 'red', label = 'The Number of Total Confirmed Fatalities for US', linewidth = 2, alpha = 0.8, grid = True,
                 linestyle = ':', figsize = (20,20), fontsize=15)
df.nwoundus.plot(color = "green", label = 'The Number of Confirmed Non-Fatal Injuries for US', linewidth = 2, alpha = 0.8, grid = True,
                 linestyle = '-.', figsize = (20,20), fontsize=15)

plt.legend(loc='upper right')     # legend = puts label into plot
plt.xlabel('Database Index', fontsize=10)              # label = name of label
plt.ylabel('Number of Dead or Injuries', fontsize=15)

plt.title('Confirmed Fatalities & Non-Fatal Injuries for US')            #plot title
plt.show()

# Given that the data is sorted by dates, attacks on US citizens seem to be very rare in a given date range.

# But the terrorist act against the citizens of US has been increasingly in the following year after this rare date range.

# By finding the date of the start of the increase, the factors in increasing terrorist acts can be easily identified by
# taking into account the changes and developments in the country after this date.

# # Death and Injuries at all time

# In[14]:

df.plot(kind = 'scatter', x = 'nkill', y = 'nwound', alpha = 0.5, color = 'red', figsize = (20,20), fontsize=15)
plt.xlabel('Kill', fontsize=15)
plt.ylabel('Wound', fontsize=15)
plt.title('Kill - Wound Scatter Plot')
plt.show()

# # Frequency of Terrorist Actions in Customized Region

# Let's analyze in Middle East and North Africa

# In[15]:

middleEastData = df[df['region'] == 10]
middleEastData.iyear.plot(kind = 'hist', bins = 30, figsize = (20,20), color = 'red', fontsize=15)
plt.xlabel('Year', fontsize=15)
plt.ylabel('Frequency', fontsize=15)
plt.title('Frequency of Middle East & North Africa Terrorism Actions by Years')
plt.show()

# Terrorist attacks have increased in recent years.

# In[16]:

df.nkillus.plot(kind = 'line', color = 'red', label = 'People Dead of Terrorism in the World', linewidth = 1.5, alpha = 0.8, grid = True,
                 linestyle = ':', figsize = (20,20), fontsize=15)
middleEastData.nkillus.plot(color = "green", label = 'People Dead of Terrorism in the Middle East & North Africa', linewidth = 1.5, alpha = 0.8,
                            grid = True, linestyle = '-.', figsize = (20,20), fontsize=15)

plt.legend(loc='upper right')     # legend = puts label into plot
plt.xlabel('Database Index', fontsize=15)              # label = name of label
plt.ylabel('Number of Dead', fontsize=15)

plt.title('Comparing those who died in terror attacks in the World and Middle East & North Africa')            #plot title
plt.show()

# # Terrorist Attacks of a Particular year and their Locations.
# Let's look at the terrorist acts in the world over a certain year.
#

# In[17]:

filterYear = df['iyear'] == 1970

# In[18]:

filterData = df[filterYear] # filter data
# filterData.info()
reqFilterData = filterData.loc[:,'city':'longitude'] #Required fields
reqFilterData = reqFilterData.dropna() # drop NaN values in latitude and longitude
reqFilterDataList = reqFilterData.values.tolist()
# reqFilterDataList

# In[19]:

map = folium.Map(location = [0, 30], tiles='CartoDB positron', zoom_start=2)
# Clustered Marker
markerCluster = folium.plugins.MarkerCluster().add_to(map)
for point in range(0, len(reqFilterDataList)):
    folium.Marker(location=[reqFilterDataList[point][1],reqFilterDataList[point][2]], popup = reqFilterDataList[point][0]).add_to(markerCluster)
map

# In[20]:

killData = df.loc[:,'nkill']
print('Number of people killed by terror attack:', int(sum(killData.dropna())))# drop the NaN values

# # Types of Attacks these deaths were made of.

# In[21]:

killData = df.loc[:,'nkill']
print('Number of people killed by terror attack:', int(sum(killData.dropna())))# drop the NaN values

# In[22]:

attackData = df.loc[:,'attacktype1':'attacktype1_txt']
# attackData
typeKillData = pd.concat([attackData, killData], axis=1)

# In[23]:

typeKillFormatData = typeKillData.pivot_table(columns='attacktype1_txt', values='nkill', aggfunc='sum')
typeKillFormatData

# In[24]:

typeKillFormatData.info()

# In[28]:

countryData = df.loc[:,'country':'country_txt']
# countyData
countryKillData = pd.concat([countryData, killData], axis=1)

# In[29]:

countryKillFormatData = countryKillData.pivot_table(columns='country_txt', values='nkill', aggfunc='sum')
countryKillFormatData

# In[30]:

countryKillFormatData.info()

# In[31]:

fig_size = plt.rcParams["figure.figsize"]
fig_size[0]=25
fig_size[1]=25
plt.rcParams["figure.figsize"] = fig_size

# In[32]:

labels = countryKillFormatData.columns.tolist()
labels = labels[:50] #50 bar provides nice view
index = np.arange(len(labels))
transpoze = countryKillFormatData.T
values = transpoze.values.tolist()
values = values[:50]
values = [int(i[0]) for i in values] # convert float to int
colors = ['red', 'green', 'blue', 'purple', 'yellow', 'brown', 'black', 'gray', 'magenta', 'orange'] # color list for bar chart bar color
fig, ax = plt.subplots(1, 1)
ax.yaxis.grid(True)
fig_size = plt.rcParams["figure.figsize"]
fig_size[0]=25
fig_size[1]=25
plt.rcParams["figure.figsize"] = fig_size
plt.bar(index, values, color = colors, width = 0.9)
plt.ylabel('Killed People', fontsize=15)
plt.xticks(index, labels, fontsize=12, rotation=90)
plt.title('Number of people killed by countries')
# print(fig_size)
plt.show()

# In[33]:

labels = countryKillFormatData.columns.tolist()
labels = labels[50:101]
index = np.arange(len(labels))
transpoze = countryKillFormatData.T
values = transpoze.values.tolist()
values = values[50:101]
values = [int(i[0]) for i in values]
colors = ['red', 'green', 'blue', 'purple', 'yellow', 'brown', 'black', 'gray', 'magenta', 'orange']
fig, ax = plt.subplots(1, 1)
ax.yaxis.grid(True)
fig_size = plt.rcParams["figure.figsize"]
fig_size[0]=25
fig_size[1]=25
plt.rcParams["figure.figsize"] = fig_size
plt.bar(index, values, color = colors, width = 0.9)
plt.ylabel('Killed People', fontsize=15)
plt.xticks(index, labels, fontsize=12, rotation=90)
plt.title('Number of people killed by countries')
plt.show()

# In[34]:

labels = countryKillFormatData.columns.tolist()
labels = labels[152:206]
index = np.arange(len(labels))
transpoze = countryKillFormatData.T
values = transpoze.values.tolist()
values = values[152:206]
values = [int(i[0]) for i in values]
colors = ['red', 'green', 'blue', 'purple', 'yellow', 'brown', 'black', 'gray', 'magenta', 'orange']
fig, ax = plt.subplots(1, 1)
ax.yaxis.grid(True)
fig_size = plt.rcParams["figure.figsize"]
fig_size[0]=25
fig_size[1]=25
plt.rcParams["figure.figsize"] = fig_size
plt.bar(index, values, color = colors, width = 0.9)
plt.ylabel('Killed People', fontsize=15)
plt.xticks(index, labels, fontsize=12, rotation=90)
plt.title('Number of people killed by countries')
plt.show()

# # Observation:
# Terrorist acts in the Middle East and northern Africa have been seen to have fatal consequences.

# The Middle East and North Africa are seen to be the places of serious terrorist attacks.

# In addition, even though there is a perception that Muslims are supporters of terrorism, Muslims are the people who are most damaged by terrorist attacks.

# If you look at the graphics, it appears that Iraq, Afghanistan and Pakistan are the most damaged countries.

# All of these countries are Muslim countries.

Exploratory Data Analysis - Terrorism/Exploratory Data Analysis - Terrorism.py