__author__ = 'jgwall'

import argparse
import networkx as nx
import matplotlib.pyplot as plt
import math
import pandas as pd
import re

debug = False

root_key={
    # KO terms
    "Metabolism" : {'color': {'r': 0, 'g': 0, 'b': 255, 'a': 0}},
    "Unclassified" :    {'color': {'r': 255, 'g': 255, 'b': 0, 'a': 0}},
    "Cellular Processes"  : {'color': {'r': 0, 'g': 255, 'b': 0, 'a': 0}} ,
    "Environmental Information Processing" : {'color': {'r': 0, 'g': 255, 'b': 255, 'a': 0}},
    "Genetic Information Processing" : {'color': {'r': 255, 'g': 0, 'b': 255, 'a': 0}},
    "Human Diseases": {'color': {'r': 0, 'g': 0, 'b': 0, 'a': 0}},

    # COG terms
    "POORLY CHARACTERIZED": {'color': {'r': 255, 'g': 255, 'b': 0, 'a': 0}},
    "CELLULAR PROCESSES AND SIGNALING": {'color': {'r': 0, 'g': 255, 'b': 0, 'a': 0}} ,
    "METABOLISM": {'color': {'r': 0, 'g': 0, 'b': 255, 'a': 0}},
    "INFORMATION STORAGE AND PROCESSING":  {'color': {'r': 255, 'g': 0, 'b': 255, 'a': 0}}
}

def main():
    args = parse_args()
    print("Making subgraph of significant hits in",args.infile)

    # Load heritability and graph
    targets = load_herits(args.infile, args.max_herit, args.pval_cutoff)
    graph = nx.read_gexf(args.graphfile)

    # Find targets among renamed nodes with full hierarchy
    targets = find_targets(graph, targets)

    # Get sets of terminal significant ones and their ancestors
    ancestors = set()
    nodes = graph.nodes()
    added=0
    for t in targets:
        if t not in nodes: continue # Skip any hits that aren't targets
        added+=1
        for a in nx.ancestors(graph, t):
            ancestors.add(a)
    print("\tAdded ancestors of",added,"nodes to look for")

    # Make subgraph
    subgraph = graph.subgraph(targets | ancestors)
    print("Subgraph has",len(subgraph.nodes()),"nodes of the original",len(graph.nodes()))

    # Prettify subgraph
    subgraph = prettify_subgraph(subgraph, targets)

    # Output
    #subgraph = subgraph.reverse()   #Flip so individual nodes have arrows toward their supercategory, not vice-versa
    nx.write_gexf(subgraph, args.outprefix + ".gexf", version="1.2draft")
    nx.draw_networkx(subgraph)
    plt.savefig(args.outprefix + ".png")

    # Output version without terminal leaves
    remove_terminal_leaves(subgraph)
    nx.write_gexf(subgraph, args.outprefix + ".trimmed.gexf", version="1.2draft")
    nx.draw_networkx(subgraph)
    plt.savefig(args.outprefix + "trimmed.png")

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--infile", help="Parsed heritabilities from 5_AssembleMetagenomeHeritData.py")
    parser.add_argument("-g", "--graphfile", help="GEXF-formatted graph file of the annotation hierarchy")
    parser.add_argument("-f", "--fisher", help="P-value results for Fisher-exact test of term enrichment")
    parser.add_argument("-o", "--outprefix", help="Output file prefix")
    parser.add_argument("-p", "--pval-cutoff", type=float, default=1, help="P-value cutoff for including in the signature")
    parser.add_argument("-m", "--max-herit", type=float, default=1, help="Exclude heritability above this (b/c is probably a artifact)")
    parser.add_argument("--debug", default=False, action="store_true")
    args = parser.parse_args()

    # Handle debug flag
    global debug
    debug = args.debug

    return parser.parse_args()

def load_herits(infile, max_herit, pval_cutoff):
    data = pd.read_table(infile, index_col=0)

    # Reformat trait names
    traits = data['trait']
    traits = [re.sub(pattern="^log_", string=t, repl="") for t in traits]
    traits = [re.sub(pattern="_[0-9]+$", string=t, repl="") for t in traits]
    traits = [re.sub(pattern="___", string=t, repl=" - ") for t in traits]
    traits = [re.sub(pattern="_", string=t, repl=" ") for t in traits]
    data['trait'] = traits

    # Identify the enriched, non-enriched, and bad traits based on heritability and empirical p-value
    good_pval = data['empirical_pval'] <= pval_cutoff
    good_herit = data['h2'] <= max_herit
    data['classification'] = 'unknown'
    data.loc[good_pval & good_herit, 'classification'] = "target"
    data.loc[~good_pval & good_herit, 'classification'] = "background"
    data.loc[~good_herit, 'classification'] = "error"

    # Subset
    targets = data['trait'].loc[data['classification'] == 'target']
    background = data['trait'].loc[data['classification'] == 'background']
    errors = data['trait'].loc[data['classification'] == 'error']

    # Status report and sanity check
    print("Of", len(data), "base traits, identified", len(targets), "as good/enriched", len(background),
          "as non-enriched/background, and",
          len(errors), "as bad due to too high heritability")
    if len(data) != (len(targets) + len(background) + len(errors)):
        print("\tWARNING!! These three categories should sum to the total but they don't!!!")

    # Convert to sets and return
    targetset = {t for t in targets}
    if len(targetset) != len(targets): print("WARNING! Duplicate target names detected!!")
    return targetset

def find_targets(graph, targets):
    prettykey = {pretty_name(node):node for node in graph.nodes()}
    new_targets=set()
    for t in targets:
        if t in prettykey:
            new_targets.add(prettykey[t])
        else:
            print("Warning! Unable to find",t)
    return(new_targets)

# Make graph pretty
def prettify_subgraph(graph, hits):

    # Relabel nodes with shorter names
    label_key = {n:pretty_name(n) for n in graph.nodes()}   # Remove higher-level pathway info
    label_key = {n: remove_base_names(label_key[n]) for n in label_key} # Strip base COG/KO terms
    nx.set_node_attributes(graph, name="label", values=label_key)

    # Colorize nodes by their core region (metabolism, etc)
    for n in graph.nodes():
        found = 0
        colorkey = {'color': {'r': 125, 'g': 125, 'b': 125, 'a': 0}}    # Default gray
        # Colorize hits red
        if n in hits or pretty_name(n) in hits:
            colorkey = {'color': {'r': 255, 'g': 0, 'b': 0, 'a': 0}}    # Red if hit
            found+=1
        # Colorize other nodes according to their ancestral group
        else:
            ancestors = nx.ancestors(graph, n)
            for root in root_key:
                if n == root or root in ancestors:
                    colorkey = root_key[root]
                    found+=1
        if found != 1:
            print("!!WARNING!! Found",found,"root nodes for",n,"with ancestors",ancestors)
        graph.node[n]['viz'] = colorkey
    # print("\tFound",found,"hits that passed heritability")

    # Set size based on cumulative number of hits
    countkey = dict()
    for n in graph.nodes():
        desc = nx.descendants(graph, n) # Set of descendant nodes
        myhits = desc & hits
        mysize = (len(myhits) + 1) * 5
        graph.node[n]['viz']['size'] = math.sqrt(mysize)
        countkey[n] = len(myhits)
    nx.set_node_attributes(graph, name="hit_count", values=countkey)

    return graph

def pretty_name(node):
    if "->" in node:
        node = node.split("->")[-1]
    if node == "D-Arginine and D-ornithine metabolism":
        node = "D Arginine and D ornithine metabolism"
    return node

def remove_base_names(node):
    if node.startswith("K0") or node.startswith("K1") or node.startswith("COG"):
        node=""
    return node

def remove_terminal_leaves(graph):
    for node in graph.nodes():
        if (node.startswith("K0") or node.startswith("K1") or node.startswith("COG")) and len(nx.descendants(graph, node)) ==0:
            graph.remove_node(node)

if __name__ == '__main__': main()


"""
Module for handling CAMUNDA tasks.
"""
import asyncio
import logging
import os
from pyzeebe import ZeebeWorker, ZeebeClient, create_insecure_channel

from ticket_broker._version import NAME

ENDPOINT_HOST_KEY = "ENDPOINT_HOST"
ENDPOINT_PORT_KEY = "ENDPOINT_PORT"

logger = logging.getLogger(__name__)

worker: ZeebeWorker = None
client: ZeebeClient = None

def create_client():
    global worker, client
    print("Creating worker/client instance")
    env_vars = os.environ.keys()
    if ENDPOINT_HOST_KEY in env_vars and ENDPOINT_PORT_KEY in env_vars:
        channel = create_insecure_channel(
            hostname=os.environ[ENDPOINT_HOST_KEY],
            port=int(os.environ[ENDPOINT_PORT_KEY]))
    else:
        logging.warning("No endpoint specified. Using defaults!")
        channel = create_insecure_channel()
    worker = ZeebeWorker(channel)
    client = ZeebeClient(channel)

def run_loop():
    loop = asyncio.get_event_loop()
    loop.run_until_complete(worker.work())


import typing as t

import click

from tutor import config as tutor_config
from tutor import env as tutor_env
from tutor import exceptions, fmt, hooks
from tutor import interactive as interactive_config
from tutor import utils
from tutor.commands import compose
from tutor.types import Config, get_typed

class DevJobRunner(compose.ComposeJobRunner):
    def __init__(self, root: str, config: Config):
        """
        Load docker-compose files from dev/ and local/
        """
        super().__init__(root, config)
        self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str)
        self.docker_compose_tmp_path = tutor_env.pathjoin(
            self.root, "dev", "docker-compose.tmp.yml"
        )
        self.docker_compose_jobs_tmp_path = tutor_env.pathjoin(
            self.root, "dev", "docker-compose.jobs.tmp.yml"
        )
        self.docker_compose_files += [
            tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
            tutor_env.pathjoin(self.root, "dev", "docker-compose.yml"),
            self.docker_compose_tmp_path,
            tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"),
            tutor_env.pathjoin(self.root, "dev", "docker-compose.override.yml"),
        ]
        self.docker_compose_job_files += [
            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
            tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.yml"),
            self.docker_compose_jobs_tmp_path,
            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"),
            tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.override.yml"),
        ]

class DevContext(compose.BaseComposeContext):
    def job_runner(self, config: Config) -> DevJobRunner:
        return DevJobRunner(self.root, config)

@click.group(help="Run Open edX locally with development settings")
@click.pass_context
def dev(context: click.Context) -> None:
    context.obj = DevContext(context.obj.root)

@click.command(help="Configure and run Open edX from scratch, for development")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.option("-p", "--pullimages", is_flag=True, help="Update docker images")
@click.pass_context
def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) -> None:
    try:
        utils.check_macos_docker_memory()
    except exceptions.TutorError as e:
        fmt.echo_alert(
            f"""Could not verify sufficient RAM allocation in Docker:
    {e}
Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from:
    https://docs.tutor.overhang.io/install.html"""
        )

    click.echo(fmt.title("Interactive platform configuration"))
    config = tutor_config.load_minimal(context.obj.root)
    if not non_interactive:
        interactive_config.ask_questions(config, run_for_prod=False)
    tutor_config.save_config_file(context.obj.root, config)
    config = tutor_config.load_full(context.obj.root)
    tutor_env.save(context.obj.root, config)

    click.echo(fmt.title("Stopping any existing platform"))
    context.invoke(compose.stop)

    if pullimages:
        click.echo(fmt.title("Docker image updates"))
        context.invoke(compose.dc_command, command="pull")

    click.echo(fmt.title("Building Docker image for LMS and CMS development"))
    context.invoke(compose.dc_command, command="build", args=["lms"])

    click.echo(fmt.title("Starting the platform in detached mode"))
    context.invoke(compose.start, detach=True)

    click.echo(fmt.title("Database creation and migrations"))
    context.invoke(compose.init)

    fmt.echo_info(
        """The Open edX platform is now running in detached mode
Your Open edX platform is ready and can be accessed at the following urls:
    {http}://{lms_host}:8000
    {http}://{cms_host}:8001
    """.format(
            http="https" if config["ENABLE_HTTPS"] else "http",
            lms_host=config["LMS_HOST"],
            cms_host=config["CMS_HOST"],
        )
    )

@click.command(
    help="DEPRECATED: Use 'tutor dev start ...' instead!",
    context_settings={"ignore_unknown_options": True},
)
@compose.mount_option
@click.argument("options", nargs=-1, required=False)
@click.argument("service")
@click.pass_context
def runserver(
    context: click.Context,
    mounts: t.Tuple[t.List[compose.MountParam.MountType]],
    options: t.List[str],
    service: str,
) -> None:
    depr_warning = "'runserver' is deprecated and will be removed in a future release. Use 'start' instead."
    for option in options:
        if option.startswith("-v") or option.startswith("--volume"):
            depr_warning += " Bind-mounts can be specified using '-m/--mount'."
            break
    fmt.echo_alert(depr_warning)
    config = tutor_config.load(context.obj.root)
    if service in ["lms", "cms"]:
        port = 8000 if service == "lms" else 8001
        host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"]
        fmt.echo_info(
            f"The {service} service will be available at http://{host}:{port}"
        )
    args = ["--service-ports", *options, service]
    context.invoke(compose.run, mounts=mounts, args=args)

@hooks.Actions.COMPOSE_PROJECT_STARTED.add()
def _stop_on_local_start(root: str, config: Config, project_name: str) -> None:
    """
    Stop the dev platform as soon as a platform with a different project name is
    started.
    """
    runner = DevJobRunner(root, config)
    if project_name != runner.project_name:
        runner.docker_compose("stop")

dev.add_command(quickstart)
dev.add_command(runserver)
compose.add_commands(dev)


import MetaTrader5 as mt5
import pandas as pd
import numpy as np
import plotly.graph_objects as go
from IPython.display import display
import time
from datetime import datetime, timedelta

mt5.initialize()

status = 'open'
symbols = mt5.symbols_get()

price = mt5.symbol_info_tick('EURUSD') # Make a choice of your currency pair
print(datetime.fromtimestamp(price.time))
print(price.bid)

class SupportResistance:
    def __init__(self):
        self.status = None
        self.symbol = 'EURUSD' # Use the currency pair you have chosen
        self.year = int(str(datetime.now())[:-22])
        self.month = int(str(datetime.now())[5:-19])
        self.day = int(str(datetime.now())[8:-16])
        self.price = mt5.symbol_info_tick(self.symbol)
        self.timeframe = mt5.TIMEFRAME_M1
        pass

    def checkgraph(self):
        while self.status is None:
            start_dt = datetime(self.year, self.month, self.day)
            end_dt = datetime.now()

            bars = mt5.copy_rates_range(self.symbol, self.timeframe, start_dt, end_dt)
            df = pd.DataFrame(bars)[['time', 'open', 'high', 'low', 'close']]
            df['time'] = pd.to_datetime(df['time'], unit='s')

            fig = go.Figure(data=go.Ohlc(
                x=df['time'], open=df['open'], high=df['high'], low=df['low'], close=df['close']))

            def specify_candle_type(open_price, close_price):
                if close_price > open_price:
                    return 'bullish'
                elif close_price < open_price:
                    return 'bearish'
                else:
                    return 'doji'

            df['candle_type'] = np.vectorize(specify_candle_type)(df['open'], df['close'])

            df['candle_1'] = df['candle_type'].shift(1)
            df['candle_2'] = df['candle_type'].shift(2)
            df['candle_3'] = df['candle_type'].shift(3)
            df['prev_close'] = df['close'].shift(1)

            df_bullish = df[
                (df['candle_1'] == 'bullish') & (df['candle_2'] == 'bullish') & (df['candle_3'] == 'bullish')].copy()
            df_bullish['points'] = df_bullish['close'] - df['prev_close']

            df_bearish = df[
                (df['candle_1'] == 'bearish') & (df['candle_2'] == 'bearish') & (df['candle_3'] == 'bearish')].copy()
            df_bullish['points'] = df_bullish['close'] - df['prev_close']
            print(df_bearish)
            print(df_bullish)

            last_value = str(df_bearish['time'].tail(1))[6:-37]
            time_now = str(datetime.now())[:-10]
            print(last_value)
            print(time_now)
            print(last_value == time_now)
            if last_value == time_now:
                self.status = 'open_bearish'
                trade = SupportResistance()
                trade.buy()

            else:
                last_value = str(df_bullish['time'].tail(1))[6:-37]
                time_now = str(datetime.now())[:-10]
                print(last_value)
                print(time_now)
                print(last_value == time_now)
                if last_value == time_now:
                    self.status = 'open_bullish'
                    SupportResistance.sell()
                    trade = SupportResistance()
                    trade.buy()

            time.sleep(5)
        pass

    def analysis(self):
        lines = [] # Enter the value of support and resistance lines here
        lines = sorted(lines)
        lines_btn = []
        iterator = 0
        least_difference = lines[0] - self.price.bid

        while iterator < len(lines):
            least_difference_test = lines[iterator] - self.price.bid
            if 0 < least_difference_test <= least_difference:
                least_difference = least_difference_test
            iterator += 1

        for line in lines:
            difference = line - self.price.bid
            if difference == least_difference:
                lines_btn.append(line)
                break
        else:
            SupportResistance().closingpositions()
            pass

        iterator = 0
        least_difference = lines[0] - self.price.bid
        while iterator < len(lines):
            least_difference_test = lines[iterator] - self.price.bid
            if least_difference_test <= least_difference and least_difference_test < 0:
                least_difference = least_difference_test
            iterator += 1

        for line in lines:
            difference = line - self.price.bid
            if difference == least_difference:
                lines_btn.append(line)
                break
        else:
            SupportResistance().closingpositions()
            pass

        start_dt = datetime(self.year, self.month, self.day)
        end_dt = datetime.now()

        bars = mt5.copy_rates_range(self.symbol, self.timeframe, start_dt, end_dt)
        df = pd.DataFrame(bars)[['time', 'open', 'high', 'low', 'close']]
        df['time'] = pd.to_datetime(df['time'], unit='s')
        df['sma_200'] = df['open'].rolling(200).mean()

        while lines_btn[0] <= self.price.bid <= lines_btn[1] and lines_btn[0] <= float(df['sma_200'].tail(1)) <= \
                lines_btn[1]:
            start_dt = datetime(self.year, self.month, self.day)
            end_dt = datetime.now()

            bars = mt5.copy_rates_range(self.symbol, self.timeframe, start_dt, end_dt)
            df = pd.DataFrame(bars)[['time', 'open', 'high', 'low', 'close']]
            df['time'] = pd.to_datetime(df['time'], unit='s')
            df['sma_200'] = df['open'].rolling(200).mean()
            print('check 3')
            continue
        else:
            SupportResistance().closingpositions()
        pass

    def buy(self):
        request = {
            'action': mt5.TRADE_ACTION_DEAL,
            'symbol': self.symbol,
            'volume': 0.1,
            'type': mt5.ORDER_TYPE_BUY,
            'price': mt5.symbol_info_tick(self.symbol).ask,
            'deviation': 20,
            'magic': 100,
            'comment': 'python script open',
            'type-time': mt5.ORDER_TIME_GTC,
            'type_filling': mt5.ORDER_FILLING_IOC
        }

        mt5.order_send(request)
        self.status = 'open_bullish'
        SupportResistance().analysis()
        pass

    def sell(self):
        request = {
            'action': mt5.TRADE_ACTION_DEAL,
            'symbol': self.symbol,
            'volume': 0.1,
            'type': mt5.ORDER_TYPE_SELL,
            'price': mt5.symbol_info_tick(self.symbol).bid,
            'deviation': 20,
            'magic': 100,
            'comment': 'python script open',
            'type-time': mt5.ORDER_TIME_GTC,
            'type_filling': mt5.ORDER_FILLING_IOC
        }

        mt5.order_send(request)
        self.status = 'open_bullish'
        SupportResistance().analysis()
        pass

    def closingpositions(self):
        positions = mt5.positions_get()

        for position in positions:
            tick = mt5.symbol_info_tick(position.self.symbol)

            request = {
                'action': mt5.TRADE_ACTION_DEAL,
                'position': position.ticket,
                'symbol': position.symbol,
                'volume': position.volume,
                'type': mt5.ORDER_TYPE_SELL if position.type == 0 else mt5.ORDER_TYPE_BUY,
                'price': tick.ask if position.type == 1 else tick.bid,
                'deviation': 20,
                'magic': 100,
                'comment': 'python script open',
                'type-time': mt5.ORDER_TIME_GTC,
                'type_filling': mt5.ORDER_FILLING_IOC
            }
            mt5.order_send(request)

        self.status = None
        print('check 4')
        SupportResistance().checkgraph()
        pass

trade = SupportResistance()
trade.checkgraph()


import random

import torch

def _gen_cutout_coord(height, width, size):
    height_loc = random.randint(0, height - 1)
    width_loc = random.randint(0, width - 1)

    upper_coord = (max(0, height_loc - size // 2),
                    max(0, width_loc - size // 2))
    lower_coord = (min(height, height_loc + size // 2),
                    min(width, width_loc + size // 2))

    return upper_coord, lower_coord

class Cutout(torch.nn.Module):
    def __init__(self, size=16):
        super().__init__()
        self.size = size

    def forward(self, img):
        h, w = img.shape[-2:]
        upper_coord, lower_coord  = _gen_cutout_coord(h, w, self.size)

        mask_height = lower_coord[0] - upper_coord[0]
        mask_width = lower_coord[1] - upper_coord[1]
        assert mask_height > 0
        assert mask_width > 0

        mask = torch.ones_like(img)
        mask[..., upper_coord[0]:lower_coord[0], upper_coord[1]:lower_coord[1]] = 0
        return img * mask


from epg_grabber.models import Programme, ChannelMetadata, Channel

from datetime import date, datetime, timedelta
import requests

KALSIG = "1c9a9da646d991758f659424dccec62f"

LOGIN_URL = "https://app-kaltura-proxy.sooka.my/prod/api/v1/api_v3/service/ottuser/action/anonymousLogin"
LIST_URL = "https://app-kaltura-proxy.sooka.my/prod/api/v1/api_v3/service/asset/action/list"
DEFAULT_HEADERS = {
    "authority": "app-kaltura-proxy.sooka.my",
    "origin": "https://sooka.my",
    "referer": "https://sooka.my/",
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
    'content-type': 'application/json'
}

def login() -> dict:

  try:
    response = requests.post(LOGIN_URL, json={
        "partnerId": "3209",
        "udid": "WEB-6764bc29-d82d-3f3b-f9ac-bc0613757dca",
        "format": 1,
        "clientTag": "AstroQA",
        "apiVersion": "6.1.0.28839",
        "language": "en",
        "kalsig": KALSIG,
    },
    headers=DEFAULT_HEADERS)
  except Exception as e:
    raise e

  output = response.json()
  ks = output["result"]["ks"]

  return {
    'ks': ks,
    'kalsig': KALSIG
  }

session = login()

def generate() -> ChannelMetadata:

    try:
        response = requests.post(LIST_URL, json={
           "filter": {
                "objectType": "KalturaChannelFilter",
                "idEqual": "339523",
                "kSql": "(and (or (and asset_type = 'epg' ) (and Catalogue = 'sottott') ) (and name ~'' (or )) )"
           },
            "pager": {
                "objectType": "KalturaFilterPager",
                "pageIndex": 1,
                "pageSize": 21
            },
            "format": 1,
            "clientTag": "AstroQA",
            "apiVersion": "6.1.0.28839",
            "language":"en",
            "ks": session.get('ks'),
            "kalsig": session.get('kalsig')
        },
        headers=DEFAULT_HEADERS
    )
    except Exception as e:
        raise e

    output = response.json()
    objects = output['result']['objects']

    channels = []

    for obj in objects:
       channel_obj = Channel(
          id=obj['externalIds'],
          display_name=obj['name'],
          icon=obj['images'][0]['url'],
       )

       channels.append(channel_obj)

    return ChannelMetadata(channels=channels)

def get_programs(channel_id: str, days: int = 1, channel_xml_id: str = None):

    channel_name = channel_xml_id if channel_xml_id else channel_id

    start_dt = date.today()
    start_date = datetime.combine(start_dt, datetime.min.time())

    end_dt = start_dt + timedelta(days=days)
    end_date = datetime.combine(end_dt, datetime.max.time())

    # The KSQL is buggy, it expected end_date input for a start_date and the other way around
    end_timestamp = str(int(start_date.timestamp()))
    start_timestamp = str(int(end_date.timestamp()))

    epg_ksql = f"(and epg_channel_id = \'{channel_id}\' end_date>=\'{end_timestamp}\' start_date<\'{start_timestamp}\')"

    try:
        response = requests.post(LIST_URL, json={
           "filter": {
                "objectType": "KalturaSearchAssetFilter",
                "orderBy": "START_DATE_ASC",
                "kSql": epg_ksql
           },
            "pager": {
                "objectType": "KalturaFilterPager",
                "pageIndex": 1,
                "pageSize": 50
            },
            "format": 1,
            "clientTag": "AstroQA",
            "apiVersion": "6.1.0.28839",
            "language":"en",
            "ks": session.get('ks'),
            "kalsig": session.get('kalsig')
        },
        headers=DEFAULT_HEADERS
    )
    except Exception as e:
        raise e

    output = response.json()

    programs = []

    for obj in output['result']['objects']:
        title = obj['metas']['TitleSortName']['value']
        description = obj['metas']['LongSynopsis']['value']
        start_date = datetime.fromtimestamp(obj['startDate'])
        end_date = datetime.fromtimestamp(obj['endDate'])

        program_obj = Programme(
            start=start_date,
            stop=end_date,
            channel=channel_name,
            title=title,
            desc=description
        )

        programs.append(program_obj)

    return programs


# - * - coding: utf-8 - * -
from ZODB import FileStorage, DB
import transaction

class MyZODB(object):
    def __init__(self, path):
        self.storage = FileStorage.FileStorage(path)
        self.db = DB(self.storage)
        self.connection = self.db.open()
        self.db_root = self.connection.root()

    def close(self):
        self.connection.close()
        self.db.close()
        self.storage.close()

def create_data():
    db = MyZODB('./db_location/data.fs')
    db_root = db.db_root
    db_root['a_number'] = 3
    db_root['a_string'] = 'Gift'
    db_root['a_list'] = [1, 2, 3, 5, 7, 12]
    db_root['a_dictionary'] = {1918: 'Red Sox', 1919: 'Reds'}
    db_root['deeply_nested'] = {
        1918: [('Red, Sox', 4), ('Cubs', 2)],
        1919: [('Reds', 5), ('White Sox', 3)]
    }
    transaction.commit()
    db.close()

def get_zodb_data():
    db = MyZODB('./db_location/data.fs')
    db_root = db.db_root
    for key in db_root.keys():
        print(f"{key}: {db_root[key]}")
    db.close()

def alter_zodb_data():
    db = MyZODB('./db_location/data.fs')
    db_root = db.db_root
    db_root['a_string'] = 'Something Else'
    transaction.commit()
    db.close()

    # 但是需要显式地将对列表或字典的更改告诉zodb，下面代码将不会修改zodb中对应的内容
    a_dictionary = db_root['a_dictionary']
    a_dictionary[1920] = 'Indians'
    transaction.commit()
    db.close()

    # 如果打算更改而不是完全替换，则需要设置数据库根的属性_p_changed,以通知它需要重新存储其下的属性
    a_dictionary = db_root['a_dictionary']
    a_dictionary[1920] = 'Indians'
    db._p_changed = 1
    transaction.commit()
    db.close()

def delete_zodb_data():
    db = MyZODB('./db_location/data.fs')
    db_root = db.db_root
    del db_root['a_number']
    transaction.commit()
    db.close()

create_data()
get_zodb_data()
alter_zodb_data()
delete_zodb_data()


#!/usr/bin/python3

from langtag import lookup
from palaso.sldr.ldml import Ldml, iterate_files
import argparse, os
import palaso.sldr.UnicodeSets as usets

parser = argparse.ArgumentParser()
parser.add_argument("indir",help="Root of SLDR file tree")
parser.add_argument("-l","--ldml",help="ldml file identifier (without .xml)")
args = parser.parse_args()

if args.ldml:
    allfiles = [os.path.join(args.indir, args.ldml[0], args.ldml+".xml")]
else:
    allfiles = iterate_files(args.indir)

for f in allfiles:
    l = Ldml(f)
    if len(l.root) == 1 and l.root[0].tag == "identity":
        continue
    ident = l.find(".//identity/special/sil:identity")
    if ident is None or ident.get("source", "") == "cldr":
        continue
    name = os.path.splitext(os.path.basename(f))[0].replace("_", "-")
    tagset = lookup(name, "")
    if tagset == "":
        print("No langtag for " + name)
        continue
    ename = getattr(tagset, "name", None)
    if ename is not None:
        nameel = l.ensure_path('localeDisplayNames/special/sil:names/sil:name[@xml:lang="en"]')[0]
        if nameel.text is None:
            nameel.text = ename
    lnames = getattr(tagset, "localnames", [getattr(tagset, 'localname', None)])
    if lnames != [None]:
        main = ""
        for e in l.findall('.//characters/exemplarCharacters'):
            t = e.get('type', None)
            if t or not e.text: continue
            main = usets.parse(e.text, 'NFD')[0].asSet()
            break
        for c in ("\uA78C", "\u02BC"):
            if c in main:
                lnames = [s.replace("'", c) for s in lnames]
                break
    lname = lnames[0]
    if lname is not None:
        nameel = l.ensure_path('localeDisplayNames/languages/language[@type="{}"]'.format(name))[0]
        if nameel.text is None:
            nameel.text = lname
        elif nameel.text != lname and nameel.text not in lnames:
            print("Name difference for {} has {}, want to add {}".format(name, nameel.text, lname))
            nameel.text = lname
    l.normalise()
    l.save_as(f)


import logging

import aioredis
import uvicorn as uvicorn
from elasticsearch import AsyncElasticsearch
from fastapi import FastAPI
from fastapi.responses import ORJSONResponse

from api.v1 import films, genres, persons
from core.config import get_settings
from core.logger import LOGGING
from db import elastic
from db import redis

conf = get_settings()

app = FastAPI(
    title=conf.PROJECT_NAME,
    docs_url='/api/openapi',
    openapi_url='/api/openapi.json',
    default_response_class=ORJSONResponse,
)

@app.on_event('startup')
async def startup():
    """Метод, выполняющий инициализацию компонентов приложения при старте."""
    redis.cache = aioredis.from_url(
        f"redis://{conf.CACHE_HOST}:{conf.CACHE_PORT}", encoding="utf-8", decode_responses=True
    )
    elastic.es = AsyncElasticsearch(hosts=[f'http://{conf.ELASTIC_HOST}:{conf.ELASTIC_PORT}'])

@app.on_event('shutdown')
async def shutdown():
    """Метод, выполняющий утилизацию компонентов приложения после завершения работы  приложения."""
    await redis.cache.close()
    await elastic.es.close()

app.include_router(films.router, prefix='/api/v1/films', tags=['films'])
app.include_router(genres.router, prefix='/api/v1/genres', tags=['genres'])
app.include_router(persons.router, prefix='/api/v1/persons', tags=['persons'])

if __name__ == '__main__':
    uvicorn.run(
        'main:app',
        host='0.0.0.0',
        port=8000,
        log_config=LOGGING,
        log_level=logging.DEBUG,
    )


import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import os

def number_loans_per_gender(df):

    df['mnth_yr2'] = df['date'].apply(lambda x: x.strftime('%Y-%m'))

    # Add a column for quarter.
    df['Qtr'] = pd.PeriodIndex(pd.to_datetime(df.mnth_yr2), freq='Q')

    genderdf = df.groupby(['mnth_yr2', 'gender']).agg('count')[['id']]

    gd_df = df.groupby(['Qtr', 'gender'])['id'].count().reset_index()
    gd_df = gd_df[gd_df['Qtr'] < pd.Period('2017Q3')]
    gd_df = gd_df.pivot(index='Qtr', columns='gender')

    gd_df.plot.line(
        color=['green', 'steelblue'],
        alpha = 0.8,
        figsize=(14,8),
        marker='.',
        markersize=12,
    )
    plt.legend(labels=['Female', 'Male'], title='')
    plt.title('Number of Loans per Gender and Quarter')
    plt.ylabel('Number of Loans')
    plt.xlabel('Quarter')
    plt.savefig(os.path.join('image', 'num_loans_per_gender.png'))
    plt.show()


# simple script to train a model
import torch
import torch.nn.functional as F

def train_model(model, params, train_loader):
  model.train()
  # negative log likelihood loss
  # https://pytorch.org/docs/stable/generated/torch.nn.functional.nll_loss.html
  loss_function = F.nll_loss
  # optimizer: standard SGD
  optimizer = torch.optim.SGD(model.parameters(), lr=params.lr)
  for epoch in range(params.epochs):
    # monitor training loss
    train_loss = 0.0
    for data, target in train_loader:
        optimizer.zero_grad()
        # apply model to data & calculate loss
        output = model(data)
        loss = loss_function(output, target)
        # calculate gradients & perform optim.step
        loss.backward()
        optimizer.step()
        train_loss += loss.item()*data.size(0)
    # print training statistics
    # calculate average loss
    train_loss = train_loss/len(train_loader.dataset)

    print('Epoch: {} \tTraining Loss: {:.4f}'.format(
        epoch+1,
        train_loss
        ))
  # operates on model parameters in place returns nothing


import os
import shutil
import tempfile
from typing import Any, Optional
from urllib import parse as url_parse
from urllib import request as url_request
from zipfile import ZipFile, is_zipfile

import boto3
import requests

class ThrowOnErrorOpener(url_request.FancyURLopener):
    def http_error_default(
        self, url: str, fp: Any, errcode: int, errmsg: str, headers: Any
    ) -> Any:
        raise Exception(f"{errcode}: {errmsg}")

class FileDownloader:
    """
    Module to download files from urls
    Inspired from python wget module: https://github.com/steveeJ/python-wget
    """

    def __init__(
        self,
        url: str,
        download_path: str,
        extract_archives: Optional[bool] = False,
    ):
        self.url = url
        self.download_to_path = download_path
        self.extract_archives = extract_archives

    @staticmethod
    def check_if_path_exists(path: str) -> bool:
        if not os.path.exists(path):
            raise FileNotFoundError(f"File does not exists at this path: {path}")
        return True

    @staticmethod
    def get_filename_from_url(url: str) -> str:
        filename = os.path.basename(url_parse.urlparse(url).path)
        filename = filename.strip(" \n\t.")
        if len(filename) == 0:
            return None  # type: ignore
        return filename

    @staticmethod
    def rename_filename_if_exists(filename: str) -> str:
        """Expands name portion of filename with numeric ' (x)' suffix to
        return filename that doesn't exist already.
        """
        dirname = os.path.dirname(filename)
        name, ext = os.path.basename(filename).rsplit(".", 1)
        names = [x for x in os.listdir(dirname) if x.startswith(name)]
        names = [x.rsplit(".", 1)[0] for x in names]
        suffixes = [x.replace(name, "") for x in names]
        suffixes = [x[2:-1] for x in suffixes if x.startswith(" (") and x.endswith(")")]
        indexes = [int(x) for x in suffixes if set(x) <= set("0123456789")]
        idx = 1
        if indexes:
            idx += sorted(indexes)[-1]
        return os.path.join(dirname, f"{name} ({idx}).{ext}")

    def extract_zip_files(
        self,
        filename: str,
        path: str,
        out_path: Optional[str] = None,
    ) -> Optional[str]:
        """
        This function extracts the archive files and returns the corresponding file path.
        Returns null if the specified path does not exists
        :param filename: str: Name of the archive file
        :param path: str: File path of the archive file
        :param out_path: optional(str): Destination file path where the extracted files should be stored (default is same as path)
        :returns filename: str: Output file path
        """
        file_loc = os.path.join(path, filename)
        out_path = out_path or path

        if self.check_if_path_exists(path):
            if is_zipfile(file_loc):
                with ZipFile(file_loc, "r") as zip_ref:
                    zip_ref.extractall(out_path)
                return out_path

        return None

    def download_files_locally(self, filename: str) -> str:
        local_file_protocol_length: int = len("file://")
        download_to_path = self.download_to_path[local_file_protocol_length:]
        prefix = (filename or download_to_path or "") + ".."
        (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=prefix, dir=".")
        os.close(fd)
        os.unlink(tmpfile)

        # Define callbacks in this code block
        # callback = None

        (tmpfile, headers) = ThrowOnErrorOpener().retrieve(self.url, tmpfile)

        if os.path.isdir(download_to_path):
            file_path = filename
            file_path = os.path.join(download_to_path, file_path)
        else:
            file_path = download_to_path or filename
        if os.path.exists(file_path):
            file_path = self.rename_filename_if_exists(file_path)
        shutil.move(tmpfile, file_path)

        if self.extract_archives:
            file_path = (
                self.extract_zip_files(
                    filename=file_path,
                    path=download_to_path,
                )
                or file_path
            )

        return file_path

    def download_files_to_s3(
        self, filename: str, bucket: str, download_path: str
    ) -> str:
        file_request = requests.get(self.url, stream=True)
        s3_resource = boto3.resource("s3")

        bucket = s3_resource.Bucket(bucket)
        s3_location = f"{download_path}/{filename}"

        if bucket.creation_date is not None:  # type: ignore
            return bucket.upload_fileobj(file_request.raw, s3_location)  # type: ignore
        else:
            raise s3_resource.exception.NoSuchBucket(f"{bucket} does not exists!!")

    def download_files_from_url(self) -> Optional[str]:
        """
        Function to download files from a url

        """
        filename = self.get_filename_from_url(self.url)
        url_segments = url_parse.urlparse(self.download_to_path)

        if url_segments.scheme in ["s3", "s3a"]:
            file_path = os.path.join(
                self.url,
                self.download_files_to_s3(
                    str(filename),
                    str(url_segments.netloc),
                    str(url_segments.path).strip("/"),
                ),
            )
        elif url_segments.scheme in ["file"]:
            file_path = self.download_files_locally(filename)
        else:
            raise NotImplementedError

        if file_path:
            return filename

        return None


import logging as lg
import  logging , platform
import socket

class HostnameFilter(logging.Filter):
    hostname = platform.node()

    def filter(self, record):
        record.hostname = HostnameFilter.hostname
        return True

class Logger:
    def __init__(self):
        pass

    def log(self, from_file, log_type, log_message):

        hostname = socket.gethostname()
        IPAddr = socket.gethostbyname(hostname)
        lg.basicConfig(filename="Log_Files/main.log", level=lg.INFO,format='{} - {} - %(asctime)s - %(levelname)s -%(name)s -  %(message)s'.format(hostname,IPAddr),datefmt='%d-%b-%y %H:%M:%S')
        logger = lg.getLogger(from_file)

        if log_type == "INFO":
            logger.info( log_message + "\n")
        elif log_type == "ERROR":
            logger.error( log_message + "\n")


"""The fsl module provides classes for interfacing with the `FSL
<http://www.fmrib.ox.ac.uk/fsl/index.html>`_ command line tools.  This
was written to work with FSL version 5.0.4.

    Change directory to provide relative paths for doctests
    >>> import os
    >>> filepath = os.path.dirname(os.path.realpath(__file__))
    >>> datadir = os.path.realpath(os.path.join(filepath,
    ...                            '../../testing/data'))
    >>> os.chdir(datadir)
"""
from __future__ import print_function, division, unicode_literals, absolute_import

import os
import warnings
from builtins import str

from nipype.interfaces.base import (traits, TraitedSpec, File, isdefined)
from nipype.interfaces.fsl.base import FSLCommand, FSLCommandInputSpec

class EddyInputSpec(FSLCommandInputSpec):
    in_file = File(exists=True, mandatory=True, argstr='--imain=%s',
                   desc=('File containing all the images to estimate '
                         'distortions for'))
    in_mask = File(exists=True, mandatory=True, argstr='--mask=%s',
                   desc='Mask to indicate brain')
    in_index = File(exists=True, mandatory=True, argstr='--index=%s',
                    desc=('File containing indices for all volumes in --imain '
                          'into --acqp and --topup'))
    in_acqp = File(exists=True, mandatory=True, argstr='--acqp=%s',
                   desc='File containing acquisition parameters')
    in_bvec = File(exists=True, mandatory=True, argstr='--bvecs=%s',
                   desc=('File containing the b-vectors for all volumes in '
                         '--imain'))
    in_bval = File(exists=True, mandatory=True, argstr='--bvals=%s',
                   desc=('File containing the b-values for all volumes in '
                         '--imain'))
    out_base = traits.Str('eddy_corrected', argstr='--out=%s',
                          usedefault=True,
                          desc=('basename for output (warped) image'))
    session = File(exists=True, argstr='--session=%s',
                   desc=('File containing session indices for all volumes in '
                         '--imain'))
    in_topup_fieldcoef = File(exists=True, argstr="--topup=%s",
                              requires=['in_topup_movpar'],
                              desc=('topup file containing the field '
                                    'coefficients'))
    in_topup_movpar = File(exists=True, requires=['in_topup_fieldcoef'],
                           desc='topup movpar.txt file')

    flm = traits.Enum('linear', 'quadratic', 'cubic', argstr='--flm=%s',
                      desc='First level EC model')

    fwhm = traits.Float(desc=('FWHM for conditioning filter when estimating '
                              'the parameters'), argstr='--fwhm=%s')

    niter = traits.Int(5, argstr='--niter=%s', desc='Number of iterations')

    method = traits.Enum('jac', 'lsr', argstr='--resamp=%s',
                         desc=('Final resampling method (jacobian/least '
                               'squares)'))
    repol = traits.Bool(False, argstr='--repol',
                        desc='Detect and replace outlier slices')
    num_threads = traits.Int(1, usedefault=True, nohash=True,
                             desc="Number of openmp threads to use")
    is_shelled = traits.Bool(False, argstr='--data_is_shelled',
                             desc="Override internal check to ensure that "
                                  "date are acquired on a set of b-value "
                                  "shells")
    field = traits.Str(argstr='--field=%s',
                       desc="NonTOPUP fieldmap scaled in Hz - filename has "
                            "to be provided without an extension. TOPUP is "
                            "strongly recommended")
    field_mat = File(exists=True, argstr='--field_mat=%s',
                     desc="Matrix that specifies the relative locations of "
                          "the field specified by --field and first volume "
                          "in file --imain")
    use_cuda = traits.Bool(False, desc="Run eddy using cuda gpu")

class EddyOutputSpec(TraitedSpec):
    out_corrected = File(exists=True,
                         desc=('4D image file containing all the corrected '
                               'volumes'))
    out_parameter = File(exists=True,
                         desc=('text file with parameters definining the '
                               'field and movement for each scan'))
    out_rotated_bvecs = File(exists=True,
                             desc=('File containing rotated b-values for all volumes'))
    out_movement_rms = File(exists=True,
                            desc=('Summary of the "total movement" in each volume'))
    out_restricted_movement_rms = File(exists=True,
                                       desc=('Summary of the "total movement" in each volume '
                                             'disregarding translation in the PE direction'))
    out_shell_alignment_parameters = File(exists=True,
                                          desc=('File containing rigid body movement parameters '
                                                'between the different shells as estimated by a '
                                                'post-hoc mutual information based registration'))
    out_outlier_report = File(exists=True,
                              desc=('Text-file with a plain language report '
                                    'on what outlier slices eddy has found'))

class Eddy(FSLCommand):
    """
    Interface for FSL eddy, a tool for estimating and correcting eddy
    currents induced distortions. `User guide
    <http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Eddy/UsersGuide>`_ and
    `more info regarding acqp file
    <http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/eddy/Faq#How_do_I_know_what_to_put_into_my_--acqp_file>`_.

    Examples
    --------

    >>> from nipype.interfaces.fsl import Eddy
    >>> eddy = Eddy()
    >>> eddy.inputs.in_file = 'epi.nii'
    >>> eddy.inputs.in_mask  = 'epi_mask.nii'
    >>> eddy.inputs.in_index = 'epi_index.txt'
    >>> eddy.inputs.in_acqp  = 'epi_acqp.txt'
    >>> eddy.inputs.in_bvec  = 'bvecs.scheme'
    >>> eddy.inputs.in_bval  = 'bvals.scheme'
    >>> eddy.cmdline # doctest: +ELLIPSIS +ALLOW_UNICODE
    'eddy_openmp --acqp=epi_acqp.txt --bvals=bvals.scheme --bvecs=bvecs.scheme \
--imain=epi.nii --index=epi_index.txt --mask=epi_mask.nii \
--out=.../eddy_corrected'
    >>> res = eddy.run() # doctest: +SKIP

    """
    _cmd = 'eddy'
    input_spec = EddyInputSpec
    output_spec = EddyOutputSpec

    _num_threads = 1

    def __init__(self, **inputs):
        super(Eddy, self).__init__(**inputs)
        self.inputs.on_trait_change(self._num_threads_update, 'num_threads')
        if isdefined(self.inputs.use_cuda):
            self._use_cuda()
        if not isdefined(self.inputs.num_threads):
            self.inputs.num_threads = self._num_threads
        else:
            self._num_threads_update()

    def _num_threads_update(self):
        self._num_threads = self.inputs.num_threads
        if not isdefined(self.inputs.num_threads):
            if 'OMP_NUM_THREADS' in self.inputs.environ:
                del self.inputs.environ['OMP_NUM_THREADS']
        else:
            self.inputs.environ['OMP_NUM_THREADS'] = str(
                self.inputs.num_threads)

    def _use_cuda(self):
        if self.inputs.use_cuda:
            _cmd = 'eddy_cuda'
        else:
            _cmd = 'eddy_openmp'

    def _format_arg(self, name, spec, value):
        if name == 'in_topup_fieldcoef':
            return spec.argstr % value.split('_fieldcoef')[0]
        if name == 'out_base':
            return spec.argstr % os.path.abspath(value)
        return super(Eddy, self)._format_arg(name, spec, value)

    def _list_outputs(self):
        outputs = self.output_spec().get()
        outputs['out_corrected'] = os.path.abspath(
            '%s.nii.gz' % self.inputs.out_base)
        outputs['out_parameter'] = os.path.abspath(
            '%s.eddy_parameters' % self.inputs.out_base)

        # File generation might depend on the version of EDDY
        out_rotated_bvecs = os.path.abspath(
            '%s.eddy_rotated_bvecs' % self.inputs.out_base)
        out_movement_rms = os.path.abspath(
            '%s.eddy_movement_rms' % self.inputs.out_base)
        out_restricted_movement_rms = os.path.abspath(
            '%s.eddy_restricted_movement_rms' % self.inputs.out_base)
        out_shell_alignment_parameters = os.path.abspath(
            '%s.eddy_post_eddy_shell_alignment_parameters' % self.inputs.out_base)
        out_outlier_report = os.path.abspath(
            '%s.eddy_outlier_report' % self.inputs.out_base)

        if os.path.exists(out_rotated_bvecs):
            outputs['out_rotated_bvecs'] = out_rotated_bvecs
        if os.path.exists(out_movement_rms):
            outputs['out_movement_rms'] = out_movement_rms
        if os.path.exists(out_restricted_movement_rms):
            outputs['out_restricted_movement_rms'] = out_restricted_movement_rms
        if os.path.exists(out_shell_alignment_parameters):
            outputs['out_shell_alignment_parameters'] = out_shell_alignment_parameters
        if os.path.exists(out_outlier_report):
            outputs['out_outlier_report'] = out_outlier_report

        return outputs

class EddyCorrectInputSpec(FSLCommandInputSpec):
    in_file = File(exists=True, desc='4D input file', argstr='%s', position=0,
                   mandatory=True)
    out_file = File(desc='4D output file', argstr='%s', position=1,
                    name_source=['in_file'], name_template='%s_edc',
                    output_name='eddy_corrected')
    ref_num = traits.Int(0, argstr='%d', position=2, desc='reference number',
                         mandatory=True, usedefault=True)

class EddyCorrectOutputSpec(TraitedSpec):
    eddy_corrected = File(exists=True,
                          desc='path/name of 4D eddy corrected output file')

class EddyCorrect(FSLCommand):
    """

    .. warning:: Deprecated in FSL. Please use
      :class:`nipype.interfaces.fsl.epi.Eddy` instead

    Example
    -------

    >>> from nipype.interfaces.fsl import EddyCorrect
    >>> eddyc = EddyCorrect(in_file='diffusion.nii',
    ...                     out_file="diffusion_edc.nii", ref_num=0)
    >>> eddyc.cmdline # doctest: +ALLOW_UNICODE
    'eddy_correct diffusion.nii diffusion_edc.nii 0'
    """
    _cmd = 'eddy_correct'
    input_spec = EddyCorrectInputSpec
    output_spec = EddyCorrectOutputSpec

    def __init__(self, **inputs):
        warnings.warn(("Deprecated: Please use nipype.interfaces.fsl.epi.Eddy "
                       "instead"), DeprecationWarning)
        super(EddyCorrect, self).__init__(**inputs)

    def _run_interface(self, runtime):
        runtime = super(EddyCorrect, self)._run_interface(runtime)
        if runtime.stderr:
            self.raise_exception(runtime)
        return runtime


from pathlib import Path
from typing import Tuple

# Get git root
git_root = Path(__file__).parent.parent

no_git_str = '.git structure does not exist'

class GitInfo:
    """
    Provides ``git`` repo information

    :param input_path: Full path containing the ``.git`` contents

    :ivar input_path: Full path containing the ``.git`` contents
    :ivar head_path: Full path of the ``.git`` HEAD
    :ivar branch: Active branch name
    :ivar commit: Full hash
    :ivar short_commit: short hash commit
    """
    def __init__(self, input_path: str):
        self.input_path: str = input_path
        self.head_path: Path = Path(self.input_path) / ".git" / "HEAD"
        self.branch: str = self.get_active_branch_name()
        commit_tuple: tuple = self.get_latest_commit()
        self.commit: str = commit_tuple[0]
        self.short_commit: str = commit_tuple[1]

    def get_active_branch_name(self) -> str:
        """Retrieve active branch name"""
        if self.head_path.exists():
            with self.head_path.open("r") as f:
                content = f.read().splitlines()

            for line in content:
                if line[0:4] == "ref:":
                    return line.partition("refs/heads/")[2]
                else:
                    return f"HEAD detached : {content[0]}"
        else:
            return no_git_str

    def get_latest_commit(self) -> Tuple[str, str]:
        """Retrieve latest commit hash"""
        if self.head_path.exists():
            with self.head_path.open("r") as f:
                content = f.read().splitlines()

            for line in content:
                if line[0:4] == "ref:":
                    ref_path = Path(self.input_path) / ".git" / f"{line.partition(' ')[2]}"
                    with ref_path.open('r') as g:
                        commit = g.read().splitlines()
                else:
                    commit = content

            return commit[0], commit[0][:7]  # full and short hash
        else:
            return no_git_str, ''


from src.communicator.Android_com import Android
from src.communicator.Arduino_com import Arduino
from src.communicator.Algorithm_com import Algorithm
from src.config import STOPPING_IMAGE, IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_FORMAT
from src.protocols import *

from PIL import Image
import numpy as np
import imagezmq
from picamera import PiCamera
from picamera.array import PiRGBArray

import collections
from multiprocessing import Process, Value
from multiprocessing.managers import BaseManager

img_count = 0
class DequeProxy(object):
    def __init__(self, *args):
        self.deque = collections.deque(*args)
    def __len__(self):
        return self.deque.__len__()
    def append(self, x):
        self.deque.append(x)
    def appendleft(self, x):
        self.deque.appendleft(x)
    def popleft(self):
        return self.deque.popleft()
    def empty(self):
        if self.deque:
            return False
        else:
            return True

class DequeManager(BaseManager):
    pass

DequeManager.register('DequeProxy', DequeProxy,
                      exposed=['__len__', 'append', 'appendleft', 'popleft', 'empty'])

class MultiProcessCommunicator:
    """
    This class will carry out multi-processing communication process between Algorithm, Android and Arduino.
    """
    def __init__(self, image_processing_server_url: str=None):
        """
        Start the multiprocessing process and set up all the relevant variables

        Upon starting, RPi will connect to individual devices in the following order
        - Algorithm
        - Arduino
        - Android

        The multiprocessing queue will also be instantiated
        """
        print('Starting Multiprocessing Communication')

        self.algorithm = Algorithm_communicator()  # handles connection to Algorithm
        self.arduino = Arduino_communicator()  # handles connection to Arduino
        self.android = Android_communicator()  # handles connection to Android

        self.manager = DequeManager()
        self.manager.start()

        # messages from Arduino, Algorithm and Android are placed in this queue before being read
        self.message_deque = self.manager.DequeProxy()
        self.to_android_message_deque = self.manager.DequeProxy()

        self.read_arduino_process = Process(target=self._read_arduino)
        self.read_algorithm_process = Process(target=self._read_algorithm)
        self.read_android_process = Process(target=self._read_android)

        self.write_process = Process(target=self._write_target)
        self.write_android_process = Process(target=self._write_android)

        # the current action / status of the robot
        self.status = Status.IDLE  # robot starts off being idle

        self.dropped_connection = Value('i',0) # 0 - arduino, 1 - algorithm

        # for image recognition
        self.image_process = None
        self.image_deque = None

        if image_processing_server_url is not None:
            print('------Start Exploration with image recognition-----')
            self.image_process = Process(target=self._process_pic)
            # pictures taken using the PiCamera are placed in this queue
            self.image_deque = self.manager.DequeProxy()

            self.image_processing_server_url = image_processing_server_url
            self.image_count = Value('i',0)
            print("self.image_deque:", self.image_deque)

    def start(self):
        try:
            self.algorithm.connect_arduino()
            self.arduino.connect_arduino()
            self.android.connect_arduino()

            print('Connected to Algorithm, Arduino and Android')

            self.read_algorithm_process.start_algo()
            self.read_arduino_process.start_arduino()
            self.read_android_process.start_android()
            self.write_process.start()
            self.write_android_process.start()

            if self.image_process is not None:
                self.image_process.start()
                print('All processes have started : read-arduino, read-algorithm, read-android, write-android, image_processing')
            else:
                print('All processes have started : read-arduino, read-algorithm, read-android, write-android')

            print('Multiprocess communication session started')

        except Exception as error:
            raise error

        self._allow_reconnection()

    def end(self):

        self.algorithm.disconnect_all_algo()
        self.android.disconnect_all()
        print('Multiprocess communication has just stopped')

    def _allow_reconnection(self):
        print('RPi can reconnect now')

        while True:
            try:
                if not self.read_algorithm_process.is_alive():
                    self._reconnect_algorithm()

                if not self.read_arduino_process.is_alive():
                    self._reconnect_arduino()

                if not self.read_android_process.is_alive():
                    self._reconnect_android()

                if not self.write_process.is_alive():
                    if self.dropped_connection.value == 0:
                        self._reconnect_arduino()
                    elif self.dropped_connection.value == 1:
                        self._reconnect_algorithm()

                if not self.write_android_process.is_alive():
                    self._reconnect_android()

                if self.image_process is not None and not self.image_process.is_alive():
                   self.image_process.terminate()

            except Exception as error:
                print("Error during reconnection: ",error)
                raise error

    def _reconnect_algorithm(self):
        self.algorithm.disconnect()

        self.read_algorithm_process.terminate()
        self.write_process.terminate()
        self.write_android_process.terminate()

        self.algorithm.connect()

        self.read_algorithm_process = Process(target=self._read_algorithm)
        self.read_algorithm_process.start()

        self.write_process = Process(target=self._write_target)
        self.write_process.start()

        self.write_android_process = Process(target=self._write_android)
        self.write_android_process.start()

        print('Successfully reconnected to Algorithm')

    def _reconnect_arduino(self):
        self.arduino.disconnect()

        self.read_arduino_process.terminate()
        self.write_process.terminate()
        self.write_android_process.terminate()

        self.arduino.connect()

        self.read_arduino_process = Process(target=self._read_arduino)
        self.read_arduino_process.start()

        self.write_process = Process(target=self._write_target)
        self.write_process.start()

        self.write_android_process = Process(target=self._write_android)
        self.write_android_process.start()

        print('Successfully reconnected to Arduino')

    def _reconnect_android(self):
        self.android.disconnect()

        self.read_android_process.terminate()
        self.write_process.terminate()
        self.write_android_process.terminate()

        self.android.connect()

        self.read_android_process = Process(target=self._read_android)
        self.read_android_process.start()

        self.write_process = Process(target=self._write_target)
        self.write_process.start()

        self.write_android_process = Process(target=self._write_android)
        self.write_android_process.start()

        print('Successfully reconnected to Android')

    def _read_arduino(self):
        while True:
            try:
                messages = self.arduino.read()

                if messages is None:
                    continue
                message_list = messages.splitlines()

                for message in message_list:

                    if len(message) <= 0:
                        continue

                    self.message_deque.append(self._format_for(
                        ALGORITHM_HEADER,
                        message + NEWLINE
                    ))

            except Exception as error:
                print('Process read_arduino failed: ' + str(error))
                break

    def _read_algorithm(self):
        while True:
            try:
                messages = self.algorithm.read()

                if messages is None:
                    continue

                message_list = messages.splitlines()

                for message in message_list:

                    if len(message) <= 0:
                        continue

                    # image recognition
                    elif message[0] == Algorithm-RPi.TAKE_PICTURE:

                        if self.image_count.value >= 5:
                            self.message_deque.append(self._format_for(
                            ALGORITHM_HEADER,
                            RPi-Algorithm.DONE_IMG_REC + NEWLINE
                        ))

                        else:

                            message = message[2:-1]  # to remove 'C[' and ']'
                            self.to_android_message_deque.append(
                                RPi-Android.STATUS_TAKING_PICTURE + NEWLINE
                            )
                            image = self._take_pic()
                            print('Picture taken')
                            self.message_deque.append(self._format_for(
                                ALGORITHM_HEADER,
                                RPi-Algorithm.DONE_TAKING_PICTURE + NEWLINE
                            ))
                            self.image_deque.append([image,message])

                    elif message == Algorithm-RPi.EXPLORATION_COMPLETE:
                        # to let image processing server end all processing and display all images
                        self.status = Status.IDLE

                        # to use pillow instead of cv2
                        pil_img = Image.open(STOPPING_IMAGE).convert('RGB')
                        cv_img = np.array(pil_img)
                        cv_img = cv_img[:,:,::-1].copy()
                        self.image_deque.append([cv_img,"-1,-1|-1,-1|-1,-1"])

                    elif message[0] == Algorithm-Android.MDF_STRING:
                        self.to_android_message_deque.append(
                            message[1:] + NEWLINE
                        )

                    else:  # (message[0]=='W' or message in ['D|', 'A|', 'Z|']):
                        self._forward_message_algorithm_to_android(message)
                        self.message_deque.append(self._format_for(
                            ARDUINO_HEADER,
                            message + NEWLINE
                        ))

            except Exception as error:
                print('Process read_algorithm failed: ' + str(error))
                break

    def _forward_message_algorithm_to_android(self, message):
        messages_for_android = message.split(MESSAGE_SEPARATOR)

        for message_for_android in messages_for_android:

            if len(message_for_android) <= 0:
                continue

            elif message_for_android[0] == Algorithm-Android.CALIBRATING_CORNER:
                self.to_android_message_deque.append(
                    RPi-Android.STATUS_CALIBRATING_CORNER + NEWLINE
                )

            elif message_for_android[0] == Algorithm-Android.SENSE_ALL:
                self.to_android_message_deque.append(
                    RPi-Android.STATUS_SENSE_ALL + NEWLINE
                )

            elif message_for_android[0] == Algorithm-Android.ALIGN_RIGHT:
                self.to_android_message_deque.append(
                    RPi-Android.STATUS_ALIGN_RIGHT + NEWLINE
                )

            elif message_for_android[0] == Algorithm-Android.ALIGN_FRONT:
                self.to_android_message_deque.append(
                    RPi-Android.STATUS_ALIGN_FRONT + NEWLINE
                )

            elif message_for_android[0] == Algorithm-Android.MOVE_FORWARD:
                if self.status == Status.EXPLORING:
                    self.to_android_message_deque.append(
                        RPi-Android.STATUS_EXPLORING + NEWLINE
                    )

            elif message_for_android[0] == Algorithm-Android.TURN_LEFT:
                self.to_android_message_deque.append(
                    RPi-Android.TURN_LEFT + NEWLINE
                )

                self.to_android_message_deque.append(
                   RPi-Android.STATUS_TURNING_LEFT + NEWLINE
                )

            elif message_for_android[0] == Algorithm-Android.TURN_RIGHT:
                self.to_android_message_deque.append(
                    RPi-Android.TURN_RIGHT + NEWLINE
                )

                self.to_android_message_deque.append(
                   RPi-Android.STATUS_TURNING_RIGHT + NEWLINE
                )

                forward_steps_no = int(message_for_android.decode()[1:])

                print('Move forward by this number of steps:', forward_steps_no)
                for _ in range(forward_steps_no):
                    self.to_android_message_deque.append(
                        RPi-Android.MOVE_UP + NEWLINE
                    )

                    self.to_android_message_deque.append(
                        RPi-Android.STATUS_MOVING_FORWARD + NEWLINE
                    )

    def _read_android(self):
        while True:
            try:
                messages = self.android.read()

                if messages is None:
                    continue

                message_list = messages.splitlines()

                for message in message_list:
                    if len(message) <= 0:
                        continue

                    elif message in (Android-Arduino.ALL_MESSAGES + [Android-RPi.CALIBRATE_SENSOR]):
                        if message == Android-RPi.CALIBRATE_SENSOR:
                            self.message_deque.append(self._format_for(
                                ARDUINO_HEADER,
                                RPi-Arduino.CALIBRATE_SENSOR + NEWLINE
                            ))

                        else:
                            self.message_deque.append(self._format_for(
                                ARDUINO_HEADER, message + NEWLINE
                            ))

                    else:
                        if message == Android-Algorithm.START_SHORTEST_PATH:
                            self.status = Status.SHORTEST_PATH
                            self.message_deque.append(self._format_for(
                                ARDUINO_HEADER,
                                RPi-Arduino.START_SHORTEST_PATH + NEWLINE
                            ))

                        elif message == Android-Algorithm.START_EXPLORATION:
                            self.status = Status.EXPLORING
                            time.sleep(0.5)
                            self.message_deque.append(self._format_for(
                                ARDUINO_HEADER,
                                RPi-Arduino.START_EXPLORATION + NEWLINE
                            ))

                        self.message_deque.append(self._format_for(
                            ALGORITHM_HEADER,
                            message + NEWLINE
                        ))

            except Exception as error:
                print('Process read_android failed: ' + str(error))
                break

    def _write_target(self):
        while True:
            target = None
            try:
                if len(self.message_deque)>0:
                    message = self.message_deque.popleft()
                    target, payload = message['target'], message['payload']

                    if target == ARDUINO_HEADER:
                        self.arduino.write(payload)

                    elif target == ALGORITHM_HEADER:
                        self.algorithm.write(payload)

                    else:
                        print("Invalid header", target)

            except Exception as error:
                print('Process write_target failed: ' + str(error))

                if target == ARDUINO_HEADER:
                    self.dropped_connection.value = 0

                elif target == ALGORITHM_HEADER:
                    self.dropped_connection.value = 1

                self.message_deque.appendleft(message)

                break

    def _write_android(self):
        while True:
            try:
                if len(self.to_android_message_deque)>0:
                    message = self.to_android_message_deque.popleft()

                    self.android.write(message)

            except Exception as error:
                print('Process write_android failed: ' + str(error))
                self.to_android_message_deque.appendleft(message)
                break

    def _take_pic(self):
        global img_count
        try:

            # initialise pi camera
            camera = PiCamera(resolution=(IMAGE_WIDTH, IMAGE_HEIGHT))  # '640x360'
            camera.awb_mode = 'horizon'

            # allow the camera to warmup
            time.sleep(3)
            rawCapture = PiRGBArray(camera)

            # grab an image from the camera
            camera.capture(rawCapture, format=IMAGE_FORMAT)
            image = rawCapture.array
            pil_img = np.asarray(image)
            pil_img = Image.fromarray(pil_img[:,:,::-1])
            pil_img.save('./frame_{}.jpg'.format(img_count))
            img_count += 1

            camera.close()

        except Exception as error:
            print('Picture taking process failed: ' + str(error))

        return image

    def _process_pic(self):
        print("----------------------- Image Processing Starts -------------------")
        print(self.image_processing_server_url)
        print(type(self.image_processing_server_url))
        image_sender = imagezmq.ImageSender(
            connect_to=self.image_processing_server_url)
        print(type(self.image_deque))
        image_id_list = []
        while True:
            try:
                if not self.image_deque.empty():

                    message_for_image =  self.image_deque.popleft()
                    # format: 'x,y|x,y|x,y'
                    obstacle_coordinates = message_for_image[1]

                    #print("-----------Prepareing to send image---------")
                    reply = image_sender.send_image(
                        'image message from RPi',
                        message_for_image[0]
                    )

                    reply = reply.decode('utf-8')
                    print("Reply:" + reply)

                    if reply == 'End':
                        break  # stop sending images

                    # example replies
                    # "1|2|3" 3 symbols in order from left to right
                    # "1|-1|3" 2 symbols, 1 on the left, 1 on the right
                    # "1" 1 symbol either on the left, middle or right
                    else:
                        detections = reply.split(MESSAGE_SEPARATOR) #from server
                        coordinate_list_obstacle = obstacle_coordinates.split(MESSAGE_SEPARATOR) #from algo

                        for detection, coordinates in zip(detections, coordinate_list_obstacle):
                            print("Coordinate:",coordinates)
                            print("Detection:",detection)

                            if detection == '-1':
                                continue  # if there isn't any  symbol detected, mapping of symbol id will  be skipped
                            elif coordinates == '-1,-1':
                                continue  # if there isn't any obstacle detected, mapping of id symbol will skipped

                            else:
                                id_string_to_android = '{"image":[' + coordinates + \
                                ',' + detection + ']}'
                                print(id_string_to_android)

                                if detection not in image_id_list:
                                    self.image_count.value += 1
                                    image_id_list.append(detection)

                                self.to_android_message_deque.append(
                                    id_string_to_android + NEWLINE
                                )

            except Exception as error:
                print('Image processing process has failed: ' + str(error))

    def _format_for(self, target, payload):
        return {
            'target': target,
            'payload': payload,
        }


# coding:utf-8
import unittest
from selenium import webdriver
from AW.common.screenShot import *
from AW.service.Wms import Wms
from AW.service.BaseData import BaseData

class TestAddWarehouse_001(unittest.TestCase):
    def setUp(self):
        # 选择浏览器
        self.driver = webdriver.Chrome()
        # 要打开的链接地址
        self.driver.get('http://172.31.75.173:8080/login.html')
        # 放大窗口
        self.driver.maximize_window()

    def testAddWarehouse(self):
        # 实例化对象
        comWms = Wms(self.driver)
        comBaseData = BaseData(self.driver)
        time.sleep(2)
        # 登陆WMS后选择基础资料
        comWms.login_and_choose_business("zouyy", "123", u"基础资料")
        time.sleep(2)
        # 进入商品资料-仓库管理-新增仓库
        comBaseData.add_warehouse()
        time.sleep(2)
        # 校验进入新增仓库界面成功
        ele = comWms.action_wms.findElementsByWay("layui-form-label", "class_name")
        self.assertEqual(len(ele), 11, u"没有进入新增仓库界面")

    def tearDown(self):
        self.driver.quit()


import os
from  PIL import Image as im
import pandas as pd

location = os.path.dirname(__file__)
os.chdir(location)
photo_folder=location+'\\Pics'

photo_list = os.listdir(photo_folder)
Main=im.open('Pics\\'+ photo_list[-1])

def img_ref_assigner(refno):
    for photo in photo_list:
        if str(refno) in photo:
            return (photo[:1])
            break

def photo_selector(img_ref):
    if img_ref == None: #Thus photo_selector should not return anything
        return None
    else:
        for photo in photo_list:
            if photo[:1] == img_ref:
                return(photo)
            else:
                continue

class Component:
    def __init__(self,name,description,ref_no):
        self.name =  name
        self.ds =  description
        self.refno = ref_no
        self.img_ref = img_ref_assigner(ref_no)

    def image(self):
        photo_name = photo_selector(self.img_ref)
        if photo_name == None:
            print('NO ASSOCIATED PICTURE, NOT TESTED') #And no photo.
        else:
            img_file = im.open('Pics\\'+ photo_name)
            img_file.show()

#Initialization Sequence
data = pd.read_csv("Component Data.csv")

Name_list_1 = data.iloc[:,0]
Name_list = Name_list_1.values.tolist()

untested_bool = data['Ref.']=='-'
Untested_list = data[untested_bool].iloc[:,0].reset_index()

tested_bool = data['Ref.']!='-'
Tested_list = data[tested_bool].iloc[:,0].reset_index()

Component_list=[]
index = data.index

Component_count = 0
for i in index: #Create list of components
    Component_list.append(Component(data.iloc[i,0],data.iloc[i,1],data.iloc[i,2]))
    Component_count += 1
print(Component_count, 'Components created')


import torch

import alf
import alf.utils.data_buffer as db

from . import metric

class MetricBuffer(torch.nn.Module):
    """A metric buffer for computing average metric values. The buffer is assumed
    to store only scalar values."""

    def __init__(self, max_len, dtype):
        """
        Args:
            max_len (int): maximum length of the buffer
            dtype (torch.dtype): dtype of the content of the buffer
        """
        super().__init__()
        self._dtype = dtype
        self._buf = db.DataBuffer(
            data_spec=alf.tensor_specs.TensorSpec((), dtype=dtype),
            capacity=max_len,
            device='cpu')

    def append(self, value):
        """Append multiple values to the buffer.

        Args:
            value (Tensor): a batch of scalars with the shape :math:`[B]`.
        """
        self._buf.add_batch(value)

    def mean(self):
        if self._buf.current_size == 0:  # avoid nan
            return torch.tensor(0, dtype=self._dtype, device='cpu')
        return self._buf.get_all()[:self._buf.current_size].mean()

    def clear(self):
        return self._buf.clear()

class EnvironmentSteps(metric.StepMetric):
    """Counts the number of steps taken in the environment after FrameSkip.

    If Frames are skipped by any of the environment wrappers, a separate metric
    AverageEnvInfoMetric['num_env_frames'] will report the actual frame count including
    skipped ones.
    """

    def __init__(self,
                 name='EnvironmentSteps',
                 prefix='Metrics',
                 dtype=torch.int64):
        super().__init__(name=name, dtype=dtype, prefix=prefix)
        self.register_buffer('_environment_steps', torch.zeros((),
                                                               dtype=dtype))

    def call(self, time_step):
        """Increase the number of environment_steps according to ``time_step``.
        Step count is not increased on ``time_step.is_first()`` since that step
        is not part of any episode.

        Args:
            time_step (alf.data_structures.TimeStep): batched tensor
        Returns:
            The arguments, for easy chaining.
        """
        steps = (torch.logical_not(time_step.is_first())).type(self._dtype)
        num_steps = torch.sum(steps)
        self._environment_steps.add_(num_steps)
        return time_step

    def result(self):
        return self._environment_steps

    def reset(self):
        self._environment_steps.fill_(0)

class NumberOfEpisodes(metric.StepMetric):
    """Counts the number of episodes in the environment."""

    def __init__(self,
                 name='NumberOfEpisodes',
                 prefix='Metrics',
                 dtype=torch.int64):
        super(NumberOfEpisodes, self).__init__(
            name=name, dtype=dtype, prefix=prefix)
        self.register_buffer('_number_episodes', torch.zeros((), dtype=dtype))

    def call(self, time_step):
        """Increase the number of number_episodes according to ``time_step``.
        It would increase for all ``time_step.is_last()``.

        Args:
            time_step (alf.data_structures.TimeStep): batched tensor
        Returns:
            The arguments, for easy chaining.
        """
        episodes = time_step.is_last().type(self._dtype)
        num_episodes = torch.sum(episodes)
        self._number_episodes.add_(num_episodes)
        return time_step

    def result(self):
        return self._number_episodes

    def reset(self):
        self._number_episodes.fill_(0)

class AverageEpisodicSumMetric(metric.StepMetric):
    """A base metric to sum up quantities over an episode. It supports accumulating
    a nest of scalar values.
    """

    def __init__(self,
                 name="AverageEpisodicSumMetric",
                 prefix='Metrics',
                 dtype=torch.float32,
                 batch_size=1,
                 buffer_size=10,
                 example_metric_value=None):
        """
        Args:
            name (str):
            prefix (str): a prefix indicating the category of the metric
            dtype (torch.dtype): dtype of metric values. Should be floating types
                in order to be averaged.
            batch_size (int): the number of metric values generated in parallel
            buffer_size (int): number of episodes the metric value will be averaged
                across
            example_metric_value (nest): an example of metric value to be summarized;
                if ``None``, a zero scalar is used.
        """
        super(AverageEpisodicSumMetric, self).__init__(
            name=name, dtype=dtype, prefix=prefix)
        if example_metric_value is None:
            example_metric_value = torch.zeros((), device='cpu')
        self._batch_size = batch_size
        self._buffer_size = buffer_size
        self._initialize(example_metric_value)

    def _extract_metric_values(self, time_step):
        """Extract metrics from the time step. The return can be a nest."""
        raise NotImplementedError()

    def _initialize(self, example_metric_value):
        counter = [0]

        def _init_buf(val):
            return MetricBuffer(max_len=self._buffer_size, dtype=self._dtype)

        def _init_acc(val):
            accumulator = torch.zeros(
                self._batch_size, dtype=self._dtype, device='cpu')
            self.register_buffer('_accumulator%d' % counter[0], accumulator)
            counter[0] += 1
            return accumulator

        self._buffer = alf.nest.map_structure(_init_buf, example_metric_value)
        self._accumulator = alf.nest.map_structure(_init_acc,
                                                   example_metric_value)

    def call(self, time_step):
        """Accumulate values from the time step. The values are defined by
        subclasses' ``_extract_metric_values()``. It will ignore the values of
        first time steps.

        Args:
            time_step (alf.data_structures.TimeStep): batched tensor
        Returns:
            The arguments, for easy chaining.
        """

        values = self._extract_metric_values(time_step)

        assert all(
            alf.nest.flatten(
                alf.nest.map_structure(
                    lambda val: list(val.shape) == [self._batch_size],
                    values))), ("Value shape is not correct "
                                "(only scalar values are supported).")

        is_first = time_step.is_first()

        def _update_accumulator_(acc, val):
            """In-place update of the accumulators."""
            # Zero out batch indices where a new episode is starting.
            # Update with new values; Ignores first step whose reward comes from
            # the boundary transition of the last step from the previous episode.
            acc[:] = torch.where(is_first, torch.zeros_like(acc),
                                 acc + val.to(self._dtype))

        alf.nest.map_structure(_update_accumulator_, self._accumulator, values)

        # Add final accumulated value to buffer.
        last_episode_indices = torch.where(time_step.is_last())[0]

        if len(last_episode_indices) > 0:
            alf.nest.map_structure(
                lambda buf, acc: buf.append(acc[last_episode_indices]),
                self._buffer, self._accumulator)

        return time_step

    def result(self):
        return alf.nest.map_structure(lambda buf: buf.mean(), self._buffer)

    def reset(self):
        alf.nest.map_structure(lambda buf: buf.clear(), self._buffer)
        alf.nest.map_structure(lambda acc: acc.fill_(0), self._accumulator)

class AverageReturnMetric(AverageEpisodicSumMetric):
    """Metric for computing the average return."""

    def __init__(self,
                 name='AverageReturn',
                 prefix='Metrics',
                 reward_shape=(),
                 dtype=torch.float32,
                 batch_size=1,
                 buffer_size=10):
        if reward_shape == ():
            example_metric_value = torch.zeros((), device='cpu')
        else:
            example_metric_value = torch.zeros(
                reward_shape, device='cpu').reshape(-1)
            example_metric_value = list(example_metric_value)

        super(AverageReturnMetric, self).__init__(
            name=name,
            dtype=dtype,
            prefix=prefix,
            batch_size=batch_size,
            buffer_size=buffer_size,
            example_metric_value=example_metric_value)

    def _extract_metric_values(self, time_step):
        """Accumulate immediate rewards to get episodic return."""
        ndim = time_step.step_type.ndim
        if time_step.reward.ndim == ndim:
            return time_step.reward
        else:
            reward = time_step.reward.reshape(*time_step.step_type.shape, -1)
            return [reward[..., i] for i in range(reward.shape[-1])]

class AverageEpisodeLengthMetric(AverageEpisodicSumMetric):
    """Metric for computing the average episode length."""

    def __init__(self,
                 name='AverageEpisodeLength',
                 prefix='Metrics',
                 dtype=torch.float32,
                 batch_size=1,
                 buffer_size=10):
        super(AverageEpisodeLengthMetric, self).__init__(
            name=name,
            dtype=dtype,
            prefix=prefix,
            batch_size=batch_size,
            buffer_size=buffer_size)

    def _extract_metric_values(self, time_step):
        """Return a constant of 1 each time, except for ``time_step.is_first()``.
        The first time step is the boundary step and needs to be ignored, different
        from ``tf_agents``
        """
        return torch.where(time_step.is_first(),
                           torch.zeros_like(time_step.step_type),
                           torch.ones_like(time_step.step_type))

class AverageEnvInfoMetric(AverageEpisodicSumMetric):
    """Metric for computing average quantities contained in the environment info.
    An example of env info (which can be a nest) has to be provided when constructing
    an instance in order to initialize the accumulator and buffer with the same
    nested structure.
    """

    def __init__(self,
                 example_env_info,
                 name="AverageEnvInfoMetric",
                 prefix="Metrics",
                 dtype=torch.float32,
                 batch_size=1,
                 buffer_size=10):
        super(AverageEnvInfoMetric, self).__init__(
            name=name,
            dtype=dtype,
            prefix=prefix,
            batch_size=batch_size,
            buffer_size=buffer_size,
            example_metric_value=example_env_info)

    def _extract_metric_values(self, time_step):
        return time_step.env_info


import torch
from torch import nn, optim
from torch.autograd import Variable
import AENet
import argparse
import utils as utils
import visualize
import numpy as np
import sys

torch.manual_seed(1)

# parse arguments
def setup_args():

    options = argparse.ArgumentParser()

    options.add_argument('-sf', action="store", dest="save_file", default = "num_lamb_0.001")
    options.add_argument('-pt', action="store", dest="pretrained_file", default=None)
    options.add_argument('-bs', action="store", dest="batch_size", default = 128, type = int)
    options.add_argument('-env', action="store", dest="env", default="VAE_MNIST_USPS")

    options.add_argument('-iter', action="store", dest="max_iter", default = 200, type = int)
    options.add_argument('-lr', action="store", dest="lr", default=1e-3, type = float)
    options.add_argument('-nz', action="store", dest="nz", default=20, type = int)
    options.add_argument('-lamb', action="store", dest="lamb", default=0.001, type = float)

    return options.parse_args()

args = setup_args()
print(args)
sys.stdout.flush()

# retrieve dataloaders
train_loader, test_loader = utils.setup_data_loaders(args.batch_size)
print('Data loaded')

model = AENet.VAE(nc=1, latent_size=args.nz)
if args.pretrained_file is not None:
    model.load_state_dict(torch.load(args.pretrained_file))
    print("Pre-trained model loaded")
    sys.stdout.flush()

if torch.cuda.is_available():
    print('Using GPU')
    model.cuda()

optimizer = optim.Adam([
    {'params': model.parameters()}],
    lr = args.lr)

def loss_function(recon_x, x, mu, logvar, latents):
    MSE = nn.MSELoss()
    lloss = MSE(recon_x,x)
    if args.lamb>0:
        KL_loss = -0.5*torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
        lloss = lloss + args.lamb*KL_loss
    return lloss

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (inputs, _) in enumerate(train_loader):

        inputs = Variable(inputs)
        if torch.cuda.is_available():
            inputs = inputs.cuda()

        optimizer.zero_grad()
        recon_inputs, latents, mu, logvar = model(inputs)

        loss = loss_function(recon_inputs, inputs, mu, logvar, latents)
        train_loss += loss.data[0]

        loss.backward()
        optimizer.step()

    print('Epoch: {} Average loss: {:.15f}'.format(epoch, train_loss / len(train_loader.dataset)))

def test(epoch):
    model.eval()
    test_loss = 0
    for i, (inputs, _) in enumerate(test_loader):

        inputs = Variable(inputs)
        if torch.cuda.is_available():
            inputs = inputs.cuda()

        recon_inputs, latents, mu, logvar = model(inputs)

        loss = loss_function(recon_inputs, inputs, mu, logvar, latents)
        test_loss += loss.data[0]

    test_loss /= len(test_loader.dataset)
    print('Test set loss: {:.15f}'.format(test_loss))
    return test_loss

def save(epoch):
    torch.save(model.cpu().state_dict(), args.save_file+'_'+str(epoch)+".pth")
    if torch.cuda.is_available():
        model.cuda()

def generate_image(epoch):

    z = torch.normal(torch.zeros(args.batch_size, args.nz), 1)

    if torch.cuda.is_available():
        z = z.cuda()

    z = Variable(z)

    samples = model.generate(z)
    visualize.visualize_samples("Generated", samples.cpu().data, args.env)

    for inputs, _ in train_loader:
        inputs = Variable(inputs)
        if torch.cuda.is_available():
            inputs = inputs.cuda()

        recon_inputs, _, _, _ = model(inputs)

        visualize.visualize_samples("Train "+str(epoch)+" inputs", inputs.cpu().data, args.env)
        visualize.visualize_samples("Train "+str(epoch)+" recon", recon_inputs.cpu().data, args.env)
        break

    for inputs, _ in test_loader:
        inputs = Variable(inputs)
        if torch.cuda.is_available():
            inputs = inputs.cuda()

        recon_inputs, _, _, _ = model(inputs)

        visualize.visualize_samples("Test "+str(epoch)+" inputs", inputs.cpu().data, args.env)
        visualize.visualize_samples("Test "+str(epoch)+" recon", recon_inputs.cpu().data, args.env)
        break

# main training loop
visualize.reset(args.env)
generate_image(0)
save(0)

for epoch in range(args.max_iter):
    train(epoch)
    _ = test(epoch)

    if epoch % 10 == 0:
        generate_image(epoch)
        save(epoch)


import logging

import cyrusbus
from gi.repository import Gtk

import application.event
from application.note import NoteOpened

class WindowTitleHandler(object):
    """Controls the title of a Gtk.Window."""

    def __init__(
            self,
            bus: cyrusbus.bus.Bus,
            window: Gtk.Window,
    ):
        self.log = logging.getLogger('{m}.{c}'.format(m=self.__class__.__module__, c=self.__class__.__name__))
        self.window = window

        bus.subscribe(application.event.APPLICATION_TOPIC, self.on_application_event)

        self.set_title(None)

    def on_application_event(self, bus, event):
        self.log.debug(u'Event received: {event}'.format(event=event))

        if isinstance(event, NoteOpened):  # type: NoteOpened
            self.set_title(event.note)
        else:
            self.log.debug(u'Unhandled event: {event}'.format(event=event))

    def set_title(self, note):
        if note is not None:
            self.window.set_title('WMS Notes - {title}'.format(title=note.title))
        else:
            self.window.set_title('WMS Notes')


'''
Created on Jul 8, 2011

@author: christopherreilly
'''
from PyQt4 import QtGui, QtCore

from component import updateModel
from view import BaseListView
from reorderlist import IReorderList
from zope.interface import implements

class ComponentReorderList( QtCore.QAbstractListModel ):
    """
    Takes in a python list and a Qt view.  View must implement currentIndex().

    Implementation of IReorderList for lists of lrexp components.
    """
    implements( IReorderList )
    def __init__( self, pyList ):
        super( ComponentReorderList, self ).__init__()
        self.pyList = pyList
        self.view = self.widget = BaseListView()
        self.view.setModel( self )
    def rowCount( self, parent = QtCore.QModelIndex() ):
        if not parent.isValid():
            return len( self.pyList )
        return 0
    def data( self, index, role ):
        if not index.isValid(): return QtCore.QVariant()
        if role == QtCore.Qt.DisplayRole:
            return repr( self.pyList[index.row()] )
        return QtCore.QVariant()
    def getRow( self ):
        index = self.view.currentIndex()
        if not index.isValid(): return None
        return index.row()
    @updateModel
    def raiseItem( self ):
        row = self.getRow()
        if row is None or row is 0: return
        self.pyList[row], self.pyList[row - 1] = self.pyList[row - 1], self.pyList[row]
        self.dataChanged.emit( self.index( row - 1, 0 ), self.index( row, 0 ) )
        self.view.setCurrentIndex( self.index( row - 1 ) )
    @updateModel
    def lowerItem( self ):
        row = self.getRow()
        if row is None or row + 1 >= len( self.pyList ): return
        self.pyList[row], self.pyList[row + 1] = self.pyList[row + 1], self.pyList[row]
        self.dataChanged.emit( self.index( row, 0 ), self.index( row + 1, 0 ) )
        self.view.setCurrentIndex( self.index( row + 1 ) )
    @updateModel
    def removeItem( self ):
        row = self.getRow()
        if row is None: return
        self.beginRemoveRows( QtCore.QModelIndex(), row, row )
        self.pyList.pop( row )
        self.endRemoveRows()
    @updateModel
    def addItem( self ):
        pass

    @updateModel
    def removeAll( self ):
        self.beginRemoveRows( QtCore.QModelIndex(), 0, len( self.pyList ) - 1 )
        while self.pyList: self.pyList.pop()
        self.endRemoveRows()

    def appendObject( self, obj ):
        l = len( self.pyList )
        self.beginInsertRows( QtCore.QModelIndex(), l, l )
        self.pyList.append( obj )
        self.endInsertRows()