import os
import time
import pytest
import socket

from knit import Knit
from knit.exceptions import HDFSConfigException, KnitException

def check_docker():
    """check if inside docker container"""
    return os.path.exists('/.dockerenv')

inside_docker = check_docker

def wait_for_status(k, status, timeout=30):
    cur_status = k.runtime_status()
    while cur_status != status and timeout > 0:
        print("Current status is {0}, waiting for {1}".format(cur_status, status))

        time.sleep(2)
        timeout -= 2
        cur_status = k.runtime_status()

    return timeout > 0

def wait_for_containers(k, running_containers, timeout=30):
    cur_running_containers = k.status()['app']['runningContainers']
    while cur_running_containers != running_containers and timeout > 0:
        print("Current number of containers is {0}, waiting for {1}".format(cur_running_containers, running_containers))

        time.sleep(2)
        timeout -= 2
        cur_running_containers = k.status()['app']['runningContainers']

    return timeout > 0

@pytest.yield_fixture
def k():
    knitter = Knit(nn_port=8020, rm_port=8088)
    yield knitter

def test_port():
    with pytest.raises(HDFSConfigException):
        Knit(nn_port=90000, rm_port=90000)
    with pytest.raises(HDFSConfigException):
        Knit(nn_port=90000, rm_port=8088)
    with pytest.raises(HDFSConfigException):
        Knit(nn_port=9000, rm_port=5000)

    if inside_docker:
        # should pass without incident
        Knit(nn_port=8020, rm_port=8088)

def test_hostname(k):
    with pytest.raises(HDFSConfigException):
        Knit(nn="foobarbiz")
    with pytest.raises(HDFSConfigException):
        Knit(rm="foobarbiz")

    if inside_docker:
        # should pass without incident
        Knit(nn="localhost")
        k = Knit(autodetect=True)
        str(k) == 'Knit<NN=localhost:8020;RM=localhost:8088>'

def test_argument_parsing(k):
    cmd = "sleep 10"
    with pytest.raises(KnitException):
        k.start(cmd, files='a,b,c')

    with pytest.raises(KnitException):
        k.start(cmd, memory='128')

def test_cmd(k):
    cmd = "python -c 'import socket; print(socket.gethostname()*2)'"
    k.start(cmd, memory=128)

    if not k.wait_for_completion(30):
        k.kill()

    hostname = socket.gethostname() * 2
    logs = k.logs(shell=True)
    print(logs)

    assert hostname in logs, logs

def test_multiple_containers(k):
    cmd = "sleep 30"
    k.start(cmd, num_containers=2)

    wait_for_status(k, 'RUNNING')

    got_containers = wait_for_containers(k, 3)

    # wait for job to finish
    if not k.wait_for_completion(30):
        k.kill()

    if not got_containers:
        logs = k.logs(shell=True)
        print(logs)

    assert got_containers

def test_add_remove_containers(k):
    cmd = "sleep 60"
    k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    got_containers = wait_for_containers(k, 2)

    containers = k.get_containers()
    assert len(containers) == 2

    k.add_containers(num_containers=1)

    got_more_containers = wait_for_containers(k, 3)

    containers = k.get_containers()
    assert len(containers) == 3
    k.remove_containers(containers[1])

    got_more_containers = wait_for_containers(k, 2)
    containers = k.get_containers()
    assert len(containers) == 2

    # wait for job to finish
    if not k.wait_for_completion(30):
        k.kill()

    if not (got_containers and got_more_containers):
        logs = k.logs(shell=True)
        print(logs)

    assert got_containers and got_more_containers

def test_memory(k):
    cmd = "sleep 10"
    k.start(cmd, num_containers=2, memory=300)

    wait_for_status(k, 'RUNNING')

    time.sleep(2)
    status = k.status()

    # not exactly sure on getting an exact number
    # 300*2+128(AM)
    assert status['app']['allocatedMB'] >= 728, status['app']['allocatedMB']

    # wait for job to finish
    if not k.wait_for_completion(30):
        k.kill()

def test_cmd_w_conda_env(k):
    env_zip = k.create_env(env_name='dev', packages=['python=2.6'], remove=True)
    cmd = "$PYTHON_BIN -c 'import sys; print(sys.version_info); import random; print(str(random.random()))'"
    k.start(cmd, env=env_zip)

    if not k.wait_for_completion(30):
        k.kill()

    logs = k.logs(shell=True)
    assert "(2, 6, 9, 'final', 0)" in logs, logs

cur_dir = os.path.dirname(__file__)
txt_file = os.path.join(cur_dir, 'files', 'upload_file.txt')
py_file = os.path.join(cur_dir, 'files', 'read_uploaded.py')

def test_file_uploading(k):
    cmd = 'python ./read_uploaded.py'
    k.start(cmd, files=[txt_file, py_file])

    if not k.wait_for_completion(30):
        k.kill()

    logs = k.logs(shell=True)
    assert "rambling on" in logs, logs

def test_kill_status(k):
    cmd = "sleep 10"
    k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    assert k.kill()

    status = k.runtime_status()
    assert status == 'KILLED'

def test_yarn_kill_status(k):
    cmd = "sleep 10"
    app_id = k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    assert k.yarn_api.kill(app_id)

    status = k.runtime_status()
    assert status == 'KILLED'

def test_logs(k):
    cmd = "sleep 10"
    k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    assert k.logs()

    if not k.wait_for_completion(30):
        k.kill()

# temporarily removing test until vCore handling is better resolved in the core
# def test_vcores(k):
#     cmd = "sleep 10"
#     appId = k.start(cmd, num_containers=1, memory=300, virtual_cores=2)
#
#     time.sleep(2)
#     status = k.status(appId)
#
#     while status['app']['state'] != 'RUNNING':
#         status = k.status(appId)
#         time.sleep(2)
#
#     time.sleep(2)
#     status = k.status(appId)
#
#     assert status['app']['allocatedVCores'] == 3
#


"""
   Copyright (c) 2015, Mariano Tepper, Duke University.
   All rights reserved.

   This file is part of RCNMF and is under the BSD 3-Clause License,
   which can be found in the LICENSE file in the root directory, or at
   http://opensource.org/licenses/BSD-3-Clause
"""
from __future__ import absolute_import
import math
import numpy as np
import rcnmf.compression as randcomp

def compute(mat, q, n_iter=1e5, n_power_iter=4):

    n_iter = int(n_iter)

    m = mat.shape[0]
    n = mat.shape[1]

    mat_l, left_comp = randcomp.compress(mat, q, n_power_iter=n_power_iter)
    mat_r, right_comp = randcomp.compress(mat.T, q, n_power_iter=n_power_iter)
    mat_r = np.transpose(mat_r)
    right_comp = np.transpose(right_comp)

    mat_lr = left_comp.dot(mat.dot(right_comp))

    u = np.fabs(np.random.randn(m, q))
    v = np.fabs(np.random.randn(q, n))

    u_comp = left_comp.dot(u)

    err = np.zeros((n_iter, 1))
    for i in xrange(n_iter):

        temp1 = u_comp.T.dot(mat_l)
        temp2 = u_comp.T.dot(u_comp)
        num = projection_pos(temp1) + projection_neg(temp2).dot(v)
        denominator = projection_neg(temp1) + projection_pos(temp2).dot(v)
        v = np.sqrt(num / denominator) * v

        v_comp = v.dot(right_comp)

        temp1 = mat_r.dot(v_comp.T)
        temp2 = v_comp.dot(v_comp.T)
        num = projection_pos(temp1) + u.dot(projection_neg(temp2))
        denominator = projection_neg(temp1) + u.dot(projection_pos(temp2))
        u = u * np.sqrt(num / denominator)

        u_comp = left_comp.dot(u)

        err[i] = math.log10(np.linalg.norm(mat_lr - u_comp.dot(v_comp), 'fro'))

        if i >= 20 and (err[i] < -10 or math.fabs(err[i] - err[i-1]) <= 1e-10):
            err.reshape((i+1, 1))
            break

    return u, v, err

def projection_neg(mat):
    return (np.fabs(mat) - mat) / 2

def projection_pos(mat):
    return (np.fabs(mat) + mat) / 2

if __name__ == "__main__":

    import matplotlib.pyplot as plt
    import time

    m = 1000
    n = 1000
    q = 5
    X = np.fabs(np.random.rand(m, q))
    Y = np.fabs(np.random.rand(q, n))
    M = X.dot(Y)

    start = time.clock()
    U, V, err = compute(M, q, 1e3)
    end = time.clock()
    print(end-start)

    plt.figure()
    plt.plot(err)

    plt.show()


from typing import List

class Solution:
    def maxProfit(self, prices: List[int]) -> int:
        if not prices:
            return 0

        res, min_prices = 0, prices[0]
        for i in range(1, len(prices)):
            res = max(res, prices[i]-min_prices)
            min_prices = min(min_prices, prices[i])

        return res

def test(test_name, prices, expected):
    res = Solution().maxProfit(prices)
    if res == expected:
        print(test_name + ' success.')
    else:
        print(test_name + ' failed.')

if __name__ == "__main__":
    prices1 = [7,1,5,3,6,4]
    expected1 = 5
    test('test1', prices1, expected1)

    prices2 = [7,6,4,3,1]
    expected2 = 0
    test('test2', prices2, expected2)

    prices3 = []
    expected3 = 0
    test('test3', prices3, expected3)


import sys

from honcho.environ import Env

class Printer(object):
    def __init__(self,
                 output=sys.stdout,
                 name='unknown',
                 colour=None,
                 width=0,
                 env=None):
        self._env = env if env is not None else Env()
        self.output = output
        self.name = name
        self.colour = colour
        self.width = width

        self._write_prefix = True

    def write(self, string):
        lines = string.split('\n')
        lines = [self._prefix() + l if l else l for l in lines]
        new_string = '\n'.join(lines)

        self.output.write(new_string)

    def _prefix(self):
        time = self._env.now().strftime('%H:%M:%S')
        name = self.name.ljust(self.width)
        prefix = '{time} {name} | '.format(time=time, name=name)
        if self.colour:
            return _colour_string(self.colour, prefix)
        else:
            return prefix

def _ansi(code):
    return '\033[{0}m'.format(code)

def _colour_string(colour, s):
    return '{0}{1}{2}'.format(_ansi(colour), s, _ansi(0))


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.9.1+dev
#   kernelspec:
#     display_name: Python [conda env:generic_expression] *
#     language: python
#     name: conda-env-generic_expression-py
# ---

# # Identify generic genes and pathways
#
# Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These generic genes and subsequent pathways are not necessarily specific to the biological process being studied but instead represent a more systematic change.
#
# This notebook identifies generic genes and pathways and then evaluates if those identified are consistent with published findings.
#
# **Steps to identify generic genes:**
# 1. Simulates N gene expression experiments using [ponyo](https://github.com/ajlee21/ponyo)
# 2. Perform DE analysis to get association statistics for each gene
# 3. For each gene, aggregate statsitics across all simulated experiments
# 4. Rank genes based on this aggregated statistic (i.e. log fold change, or p-value)
#
#
# **Steps to identify generic gene sets (pathways):**
# 1. Using the same simulated experiments from above, perform GSEA analysis. This analysis will determine whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list of genes, where genes are ranked by log fold change, indicating a correlation with change in expression.
# 2. For each gene set (pathway), aggregate statistics across all simulated experiments
# 3. Rank gene sets based on this aggregated statistic
#
# **Evaluation:**
# * We want to compare the ranking of genes identified using the above method with the ranking found from [Crow et. al.](https://www.pnas.org/content/pnas/116/13/6491.full.pdf), which identified a set of genes as generic based on how frequently they were found to be DE across 600 experiments
# * We want to compare the ranking of pathways identified using the above method with the ranking based on the [Powers et. al.](https://www.biorxiv.org/content/10.1101/259440v1.full.pdf) data, where ranking was determined based on the fraction of 442 experiments a pathway was found to be enriched
# * This comparison will validate our method being used as a way to automatically identify generic genes and pathways.

# +
# %load_ext autoreload
# %load_ext rpy2.ipython
# %autoreload 2

import os
import sys
import pandas as pd
import numpy as np
import pickle
import glob
import scipy.stats as ss
from keras.models import load_model
from rpy2.robjects import pandas2ri
from ponyo import utils
from generic_expression_patterns_modules import process, stats, ranking

pandas2ri.activate()

np.random.seed(123)

# +
# Read in config variables
base_dir = os.path.abspath(os.path.join(os.getcwd(), "../"))

config_filename = os.path.abspath(
    os.path.join(base_dir, "configs", "config_human_cancer.tsv")
)

params = utils.read_config(config_filename)

# +
# Load params
local_dir = params["local_dir"]
dataset_name = params["dataset_name"]
NN_architecture = params["NN_architecture"]
num_runs = params["num_simulated"]
project_id = params["project_id"]
metadata_col_id = params["metadata_colname"]
mapped_template_filename = params["mapped_template_filename"]
processed_template_filename = params["processed_template_filename"]
normalized_compendium_filename = params["normalized_compendium_filename"]
scaler_filename = params["scaler_filename"]
col_to_rank_genes = params["rank_genes_by"]
col_to_rank_pathways = params["rank_pathways_by"]
statistic = params["gsea_statistic"]
logFC_name = params["DE_logFC_name"]
pvalue_name = params["DE_pvalue_name"]

# Load metadata file with grouping assignments for samples
sample_id_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_process_samples.tsv"
)

# Load metadata file with grouping assignments for samples
grp_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_groups.tsv"
)

# Load pickled file
with open(scaler_filename, "rb") as scaler_fh:
    scaler = pickle.load(scaler_fh)

# Percentile threshold to identify generic genes
percentile_threshold = 80.0

# +
# Output files
gene_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_gene_summary_{project_id}.tsv"
)

pathway_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_pathway_summary_{project_id}.tsv"
)

# -

# ## Need to customize code from ponyo
#
# The current simulation-related function in ponyo, `get_sample_ids` assumes that the user is using one of two different metadata files (one associated with the pseudomonas compendium and another associated with recount2). The compendium dataset we are using here has a slightly different format for their metadata file.
#
# Here we are temporarily writing our own function customized for this Powers et. al. dataset. But we will be updating ponyo to allow for different metadata files in the future. Issue in ponyo is [here](https://github.com/greenelab/ponyo/issues/18)

def get_sample_ids(experiment_id, dataset_name, sample_id_colname):
    """
    Returns sample ids (found in gene expression df) associated with
    a given list of experiment ids (found in the metadata)

    Arguments
    ----------
    experiment_ids_file: str
        File containing all cleaned experiment ids

    dataset_name: str
        Name for analysis directory. Either "Human" or "Pseudomonas"

    sample_id_colname: str
        Column header that contains sample id that maps expression data
        and metadata

    """
    # Read in metadata
    metadata = pd.read_csv(metadata_filename, header=0)
    metadata.set_index("gse", inplace=True)

    selected_metadata = metadata.loc[experiment_id]
    sample_ids = list(selected_metadata[sample_id_colname])

    return sample_ids

def shift_template_experiment_with_metadatafile(
    normalized_data_file,
    selected_experiment_id,
    sample_id_colname,
    NN_architecture,
    dataset_name,
    scaler,
    local_dir,
    base_dir,
    run,
    metadata_filename,
):
    """
    Generate new simulated experiment using the selected_experiment_id as a template
    experiment using the same workflow as `simulate_by_latent_transform`

    This will return a file with a single simulated experiment following the workflow mentioned.
    This function can be run multiple times to generate multiple simulated experiments from a
    single selected_experiment_id.

    Arguments
    ----------
    normalized_data_file: str
        File containing normalized gene expression data

        ------------------------------| PA0001 | PA0002 |...
        05_PA14000-4-2_5-10-07_S2.CEL | 0.8533 | 0.7252 |...
        54375-4-05.CEL                | 0.7789 | 0.7678 |...
        ...                           | ...    | ...    |...

    selected_experiment_id: str
        Experiment id selected as template

    sample_id_colname: str
        Column header that contains sample id that maps expression data and metadata

    NN_architecture: str
        Name of neural network architecture to use.
        Format 'NN_<intermediate layer>_<latent layer>'

    dataset_name: str
        Name for analysis directory. Either "Human" or "Pseudomonas"

    scaler: minmax model
        Model used to transform data into a different range

    local_dir: str
        Parent directory on local machine to store intermediate results

    base_dir: str
        Root directory containing analysis subdirectories

    run: int
        Simulation run

    Returns
    --------
    simulated_data_file: str
        File containing simulated gene expression data

    """

    # Files
    NN_dir = os.path.join(base_dir, dataset_name, "models", NN_architecture)
    latent_dim = NN_architecture.split("_")[-1]

    model_encoder_file = glob.glob(os.path.join(NN_dir, "*_encoder_model.h5"))[0]

    weights_encoder_file = glob.glob(os.path.join(NN_dir, "*_encoder_weights.h5"))[0]

    model_decoder_file = glob.glob(os.path.join(NN_dir, "*_decoder_model.h5"))[0]

    weights_decoder_file = glob.glob(os.path.join(NN_dir, "*_decoder_weights.h5"))[0]

    # Load saved models
    loaded_model = load_model(model_encoder_file, compile=False)
    loaded_decode_model = load_model(model_decoder_file, compile=False)

    loaded_model.load_weights(weights_encoder_file)
    loaded_decode_model.load_weights(weights_decoder_file)

    # Read data
    normalized_data = pd.read_csv(normalized_data_file, header=0, sep="\t", index_col=0)

    # Get corresponding sample ids
    sample_ids = get_sample_ids(
        selected_experiment_id, metadata_filename, sample_id_colname
    )

    # Gene expression data for selected samples
    selected_data_df = normalized_data.loc[sample_ids]

    # Encode selected experiment into latent space
    data_encoded = loaded_model.predict_on_batch(selected_data_df)
    data_encoded_df = pd.DataFrame(data_encoded, index=selected_data_df.index)

    # Get centroid of original data
    centroid = data_encoded_df.mean(axis=0)

    # Add individual vectors(centroid, sample point) to new_centroid

    # Encode original gene expression data into latent space
    data_encoded_all = loaded_model.predict_on_batch(normalized_data)
    data_encoded_all_df = pd.DataFrame(data_encoded_all, index=normalized_data.index)

    data_encoded_all_df.head()

    # Find a new location in the latent space by sampling from the latent space
    encoded_means = data_encoded_all_df.mean(axis=0)
    encoded_stds = data_encoded_all_df.std(axis=0)

    latent_dim = int(latent_dim)
    new_centroid = np.zeros(latent_dim)

    for j in range(latent_dim):
        new_centroid[j] = np.random.normal(encoded_means[j], encoded_stds[j])

    shift_vec_df = new_centroid - centroid
    # print(shift_vec_df)

    simulated_data_encoded_df = data_encoded_df.apply(
        lambda x: x + shift_vec_df, axis=1
    )

    # Decode simulated data into raw gene space
    simulated_data_decoded = loaded_decode_model.predict_on_batch(
        simulated_data_encoded_df
    )

    simulated_data_decoded_df = pd.DataFrame(
        simulated_data_decoded,
        index=simulated_data_encoded_df.index,
        columns=selected_data_df.columns,
    )

    # Un-normalize the data in order to run DE analysis downstream
    simulated_data_scaled = scaler.inverse_transform(simulated_data_decoded_df)

    simulated_data_scaled_df = pd.DataFrame(
        simulated_data_scaled,
        columns=simulated_data_decoded_df.columns,
        index=simulated_data_decoded_df.index,
    )

    # Save template data for visualization validation
    test_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        "template_normalized_data_" + selected_experiment_id + "_test.txt",
    )

    selected_data_df.to_csv(test_file, float_format="%.3f", sep="\t")

    # Save
    out_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        "selected_simulated_data_" + selected_experiment_id + "_" + str(run) + ".txt",
    )

    simulated_data_scaled_df.to_csv(out_file, float_format="%.3f", sep="\t")

    out_encoded_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        f"selected_simulated_encoded_data_{selected_experiment_id}_{run}.txt",
    )

    simulated_data_encoded_df.to_csv(out_encoded_file, float_format="%.3f", sep="\t")

# ### Simulate experiments using selected template experiment
#
# Workflow:
# 1. Get the gene expression data for the selected template experiment
# 2. Encode this experiment into a latent space using the trained VAE model
# 3. Linearly shift the encoded template experiment in the latent space
# 4. Decode the samples. This results in a new experiment
# 5. Repeat steps 1-4 to get multiple simulated experiments

# Load metadata file with grouping assignments for samples
metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "all_experiments_sample_annotations.csv"
)

# Simulate multiple experiments
# This step creates the following files in "<local_dir>/pseudo_experiment/" directory:
#   - selected_simulated_data_SRP012656_<n>.txt
#   - selected_simulated_encoded_data_SRP012656_<n>.txt
#   - template_normalized_data_SRP012656_test.txt
# in which "<n>" is an integer in the range of [0, num_runs-1]
os.makedirs(os.path.join(local_dir, "pseudo_experiment"), exist_ok=True)
for run_id in range(num_runs):
    shift_template_experiment_with_metadatafile(
        normalized_compendium_filename,
        project_id,
        metadata_col_id,
        NN_architecture,
        dataset_name,
        scaler,
        local_dir,
        base_dir,
        run_id,
        metadata_filename,
    )

# ### Process template and simulated experiments
#
# * Remove samples not required for comparison. Since this experiment contains multiple conditions (i.e. estradiol vs EtOH at 12, 24, and 48 hrs are each considered a different comparison) being tested, we will only include those samples within the same condition.
# * Make sure ordering of samples matches metadata for proper comparison

# +
if not os.path.exists(sample_id_metadata_filename):
    sample_id_metadata_filename = None

stats.process_samples_for_limma(
    mapped_template_filename,
    grp_metadata_filename,
    processed_template_filename,
    sample_id_metadata_filename,
)

for i in range(num_runs):
    simulated_filename = os.path.join(
        local_dir, "pseudo_experiment", f"selected_simulated_data_{project_id}_{i}.txt"
    )
    out_simulated_filename = os.path.join(
        local_dir,
        "pseudo_experiment",
        f"selected_simulated_data_{project_id}_{i}_processed.txt",
    )
    stats.process_samples_for_limma(
        simulated_filename,
        grp_metadata_filename,
        out_simulated_filename,
        sample_id_metadata_filename,
    )
# -

# ### Differential expression analysis
#
# The gene expression dataset is array-based so we will use Limma in this case

# Create subdirectory: "<local_dir>/DE_stats/"
os.makedirs(os.path.join(local_dir, "DE_stats"), exist_ok=True)

# + magic_args="-i metadata_filename -i project_id -i processed_template_filename -i local_dir -i base_dir" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))
#
# # File created: "<local_dir>/DE_stats/DE_stats_template_data_SRP012656_real.txt"
# get_DE_stats_limma(metadata_filename,
#                    project_id,
#                    processed_template_filename,
#                    "template",
#                    local_dir,
#                    "real")

# +
# Check number of DEGs
template_DE_stats_filename = os.path.join(
    local_dir, "DE_stats", f"DE_stats_template_data_{project_id}_real.txt"
)

template_DE_stats = pd.read_csv(
    template_DE_stats_filename, sep="\t", header=0, index_col=0
)

selected = template_DE_stats[
    (template_DE_stats["adj.P.Val"] < 0.05) & (abs(template_DE_stats["logFC"]) > 1)
]
print(selected.shape)

# + magic_args="-i metadata_filename -i project_id -i base_dir -i local_dir -i num_runs" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))
#
# # Files created: "<local_dir>/DE_stats/DE_stats_simulated_data_SRP012656_<n>.txt"
# for (i in 0:(num_runs-1)){
#     simulated_data_filename <- paste(local_dir,
#                                      "pseudo_experiment/selected_simulated_data_",
#                                      project_id,
#                                      "_",
#                                      i,
#                                      ".txt",
#                                      sep = "")
#
#     get_DE_stats_limma(metadata_filename,
#                        project_id,
#                        simulated_data_filename,
#                        "simulated",
#                        local_dir,
#                        i)
# }
# -

# ### Rank genes

analysis_type = "DE"
template_DE_stats, simulated_DE_summary_stats = ranking.process_and_rank_genes_pathways(
    template_DE_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_genes,
    logFC_name,
    pvalue_name,
)

# ### Gene summary table

# +
summary_gene_ranks = ranking.generate_summary_table(
    template_DE_stats_filename,
    template_DE_stats,
    simulated_DE_summary_stats,
    col_to_rank_genes,
    local_dir,
    "gene",
    params,
)

summary_gene_ranks.head()
# -

# Check if there is an NaN values, there should not be
summary_gene_ranks.isna().any()

# Create `gene_summary_fielname`
summary_gene_ranks.to_csv(gene_summary_filename, sep="\t")

# ### Compare gene ranking
# Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These *generic genes* are not necessarily specific to the biological process being studied but instead represent a more systematic change.
#
# We want to compare the ability to detect these generic genes using our method vs those found by [Crow et. al. publication](https://www.pnas.org/content/pnas/116/13/6491.full.pdf). Their genes are ranked 0 = not commonly DE; 1 = commonly DE. Genes were ranked by the number differentially expressed gene sets a gene appeared in across 600 experiments.

# +
# Get generic genes identified by Crow et. al.
DE_prior_filename = params["reference_gene_filename"]
ref_gene_col = params["reference_gene_name_col"]
ref_rank_col = params["reference_rank_col"]

figure_filename = f"gene_ranking_{col_to_rank_genes}.svg"

corr, shared_ranking = ranking.compare_gene_ranking(
    summary_gene_ranks, DE_prior_filename, ref_gene_col, ref_rank_col, figure_filename
)
# -

# Hypergeometric test:
# Given N number of genes with K common genes in Crow et al.
# SOPHIE identifies n genes as being common
# What is the probability that k of the genes identified by SOPHIE
# are also common in Crow et al.? What is the probability of drawing
# k or more concordant genes?
num_Crow_genes = shared_ranking.shape[0]
num_generic_Crow_genes = shared_ranking.query(f"{ref_rank_col}>=80.0").shape[0]
num_generic_SOPHIE_genes = shared_ranking[
    shared_ranking["Percentile (simulated)"] >= percentile_threshold
].shape[0]
num_concordant_generic_genes = shared_ranking[
    (shared_ranking[ref_rank_col] >= percentile_threshold)
    & (shared_ranking["Percentile (simulated)"] >= percentile_threshold)
].shape[0]

print(num_Crow_genes)
print(num_generic_Crow_genes)
print(num_generic_SOPHIE_genes)
print(num_concordant_generic_genes)

p = ss.hypergeom.sf(
    num_concordant_generic_genes,
    num_Crow_genes,
    num_generic_Crow_genes,
    num_generic_SOPHIE_genes,
)
print(p)

# **Takeaway:**
# * Previously we compared gene ranks obtained from (recount2)-trained VAE model vs gene ranks obtained from manual curation using Crow et. al data. This [PR](https://github.com/ajlee21/generic-expression-patterns/blob/807377d76f63b6282c62255d7b160feb8585e0e2/human_analysis/2_identify_generic_genes_pathways.ipynb) shows that the correlation of gene ranks are very consistent.
#
# * Here we are comparing gene ranks obtained from a (Powers et. al.)-trained VAE model vs gene ranks obtained from manual curation using Crow et. al. Based on this correlation plot there is a high correlation between those very high and low ranked genes -- high correlation at the extremes but there is a lot of noise in the middle.

# ### GSEA
# **Goal:** To detect modest but coordinated changes in prespecified sets of related genes (i.e. those genes in the same pathway or share the same GO term).
#
# 1. Rank all genes using DE association statistics.
# 2. An enrichment score (ES) is defined as the maximum distance from the middle of the ranked list. Thus, the enrichment score indicates whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list (indicating a correlation with change in expression).
# 3. Estimate the statistical significance of the ES by a phenotypic-based permutation test in order to produce a null distribution for the ES (i.e. scores based on permuted phenotype)

# Create "<local_dir>/GSEA_stats/" subdirectory
os.makedirs(os.path.join(local_dir, "GSA_stats"), exist_ok=True)

# Load pathway data
hallmark_DB_filename = params["pathway_DB_filename"]

# + magic_args="-i base_dir -i template_DE_stats_filename -i hallmark_DB_filename -i statistic -i local_dir -o template_enriched_pathways" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))
#
# out_filename <- paste(local_dir,
#                      "GSA_stats/GSEA_stats_template_data_",
#                      project_id,
#                      "_real.txt",
#                      sep = "")
#
# template_enriched_pathways <- find_enriched_pathways(template_DE_stats_filename, hallmark_DB_filename, statistic)
# template_enriched_pathways <- as.data.frame(template_enriched_pathways[1:7])
#
# write.table(template_enriched_pathways, file = out_filename, row.names = F, sep = "\t")
# -

print(template_enriched_pathways.shape)
template_enriched_pathways[template_enriched_pathways["padj"] < 0.05].sort_values(
    by="padj"
)

# **Quick check:** Looks like enriched pathways are consistent with estradiol being estrogen hormone treatment.

# + magic_args="-i project_id -i local_dir -i hallmark_DB_filename -i num_runs -i statistic -i base_dir" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))
#
# # New files created: "<local_dir>/GSEA_stats/GSEA_stats_simulated_data_<project_id>_<n>.txt"
# for (i in 0:(num_runs-1)) {
#     simulated_DE_stats_filename <- paste(local_dir,
#                                      "DE_stats/DE_stats_simulated_data_",
#                                      project_id,
#                                      "_",
#                                      i,
#                                      ".txt",
#                                      sep = "")
#
#     out_filename <- paste(local_dir,
#                      "GSA_stats/GSEA_stats_simulated_data_",
#                      project_id,
#                      "_",
#                      i,
#                      ".txt",
#                      sep = "")
#
#     enriched_pathways <- find_enriched_pathways(simulated_DE_stats_filename, hallmark_DB_filename, statistic)
#
#     write.table(as.data.frame(enriched_pathways[1:7]), file = out_filename, row.names = F, sep = "\t")
# }
# -

# ### Rank pathways

analysis_type = "GSA"
template_GSEA_stats_filename = os.path.join(
    local_dir, "GSA_stats", f"GSEA_stats_template_data_{project_id}_real.txt"
)
(
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
) = ranking.process_and_rank_genes_pathways(
    template_GSEA_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_pathways,
    logFC_name,
    pvalue_name,
    "GSEA",
)

# ### Pathway summary table

# +
# Create intermediate file: "<local_dir>/gene_summary_table_<col_to_rank_pathways>.tsv"
summary_pathway_ranks = ranking.generate_summary_table(
    template_GSEA_stats_filename,
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
    col_to_rank_pathways,
    local_dir,
    "pathway",
    params,
)

summary_pathway_ranks.sort_values(by="Rank (simulated)", ascending=False).head(10)
# -

# Create `pathway_summary_filename`
summary_pathway_ranks.to_csv(pathway_summary_filename, sep="\t")

# ### Compare pathway ranking

# Studies have found that there are some pathways (gene sets) that are more likely to be significantly enriched in DEGs across a wide range of experimental designs. These generic pathways are not necessarily specific to the biological process being studied but instead represents a more systematic change.
#
# We want to compare the ability to detect these generic pathways using our method vs those found by [Powers et. al.](https://www.biorxiv.org/content/10.1101/259440v1.full.pdf) publication.  We will use the `Hallmarks_qvalues_GSEAPreranked.csv` file from https://www.synapse.org/#!Synapse:syn11806255 as a reference. The file contains the q-value (adjusted p-value) for the test: given the enrichment score (ES) of the experiment is significant compared to the null distribution of enrichment scores, where the null set is generated from permuted gene sets. For each gene set (pathway) they calculate the q-value using this test.
#
#
# To get a `reference ranking`, we calculate the fraction of experiments that a given pathway was significant (q-value <0.05) and use this rank pathways. `Our ranking` is to rank pathways based on the median q-value across the simulated experiments. We can then compare `our ranking` versus the `reference ranking.`

# Load Powers et. al. results file
powers_rank_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "Hallmarks_qvalues_GSEAPreranked.csv"
)

# Read Powers et. al. data
# This file contains qvalue results for hallmark pathways across ~400 experiments
powers_rank_df = pd.read_csv(powers_rank_filename, header=0, index_col=0)
powers_rank_df.drop(["Category"], axis=1, inplace=True)
print(powers_rank_df.shape)
powers_rank_df.head()

# +
# Count the number of experiments where a given pathway was found to be enriched (qvalue < 0.05)
total_num_experiments = powers_rank_df.shape[1]
frac_enriched_pathways = (powers_rank_df < 0.05).sum(axis=1) / total_num_experiments

# Rank pathways from 0-50, 50 indicating that the pathways was frequently enriched
pathway_ranks = frac_enriched_pathways.rank()

powers_rank_stats_df = pd.DataFrame(
    data={
        "Fraction enriched": frac_enriched_pathways.values,
        "Powers Rank": pathway_ranks.values,
    },
    index=powers_rank_df.index,
)
powers_rank_stats_df.sort_values(by="Powers Rank", ascending=False).head()

# +
# Save reference file for input into comparison
powers_rank_processed_filename = os.path.join(
    base_dir,
    dataset_name,
    "data",
    "metadata",
    "Hallmarks_qvalues_GSEAPreranked_processed.tsv",
)

powers_rank_stats_df.to_csv(
    powers_rank_processed_filename,
    sep="\t",
)

# +
figure_filename = f"pathway_ranking_{col_to_rank_pathways}.svg"

ranking.compare_pathway_ranking(
    summary_pathway_ranks, powers_rank_processed_filename, figure_filename
)
# -

# * Our method ranked pathways using median adjusted p-value score across simulated experiments.
# * Powers et. al. ranked pathways based on the fraction of experiments they had adjusted p-value < 0.05.
#
# **Takeaway:**
# * Previously we compared pathway ranks obtained from (recount2)-trained VAE model vs pathway ranking based on manual curation using Powers et. al. This [PR](https://github.com/ajlee21/generic-expression-patterns/blob/807377d76f63b6282c62255d7b160feb8585e0e2/human_analysis/2_identify_generic_genes_pathways.ipynb) shows that there was no correlation.
#
# * Here we validated that our analysis pipeline is working correctly by comparing pathway ranks obtained from a (Powers et. al.)-trained VAE model vs pathway ranking based on manual curation using Powers et. al datasets. We expect to see a high correlation between pathway ranks given that we are using the same training dataset. Indeed that is what we find

# **Conclusion:**
#
# * We find relatively similar generic genes using our simulation approach (i.e. VAE model trained on a cancer-specific dataset, Powers et. al.) compared to generic genes found from real general experiments from Crow et. al. These generic genes are not *that* context-specific at the extremes.
#
# * We found very different generic pathways training using our simulation approach trained on a general dataset (recount2) compared to generic pathways found from real cancer-specific experiments from Powers et. al. See [analysis](../human_general_analysis/2_identify_generic_genes_pathways.ipynb). But we get very similar generic pathways using our simulation approach trained on a cancer-specific dataset (Powers et. al.) compared with generic pathways found from cancer-specific dataset (Powers et. al.). This indicates that generic pathways are more context specific.
#
# * Why would the context matter more for pathways as opposed to genes? One way to think about this is using this figure from a recent [preprint](https://www.biorxiv.org/content/10.1101/2020.07.30.228296v1).Information flows from a stimulation that activates proteins within pathways and these proteins regulate gene expression. Say we have a context specific signal, that changes the TF within some pathways, this eventually trickles down to changes in gene expression. So if we think about flow of information, measuring pathway activity (or pathway enrichment, etc) will be more sensitive to our context compared to measuring DE in individual genes. Since the genes are regulated as a group, you'd see coordinated changes in expression that are correlated with your condition but looking at the expression of individual genes you wouldn’t necessarily see this correlation with condition.


import parsy
import logging
import refpy.constraints
import mmap

from functools import partial
from refpy.exceptions import ParseError

class RuleParser():
    def checkIdentifier(self, Id):
        if not Id in self.rules:
            return parsy.fail("Unsuported rule.")

        return parsy.success(Id)

    def _parse(self, rules, stream):
        logging.info("Available Rules: %s"%(", ".join(["%s"%(rule.Id) for rule in rules])))

        self.identifier = set([rule.Id for rule in rules])

        skipBlankLines = parsy.regex(r"\s*")
        singleRules = [skipBlankLines >> parsy.regex(rule.Id).desc("rule identifier") >> rule.getParser() << skipBlankLines for rule in rules]
        anyRuleParser = parsy.alt(*singleRules)

        parser = header >> anyRuleParser.many()

        return parser.parse(stream)

    def parseHeader(self, line, lineNum):
        headerParser = parsy.regex(r"refutation using").\
            then(
                parsy.regex(r" +").desc("space").\
                    then(
                        parsy.regex(r"[^0 ]+").desc("rule identifier").\
                        bind(partial(RuleParser.checkIdentifier, self))
                    ).\
                    many()
            ).skip(
                parsy.regex(r" *0 *\n?").desc("0 at end of line")
            )

        try:
            headerParser.parse(line)
        except parsy.ParseError as e:
            raise ParseError(e, line = lineNum)

    def isEmpty(self, line):
        if  len(line) == 0 \
            or line == "\n" \
            or line[0] == "*" \
            or ((line[0] == " " or line[0] == "\t")
                and line.strip() == ""):

            return True
        else:
            return False

    def parse(self, rules, file):
        self.rules = {rule.Id: rule for rule in rules}
        result = list()

        lineNum = 1
        lines = iter(file)
        # the first line is not allowed to be comment line or empty but must be the header
        self.parseHeader(next(lines), lineNum)

        for line in lines:
            lineNum += 1

            if not self.isEmpty(line):
                try:
                    rule = self.rules[line[0]]
                except KeyError as e:
                    raise ParseError("Unsupported rule '%s'"%(line[0]), line = lineNum)

                try:
                    result.append(rule.parse(line[1:]))
                except parsy.ParseError as e:
                    raise ParseError(e, line = lineNum)
                except ValueError as e:
                    raise ParseError(e, line = lineNum)

        return result

def getCNFConstraintParser():
    space    = parsy.regex(r" +").desc("space")
    literal  = parsy.regex(r"[+-]?[1-9][0-9]*").desc("literal").map(int)
    eol      = parsy.regex(r"0").desc("end of line zero")

    return (literal << space).many() << eol

def getOPBConstraintParser(allowEq = True):
    def lit2int(sign, num):
        if sign == "~":
            return -int(num)
        else:
            return int(num)

    space    = parsy.regex(r" +").desc("space").optional()
    coeff    = parsy.regex(r"[+-]?[0-9]+").map(int).desc("integer for the coefficient (make sure to not have spaces between the sign and the degree value)")
    variable = parsy.seq(parsy.regex(r"~").optional(), parsy.regex(r"x") >> parsy.regex(r"[1-9][0-9]*")).combine(lit2int).desc("variable in the form '~?x[1-9][0-9]*'")
    term     = parsy.seq(space >> coeff, space >> variable).map(tuple)

    if allowEq:
        equality = space >> parsy.regex(r"(=|>=)").desc("= or >=")
    else:
        equality = space >> parsy.regex(r">=").desc(">=")

    degree   = space >> parsy.regex(r"[+-]?[0-9]+").map(int).desc("integer for the degree")

    end      = parsy.regex(r";").desc("termination of rhs with ';'")
    finish   = space >> end

    return parsy.seq(term.many(), equality, degree << finish).map(tuple)

def flatten(constraintList):
    result = list()
    for oneOrTwo in constraintList:
        for c in oneOrTwo:
            result.append(c)
    return result

def getOPBParser():
    numVar = (parsy.regex(r" *#variable *= *") >> parsy.regex(r"[0-9]+")) \
                    .map(int) \
                    .desc("Number of variables in the form '#variable = [0-9]+'")
    numC = (parsy.regex(r" *#constraint *= *") >> parsy.regex(r"[0-9]+")) \
                    .map(int) \
                    .desc("Number of constraints in the form '#constraint = [0-9]+'")

    eol = parsy.regex(" *\n").desc("return at end of line")
    emptyLine = parsy.regex(r"(\s*)").desc("empty line")
    commentLine = parsy.regex(r"(\s*\*.*)").desc("comment line starting with '*'")
    header = (parsy.regex(r"\* ") >> parsy.seq(numVar, numC) << eol) \
                    .desc("header line in form of '* #variable = [0-9]+ #constraint = [0-9]+'")

    nothing = (emptyLine << eol | commentLine << eol).map(lambda x: [])

    constraint = getOPBConstraintParser().bind(refpy.constraints.Inequality.fromParsy) << eol

    return parsy.seq(
            header,
            (nothing | constraint).many().map(flatten)
        )


"""
@ ----------------------------------------------------------------
@ This module will execute all test cases for ota smoke automation.
@ Framework: pytest (version:6.2.1)
@ Python : 3.7.1
@ IDE: Pycharm 202.8194
@ ----------------------------------------------------------------
"""

import sys
sys.path.append('/home/tiankang/Downloads/OtaSmoke/common')
sys.path.append('/home/tiankang/Downloads/OtaSmoke/Mylog')
from logs import GetLogs
import pytest
from common_helper import *
from file_helper import *
import datetime
from time import sleep

mylogs = GetLogs()

class TestOtaSmokeClass(object):

    def setup_class(self):
        """
        Execute one time before run all test cases
        """
        now_time = datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
        mylogs.log_info('--------------------test_4check_extract_folder-----------------------------------')

        mylogs.log_info('Start to execute check extract release at {}'.format(now_time))

    def test_check_extract_release(self):
        """
        Usage: Create doip folder and update.zip before executing udsserver and esyncclient.
        """
        bin_path = '/home/tiankang/Downloads/Excelforepackage/excelfore/esync/bin'
        try:
            extract_result = OtaFileOperate.check_extract_package(bin_path)
            assert extract_result
        except Exception as msg:
            mylogs.log_error('Error occurs while exucuting test launch ota due to {}'.format(msg))
            raise msg

    def teardown_class(self):
        """
        Execute one time after run all test cases
        """
        now_time = datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
        # print('------------------------------------------')
        mylogs.log_info('Check extracted release successfully at {}'.format(now_time))

if __name__ == "__main__":

    try:
        pytest.main(["test_4check_extract_folder.py", "-s", '--html=test_4check_extract_folder.html'])
    except Exception as e:
        raise e
    # finally:
    #     print('ota smoke auto test completed')


# -*- coding: utf-8 -*-
from osv import fields, osv
from tools.translate import _
import openerp.addons.decimal_precision as dp
import time
from openerp.osv import fields, orm

class stock_picking_out(osv.osv):
    _inherit = 'stock.picking.out'

    def write(self, cr, uid, ids, vals, context=None):
        stock_picking_objs = self.browse(cr, uid, ids)
        for stock_picking_obj in stock_picking_objs:
            model_data_class = self.pool.get('ir.model.data')
            warehouse_group_id = model_data_class.get_object(cr, uid, 'stock', 'group_stock_user').id
            accounting_group_id = model_data_class.get_object(cr, uid, 'account', 'group_account_user').id
            user_obj = self.pool.get('res.users').browse(cr, uid, uid)
            user_group_list = []
            for user_group in user_obj.groups_id:
                user_group_list.append(user_group.id)
            if stock_picking_obj.state in ['auto', 'confirmed', 'assigned', 'approved', 'done'] and warehouse_group_id not in user_group_list and accounting_group_id not in user_group_list:
                raise osv.except_osv(_('Error!'), _("You can't edit Delivery Order that in state Waiting another Operation, Waiting Availability, Waiting Approval, Ready to Transfer, Transferred."))
        return super(stock_picking_out, self).write(cr, uid, ids, vals, context=context)

stock_picking_out()


import sys
from pprint import pprint
from collections import deque
sys.stdin = open('./ex1.txt')
origin_loc = []
mat = [[[] for _ in range(4)] for _ in range(4)]
m, s = map(int,sys.stdin.readline().strip().split())

for _ in range(m):
    fx, fy, d = map(int, sys.stdin.readline().strip().split())
    fx, fy, d = fx-1, fy-1, d
    mat[fx][fy] = [(d,s)]
    origin_loc.append((fx,fy,d))
# pprint(mat)
# print(' ')
sx,sy = map(int, sys.stdin.readline().strip().split())
sx, sy = sx-1, sy-1
mat[sx][sy] = 's'
# pprint(mat)
# if sum(mat[0][2]) != 0:
#     mat[0][2].append(5)
# else:
#     mat[0][2] = 5
# print(sum(mat[0][2]))
# pprint(mat)
def change_direction(d):
    d -= 1
    if d == 0:
        d = 8
    return d

def fish_move(mat, fx, fy, d, s):
    """
    물고기 움직임 정의
    ←, ↖, ↑, ↗, →, ↘, ↓, ↙
    1부터 순서대로 idx가짐
    """
    origin_d = str(d)
    dx = [0,0, -1, -1, -1, 0, 1, 1, 1]
    dy = [0,-1, -1, 0, 1, 1, 1, 0, -1]
    state = False
    cnt = 0
    while state == False:
        nx = fx + dx[d]
        ny = fy + dy[d]
        if 0<=nx<4 and 0<=ny<4 and ['s','ds'] not in mat[nx][ny]:
            if len(mat[nx][ny]) != 0:
                mat[nx][ny].append((d, s+1))
                state = True
            else:
                mat[nx][ny] = [(d, s+1)]
                state = True
        if cnt == 8:
            break
        else:
            change_direction(d)
            cnt += 1
    return mat

def shark_move(mat,sx,sy):
    dx = [0,0,1,-1]
    dy = [1,-1,0,0]
    queue = deque((mat,sx,sy))
    result = []
    eat_cnt = 0
    while queue:
        que = queue.popleft()
        for q in que:
            m, x, y = q
            for i in range(len(dx)):
                nx = x + dx[i]
                ny = y + dy[i]

for i in range(len(mat)):
    for j in range(len(mat)):
        # print(mat[i][j])
        # print(list(map(lambda x: x[0], mat[i][j])))
        # print(' ')
        if len(mat[i][j]) > 0:
            if mat[i][j] != 's':
                for f in mat[i][j]:
                    f_cnt = 0

                    while f_cnt < len(mat[i][j]):
                        if f[0] > 0 and f[1] == s:
                            mat = fish_move(mat,i,j,f[0],s)
                            del mat[i][j][f_cnt]
                            f_cnt += 1
                        else:
                            f_cnt += 1

# 상어 움직여야함

pprint(mat)
# a = [[[1, 2],[1, 3]],[[3,4],[4,5]]]
# print(a[0])
# print(list(map(lambda x: x[0], a[1])))


from sklearn.feature_extraction.text import CountVectorizer
import csv
import numpy as np
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
import pandas as pd
from sklearn.metrics import confusion_matrix

if __name__ == '__main__':
    # Reading in the file via csv library
    filepath = 'C:\\Users\\Joash\\Desktop\\University Stuff\\4B uni stuff\\SYDE 522\\522 Project\\SMS_spam_or_ham' \
               '\\spam_result'
    csvfile = open(filepath + '.csv', "rt", encoding="utf8")
    reader = csv.reader(csvfile)
    sms_stemmed = []
    classification = []
    sms = []
    for row in reader:
        if len(row[2]) != 0:
            sms_stemmed.append(row[2])
            sms.append(row[1])
            if row[0] == "spam":
                classification.append(1)
            elif row[0] == "ham":
                classification.append(0)
    sms_stemmed = sms_stemmed[1:]
    sms = sms[1:]
    print(len(sms_stemmed), len(classification))

    hidden_1st = [5,10,20,30,40,50,60,70,80,90,100,200,300,400,500,1000,1500,2000,2500,3000]
    # hidden_1st = [5]
    pre_score = []
    acc_score = []
    scores = []

    from sklearn.feature_extraction.text import TfidfVectorizer

    max_features = 1000
    tfidf = TfidfVectorizer(max_features=max_features)
    x_tfidf = tfidf.fit_transform(sms_stemmed).toarray()
    classification = np.asarray(classification)
    print(type(x_tfidf), x_tfidf.shape, classification.shape)  # 5572 doc, tfidf 100 dimension

    # split into train and test
    X_train, X_test, y_train, y_test = train_test_split(x_tfidf, classification, test_size=0.30, random_state=13)

    kfold = StratifiedKFold(n_splits=4, shuffle=True, random_state=42)

    for i in hidden_1st:
        count = 1
        for train, validate in kfold.split(X_train, y_train):
            print('This is', count, 'fold!')
            model = Sequential()
            model.add(Dense(i, input_shape=(max_features,)))
            model.add(Activation('relu'))
            model.add(Dense(1))
            model.add(Activation('sigmoid'))
            model.summary()
            model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])
            model.fit(X_train[train], y_train[train], batch_size=32, epochs=10, verbose=1)

            test_pred = model.predict(X_train[validate])
            # print(test_pred)
            predicted_classes = np.around(test_pred, decimals=0)
            cm = confusion_matrix(y_train[validate], predicted_classes)
            print(cm)
            accuracy = (cm[0, 0] + cm[1, 1]) / X_train[validate].shape[0]
            precision = cm[0, 0] / (cm[0, 0] + cm[1, 0])
            print('The accuracy and precision of the model is:', round(accuracy, 3), 'and', round(precision, 3))

            acc_score.append(accuracy)
            pre_score.append(precision)
            count += 1

        scores.append([i, (sum(acc_score) / len(acc_score)), (sum(pre_score) / len(pre_score))])

    print(scores)
    scores_pd = pd.DataFrame(scores, columns=['1st Hidden # Neuron', 'Test Accuracy', 'Test Precision'])
    scores_pd.to_csv('Pick Hidden 1st NN.csv', index=False)

    import winsound

    frequency = 2500  # Set Frequency To 2500 Hertz
    duration = 1000  # Set Duration To 1000 ms == 1 second
    winsound.Beep(frequency, duration)


import collections

def found_target(to_find):
    return to_find == 0

def minWindow(s, t):
    target_count = collections.Counter(t)
    start, end, to_find = 0, 0, len(t)
    min_window = ""

    for end in range(len(s)):

        # If we see the target letter, decrease the number of chars to find
        if target_count[s[end]] > 0:
            to_find -= 1

        # Decrease the target count for the current letter
        target_count[s[end]] -= 1

        # if all leetters in the target are found
        while found_target(to_find):
            window_len = end - start + 1
            if not min_window or window_len < len(min_window):
                min_window = s[start:end+1]

            target_count[s[start]] += 1
            if target_count[s[start]] > 0:
                to_find += 1
            start += 1
    return min_window

if __name__ == "__main__":
    s = "ad0becodebanc"
    t = 'bc'
    # s = "a"
    # t = "a"
    print(minWindow(s, t))


import json

import requests

from flask import (
    abort,
    render_template,
    request,
    redirect,
    url_for,
    flash,
    session,
    current_app
)

from flask.ext.login import (
    login_required,
    current_user
)

from forms import (
    ChangeForm,
    ConfirmForm
)

from application.services import (
    post_to_cases,
    is_matched,
    is_owner,
    get_lrid_and_roles,
    get_client_lrid,
    is_allowed_to_see_title,
    view_count_limited
)

from application import (
    app
)

from .utils import get_or_log_error

@app.route('/')
@login_required
def index():
    lrid, roles = get_lrid_and_roles(session)
    return render_template('index.html', roles=roles, lrid=lrid)

@app.route('/property/<title_number>')
@login_required
@view_count_limited
def property_by_title(title_number):
    title_url = "%s/%s/%s" % (
       app.config['AUTHENTICATED_SEARCH_API'],
       'auth/titles',
       title_number)
    app.logger.debug("Requesting title url : %s" % title_url)
    response = get_or_log_error(title_url)
    title = response.json()

    app.logger.debug("Found the following title: %s" % title)
    owner = is_owner(current_user, title_number)
    return render_template(
           'view_property.html',
           title=title,
           is_owner=owner,
           apiKey=app.config['OS_API_KEY'])

# Sticking to convention, "/property/<title_number>" will show the
# resource, and "/property/<title_number>/edit" will show a form
# to edit said resource. Here we go a step further, and limit
# the form to a section on the resource, e.g. "proprietor".
@app.route('/property/<title_number>/edit/title.proprietor.<int:proprietor_index>', methods=['GET', 'POST'])
@login_required
@view_count_limited
def property_by_title_edit_proprietor(title_number, proprietor_index):
    if is_owner(current_user, title_number):
        form = ChangeForm(request.form, marriage_country='GB')
        if request.method == 'GET':
            title = _get_title(title_number)
            app.logger.debug("Found the following title: %s" % title)
            form.title_number.data = title['title_number']

            proprietor = title['proprietorship']['fields']['proprietors'][proprietor_index - 1]
            form.proprietor_full_name.data = proprietor['name']['full_name']

        if form.validate_on_submit():
            if 'confirm' in form and form.confirm.data:

                post_to_cases('change-name-marriage', form.data)
                # TODO handle non-200 responses, and ack accordingly.
                return render_template('acknowledgement.html', form=form)
            else:
                from datatypes.validators.iso_country_code_validator import countries

                country = countries.get(alpha2=form.data['marriage_country']).name
                return render_template('confirm.html', form=ConfirmForm(obj=form.data), country=country)
        return render_template('edit_property.html', form=form)
    else:
        abort(401)

def _get_title(title_number):
    title_url = "%s/%s/%s" % (
        app.config['AUTHENTICATED_SEARCH_API'],
        'auth/titles',
        title_number)
    app.logger.debug("Requesting title url : %s" % title_url)
    response = get_or_log_error(title_url)
    return response.json()

@app.route('/property/<title_number>/changes')
@login_required
@view_count_limited
def changes(title_number):
    if is_owner(current_user, title_number):
        cases_url = app.config['CASES_URL'] + '/cases/property/' + title_number
        app.logger.debug("Requesting cases from %s" % cases_url)
        cases_response = requests.get(cases_url)
        cases = cases_response.json()
        pending = []
        historical_changes_list = {}

        historian_list_url = app.config['HISTORIAN_URL'] + '/titles/' + title_number + '?version=list'
        historian_version_url = app.config['HISTORIAN_URL'] + '/titles/' + title_number + '?version='
        app.logger.debug('requesting history from ' + historian_list_url)
        historian_list_response = requests.get(historian_list_url)
        if historian_list_response:
            #version information put in a list to pass to the template.
            for version in historian_list_response.json()['versions']:
                historian_version_response = requests.get(historian_version_url + version['version_id'])
                historical_changes_list[version['version_id']] = historian_version_response.json()['contents']['last_application']

        for case in cases:
            if case['status'] != 'completed':
                pending.append(case)

        order_by_latest_version_first = list(reversed(sorted(historical_changes_list.keys())))

        return render_template('changes.html', title_number=title_number, pending=pending,
                               historical_changes=historical_changes_list,
                               order_by_latest_version_first=order_by_latest_version_first)
    else:
        abort(401)

@app.route('/property/<title_number>/changes/<version>')
@login_required
@view_count_limited
def change_version(title_number, version):

    historian_version_url = app.config['HISTORIAN_URL'] + '/titles/' + title_number + '?version='
    app.logger.debug('requesting historical version from ' + historian_version_url)
    historian_version_response = requests.get(historian_version_url + version).json()['contents']
    converted_unix_timestamp = historian_version_response['last_application']
    owner = is_owner(current_user, title_number)

    return render_template(
        'view_property.html',
        historical_view='true',
        title=historian_version_response,
        is_owner=owner,
        apiKey=app.config['OS_API_KEY'],
        change_date=converted_unix_timestamp)


from __future__ import absolute_import
from apscheduler.job import Job
from apscheduler.jobstores.base import JobLookupError
from apscheduler.jobstores.mongodb import MongoDBJobStore
from apscheduler.schedulers.base import BaseScheduler, STATE_STOPPED, ConflictingIdError
from apscheduler.triggers.base import BaseTrigger
from flask_apscheduler import APScheduler
from threading import Thread, Event
from apscheduler.util import undefined, datetime_to_utc_timestamp, TIMEOUT_MAX, asbool
from inspect import ismethod, isclass
import six
from apscheduler.util import ref_to_obj, obj_to_ref, get_callable_name, check_callable_args, convert_to_datetime
from hdy_flask_apschnduler.utils import get_host_ip

try:
    from collections.abc import Iterable, Mapping
except ImportError:
    from collections import Iterable, Mapping

try:
    import cPickle as pickle
except ImportError:  # pragma: nocover
    import pickle

try:
    from bson.binary import Binary
    from pymongo.errors import DuplicateKeyError
    from pymongo import MongoClient, ASCENDING
except ImportError:  # pragma: nocover
    raise ImportError('MongoDBJobStore requires PyMongo installed')

"""
说明： 通过继承和调整部分方法，使flask_apschnduler支持分布式，动态修改mongo表任务对应的ip可支持任务自动分发到对应的机器
实现原理： 增加ip字段，判断配置的ip是否和服务器真实ip一致，如果是则可以执行定时任务
局限： 1.基于 Flask-APScheduler==1.12.1 APScheduler==3.7.0 版本的
      2.SCHEDULER_JOBSTORES只支持mongodb的存储，其他类型都不支持
"""

class HdyAPScheduler(APScheduler):
    def __init__(self, scheduler=None, app=None):
        super(HdyAPScheduler, self).__init__(scheduler, app)
        self._scheduler = HdyBackgroundScheduler()

class HdyBaseScheduler(BaseScheduler):
    def add_job(self, func, ip=None, trigger=None, args=None, kwargs=None, id=None, name=None,
                misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
                next_run_time=undefined, jobstore='default', executor='default',
                replace_existing=False, **trigger_args):

        job_kwargs = {
            'trigger': self._create_trigger(trigger, trigger_args),
            'executor': executor,
            'func': func,
            'args': tuple(args) if args is not None else (),
            'kwargs': dict(kwargs) if kwargs is not None else {},
            'id': id,
            'name': name,
            'misfire_grace_time': misfire_grace_time,
            'coalesce': coalesce,
            'max_instances': max_instances,
            'next_run_time': next_run_time,
            'ip': ip
        }
        job_kwargs = dict((key, value) for key, value in six.iteritems(job_kwargs) if
                          value is not undefined)
        job = HdyJob(self, **job_kwargs)

        # Don't really add jobs to job stores before the scheduler is up and running
        with self._jobstores_lock:
            if self.state == STATE_STOPPED:
                self._pending_jobs.append((job, jobstore, replace_existing))
                self._logger.info('Adding job tentatively -- it will be properly scheduled when '
                                  'the scheduler starts')
            else:
                self._real_add_job(job, jobstore, replace_existing)

        return job

class HdyBlockingScheduler(HdyBaseScheduler):
    _event = None

    def start(self, *args, **kwargs):
        if self._event is None or self._event.is_set():
            self._event = Event()

        super(HdyBlockingScheduler, self).start(*args, **kwargs)
        self._main_loop()

    def shutdown(self, wait=True):
        super(HdyBlockingScheduler, self).shutdown(wait)
        self._event.set()

    def _main_loop(self):
        wait_seconds = TIMEOUT_MAX
        while self.state != STATE_STOPPED:
            self._event.wait(wait_seconds)
            self._event.clear()
            wait_seconds = self._process_jobs()

    def wakeup(self):
        self._event.set()

class HdyBackgroundScheduler(HdyBlockingScheduler):
    _thread = None

    def _configure(self, config):
        self._daemon = asbool(config.pop('daemon', True))
        super(HdyBackgroundScheduler, self)._configure(config)

    def start(self, *args, **kwargs):
        if self._event is None or self._event.is_set():
            self._event = Event()

        HdyBaseScheduler.start(self, *args, **kwargs)
        self._thread = Thread(target=self._main_loop, name='APScheduler')
        self._thread.daemon = self._daemon
        self._thread.start()

    def shutdown(self, *args, **kwargs):
        super(HdyBackgroundScheduler, self).shutdown(*args, **kwargs)
        self._thread.join()
        del self._thread

class HdyJob(Job):
    __slots__ = ('ip',)

    def _modify(self, **changes):
        """
        Validates the changes to the Job and makes the modifications if and only if all of them
        validate.

        """
        approved = {}

        if 'id' in changes:
            value = changes.pop('id')
            if not isinstance(value, six.string_types):
                raise TypeError("id must be a nonempty string")
            if hasattr(self, 'id'):
                raise ValueError('The job ID may not be changed')
            approved['id'] = value

        if 'func' in changes or 'args' in changes or 'kwargs' in changes:
            func = changes.pop('func') if 'func' in changes else self.func
            args = changes.pop('args') if 'args' in changes else self.args
            kwargs = changes.pop('kwargs') if 'kwargs' in changes else self.kwargs

            if isinstance(func, six.string_types):
                func_ref = func
                func = ref_to_obj(func)
            elif callable(func):
                try:
                    func_ref = obj_to_ref(func)
                except ValueError:
                    # If this happens, this Job won't be serializable
                    func_ref = None
            else:
                raise TypeError('func must be a callable or a textual reference to one')

            if not hasattr(self, 'name') and changes.get('name', None) is None:
                changes['name'] = get_callable_name(func)

            if isinstance(args, six.string_types) or not isinstance(args, Iterable):
                raise TypeError('args must be a non-string iterable')
            if isinstance(kwargs, six.string_types) or not isinstance(kwargs, Mapping):
                raise TypeError('kwargs must be a dict-like object')

            check_callable_args(func, args, kwargs)

            approved['func'] = func
            approved['func_ref'] = func_ref
            approved['args'] = args
            approved['kwargs'] = kwargs

        if 'name' in changes:
            value = changes.pop('name')
            if not value or not isinstance(value, six.string_types):
                raise TypeError("name must be a nonempty string")
            approved['name'] = value

        if 'misfire_grace_time' in changes:
            value = changes.pop('misfire_grace_time')
            if value is not None and (not isinstance(value, six.integer_types) or value <= 0):
                raise TypeError('misfire_grace_time must be either None or a positive integer')
            approved['misfire_grace_time'] = value

        if 'coalesce' in changes:
            value = bool(changes.pop('coalesce'))
            approved['coalesce'] = value

        if 'max_instances' in changes:
            value = changes.pop('max_instances')
            if not isinstance(value, six.integer_types) or value <= 0:
                raise TypeError('max_instances must be a positive integer')
            approved['max_instances'] = value

        if 'trigger' in changes:
            trigger = changes.pop('trigger')
            if not isinstance(trigger, BaseTrigger):
                raise TypeError('Expected a trigger instance, got %s instead' %
                                trigger.__class__.__name__)

            approved['trigger'] = trigger

        if 'executor' in changes:
            value = changes.pop('executor')
            if not isinstance(value, six.string_types):
                raise TypeError('executor must be a string')
            approved['executor'] = value

        if 'next_run_time' in changes:
            value = changes.pop('next_run_time')
            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone,
                                                            'next_run_time')

        if 'ip' in changes:
            value = changes.pop('ip')
            if not isinstance(value, six.string_types):
                raise TypeError('ip must be a string')
            approved['ip'] = value

        if changes:
            raise AttributeError('The following are not modifiable attributes of Job: %s' %
                                 ', '.join(changes))

        for key, value in six.iteritems(approved):
            setattr(self, key, value)

    def __getstate__(self):
        # Don't allow this Job to be serialized if the function reference could not be determined
        if not self.func_ref:
            raise ValueError(
                'This Job cannot be serialized since the reference to its callable (%r) could not '
                'be determined. Consider giving a textual reference (module:function name) '
                'instead.' % (self.func,))

        # Instance methods cannot survive serialization as-is, so store the "self" argument
        # explicitly
        func = self.func
        if ismethod(func) and not isclass(func.__self__) and obj_to_ref(func) == self.func_ref:
            args = (func.__self__,) + tuple(self.args)
        else:
            args = self.args

        return {
            'version': 1,
            'id': self.id,
            'func': self.func_ref,
            'trigger': self.trigger,
            'executor': self.executor,
            'args': args,
            'kwargs': self.kwargs,
            'name': self.name,
            'misfire_grace_time': self.misfire_grace_time,
            'coalesce': self.coalesce,
            'max_instances': self.max_instances,
            'next_run_time': self.next_run_time,
            'ip': self.ip
        }

    def __setstate__(self, state):
        if state.get('version', 1) > 1:
            raise ValueError('Job has version %s, but only version 1 can be handled' %
                             state['version'])

        self.id = state['id']
        self.func_ref = state['func']
        self.func = ref_to_obj(self.func_ref)
        self.trigger = state['trigger']
        self.executor = state['executor']
        self.args = state['args']
        self.kwargs = state['kwargs']
        self.name = state['name']
        self.misfire_grace_time = state['misfire_grace_time']
        self.coalesce = state['coalesce']
        self.max_instances = state['max_instances']
        self.next_run_time = state['next_run_time']
        self.ip = state['ip']

class HdyMongoDBJobStore(MongoDBJobStore):
    def get_due_jobs(self, now):
        timestamp = datetime_to_utc_timestamp(now)
        return self._get_jobs({'next_run_time': {'$lte': timestamp}, 'ip': self.ip})

    def _get_jobs(self, conditions):
        jobs = []
        failed_job_ids = []
        for document in self.collection.find(conditions, ['_id', 'job_state', 'ip'],
                                             sort=[('next_run_time', ASCENDING)]):
            try:
                jobs.append(self._reconstitute_job(document['job_state']))
            except BaseException:
                self._logger.exception('Unable to restore job "%s" -- removing it',
                                       document['_id'])
                failed_job_ids.append(document['_id'])

        # Remove all the jobs we failed to restore
        if failed_job_ids:
            self.collection.remove({'_id': {'$in': failed_job_ids}})

        return jobs

    def _reconstitute_job(self, job_state, ip=None):
        job_state = pickle.loads(job_state)
        job = HdyJob.__new__(HdyJob)
        if 'ip' not in job_state and ip:
            job_state[ip] = ip
        job.__setstate__(job_state)
        job._scheduler = self._scheduler
        job._jobstore_alias = self._alias
        return job

    @property
    def ip(self):
        if not hasattr(self, '_ip'):
            self._ip = get_host_ip()
        return self._ip

    def add_job(self, job):
        try:
            self.collection.insert_one({
                '_id': job.id,
                'ip': job.ip,
                'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
                'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
            })
        except DuplicateKeyError:
            raise ConflictingIdError(job.id)

    def lookup_job(self, job_id):
        document = self.collection.find_one(job_id, ['job_state', 'ip'])
        return self._reconstitute_job(document['job_state'], document['ip']) if document else None

    def update_job(self, job):
        """
        当replace_existing设置为True时，默认是没有更新ip地址的，如果需要更新ip则在changes中添加:
            'ip': job.ip
        """
        try:
            changes = {
                'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
                'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol)),
                # 'ip': job.ip  # 新加的任务替换时是否替换已有的ip
            }
            result = self.collection.update_one({'_id': job.id}, {'$set': changes})
            if result and result.matched_count == 0:
                raise JobLookupError(job.id)
        except Exception as e:
            print(e)


from flask import Flask, Blueprint, request, jsonify
from flask_restful import Api, Resource
from .models import all_Products, all_Sales, Products, Sales

app = Flask(__name__)
api = Api(app)

product = Products()
sale = Sales()

class All_Products_Endpoint(Resource):

  def post(self):
    data = request.get_json()

    product_id = len(all_Products) + 1
    product_name = data["product_name"]
    product_price = data["price"]
    quantity = data["quantity"]
    category = data["category"]

    response = jsonify(product.create_new_product(product_name, product_id, product_price, quantity, category))
    response.status_code = 201
    return response

  def get(self):
    response = jsonify({"products": product.get_all_Products(), "message":"success"})
    response.status_code = 200
    return response

class One_Product_Endpoint(Resource):

  def get(self, product_id):
    response = jsonify(product.get_one_product(product_id))
    response.status_code = 200
    return response

class All_Sales_Endpoint(Resource):    #fetch all sales records

  def post(self):
    data = request.get_json()

    sale_id = len(all_Sales) + 1
    attendant_name = data["attendant_name"]
    total_worth = data["total_worth"]
    profit = data["profit"]

    response = jsonify(sale.create_new_sale_record(attendant_name,sale_id, total_worth, profit))
    response.status_code = 201
    return response

  def get(self):
    response = jsonify(sale.get_all_Sales())
    response.status_code = 200
    return response

class One_Sale_Endpoint(Resource):      #fetch one sale record

  def get(self, sale_id):
    response = jsonify(sale.get_one_sale(sale_id))
    response.status_code = 200
    return response


#!/usr/bin/env python3
# Use to connect to the Agilent Multimeter
# Coded by David on Jul 19 2019
# Only methods to query ac/dc current or voltage are currently included

#     _         _ _            _     _____ _  _   _  _    ___  _    _
#    / \   __ _(_) | ___ _ __ | |_  |___ /| || | | || |  / _ \/ |  / \
#   / _ \ / _` | | |/ _ \ '_ \| __|   |_ \| || |_| || |_| | | | | / _ \
#  / ___ \ (_| | | |  __/ | | | |_   ___) |__   _|__   _| |_| | |/ ___ \
# /_/   \_\__, |_|_|\___|_| |_|\__| |____/   |_|    |_|  \___/|_/_/   \_\
#         |___/

import serial
import sys

class Agilent34401A():
    def __init__(self, settings={}):
        self.defaults = {'port':'/dev/ttyUSB0',
            'baudrate':9600,
            'lf':'\r\n',
            'timeout': 0.5,
            'mode':'DC'}
        self.shortname = 'Multimeter'
        self.fullname = 'Agilent - 34401A Multimeter'
        self.manual_fname = './zialab/man/'+self.fullname + '.pdf'
        self.platform = sys.platform
        self.notes = '''This to read voltages, currents, and the like.'''
        if settings: # if settings are given, then set it up like so
            self.settings = settings
        else: # else use defaults
            print("Using default settings:")
            print(self.defaults)
            self.settings = self.defaults
        self.baudrate = self.settings['baudrate']
        self.port = self.settings['port']
        self.timeout = self.settings['timeout']
        self.lf = self.settings['lf']
        self.mode = self.settings['mode']
        try:
            print("Setting up the serial connection.")
            self.serialconn = serial.Serial(port=self.port,
                baudrate=self.baudrate,
                timeout=self.timeout)
            print("Setting up the system to remote mode...")
            self.sendtodev('SYST:REMOTE')
            print("Setting up the AC/DC mode...")
            self.sendtodev('CONF:VOLT:%s' % self.mode.upper())
        except:
            print("There was a problem setting up the serial connection.")
    def makecmd(self,command):
        '''composes command according to serial config'''
        return command+self.lf
    def sendtodev(self,command):
        '''sends command through the serial connection'''
        self.serialconn.write(self.makecmd(command).encode())
        return '\n'.join([s.decode()[:-2] for s in self.serialconn.readlines()])
    def get_dc_voltage(self):
        '''get DC voltage'''
        if self.mode.upper() != 'DC':
            print("Changing mode to DC.")
            self.sendtodev('CONF:VOLT:DC')
        return float(self.sendtodev('MEAS:VOLT:DC?'))
    def get_dc_current(self):
        '''get DC current'''
        if self.mode.upper() != 'DC':
            print("Changing mode to DC.")
            self.sendtodev('CONF:CURR:DC')
            self.mode = 'DC'
        return float(self.sendtodev('MEAS:CURR:DC?'))
    def get_ac_voltage(self):
        '''get AC voltage'''
        if self.mode.upper() != 'AC':
            print("Changing mode to AC.")
            self.sendtodev('CONF:VOLT:AC')
        return float(self.sendtodev('MEAS:VOLT:AC?'))
    def get_ac_current(self):
        '''get AC current'''
        if self.mode.upper() != 'AC':
            print("Changing mode to AC.")
            self.sendtodev('CONF:CURR:AC')
            self.mode = 'AC'
        return float(self.sendtodev('MEAS:CURR:AC?'))


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required, user_passes_test
from holidays.models import Holiday
from datetime import date, timedelta

def is_admin(user):
    return user.groups.filter(name='訂餐系統管理者').exists()

def convert_date_format(date_list):
    convert_dates = []
    for the_date in date_list:
        year = the_date.split("-")[0]
        month = the_date.split("-")[1]
        day = the_date.split("-")[2]
        convert_dates.append(month + "/" + day + "/" +year)
    return convert_dates

# 設定假日
@login_required(login_url='/login/')
@user_passes_test(is_admin, login_url='/login/')
def list_Holidays(request, the_year=None):
    if not the_year:
        the_year = date.today().year
    minDate = date(int(the_year), 1, 1) - date.today()
    maxDate = date(int(the_year), 12, 31) - date.today()
    next_year = date.today().year + 1
    this_year = date.today().year
    holidays = []
    if Holiday.objects.filter(holidays__year=the_year):
        for holiday in Holiday.objects.filter(holidays__year=the_year).order_by('holidays'):
            holidays.append(str(holiday))
        holidays = convert_date_format(holidays)

    if 'ok' in request.POST and request.POST['ok'] != '':
        Holiday.objects.filter(holidays__year=the_year).delete()
        input_dates = request.POST['ok'].split(",")
        convert_dates = []
        for i in input_dates:
            year = int(i.split("/")[2])
            month = int(i.split("/")[0])
            day = int(i.split("/")[1])
            convert_dates.append(date(year, month, day))
        for i in convert_dates:
            holiday = Holiday(holidays=i)
            holiday.save()
        return redirect('/holidays/' + str(the_year))
    else:
        return render(
            request,
            'holidays.html', {
                '本年度':this_year,
                '明年度':next_year,
                '年度':the_year,
                '假日': holidays,
                'minDate': minDate.days,
                'maxDate': maxDate.days,})

# 將某一年度重設回只有星期六、日為休假日之狀態
@login_required(login_url='/login/')
@user_passes_test(is_admin, login_url='/login/')
def reset_default_weekends(request):
    if 'the_year' in request.POST:
        if request.POST['the_year']:
            try:
                the_year = int(request.POST['the_year'])
                if len(request.POST['the_year']) != 4:
                    errors = "請輸入4位數字"
                    return render(request, 'reset_default_weekends.html', {'errors': errors})
                existed = Holiday.objects.filter(holidays__year=the_year)
                if existed:
                    existed.delete()
                inyear_days = date(the_year, 12, 31) - date(the_year, 1, 1)
                for i in range(0, int(inyear_days.days)+1):
                    day = date(the_year, 1, 1) + timedelta(days=i)
                    if day.isoweekday() > 5:
                        weekend = Holiday(holidays=day)
                        weekend.save()
                return redirect('/holidays/' + str(the_year))
            except:
                errors = "請輸入數字"
                return render(request, 'reset_default_weekends.html', {'errors': errors})
        else:
            errors = "請輸入年度（4位數字）"
            return render(request, 'reset_default_weekends.html', {'errors': errors})
    else:
        return render(request, 'reset_default_weekends.html')


import argparse

import torch

from admm_manager_v2 import PruningPhase

parser = argparse.ArgumentParser(
    description='Simulate a checkpoint saved at the last step of ADMM pruning')
parser.add_argument("--input_checkpoint", default=None, type=str, required=True,
                    help="Path to the input checkpoint.")
parser.add_argument("--output_checkpoint", default=None, type=str, required=True,
                    help="Path to the output checkpoint.")
args = parser.parse_args()

print(f'loading checkpoint at {args.input_checkpoint}...')
checkpoint = torch.load(args.input_checkpoint, map_location='cpu')
# e.g. 'model', 'admm', 'config', 'optimizer', 'amp', 'timer'
print(f'checkpoint keys: {list(checkpoint.keys())}')
# e.g. {'cur_phase': 'masked_retrain', 'cur_rho': None, 'cur_step': 191999}
print(f'old timer status: {checkpoint["timer"]}')

assert checkpoint['timer']['cur_phase'] == PruningPhase.admm.name

config = checkpoint['config']
print(f'training configurations: \n{config}')

# see: ProximalADMMPruningManager._calc_current_rho
final_rho = config['initial_rho'] * 10 ** (config['rho_num'] - 1)
final_lambda = config['initial_lambda'] * 11 ** (config['rho_num'] - 1)
total_admm_steps = config['admm_steps'] - 1
checkpoint['timer']['cur_rho'] = final_rho
checkpoint['timer']['cur_step'] = total_admm_steps
checkpoint['admm']['rho'] = final_rho
checkpoint['admm']['lambda'] = final_lambda
checkpoint['admm']['rhos'] = { k: final_rho for k in checkpoint['admm']['rhos'] }
checkpoint['admm']['lamdas'] = { k: final_lambda for k in checkpoint['admm']['lamdas'] }

print(f'new timer status: {checkpoint["timer"]}')
print(f'saving checkpoint to {args.output_checkpoint}...')

torch.save(checkpoint, args.output_checkpoint)


import os
import sys
import random
import math
import re
import time
import numpy as np
import tensorflow as tf
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import shutil
import glob
import os.path as osp

# Root directory of the project
ROOT_DIR = os.path.abspath("../../")

# Import Mask RCNN
sys.path.append(ROOT_DIR)  # To find local version of the library
from mrcnn import utils
from mrcnn import visualize_edited as visualize
from mrcnn.visualize_edited import display_images
import mrcnn.model as modellib
from mrcnn.model import log
import skimage.draw
import cv2

# %matplotlib inline

# Directory to save logs and trained model
MODEL_DIR = os.path.join(ROOT_DIR, "logs")

# Local path to trained weights file
COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
# Download COCO trained weights from Releases if needed
if not os.path.exists(COCO_MODEL_PATH):
    utils.download_trained_weights(COCO_MODEL_PATH)

# Path to Shapes trained weights
SHAPES_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_shapes.h5")

from mrcnn.config import Config

class ObjectDetectionConfig(Config):
    """Configuration for training on the buildings and ground detection.
    Derives from the base Config class and overrides some values.
    """
    # Give the configuration a recognizable name
    NAME = "objectDetection"

    # We use a GPU with 12GB memory, which can fit two images.
    # Adjust down if you use a smaller GPU.
    IMAGES_PER_GPU = 2

    # Number of classes (including background)
    NUM_CLASSES = 1 + 5  # Background + building-ground classes

    # Number of training steps per epoch
    STEPS_PER_EPOCH = 500

    # Skip detections with < 90% confidence
    DETECTION_MIN_CONFIDENCE = 0.9

class InferenceConfig(ObjectDetectionConfig):
    # Set batch size to 1 since we'll be running inference on
    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
    GPU_COUNT = 1
    IMAGES_PER_GPU = 1

# my object detection config:
config = InferenceConfig()
data_DIR_images = "F:\\MaskRCNN\\Mask_RCNN\\myproject\\objectDetection\\finalDatasets\\valing\\images\\"
data_DIR_lables = "F:\\MaskRCNN\\Mask_RCNN\\myproject\\objectDetection\\finalDatasets\\valing\\labels\\"
data_DIR_thermals = "F:\\MaskRCNN\\Mask_RCNN\\myproject\\objectDetection\\finalDatasets\\valing\\thermals\\"
data_DIR_predicts = "F:\\MaskRCNN\\Mask_RCNN\\myproject\\objectDetection\\finalDatasets\\valing\\predicts\\"

# Override the training configurations with a few
# changes for inferencing.
class InferenceConfig(config.__class__):
    # Run detection on one image at a time
    GPU_COUNT = 1
    IMAGES_PER_GPU = 1

config = InferenceConfig()

DEVICE = "/cpu:0"  # /cpu:0 or /gpu:0

# Inspect the model in training or inference modes
# values: 'inference' or 'training'
# TODO: code for 'training' test mode not ready yet
TEST_MODE = "inference"

class_names = ['background','building_roof', 'ground_cars', 'building_facade', 'ground_cars', 'building_roof']

# Create model in inference mode
with tf.device(DEVICE):
    model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR,
                              config=config)

weights_path = model.find_last()

# Load weights
print("Loading weights ", weights_path)
model.load_weights(weights_path, by_name=True)

names = [x for x in os.listdir(data_DIR_images) if ".jpg" in x] #new

for name in names:
    name = name.split(".")[0]

    image = skimage.io.imread(data_DIR_images + name+".jpg")
    thermal = skimage.io.imread(data_DIR_thermals + name+".jpg")

    image = np.concatenate((image, thermal), axis=2)

    gt_image = cv2.imread(data_DIR_lables+name+".png")
    # Run object detection
    results = model.detect([image], verbose=1)

    # Display results

    r = results[0]
    pred_image = visualize.return_save(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], title="Predictions")

    gt_image = cv2.cvtColor(gt_image, cv2.COLOR_BGR2RGB)
    pred_image = pred_image[:, :, ::-1]
    # pred_image = np.array(pred_image,dtype=np.uint8)
    # pred_image = cv2.cvtColor(pred_image, cv2.COLOR_BGR2RGB)

    plt.imshow(pred_image)
    plt.show()
    plt.imshow(image)
    plt.show()
    plt.imshow(gt_image)
    plt.show()

    print(gt_image[900,400])
    print(pred_image[1000,400])
    # cv2.imwrite(data_DIR_predicts+name+".png", pred_image)
    skimage.io.imsave(data_DIR_predicts+name+".png", pred_image)


from dash.training_parameters import create_training_params_file
from dash.create_training_set import create_training_set_files
from dash.create_template_set import create_template_set_file
from dash.deep_learning_multilayer import train_model
import zipfile
import os
import shutil
import time

if __name__ == '__main__':
    dataFilenames = []
    classifyHost = True

    # CREATE PARAMETERS PICKLE FILE
    t1 = time.time()
    trainingParamsFilename = create_training_params_file()
    dataFilenames.append(trainingParamsFilename)
    t2 = time.time()
    print("time spent: {0:.2f}".format(t2 - t1))

    # CREATE TRAINING SET FILES
    trainingSetFilename = create_training_set_files(minZ=0, maxZ=0., redshiftPrecision=0.1, trainWithHost=True, classifyHost=classifyHost)
    dataFilenames.append(trainingSetFilename)
    t3 = time.time()
    print("time spent: {0:.2f}".format(t3 - t2))

    # CREATE TEMPLATE SET FILE
    templateSetFilename = create_template_set_file(classifyHost=classifyHost)
    dataFilenames.append(templateSetFilename)
    t4 = time.time()
    print("time spent: {0:.2f}".format(t4 - t3))

    # TRAIN TENSORFLOW MODEL
    modelFilenames = train_model(classifyHost=classifyHost)
    dataFilenames.extend(modelFilenames)
    t5 = time.time()
    print("time spent: {0:.2f}".format(t5 - t4))

    # SAVE ALL FILES TO ZIP FILE
    dataFilesZip = 'data_files_classifyHost_v01.zip'
    with zipfile.ZipFile(dataFilesZip, 'w') as myzip:
        for f in dataFilenames:
            myzip.write(f)

    modelZip = 'model_classifyHost_v01.zip'
    with zipfile.ZipFile(modelZip, 'w') as myzip:
        for f in [dataFilenames[0]] + dataFilenames[2:]:
            myzip.write(f)

    # Delete data_files folder since they are now in the zip files
    # for filename in filenames:
    #     os.remove(filename)