from transformers import (AutoTokenizer, CLIPTokenizer)
from tokenizers.processors import TemplateProcessing
from tokenizers import BertWordPieceTokenizer
from xc.libs.utils import pbar, load_file
from xc.libs.custom_dtypes import save
import scipy.sparse as sp
from tqdm import tqdm
import numpy as np
import argparse
import tempfile
import os
os.environ['TOKENIZERS_PARALLELISM'] = 'true'

def tokens(text, _tokenizer, max_len, batch_size=1024):
    text = _tokenizer.batch_encode_plus(
              text, truncation=True, padding='max_length',
              max_length=max_len, add_special_tokens=True,
              return_tensors="np", return_length=True)
    return text["input_ids"], text["attention_mask"]

def tokenize_corpus(text, _tokenizer, max_len, batch_size=32):
    num_items = len(text)
    indices, masks = [], []
    for start in tqdm(np.arange(0, num_items, batch_size)):
        end = min(start + batch_size, num_items)
        temp = _tokenizer.batch_encode_plus(
                  text[start:end], truncation=True, padding='max_length',
                  max_length=max_len, add_special_tokens=True,
                  return_tensors="np", return_length=True)

        indices.append(np.uint16(temp["input_ids"]))
        masks.append(np.uint16(temp["attention_mask"]))
        del temp
    indices = np.vstack(indices)
    masks = np.vstack(masks)
    return indices, masks

def to_sparse(index, mask, max_vocab):
    num_docs = len(index)
    rows = list(map(lambda x: [x[0]]*x[1].size, enumerate(mask)))
    cols = np.asarray(np.concatenate(index), np.int32)
    rows = np.asarray(np.concatenate(rows), np.int32)
    data = np.asarray(np.concatenate(mask), np.int32)
    smat_data = sp.coo_matrix((data, (rows, cols)),
                              shape=(num_docs, max_vocab))
    smat_data.sum_duplicates()
    smat_data.eliminate_zeros()
    return smat_data.tocsr()

def build_idf(lbl_map, trn_map, _tokenizer, args):
    max_vocab = _tokenizer.vocab_size
    lbl_idx, lbl_msk = tokens(lbl_map, _tokenizer, args)
    trn_idx, trn_msk = tokens(trn_map, _tokenizer, args)
    index = np.vstack([lbl_idx, trn_idx])
    masks = np.vstack([lbl_msk, trn_msk])
    print("Building IDFs")
    s_mat = to_sparse(index, masks, max_vocab)
    _idf = np.log(s_mat.shape[0]+1) - np.log(s_mat.getnnz(axis=0)+1) + 1
    _idf[np.logical_or(_idf == 1, _idf == np.log(s_mat.shape[0]+1)+1)] = 0
    print("# of useless tokens", np.where(
        _idf == 0)[0], np.where(_idf == 0)[0].size)
    idf = sp.diags(_idf, shape=(max_vocab, max_vocab)).tocsr()
    idf.eliminate_zeros()
    sp.save_npz(os.path.join(args.out_dir, "idf.npz"), idf)
    return idf

def build_vocab(args):
    _tokenizer = BertWordPieceTokenizer()
    raw_data = tempfile.NamedTemporaryFile(mode="w+")
    with open(args.trn_map, "r", encoding="latin1") as fp:
        for line in pbar(fp, desc=args.trn_map):
            line = line.split("->", 1)[1].strip()
            raw_data.write(f"{line}\n")

    with open(args.lbl_map, "r", encoding="latin1") as fp:
        for line in pbar(fp, desc=args.lbl_map):
            line = line.strip().split("->", 1)[1]
            raw_data.write(f"{line}\n")
    raw_data.seek(0)
    _tokenizer.train([raw_data.name], vocab_size=args.n_vocab)
    _tokenizer.post_processor = TemplateProcessing(
        single="[CLS] $A [SEP]", pair="[CLS] $A [SEP] $B:1 [SEP]:1",
        special_tokens=[("[CLS]", _tokenizer.token_to_id("[CLS]")),
                        ("[SEP]", _tokenizer.token_to_id("[SEP]"))])
    vocab = dict([(v, k) for (k, v) in _tokenizer.get_vocab().items()])
    num_vocab = len(vocab)
    print("Num Vocab", num_vocab)
    vocab_data = tempfile.NamedTemporaryFile(mode="w+")
    for key in range(num_vocab):
        vocab_data.write("{}\n".format(vocab[key]))
    vocab_data.seek(0)
    token = BertWordPieceTokenizer(
        vocab_data.name, model_max_length=args.max_len,
        bos_token="[CLS]", eos_token="[SEP]",
        unk_token="[UNK]", sep_token="[SEP]",
        pad_token="[PAD]", mask_token="[MASK]")
    raw_data.close()
    vocab_data.close()
    return token

def setup_tokenizer(txt_model):
    if txt_model in ["ClipViT"]:
        _tokenizer = CLIPTokenizer.from_pretrained(
            f"openai/clip-vit-base-patch32", do_lower_case=True)
    elif txt_model in ["bert-tiny", "bert-mini-mnli"]:
        _tokenizer = AutoTokenizer.from_pretrained(
            f"prajjwal1/{txt_model}", do_lower_case=True)
    elif txt_model in ["sentencebert", "custom"]:
        _tokenizer = AutoTokenizer.from_pretrained(
            "sentence-transformers/msmarco-distilbert-cos-v5",
            do_lower_case=True)
    else:
        _tokenizer = AutoTokenizer.from_pretrained(
            "bert-base-uncased", do_lower_case=True)
    # idf = build_idf(args.lbl_map, args.trn_map, _tokenizer, args)
    # _tokenizer.idf = idf
    return _tokenizer

def tokenize(text_map, text_xf, _tokenizer, args):

    if not os.path.exists(text_map):
        print(f"File {text_map} not found!!")
        return
    path = os.path.join(args.raw_dir, text_map)
    text = list(map(lambda x: x.split("->", 1)[1].strip().replace("_", " ").lower(),
                    pbar(open(path, "r", encoding="latin1"), desc=text_map)))
    input_idx, attention = tokens(text, _tokenizer, args.max_len)
    max_vocab = _tokenizer.vocab_size
    if args.txt_model in ["BoW"]:
        s_mat = to_sparse(input_idx, attention, max_vocab)
        s_mat = s_mat.dot(_tokenizer.idf).tocsr()
        s_mat.sort_indices()
        text_xf = text_xf.replace(".seq", ".npz")
        sp.save_npz(text_xf, s_mat)
    else:
        print("Average tokens / documents =",
              (np.sum(attention)-2)/attention.shape[0])
        _tokens = np.stack([input_idx, attention], axis=1)
        save(text_xf, "memmap", _tokens)

def write(tmp_mdata, labels, features):
    print("# labels:", labels.size)
    print("# features:", features.size)
    path = os.path.join(tmp_mdata, 'labels_split.txt')
    np.savetxt(path, labels, fmt='%d')
    path = os.path.join(tmp_mdata, 'features_split.txt')
    np.savetxt(path, features, fmt='%d')

def setup():
    parser = argparse.ArgumentParser("Combine evaluate")
    parser.add_argument('--data_dir', dest='data_dir',
                        action='store', type=str)
    parser.add_argument('--raw_dir', dest='raw_dir',
                        action='store', type=str)
    parser.add_argument('--out_dir', dest='out_dir',
                        action='store', type=str)
    parser.add_argument('--zsh_map', dest='zsh_map', action='store', type=str)
    parser.add_argument('--trn_map', dest='trn_map', action='store', type=str)
    parser.add_argument('--tst_map', dest='tst_map', action='store', type=str)
    parser.add_argument('--lbl_map', dest='lbl_map', action='store', type=str)
    parser.add_argument('--zsh_xf', dest='zsh_xf', action='store', type=str)
    parser.add_argument('--trn_xf', dest='trn_xf', action='store', type=str)
    parser.add_argument('--tst_xf', dest='tst_xf', action='store', type=str)
    parser.add_argument('--lbl_xf', dest='lbl_xf', action='store', type=str)
    parser.add_argument('--trn_y', dest='trn_y', action='store', type=str)

    parser.add_argument('--max_len', dest="max_len", action="store", type=int)
    parser.add_argument('--n_vocab', dest="n_vocab", action="store", type=int)
    parser.add_argument('--txt_model', dest="txt_model",
                        action="store", type=str)
    params = parser.parse_args()

    def apply_conditions(params):
        if params.txt_model == "BoW":
            for _param in ['trn_xf', 'tst_xf', 'lbl_xf', 'zsh_xf']:
                if params.__dict__[_param] is not None:
                    params.__dict__[_param] = params.__dict__[
                        _param].replace(".seq.memmap", ".npz")
        return params
    return apply_conditions(params)

if __name__ == '__main__':
    args = setup()
    print(args)
    tokenizer = setup_tokenizer(args.txt_model)
    tokenize(args.lbl_map, args.lbl_xf, tokenizer, args)
    tokenize(args.trn_map, args.trn_xf, tokenizer, args)
    tokenize(args.tst_map, args.tst_xf, tokenizer, args)
    tokenize(args.zsh_map, args.zsh_xf, tokenizer, args)
    trn_y = load_file(args.trn_y)
    trn_x_xf = load_file(args.trn_xf)
    lbl_x_xf = load_file(args.lbl_xf)
    features = np.arange(tokenizer.vocab_size)
    labels = np.arange(trn_y.shape[1])
    write(args.out_dir, labels, features)


import tensorflow as tf
from tensorflow.python.keras.layers import Embedding
from tensorflow.contrib.rnn import LSTMCell, MultiRNNCell
import itertools

import ipdb

import config

tf.enable_eager_execution()

class Policy:
    """
    Our policy. Takes as input the current state as a list of
    input/feedback indices and outputs the action distribution.
    """

    def __init__(self):
        self.guess_embedding = Embedding(config.max_guesses + 1,
                                         config.guess_embedding_size)
        self.feedback_embedding = Embedding(config.max_feedback + 1,
                                            config.feedback_embedding_size)
        self.lstm = MultiRNNCell([
            LSTMCell(config.lstm_hidden_size),
            LSTMCell(config.lstm_hidden_size)
        ])

        self.dense = tf.layers.Dense(config.max_guesses)

    @property
    def variables(self):
        """Return all the trainable parameters"""
        return [
            *self.guess_embedding.variables,
            *self.feedback_embedding.variables,
            *self.lstm.variables,
            *self.dense.variables
        ]

    @property
    def named_variables(self):
        """Method to ensure variables across different 'Policy'
        instances are named consistently"""
        return dict(zip(map(str, itertools.count()), self.variables))

    def __call__(self, game_state, with_softmax=True):
        """
        Do a forward pass to get the action distribution
        """

        state = self.lstm.zero_state(1, tf.float32)

        for guess, feedback in game_state:
            guess_tensor = tf.reshape(tf.convert_to_tensor(guess), (1,))
            feedback_tensor = tf.reshape(tf.convert_to_tensor(feedback), (1,))
            guess_embedded = self.guess_embedding(guess_tensor)
            feedback_embedded = self.feedback_embedding(feedback_tensor)

            combined_embedded = tf.concat([guess_embedded,
                                            feedback_embedded],
                                            axis=-1)
            # can I do multiple inputs to the LSTM instead of concatenating?

            output, state = self.lstm(combined_embedded, state)

        logits = self.dense(output)
        if with_softmax:
            return tf.nn.softmax(logits)
        return logits

if __name__ == "__main__":
    from episode import Episode
    import numpy as np
    np.random.seed(123)
    p = Policy()
    e = Episode(p, "0000")
    x = p(e.generate())
    print(x.numpy())


from multiprocessing import Pool
from time import sleep,ctime

def work(msg):
    sleep(2)
    print(msg)
    return msg
result= []
#创建进程池对象
pool = Pool(processes = 4)

#向进程池添加事件
for i in range(10):
    msg = "hello %d"%i
    r = pool.apply_async(func = work,args = (msg,))
    result.append(r)
#关闭进程池
pool.close()

#回收进程池

pool.join()
print("=============")
for i in result:
    print(i.get())


from behave import *

from kelpy import job
from kubernetes import config, client
from jinja2 import Environment, FileSystemLoader

@given("a Job called {job_name} does not exist")
def step_impl(context, job_name):
    job.delete(context.k8s_v1_batch_client, job_name)

@given("the Job called {job_name} exists")
@when("the Job called {job_name} is created")
def step_impl(context, job_name):
    job_namespace = "default"

    env = Environment(
        loader=FileSystemLoader("./templates"), trim_blocks=True, lstrip_blocks=True
    )
    job_tmpl = env.get_template("job.j2")
    job_spec_file = job_tmpl.render(job_name=job_name, job_namespace=job_namespace)
    context.create_resp = job.create(
        context.k8s_v1_batch_client, job_spec_file, job_namespace
    )

@when("the user attempts to retrieve the Job {job_name}")
def step_impl(context, job_name):
    context.get_resp = job.get(context.k8s_v1_batch_client, job_name)

@then("None is returned instead of the Job")
def step_impl(context):
    assert context.get_resp is None, "Did not return None"

@then("a valid Job called {job_name} can be found")
def step_impl(context, job_name):
    assert context.create_resp, f"Should've found {job_name} Job"

@then("Results for the Job {job_name} are returned")
def step_impl(context, job_name):
    assert context.get_resp, "Should've returned a valid response"


import ray
from pprint import pprint

# Import placement group APIs.
from ray.util.placement_group import (
    placement_group,
    placement_group_table,
    remove_placement_group
)

ray.init(num_gpus=2, num_cpus=4)

gpu_bundle = {"GPU": 2}
extra_resource_bundle = {"CPU": 2}

# Reserve bundles with strict pack strategy.
# It means Ray will reserve 2 "GPU" and 2 "extra_resource" on the same node (strict pack) within a Ray cluster.
# Using this placement group for scheduling actors or tasks will guarantee that they will
# be colocated on the same node.
pg = placement_group([gpu_bundle, extra_resource_bundle], strategy="SPREAD")

# Wait until placement group is created.
ray.get(pg.ready())

@ray.remote(num_gpus=1)
class GPUActor:
    def __init__(self):
        pass
    def execute(self):
        return 1 + 1

@ray.remote(num_cpus=1)
def extra_resource_task():
    import time
    # simulate long-running task.
    time.sleep(1)

# Create GPU actors on a gpu bundle.
gpu_actors = [GPUActor.options(
        placement_group=pg,
        # This is the index from the original list.
        # This index is set to -1 by default, which means any available bundle.
        placement_group_bundle_index=0) # Index of gpu_bundle is 0.
    .remote() for _ in range(2)]

# Create extra_resource actors on a extra_resource bundle.
extra_resource_actors = [extra_resource_task.options(
        placement_group=pg,
        # This is the index from the original list.
        # This index is set to -1 by default, which means any available bundle.
        placement_group_bundle_index=1) # Index of extra_resource_bundle is 1.
    .remote() for _ in range(2)]

ray.get(gpu_actors[0].execute.remote())
# This API is asynchronous.
remove_placement_group(pg)

# Wait until placement group is killed.
import time
time.sleep(1)
# Check the placement group has died.
pprint(placement_group_table(pg))

"""
{'bundles': {0: {'GPU': 2.0}, 1: {'extra_resource': 2.0}},
'name': 'unnamed_group',
'placement_group_id': '40816b6ad474a6942b0edb45809b39c3',
'state': 'REMOVED',
'strategy': 'STRICT_PACK'}
"""

ray.shutdown()


from argparse import RawDescriptionHelpFormatter
import argparse
import filecmp
import json
import os
import subprocess

# Start of our unit test
# -- main() -------------------------------------------------------------------
#
# The entry point to this script.
#
# Note: It is called at the end of this script so that the other functions
#       and classes have been defined and the syntax validated
def main():
  # -- Configure the argument parser
  parser = argparse.ArgumentParser(formatter_class=RawDescriptionHelpFormatter, description='description:\n  Unit test wrapper for the SmartNic section')
  parser.add_argument('--resource-dir', nargs='?', default=".", help='directory containing data to be used by this unit test')
  args = parser.parse_args()

  # Validate that the resource directory is valid
  if not os.path.exists(args.resource_dir):
      raise Exception("Error: The resource-dir '" + args.resource_dir +"' does not exist")

  if not os.path.isdir(args.resource_dir):
      raise Exception("Error: The resource-dir '" + args.resource_dir +"' is not a directory")

  # Prepare for testing
  xclbinutil = "xclbinutil"

  # Start the tests
  print ("Starting test")

  # ---------------------------------------------------------------------------
  step = "1) Read in a validate JSON file and validate it against the schema"

  inputJSON = os.path.join(args.resource_dir, "smartnic_full_validate_syntax.json")

  cmd = [xclbinutil, "--add-section", "SMARTNIC:JSON:" + inputJSON]
  execCmd(step, cmd)

  # ---------------------------------------------------------------------------
  step = "2) Read in a byte file and validate it was transformed correctly"

  inputJSON = os.path.join(args.resource_dir, "simple_bytefiles.json")
  outputJSON = "simple_bytefiles_output.json"
  expectedJSON = os.path.join(args.resource_dir,"simple_bytefiles_expected.json")

  cmd = [xclbinutil, "--add-section", "SMARTNIC:JSON:" + inputJSON, "--dump-section", "SMARTNIC:JSON:" + outputJSON, "--force"]
  execCmd(step, cmd)

  # Validate that the output file matches expectation
  jsonFileCompare(expectedJSON, outputJSON)

  # ---------------------------------------------------------------------------
  step = "3a) Merging Validation : Adding v++ linker contents to the xclbin"

  vppLinkerJSON = os.path.join(args.resource_dir, "vpp_linker.json")
  vppLinkerXclbin = "vpp_linker.xclbin"

  cmd = [xclbinutil, "--add-merge-section", "SMARTNIC:JSON:" + vppLinkerJSON, "--output", vppLinkerXclbin, "--force"]
  execCmd(step, cmd)

  # ...........................................................................
  step = "3b) Merging Validation : Merging Extension Metadata"

  extensionJSON = os.path.join(args.resource_dir, "extension.json")
  vppPackerXclbin1 = "vpp_packager1.xclbin"

  cmd = [xclbinutil, "--input", vppLinkerXclbin, "--add-merge-section", "SMARTNIC:JSON:" + extensionJSON, "--output", vppPackerXclbin1, "--force"]
  execCmd(step, cmd)

  # ...........................................................................
  step = "3c) Merging Validation : Merging Softhub metadata"

  softhubJSON = os.path.join(args.resource_dir, "softhub.json")
  vppPackerXclbin2 = "vpp_packager2.xclbin"

  cmd = [xclbinutil, "--input", vppPackerXclbin1, "--add-merge-section", "SMARTNIC:JSON:" + softhubJSON, "--output", vppPackerXclbin2, "--force"]
  execCmd(step, cmd)

  # ...........................................................................
  step = "3d) Merging Validation : Merging eBPF"

  ebpfJSON = os.path.join(args.resource_dir, "eBPF.json")
  expectedJSON = os.path.join(args.resource_dir,"vitis_merged_expected.json")
  vppPackerXclbin3 = "vpp_packager3.xclbin"
  outputJSON = "vitis_merged_output.json"

  cmd = [xclbinutil, "--input", vppPackerXclbin2, "--add-merge-section", "SMARTNIC:JSON:" + ebpfJSON, "--dump-section", "SMARTNIC:JSON:" + outputJSON, "--output", vppPackerXclbin3, "--force"]
  execCmd(step, cmd)

  # Validate that the output file matches expectation
  jsonFileCompare(expectedJSON, outputJSON)

  # ---------------------------------------------------------------------------

  # If the code gets this far, all is good.
  return False

def testDivider():
  print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")

def jsonFileCompare(file1, file2):
  if not os.path.isfile(file1):
    raise Exception("Error: The following json file does not exist: '" + file1 +"'")

  with open(file1) as f:
    data1 = json.dumps(json.load(f), indent=2)

  if not os.path.isfile(file2):
    raise Exception("Error: The following json file does not exist: '" + file2 +"'")

  with open(file2) as f:
    data2 = json.dumps(json.load(f), indent=2)

  if data1 != data2:
      # Print out the contents of file 1
      print ("\nFile1 : "+ file1)
      print ("vvvvv")
      print (data1)
      print ("^^^^^")

      # Print out the contents of file 1
      print ("\nFile2 : "+ file2)
      print ("vvvvv")
      print (data2)
      print ("^^^^^")

      raise Exception("Error: The two files are not the same")

def execCmd(pretty_name, cmd):
  testDivider()
  print(pretty_name)
  testDivider()
  cmdLine = ' '.join(cmd)
  print(cmdLine)
  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  o, e = proc.communicate()
  print(o.decode('ascii'))
  print(e.decode('ascii'))
  errorCode = proc.returncode

  if errorCode != 0:
    raise Exception("Operation failed with the return code: " + str(errorCode))

# -- Start executing the script functions
if __name__ == '__main__':
  try:
    if main() == True:
      print ("\nError(s) occurred.")
      print("Test Status: FAILED")
      exit(1)
  except Exception as error:
    print(repr(error))
    print("Test Status: FAILED")
    exit(1)

# If the code get this far then no errors occured
print("Test Status: PASSED")
exit(0)


import matplotlib.pyplot as plt
import numpy as np

class LogReg:
    def __init__(self, data_in, data_out, lamb=0, l_rate=0.001):  # For preprocessing data.
        self.x = np.array(data_in, dtype=np.float64)  # data_in=training input, data_out=training output
        self.y = np.array(data_out, dtype=np.float64)
        self.m = int(data_in.shape[0])  # No. of training exapmles
        self.lamb = int(lamb)
        self.j = int(np.size(self.y) / self.y.shape[1])
        self.l_rate = float(l_rate)
        try:
            self.col1 = int(data_in.shape[1]) + 1  # No. of features plus bias
        except IndexError:
            self.col1 = 2  # For one feature training set

        try:
            one = np.ones((1, self.m), dtype=np.float64)
            self.x = np.ndarray.flatten(self.x, order='C')
            self.x = np.concatenate((one, self.x.T), axis=0)
            self.x = np.reshape(self.x, (self.col1, self.m))
        except:
            one = np.ones(self.m, dtype=np.float64)
            self.x = np.concatenate((one, self.x), axis=0)
            self.x = np.reshape(self.x, (self.col1, self.m))

        self.wt1 = np.zeros((self.j, self.col1), dtype=np.float64)  # defining weights matrix OR array
        self.wt2 = np.zeros((self.j, self.col1), dtype=np.float64)
        self.wt3 = np.zeros((self.j, 1), dtype=np.float64)
        self.temp = np.zeros(self.col1, dtype=np.float64)
        self.x_prod = np.prod(self.x, axis=0)
        self.x_prod = np.reshape(self.x_prod, (1, self.m))

    def y_pred_lin(self):  # simple linear hypothesis
        t = np.empty((self.j, self.m), dtype=np.float64)
        t = np.matmul(self.wt1, self.x)
        t = t * (-1)
        return 1 / (1 + np.exp(t))

    def y_pred_quad(self):
        t = np.matmul(self.wt1, self.x) + np.matmul(self.wt2, np.square(self.x)) + np.matmul(self.wt3, self.x_prod)
        return 1 / (1 + np.e ** -t)

    def cost_quad(self):
        return sum(
            (-np.dot(self.y, np.log(self.y_pred_quad()).T)) - np.dot((1 - self.y),
                                                                     np.log(1 - self.y_pred_quad()).T)) + np.sum(
            np.square(self.wt1)) * self.lamb / 2 / self.m + np.sum(
            np.square(self.wt2)) * self.lamb / 2 / self.m + np.sum(
            np.square(self.wt3)) * self.lamb / 2 / self.m

    def cost_lin(self):
        t1 = (-np.dot(self.y, np.log(self.y_pred_lin()).T))
        t2 = - np.dot((1 - self.y), np.log(1 - self.y_pred_lin()).T)
        t3 = (np.square(self.wt1)) * self.lamb / 2 / self.m
        return sum(t1 + t2) + np.sum(t3)

    def update_lin(self):
        temp = np.dot((self.y_pred_lin() - self.y), self.x.T) / self.m + self.wt1 * self.lamb / self.m
        self.wt1 = self.wt1 - self.l_rate * temp

    def update_quad(self):
        t1 = np.dot((self.y_pred_quad() - self.y), self.x.T) / self.x.shape[1]
        t2 = np.dot((self.y_pred_quad() - self.y), np.square(self.x.T)) / self.x.shape[1]
        t3 = np.dot((self.y_pred_quad() - self.y), np.square(self.x_prod.T)) / self.x.shape[1]
        temp1 = t1 + self.wt1 * self.lamb / self.x.shape[1]
        temp2 = t2 + self.wt2 * self.lamb / self.x.shape[1]
        temp3 = t3 + self.wt3 * self.lamb / self.x.shape[1]
        self.wt1 = self.wt1 - self.l_rate * temp1
        self.wt2 = self.wt2 - self.l_rate * temp2
        self.wt3 = self.wt3 - self.l_rate * temp3

    def train_model_lin(self, epoch):
        for i in range(epoch):
            self.y_pred_lin()
            self.cost_lin()
            self.update_lin()
        return self.wt1

    def batch_train(self, epoch):
        n = int(self.m / 50)
        self.x_split = np.array_split(self.x, n, axis=1)
        self.y_split = np.array_split(self.y, n, axis=1)
        for j in range(epoch):
            for i in range(n):
                self.x = np.array(self.x_split[i])
                self.y = np.array(self.y_split[i])
                self.train_model_lin(5)

    def train_model_quad(self, epoch):
        for i in range(epoch):
            self.y_pred_quad()
            self.cost_quad()
            self.update_quad()
        print(self.wt1)
        print(self.wt2)
        print(self.wt3)

    def test_model(self, test_data_in, test_data_out):
        self.x_test = np.array(test_data_in,
                               dtype=np.float64)  # test_data_in=training input, test_data_out=training output
        self.y_test = np.array(test_data_out, dtype=np.float64)
        self.m2 = int(test_data_in.shape[0])  # No. of training exapmles
        self.j2 = int(np.size(self.y_test) / self.y_test.shape[1])
        try:
            self.col2 = int(test_data_in.shape[1]) + 1  # No. of features plus bias
        except IndexError:
            self.col2 = 2  # For one feature training set

        try:
            one = np.ones((1, self.m2), dtype=np.float64)
            self.x_test = np.ndarray.flatten(self.x_test, order='C')
            self.x_test = np.concatenate((one, self.x_test.T), axis=0)
            self.x_test = np.reshape(self.x_test, (self.col2, self.m2))
        except:
            one = np.ones(self.m2, dtype=np.float64)
            self.x_test = np.concatenate((one, self.x_test), axis=0)
            self.x_test = np.reshape(self.x_test, (self.col2, self.m2))
        self.x_prod_test = np.prod(self.x_test, axis=0)
        self.x_prod_test = np.reshape(self.x_prod_test, (1, self.m2))

    def y_test_pred(self):
        try:
            t = np.matmul(self.wt1, self.x_test)
        except:
            t = np.matmul(self.wt1, self.x_test) + np.matmul(self.wt2, np.square(self.x_test)) + np.matmul(self.wt3,
                                                                                self.x_prod_test)
        return 1 / (1 + np.e ** -t)

    def test_accu(self):
        accu = -1 * np.mean(np.abs(self.y_test_pred() - self.y_test)) * 100 + 100
        print(accu)
        return accu

    def model_accu_vs_epoch(self):
        epoch = np.arange(100, 10000, 500)
        accu = np.zeros(epoch.shape[0], dtype=float)
        try:
            for i in range(epoch.shape[0]):
                self.train_model_quad(epoch[i])
                accu[i] = np.mean(np.abs(self.y_pred_quad() - self.y)) * 100
        except:
            for i in range(epoch.shape[0]):
                self.train_model_lin(epoch[i])
                accu[i] = np.mean(np.abs(self.y_pred_lin() - self.y)) * 100

        plt.gcf().canvas.set_window_title('Learning_Curve')
        plt.xlabel = 'Epoch'
        plt.ylabel = 'Accuracy'
        plt.plot(epoch, accu)


import os
os.chdir(r"C:\Users\Satyam\Desktop\Programs\Learning tensorflow\mrf")

import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
sns.set(style='darkgrid')
#Number_of_rows = int(input("How many number of rows to be displayed"))
Number_of_rows = 5

pd.options.display.max_rows = Number_of_rows
my_data = pd.read_csv('modifiedstock.csv')
#Run the following line if running for first the Year column is not there
# my_data = my_data.assign(Year = my_data.Data.apply(lambda g: '20'+g[-2:]))
indexes = list(my_data.head(0))

#Call this functions to get graphs
def visualizedata():
	#Graph between Opening Price and High Price
	sns.lmplot(x = 'Open Price', y = 'High Price', data = my_data,fit_reg = False,hue = 'Year')
	plt.title("Open Price vs High Price")
	plt.show()
	plt.close('all')

	sns.lmplot(x = 'Open Price', y = 'Close Price', data = my_data,fit_reg = False,hue = 'Year')
	plt.title("Open Price vs Close Price")
	plt.show()
	plt.close('all')

	sns.lmplot(x = 'Open Price', y = 'Low Price', data = my_data,fit_reg = False,hue = 'Year')
	plt.title("Open Prive vs Low Price")
	plt.show()
	plt.close('all')

OpenPrice,HighPrice,LowPrice,ClosePrice = pd.DataFrame(my_data['Open Price']),my_data['High Price'],my_data['Low Price'],my_data['Close Price']
#Number of features
n = 1
#Number of samples = len(Open Price)
m = len(OpenPrice)

#Model
def Trainthemodel(x_data,y_data,numberofsteps):
	x = tf.placeholder(tf.float64,shape=[None,n])
	y_true = tf.placeholder(tf.float64,shape = None)

	with tf.name_scope('inference') as scope:
		w = tf.Variable([[1.2]],dtype = tf.float64,name = 'weights')
		bias = tf.Variable(0,dtype = tf.float64,name='bias')
		y_pred = tf.matmul(w,tf.transpose(x)) + bias

	with tf.name_scope('loss') as scope:
		loss = tf.reduce_mean(tf.square(y_true-y_pred))

	with tf.name_scope('train') as scope:
		learning_rate = 0.00000000001
		optimizer = tf.train.GradientDescentOptimizer(learning_rate)
		train = optimizer.minimize(loss)

	#This many steps will be taken by the linear regression.
	init = tf.global_variables_initializer()
	with tf.Session() as sess:
		sess.run(init)
		for step in range(numberofsteps):
			sess.run(train,{x:x_data,y_true:y_data})
		return sess.run(bias),sess.run(w)

op = np.array(OpenPrice)
bh,wh = Trainthemodel(np.array(OpenPrice),np.array(HighPrice),1000)
bl,wl = Trainthemodel(np.array(OpenPrice),np.array(LowPrice),1000)
bc,wc = Trainthemodel(np.array(OpenPrice),np.array(ClosePrice),1000)

def PredictHigh(P):
	return P*wh[0][0] + bh

def PredictLow(P):
	return P*wl[0][0] + bl

def PredictClose(P):
	return P*wc[0][0] + bc

def Accuracy(truearray,predarray):
	return (1 - sum((predarray - truearray)**2)/sum((truearray - truearray.mean())**2))*100

def visualizepredictions():
	hp = np.array(HighPrice)
	cp = np.array(ClosePrice)
	lp = np.array(LowPrice)
	#Predicting high price;
	predh = hp*wh[0] + bh
	predc = cp*wc[0] + bc
	predl = lp*wl[0] + bl
	plt.scatter(x = range(m), y = HighPrice, s =10,c = 'blue', alpha = 0.5)
	plt.scatter(x = range(m), y = predh,s = 10, c = 'red', alpha = 0.5)
	plt.legend(["Actual High Price","High Price Predicted on Open Price"])
	plt.title("Accuracy of the model is {0:.2f}".format(Accuracy(hp,predh)))
	plt.show()

	plt.scatter(x = range(m), y = ClosePrice, s = 10, c ='blue', alpha = 0.5)
	plt.scatter(x = range(m), y = (cp*wc[0]+bc), s =10 , c = 'red', alpha = 0.5)
	plt.legend(['Actual Close Price','Close Price Predicted on Open Price'])
	plt.title("Accuracy of the model is {0:.2f}".format(Accuracy(cp,predc)))
	plt.show()

	plt.scatter(x = range(m), y = LowPrice, s = 10, c = 'blue', alpha = 0.5)
	plt.scatter(x = range(m), y = (lp*wl[0]+bl), s = 10, c = 'red', alpha = 0.5)
	plt.legend(['Actual Low Price','Low Price Predicted on Open Price'])
	plt.title("Accuracy of the model is {0:.2f}".format(Accuracy(lp,predl)))
	plt.show()

def Main():
	while True:
		print("Enter The Open Price to Get todays High Price, Low Price, Close Price")
		P = float(input())
		print("High Price ", PredictHigh(P))
		print("Low Price ", PredictLow(P))
		print("Close Price ", PredictClose(P))
		print("Run Again?(Y/y)")
		ch = input()
		if ch != 'Y' and ch != 'y':
			break

Main()


import asyncio
import json

import nextcord
from nextcord.ext import commands
import tweepy
import time
loop = ""

from config import API_SECRECT, API_KEY, BEARER_TOKEN, account_list, DISCORD_TOKEN

TESTING_GUILD_ID = 721374983837974570  # Replace with your guild ID
intents = nextcord.Intents.default()
intents.message_content = True
bot = commands.Bot(intents=intents)

@bot.event
async def on_ready():
    print(f'We have logged in as {bot.user}')

@bot.slash_command(description="list of all accounts followed by bot", guild_ids=[TESTING_GUILD_ID])
async def list(interaction: nextcord.Interaction):
    with open("list.json") as f:
        jsondata = json.load(f)
    f.close()
    if str(interaction.guild.id) not in jsondata.keys():
        jsondata[str(interaction.guild.id)] = []
    if jsondata[str(interaction.guild.id)] == []:
        with open("list.json","w") as f:
            json.dump(jsondata,f)
        f.close()
        await interaction.channel.send("It's seems there aren't any followed accounts in your server. Add one with /addaccount")
        await interaction.response.send_message("here you are")
        return
    embed = nextcord.Embed(title="Followed accounts", description="Bot will send tweets from theese accounts")
    account_list = jsondata[str(interaction.guild.id)]
    for account in account_list:
        user = api.get_user(screen_name=account)
        embed.add_field(name=user.name, value=f"[@{user.screen_name}](https://twitter.com/{user.screen_name})",
                        inline=False)
    await interaction.channel.send(embed=embed)
    await interaction.response.send_message("here you are",ephermal=True)
    return 0

@commands.is_owner()
@bot.slash_command(description="Add new account to follow",guild_ids=[TESTING_GUILD_ID])
async def addaccount(interaction:nextcord.Interaction, screen_name:str):

    with open("list.json") as f:
        jsondata = json.load(f)
    f.close()
    if not str(interaction.guild.id) in jsondata.keys():
       jsondata[str(interaction.guild.id)] = []
    if screen_name in jsondata[str(interaction.guild.id)]:
        m = await interaction.send("This user is alredy followed")
        await m.delete(delay=2)
        return
    try:
        user = api.get_user(screen_name=screen_name)

    except:
        await interaction.send(f"Cannot find user:{screen_name}")
        return
    embed = nextcord.Embed(title="Adding new account")
    embed.add_field(name="username", value=user.screen_name)
    embed.add_field(name="name", value=user.name, inline=False)
    if user.description:
        embed.add_field(name="description", value=user.description, inline=False)
    message = await interaction.channel.send(embed=embed)

    await message.add_reaction(emoji="✅")
    await message.add_reaction(emoji="❌")

    if(not screen_name in account_list):
        stream.add_rules(tweepy.StreamRule(f"from: {screen_name}"))
    account_list.add(screen_name)
    temp_list.append(screen_name)
    with open("list.json", "w") as f:
        jsondata[str(str(interaction.guild.id))].append(user.screen_name)
        json.dump(jsondata, f)
    f.close()
@commands.is_owner()
@bot.slash_command(description="stop follow accaunt", guild_ids=[TESTING_GUILD_ID])
async def removeaccount(interaction: nextcord.Interaction, screen_name: str):

    with open("list.json") as f:
        jsondata = json.load(f)
    f.close()
    if not str(interaction.guild.id) in jsondata.keys():
        jsondata[str(interaction.guild.id)] = []
    if screen_name not in jsondata[str(interaction.guild.id)]:
        m = await interaction.send("This server doesn't follow this accaunt")
        await m.delete(delay=2)
        return
    try:
        user = api.get_user(screen_name=screen_name)
        print(user.id)
    except:
        await interaction.send(f"Cannot find user:{screen_name}")
        return
    embed = nextcord.Embed(title="Adding new account")
    embed.add_field(name="username", value=user.screen_name)
    embed.add_field(name="name", value=user.name, inline=False)
    if user.description:
        embed.add_field(name="description", value=user.description, inline=False)
    message = await interaction.channel.send(embed=embed)

    await message.add_reaction(emoji="✅")
    await message.add_reaction(emoji="❌")

    temp_list.remove(user.screen_name)

    global account_list
    account_list = set(temp_list)

    with open("list.json", "w") as f:
        jsondata[str(str(interaction.guild.id))].remove(user.screen_name)

        json.dump(jsondata, f)
    f.close()
    if not user.screen_name in account_list:
        for rule in stream.get_rules()[0]:
            if(rule[0] == f"from: {user.screen_name}"):
                stream.delete_rules(rule)

with open("list.json") as f:
    jsondata = json.load(f)
f.close()
temp_list =[]
account_list = []

for guild in jsondata.keys():
    for account in jsondata[guild]:
        temp_list.append(str(account))

account_list = set(temp_list)
auth = tweepy.OAuth1UserHandler(API_KEY, API_SECRECT)
api = tweepy.API(auth)

class MyStream(tweepy.StreamingClient):

    # This function gets called when the stream is working
    def on_connect(self):
        print("Connected")

    # This function gets called when a tweet passes the stream
    def on_tweet(self, tweet):
        # Displaying tweet in console
        if tweet.referenced_tweets == None:

            print(tweet.text)

            #send_tweet(tweet)

            for guild in bot.guilds:
                channel = nextcord.utils.get(guild.channels, name="political_bot")

                user = api.get_user(id=tweet.author_id)

                bot.loop.create_task(channel.send(f"https://twitter.com/{user.screen_name}/status/{tweet.id}"))
            time.sleep(0.5)

intents = nextcord.Intents.default()
intents.message_content = True
loop = ""

auth = tweepy.OAuth1UserHandler(API_KEY, API_SECRECT)
api = tweepy.API(auth)

stream = MyStream(bearer_token=BEARER_TOKEN)

def get_stream():
    return stream
print(account_list)

# for rule in stream.get_rules()[0]:
#     stream.delete_rules(rule)

for user in account_list:

    stream.add_rules(tweepy.StreamRule(f"from: {user}"))

# y = threading.Thread(target=lambda: stream.filter(is_async=True))
# y.start()

thread = stream.filter(threaded=True,tweet_fields=['author_id'])

bot.run(DISCORD_TOKEN)


from src import worker
import click
import time

@click.command()
@click.option('--address', default='localhost', help='Client address')
@click.option('--port', default=5560, type=int, help='Client port')
@click.option('--sleep_time', default=1, type=float, help='Sleep time amount')
def main(address: str, port: int, sleep_time: float):
    def length(message):
        return len(message)

    def same(message):
        time.sleep(sleep_time)
        return message

    def sleep(message):
        time.sleep(sleep_time)
        return message

    w = worker.Worker(same, address=address, port=port)
    w.run()

if __name__ == '__main__':
    main()


import sys
import datetime as dt
import numpy as np
import pandas as pd
import pdb
import csv
import logging

# types
from ibapi.common import *
from ibapi.order_condition import *
from ibapi.contract import *
from ibapi.order import *
from ibapi.order_state import *
from ibapi.execution import Execution
from ibapi.execution import ExecutionFilter
from ibapi.commission_report import CommissionReport
from ibapi.scanner import ScannerSubscription
from ibapi.ticktype import *

from App import TestApp

class Request:
    def __init__(self, symbol, callback):
        self.symbol = symbol
        self.callback = callback

global nextReqId, requestDict, fileName, contractDict

nextReqId = 1000
requestDict = {}
fileName = "contracts.csv"
contractDict = {}

logger = logging.getLogger('Contracts')
logger.setLevel(logging.DEBUG)

def loadDict():
    if len(contractDict) == 0:
        with open(fileName, mode='r') as fin:
            reader = csv.reader(fin)
            for row in reader:
                contract = Contract()
                contract.symbol = row[0]
                contract.secType = row[1]
                contract.currency = row[2]
                contract.primaryExchange = row[3]
                contract.exchange = row[4]
                contractDict[contract.symbol] = contract

def writeToDict(contract: Contract):
    contractDict[contract.symbol] = contract
    logger.debug("Saving contract: symbol:%s, secType:%s, currency:%s, primaryExchange:%s, exchange:%s",
        contract.symbol,
        contract.secType,
        contract.currency,
        contract.primaryExchange,
        contract.exchange
    )
    with open(fileName, mode='a') as fout:
        writer = csv.writer(fout)
        writer.writerow([
            contract.symbol,
            contract.secType,
            contract.currency,
            contract.primaryExchange,
            contract.exchange
        ])

def request(symbol, callback = None):
    loadDict()
    contract = contractDict.get(symbol)
    if contract != None:
        if callback != None:
            callback(contract)
        return

    logger.debug("Contract not found in local cache. Requesting contract for %s", symbol)
    nextReqId = nextReqId + 1
    req = Request(symbol, callback)
    requestDict[nextReqId] = req
    TestApp.Instance().reqMatchingSymbols(nextReqId, symbol)

def resolve(reqId, contractDescriptions: ListOfContractDescription):
    req = requestDict[reqId]
    if req != None:
        count = len(contractDescriptions)
        if count == 0:
            logger.debug("Did not find contracts for %s", req.symbol)
        elif count == 1:
            logger.debug("Exactly one contract found for %s", req.symbol)
            contract = contractDescriptions[0].contract
            requestDetail(reqId, contract)
        else:
            logger.debug("%d contracts found for %s", count, req.symbol)
            for cd in contractDescriptions:
                contract = cd.contract
                if contract.currency == 'USD':
                    logger.debug("Using first USD contract")
                    logger.debug("Contract: conId:%s, symbol:%s, secType:%s, currency:%s, primaryExchange:%s",
                        contract.conId,
                        contract.symbol,
                        contract.secType,
                        contract.currency,
                        contract.primaryExchange
                    )
                    requestDetail(reqId, contract)
                    break

def requestDetail(reqId, contract:Contract):
    TestApp.Instance().reqContractDetails(reqId, contract)

def resolveDetail(reqId, contract:ContractDetails):
    req = requestDict[reqId]
    if req != None:
        writeToDict(contract.summary)
        if req.callback != None:
            req.callback(contract.summary)


import requests
from base64 import b64encode

AUTHORIZE_URL = "https://api.enphaseenergy.com/oauth/authorize"
TOKEN_URL = "https://api.enphaseenergy.com/oauth/token"
DEFAULT_REDIRECT_URI = "https://api.enphaseenergy.com/oauth/redirect_uri"

def create_auth_header_dict(auth_string):
	header_string = f"Basic {auth_string}"
	return {"Authorization" : header_string}

class EnphaseClient:
	def __init__(self, clientID, clientSecret, redirectURI=DEFAULT_REDIRECT_URI):
		self.clientID = clientID
		self.clientSecret = clientSecret
		self.redirectURI = redirectURI

	def _generate_client_info_b64(self):
		concatenated_client_info = f"{client_id}:{client_secret}"
		client_info_bytes = concatenated_client_info.encode("utf-8")
		client_info_b64 = b64encode(client_info_bytes).decode("utf-8")
		return client_info_b64

	def generate_authorization_url(self):
		return f"{AUTHORIZE_URL}?response_type=code&client_id={self.clientID}&redirect_uri={self.redirectURI}"

	def generate_tokens(self, code):
		client_info_b64 = self._generate_client_info_b64()
		headers = create_auth_header_dict(client_info_b64)

		params = {
			"grant_type" : "authorization_code",
			"redirect_uri" : self.redirectURI,
			"code" : code,
		}

		tokens_response = requests.post(TOKEN_URL, params=params, headers=headers)
		tokens = tokens_response.json()
		return tokens

if __name__ == "__main__":
	client_id = input("Enter client id: ")
	client_secret = input("Enter client secret: ")

	client = EnphaseClient(client_id, client_secret)
	url = client.generate_authorization_url()

	print(f"Visit this URL to authorize the script: {url}")

	# Open this URL in the browser and get the code from teh web
	code = input("Enter the code from the authorization page:")
	tokens = client.generate_tokens(code)
	access_token = tokens["access_token"]
	print(access_token)

	# Future calls use the access token in the basic auth header
	# headers = create_auth_header_dict(access_token)

	# requests.get("whatever", headers=headers)


from django.urls import path
from django.contrib.auth.decorators import login_required
from .views import *

urlpatterns = [
    path('signup/', SignUpView.as_view(), name='signup'),
    path('user/<slug:slug>', login_required(ProfileView.as_view()), name='profile'),
    path('user/<slug:slug>/edit', login_required(EditProfileView.as_view()), name='edit_profile'),
    path('add_friend', AddFriendView.as_view(), name='add_friend'),
    path('friends', login_required(FriendsListView.as_view()), name='friends_list'),
    path('accept_friend_request', AcceptFriendshipView.as_view(), name='accept_friend_request'),
    path('reject_friend_request', RejectFriendshipView.as_view(), name='reject_friend_request'),
    path('cancel_friend_request', CancelFriendshipView.as_view(), name='cancel_friend_request'),
    path('remove_friend', RemoveFriendView.as_view(), name='remove_friend'),
    path('check_friend_requests', CheckFriendRequestsView.as_view(), name='check_friend_requests'),
    path('find_friends', login_required(FindFriendsView.as_view()), name='find_friends')
]


from PIL import Image
from pylab import * #import pylab中的所有模块、函数
pil_im = Image.open('empire.jpg')
pil_im_gray=Image.open('empire.jpg').convert('L') #将图像变为灰度图
figure()    #创建窗口
subplot(131)    #窗口划分为1行3列，在第1列绘图
imshow(pil_im)
subplot(132)    #在第2列绘图
imshow(pil_im_gray,cmap='gray') #图像以gray方式显示，请试试删除cmap参数，看看有什么区别
subplot(133)    #在第3列绘图
#从pil_im中剪切一块，旋转后粘贴到pil_im上，最后显示pil_im
box = (100,100,400,400)
region = pil_im.crop(box)
region = region.transpose(Image.ROTATE_180)
pil_im.paste(region,box)
imshow(pil_im)
axis('off') #第3列不显示axis
show()  #显示窗口


# Given an array nums of n integers, are there elements a, b, c in nums such that a + b + c = 0? Find all unique triplets in the array which gives the sum of zero.
#
# Notice that the solution set must not contain duplicate triplets.
#
#
#
# Example 1:
#
# Input: nums = [-1,0,1,2,-1,-4]
# Output: [[-1,-1,2],[-1,0,1]]
# Example 2:
#
# Input: nums = []
# Output: []
# Example 3:
#
# Input: nums = [0]
# Output: []
#
#
# Constraints:
#
# 0 <= nums.length <= 3000
# -10^5 <= nums[i] <= 10^5
from typing import List

class Solution:
    def threeSum(self, nums: List[int]) -> List[List[int]]:
        # suppose [a,b,c] is a valid triple, and a<=b<=c
        # then, either a=b=c=0, or we must have a<0 and c>0
        res = []
        from collections import defaultdict
        count = defaultdict(int)
        pos, neg = set(), set()
        for num in nums:
            count[num] += 1
            pos.add(num) if num > 0 else (neg.add(num) if num != 0 else ())
        if 0 in count and count[0] >= 3:
            res.append([0, 0, 0])
        for a in neg:
            for c in pos:
                b = -a - c
                # remember that a<=b<=c, b should also be in count
                if b < a or b > c or b not in count:
                    continue
                # b is in count, but there are not enough b
                if (b == a or b == c) and count[b] < 2:
                    continue
                res.append([a, b, c])
        return res

print(Solution().threeSum([-1, 0, 1, 2, -1, -4]))


import base64

from src.models.api_client import ApiClientModel

def get_api_client_from_form_data(event):
    form = event['body']
    client_id = form['client_id']
    client_secret = form['client_secret']
    client = ApiClientModel.get_by_client_data(client_id, client_secret)
    return client

def get_api_client_from_headers(event):
    headers = event['headers']

    if 'Authorization' not in headers:
        return False

    authorization = headers['Authorization']

    b64_token = authorization.split(' ')[-1]

    # decode the base64 encoded header value
    client_id, client_secret = base64.b64decode(b64_token).decode("ascii").split(':')

    # check if the given api key actually exists
    client = ApiClientModel.get_by_client_data(client_id, client_secret)

    return client


import warnings
import matplotlib.pyplot as plt
import numpy as np
import os

from stable_baselines import PPO2
from stable_baselines.common import make_vec_env
from headway import *

# numpy warnings because of tensorflow
warnings.filterwarnings("ignore", category=FutureWarning, module='tensorflow')
warnings.filterwarnings("ignore", category=FutureWarning, module='tensorboard')
warnings.filterwarnings("ignore", category=UserWarning, module='gym')

target_days = 150
max_loss = 250000
secured_percentage = .08
balance = 0

def make_env():
    return Headway(gamma=1, warmup_steps=0, max_loss=max_loss, single_position=False, trailing_end=False, debug=0)

env = make_vec_env(make_env, n_envs=16)
model = PPO2.load("headway_model")

obs = env.reset()
domain = obs.shape[0]
state = None
done = [False for _ in range(env.num_envs)]

total_reward = 0
total_diff = 0
total_secured = 0
min_pl = 0
min_balance = balance
total_orders = 0
total_bars = 0

days = 0
steps = 0

hold_action = np.array([[0, 0] for _ in range(env.num_envs)])

# fig, (prob_axis, target_axis) = plt.subplots(2)

# prob_axis.set(xlabel="Probability", ylabel="P/L")
# target_axis.set(xlabel="Target", ylabel="P/L")
# prob_axis.grid(True)
# target_axis.grid(True)

plt.grid(True)
# plt.xlabel("Days")
# plt.ylabel("USD")
# plt.title("Profit over time")

daysPlot = []
balancePlot = []
securedPlot = []

files = []

for i in range(domain):
    f = files.append(
        open(os.path.join("./studies", "study_{:d}.csv".format(i)), "w"))

while days < target_days:
    steps += domain

    action, state = model.predict(
        obs, state=state, mask=done, deterministic=True)

    prob = model.action_probability(
        obs, state=state, mask=done, actions=action).flatten()

    obs, reward, done, info = env.step(action)

    diffs = []

    for x in info:
        diffs.append(x["diff"])

    # prob_axis.plot(prob, diffs, 'go', alpha=.1)
    # target_axis.plot(action[:, 0], diffs, 'bo', alpha=.1)

    actives = 0
    warms = 0

    for j in range(domain):
        rw = info[j]["diff"]

        if info[j]["warming_up"]:
            warms += 1

            if rw != 0:
                out = ""

                for e in obs[j]:
                    out += str(e)
                    out += ","

                out += str(rw)
                out += str("\n")

                if done[j]:
                    out += "\n"

                files[j].write(out)

        if info[j]["active_order"]:
            actives += 1

        if info[j]["sequence_changed"]:
            seq = info[j]["last_sequence"]
            plt.clf()

            plt.plot(np.arange(0, seq.shape[0]), seq[:, 1])

        if rw != 0:
            total_diff += rw
            total_orders += 1
            min_pl = min(total_diff, min_pl)

            # pos_size = max((balance/(max_loss*2))/(.00001*max_loss), 1000)
            # pos_size = min(pos_size, 200*100000)
            pos_size = 1000

            profit = rw*pos_size

            if profit > 0:
                invested = profit*(1-secured_percentage)
                secured = profit*secured_percentage
                # invested = 0
                # secured = 0
            else:
                invested = profit
                secured = 0

            balance += invested
            total_secured += secured
            min_balance = min(balance, min_balance)

        if done[j]:
            days += 1
        #     # total_diff += info[j]["total_diff"]
        #     # total_reward += info[j]["total_reward"]
        #     # total_orders += info[j]["total_orders"]
        #     # total_bars += info[j]["total_bars"]

        #     min_pl = min(total_diff, min_pl)

        #     pos_size = max((balance/200)/(.00001*max_loss), 1000)
        #     pos_size = min(pos_size, 200*100000)

        #     profit = (info[j]["total_diff"])*pos_size

        #     if profit > 0:
        #         invested = profit*(1-secured_percentage)
        #         secured = profit*secured_percentage
        #     else:
        #         invested = profit
        #         secured = 0

        #     balance += invested
        #     total_secured += secured
        #     min_balance = min(balance, min_balance)

        #     if total_orders == 0:
        #         total_orders = 1

        #     print(
        #         "Day {:d}/{:d}, Rw: {:.5f}, Pts: {:.5f}, Min Pts: {:.5f}, Orders: {:d}, Mean Bars: {:.2f}, Balance: {:.2f}, Min Balance: {:.2f}, Secured: {:.2f}".format(days, target_days, total_reward, total_diff, min_pl, total_orders, total_bars/total_orders, balance, min_balance, total_secured), end='\r')

        #     daysPlot.append(days)
        #     balancePlot.append(balance)
        #     securedPlot.append(total_secured)

        # print(
        #     "Day {:d}/{:d}, Pts: {:.5f}, Min Pts: {:.5f}, Orders: {:d}, Mean Bars: {:.2f}, Balance: {:.2f}, Min Balance: {:.2f}, Secured: {:.2f}".format(days, target_days, total_reward, total_diff, min_pl, total_orders, total_bars/total_orders, balance, min_balance, total_secured), end='\r')

    print(
        "Day {:d}/{:d}, Steps: {:d}, Warm Envs: {:d}, Active Orders: {:d}, Pts: {:.5f}, Min Pts: {:.5f}, Orders: {:d}, Balance: {:.2f}, Min Balance: {:.2f}, Secured: {:.2f}".format(days, target_days, steps, warms, actives, total_diff, min_pl, total_orders, balance, min_balance, total_secured), end='\r')

plt.plot(daysPlot, balancePlot, 'b-', label="Balance")
plt.plot(daysPlot, securedPlot, 'g-', label="Secured")

plt.legend()
plt.show()


# -*- coding: utf-8 -*-
""" Calculations provided by aiida_turborvb turborvb main executable.
"""

from aiida.common import datastructures
from aiida.engine import CalcJob
from aiida.orm import SinglefileData, Dict, Str, StructureData

class TurboRVBConvertfort10molCalculationWRP(CalcJob):
    """
    AiiDA calculation plugin for TurboRVB.

    Serves as a calculation job provider for TurboRVB main executable,
    for execution of quantum monte carlo calculations
    """

    @classmethod
    def define(cls, spec):
        """Define inputs and outputs of the calculation."""
        # yapf: disable
        super().define(spec)

        # set default values for AiiDA options
        spec.inputs['metadata']['options']['resources'].default = {
            'num_machines': 1,
            'num_mpiprocs_per_machine': 1,
        }
        spec.input('fort10', valid_type=SinglefileData, help='')
        spec.input('pseudo', valid_type=SinglefileData, required=False, help='')
        spec.input('parameters', valid_type=Dict, help='')

        spec.output('fort10', valid_type=SinglefileData, help='')

        spec.inputs['metadata']['options']['parser_name'].default = 'turborvb.convertfort10molwrp'
        spec.inputs['metadata']['options']['input_filename'].default = 'execute.sh'
        spec.inputs['metadata']['options']['output_filename'].default = 'execute.out'

        spec.exit_code(300, 'ERROR_MISSING_OUTPUT_FILES', message='Calculation did not produce all expected output files.')

    def prepare_for_submission(self, folder):
        """
        Create input files.

        :param folder: an `aiida.common.folders.Folder` where the plugin should temporarily place all files
            needed by the calculation.
        :return: `aiida.common.datastructures.CalcInfo` instance
        """

        parameters = self.inputs.parameters.get_dict()

        with folder.open("fort.10_in", "w") as fhandle_out, \
             self.inputs.fort10.open() as fhandle_in:
            fhandle_out.write(fhandle_in.read())

        if "pseudo" in self.inputs:
            with folder.open("pseudo.dat", "w") as fhandle_out, \
                 self.inputs.pseudo.open() as fhandle_in:
                fhandle_out.write(fhandle_in.read())

        with folder.open(self.options.input_filename, "w") as fhandle:
            content = []
            content.append('turbo-genius.sh -j convertfort10mol -g')
            if "namelist_update" in parameters:
                content.append("cp convertfort10mol.input source")
                for key, value in parameters["namelist_update"].items():
                    content.append("sed 's/\s*!\?"+key+"\s*=.*$/"+key+"="+f"{value}"+"/g' source > dest")
                    content.append("cp dest source")
                content.append("cp dest convertfort10mol.input")
            content.append('convertfort10mol.x < convertfort10mol.input > convertfort10mol.output')

            fhandle.write("\n".join(content))

        codeinfo = datastructures.CodeInfo()
        codeinfo.code_uuid = self.inputs.code.uuid
        codeinfo.cmdline_params = []
        codeinfo.stdin_name = f"{self.inputs.metadata.options.input_filename}"
        codeinfo.stdout_name = f"{self.inputs.metadata.options.output_filename}"

        calcinfo = datastructures.CalcInfo()
        calcinfo.codes_info = [codeinfo]

        calcinfo.retrieve_list = ["fort.10_new",
                                  self.options.output_filename]

        return calcinfo


import logging

from sqlalchemy.exc import SQLAlchemyError

from database.manager import DatabaseManager
from database.models import User

logger = logging.getLogger(__name__)
SessionLocal = DatabaseManager.get_session_local

def insert_user(user_data: dict) -> User:
    """
    Insert a new user in the database
    :param user_data: dict, data of the user to insert
    :return: User, the inserted user
    """
    with SessionLocal() as db:
        user: User = User(**user_data)
        try:
            db.add(user)
            db.commit()
            db.refresh(user)
            logger.info(f'User {user.username} inserted successfully in the database.')
        except SQLAlchemyError:
            db.rollback()
            raise
    return user

def get_user(**kwargs) -> User:
    """
    Retrieve a user from the database based on the given filters
    :param kwargs: Keyword arguments, accepts either discord_id or username
    :return: User, the user
    """
    with SessionLocal() as db:
        filters: list = []
        if 'discord_id' in kwargs:
            filters.append(User.discord_id == kwargs['discord_id'])
        if 'username' in kwargs:
            filters.append(User.username == kwargs['username'])

        if not filters:
            raise ValueError('No valid filter given')

        user: User = db.query(User).filter(*filters).first()
        logger.debug(f'User retrieved from the database: {user}')

    return user

def get_all_users() -> list[User]:
    """
    Retrieve all users from the database
    :return: list[User], all users
    """
    with SessionLocal() as db:
        users = db.query(User).all()
        logger.debug(f'All users retrieved from the database: {len(users)}')
    return users

def update_user(user: User, user_data: dict) -> User:
    """
    Update a user in the database.
    :param user: User, the user to update
    :param user_data: dict, data to update
    :return: User, the updated user
    """
    with SessionLocal() as db:
        try:
            db_user = db.query(User).filter(User.discord_id == user.discord_id).first()
            if not db_user:
                raise ValueError('User not found')

            for key, value in user_data.items():
                setattr(db_user, key, value)

            db.commit()
            db.refresh(db_user)
            logger.info(f'User {db_user.username} updated successfully in the database.')
        except SQLAlchemyError:
            db.rollback()
            raise
    return db_user


import os
from flask import Flask
from api.config.database import db
from api.routes.user import user as user_blueprint

def create_app():
    app = Flask(__name__)

    app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv('DATABASE_URL', 'sqlite:///database.db')
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    db.init_app(app)

    app.register_blueprint(user_blueprint)

    with app.app_context():
        db.create_all()

    return app

if __name__ == "__main__":
    app = create_app()
    app.run(debug=True)


#!/bin/python3
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.dates as mdates

weather_df = pd.read_csv("weather_data.csv")

# Time and date separated

weather_df["date"] = pd.to_datetime(weather_df["date"], format="%m/%d/%Y_%H:%M:%S")
weather_df["time"] = weather_df["date"].dt.time
weather_df["date"] = weather_df["date"].dt.date

# Graphing change in humidity throughout the week

# Changing the percentage humidity to a int
humidity_axis = pd.to_numeric(weather_df["humidity"].str.rstrip("%"))
time_axis = np.arange(0, 10050, step=30)  # Time increments in 30 min

plt.title("Humidty/Time in one week")
plt.xlabel("Time in minutes")
plt.ylabel("Humidity in %")
plt.scatter(time_axis, humidity_axis)

plt.show(block=True)

# GraphING both actual temp and feels like to see differences

# Giving the same treatment as I did for the humidity axis
temp = pd.to_numeric(weather_df["temp"].str.rstrip("°"))
feels_like = pd.to_numeric(weather_df["feels_like"].str.rstrip("°").astype(int))

# Same time axis

# Figure set up
temp_fig, temp_ax = plt.subplots(1, figsize=(7.5, 5))
temp_fig.suptitle("Perceived Temp. v. Actual Temp.")
plt.xlabel("time")

sns.scatterplot(ax=temp_ax, y=temp, x=time_axis, label="TEMP.")
sns.scatterplot(ax=temp_ax, y=feels_like, x=time_axis, label="FEELS LIKE")

plt.show(block=True)

# Plotting to see how the sunrise and sunset times changed throughout the week

# Convert time in sunrise and sunset to pandas datetime objects
weather_df["sunrise"] = pd.to_datetime(weather_df["sunrise"].str.split(" ").str.get(0), format="%H:%M")
weather_df["sunset"] = pd.to_datetime(weather_df["sunset"].str.split(" ").str.get(0), format="%H:%M")

# Minutes for formatting and intervalling
minutes = mdates.MinuteLocator(interval=2)
min_format = mdates.DateFormatter("%H:%M:%S")
# Same for days with the date
dates = mdates.DayLocator(interval=2)
date_format = mdates.DateFormatter("%m/%d/%Y")

# Figure to plot the graphs
rise_fig, rise_ax = plt.subplots(1, figsize=(7.5, 5))

# First graph V Plot
rise_ax.plot(weather_df["date"], weather_df["sunrise"], color="pink")
# V Locators
rise_ax.yaxis.set_major_locator(minutes)
rise_ax.xaxis.set_major_locator(dates)
# V Formatters
rise_ax.yaxis.set_major_formatter(min_format)
rise_ax.xaxis.set_major_formatter(date_format)
# Title
rise_ax.set_title("Sunrise")
# Y-axis
rise_ax.set_ylabel("Time A.M.")
# X-axis
rise_ax.set_xlabel("Date")

plt.show(block=True)

# Padding to see the left graph's ticks

set_fig, set_ax = plt.subplots(1, figsize=(7.5, 5))

# Second graph V Plot
set_ax.plot(weather_df["date"], weather_df["sunset"], color="red")
# V Locators
set_ax.yaxis.set_major_locator(minutes)
set_ax.xaxis.set_major_locator(dates)
# V Formatters
set_ax.yaxis.set_major_formatter(min_format)
set_ax.xaxis.set_major_formatter(date_format)
# Title
set_ax.set_title("Sunset")
# Y-axis
set_ax.set_ylabel("Time P.M.")
# X-axis
set_ax.set_xlabel("Date")

plt.show(block=True)

# Correlation between temperature and humidity
humidity_temp_df = weather_df.loc[:, ["humidity", "temp", "feels_like"]]

# Clean up the data
humidity_temp_df["humidity"] = pd.to_numeric(humidity_temp_df["humidity"].str.rstrip("%"))
for i in ["temp", "feels_like"]:
    humidity_temp_df[i] = pd.to_numeric(humidity_temp_df[i].str.rstrip("°"))

# Correlation matrix calculation
humidity_temp_corr_matrix = humidity_temp_df.corr()

sns.heatmap(humidity_temp_corr_matrix, annot=True, cmap="coolwarm")
plt.show(block=True)

# Graph to see correlation graphically
ht_fig, ht_ax = plt.subplots(1, 2, figsize=(15,5), sharey=False, sharex=True)
ht_fig.suptitle("Temp and Feels Like Temp in Relation to Humidity")
for j in range(0,2):
    ht_ax[j].set_xlabel("Humidity")
    ht_ax[j].set_ylabel("Temperature")

# First graph
sns.scatterplot(data=humidity_temp_df, ax=ht_ax[0], x="humidity", y="temp",
        color="red", label="Temp")
sns.scatterplot(data=humidity_temp_df, ax=ht_ax[0], x="humidity", y="feels_like",
        color="blue", label="Feels Like")

# Second graph
sns.scatterplot(data=humidity_temp_df, ax=ht_ax[1], x="humidity", y="temp",
        hue="feels_like")

plt.show(block=True)


#!/usr/bin/env python3
import datetime
import statistics
import tempfile

# prompt user to select from mailroom menu: send a thank you, create a report, or quit
# send a thank you: allows user to select a name, adds donor if not in donor list, otherwise ask for donation
# and thanks donor
# create a report summarizes the donors and their donations in a table
# quit ends the program

# donors exists in global namespace
donors = {
      "Jerry Seinfeld": [
          324.33
      ],
      "John Stewart": [
          5000000.00
      ],
      "Larry David": [
          1800.18,
          36036.36
      ],
      "Gene Wilder": [
          7272.72,
          1818.18,
          545454.54
      ],
      "Jerry Lewis": [
          0.00
      ],
      "Gilda Radner": [
          324.32
      ],
      "Rodney Dangerfield": [
          363363.63,
          18181.81,
          545454.54,
          727.72,
          9090.90,
          108.18,
          126.12,
          7272.72,
          54.54,
          18.00,
          0.18
      ]
}
now = datetime.datetime.now()

def main():
    main_prompt = ("\nPlease enter an option from the following menu\n"
                   "1 - Send a Thank You to a single donor\n"
                   "2 - Create a Report\n"
                   "3 - Send letters to all donors\n"
                   "4 - Quit\n>> ")
    dispatch = {"1": send_thank_you, "2": create_report, "3": send_thank_you_all_donors, "4": quit_mailroom}
    menu_selection(main_prompt, **dispatch)

def menu_selection(prompt, **dispatch_dict):
    """
    prompts user for input and calls function object in dispatch_dict
    :param prompt: user input prompt
    :param dispatch_dict: dict of dispatch functions
    :return:
    """
    while True:
        try:
            response = get_user_input(f"{prompt}")
        except (TypeError, ValueError) as menu_err:
            print(f"Invalid option {response}\n")
            raise menu_err
        except (KeyboardInterrupt, EOFError):
            return None
        if dispatch_dict[response]() == 4:
            break

def quit_mailroom():
    """
    quit the mailroom
    :return: 4 (quit option)
    """
    return 4

def send_thank_you():
    """
    prompt user for a donor name; if name exists, prompts for donation, otherwise add name to donors
    'list' displays a list of donors
    print thank you to donor
    :return: None
    """
    donor_name = get_user_input("Please enter the name of a donor: ")
    if donor_name == 'list':
        print(f"List of donors: {', '.join(list_donors(**donors))}")
    else:
        while True:
            try:
                donation = float(get_user_input("Please enter the donation amount: ").strip('$'))
                if bool(donation) is True:
                    break
                else:
                    print(f"Invalid donation, please enter a valid donation.\n")
            except ValueError as donation_err:
                raise donation_err
                continue
        donor_info = (donor_name, donation)
        update_donor(donor_info)
        print(f"{format_thank_you(donor_info)}")

def list_donors(**donors):
    """
    :param donors: dict of key:donor and value:list of donations
    :return: list of donors
    """
    if not donors:
        raise TypeError("donors cannot be empty")
    else:
        try:
            return list(donors.keys())
        except KeyError as key_error:
            print(f"{key_error}")

def update_donor(donor_info=None):
    """
    add donor if donor does not exist, otherwise update donor's list of donations
    :param donor_info: tuple of donor name and donation amount
    :return:
    """
    if not donor_info[0]:
        raise TypeError("donor_name cannot be empty")
    else:
        if donor_info[0] in donors:
            try:
                donors.setdefault(donor_info[0], []).append(donor_info[1])
            except IndexError as donor_info_error:
                print(f"{donor_info_error}")
        else:
            donors.setdefault(donor_info[0], []).append(donor_info[1])
            print(f"Sorry, the name you entered is not in our donor list.  Adding donor to list...")

def send_thank_you_all_donors():
    """
    generate a thank you letter for all donors, write each letter to disk as .txt in tempdir
    :return: None
    """
    try:
        with tempfile.TemporaryDirectory(suffix='mailroom2', prefix='tys_all_donors', dir='.') as tempdir:
            for donor in donors:
                donor_info = (donor, format(donors.get(donor)[-1], '.2f'))
                with open(f"{(donor_info[0]).replace(' ','_')}_{str(now.date()).replace('-','_')}.txt", 'w') as f:
                    f.write(format_thank_you(donor_info))
    except (IOError,  PermissionError, NotADirectoryError, FileExistsError, FileNotFoundError) as report_err:
        print(f"{report_err}")
        raise report_err

def format_thank_you(donor_info=None):
    """
    create thank you letter
    :param donor_info: tuple of donor name and donation amount
    :return:
    """
    if not donor_info[0]:
        raise ValueError("donor name and donation cannot be empty")
    elif not donor_info[1]:
        raise ValueError("donation amount cannot be empty")
    else:
        thank_you = (
            f"\n\n{now.date()}\n\n"
            f""
            f"Dear {donor_info[0]},\n\n"
            f""
            f"On behalf of all of us at the ADL office we'd like to recognize and thank you for your generous\n"
            f"donation of ${donor_info[1]}.  "
            f"These funds will be used to further our mission in fighting hate\n"
            f"through leadership programs and education.  \n\n"
            f"We look forward to seeing you at this year's banquet.\n"
            f"\n"
            f"Sincerely, "
            f"\n"
            f"ADL")
    return str(thank_you)

def display_report(donor_summary=None):
    """
    prints formatted donor report
    :param donor_summary:
    :return:
    """
    print(f"Donor Name{'':<20} | Total Given{'':>0} | Num Gifts{'':>0} | Average Gift{'':>0}")
    print(f"-" * 72)
    for name, total, num_gift, avg_gift in donor_summary:
        print(f"{name:<32}${total:>11.2f}{num_gift:>12}  ${avg_gift:>13.2f}")

def get_report(donors=None):
    """
    data logic function for reports
    :param donors:
    :return: donor summary list
    """
    donor_summary = []
    if not donors:
        raise ValueError("donors list cannot be empty")
    else:
        for donor in donors:
            try:
                donor_summary.append([donor, sum(donors.get(donor)), len(donors.get(donor)),
                                     statistics.mean(donors.get(donor))])
            except (KeyError, IndexError) as donor_summary_err:
                raise donor_summary_err
    donor_summary.sort(key=lambda x: float(x[1]), reverse=True)
    return donor_summary

def create_report():
    """
    dispatch function for creating and displaying donor report
    :return:
    """
    display_report(get_report(donors))

def get_user_input(prompt):
    """
    :param prompt: prompt for user to respond to
    :return: response to question/prompt
    """
    return input(prompt).strip()

if __name__ == "__main__":
    print("Running", __file__)
    main()
else:
    print("Running as imported module", __file__)


from collections import Counter
import sys

def read_dictionary():
    #ソートしてある辞書を読み込む
    f = open('score_sort_dictionary.txt', 'r')
    data = f.read()
    f.close()

    list = data.split() #\nを除いて配列にする

    new = []

    for i in range(len(list)):#文字の出現回数と元の言葉をペアにする
        count_array = []
        l = list[i].split(',')
        for j in range(26):#文字の出現回数を数値の配列にする
            count_array.append(int(l[j]))
        new.append((count_array, l[26]))#l[26]は元の言葉
    return new

def read_textfile(fileName):
    #アナグラムを探すテキストファイル読み込み
    f = open(fileName, 'r')
    data = f.read()
    f.close()

    list = data.split() #\nを除いて配列にする

    return list

def count_word(word):
    #出てきた文字の回数を数値の配列にする
    #catなら、'abcdefghijklmnopqrstuvwsyz'->[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]

    #単語に出てくるアルファベットの数を調べる
    count_num = sorted(list(Counter(word).items()))

    alphabet = 'abcdefghijklmnopqrstuvwxyz'#26個
    count_array = [0]*26
    for i in range(len(count_num)):
        index = alphabet.find(count_num[i][0])
        count_array[index] += count_num[i][1]
    return count_array

def linear_search(dictionary, count_array, word):
    #上から辞書を見ていって、アナグラムを見つけたら返す
    for i in range(len(dictionary)):
        flag = 1
        for j in range(len(count_array)):
            if dictionary[i][0][j]>count_array[j]:
                flag = 0
                break
        if flag:
            if(dictionary[i][1]!=word):
                return dictionary[i][1]
    return ""

if __name__=='__main__':
    dictionary = read_dictionary()
    text = read_textfile(sys.argv[1])

    ans_arr = []

    for i in range(len(text)):
        print(i)
        counting = count_word(text[i])
        ans = linear_search(dictionary, counting, text[i])
        print(ans)
        print()

        ans_arr.append(ans)

    #ファイル書き込み
    f = open(sys.argv[2], 'w')
    f.write('\n'.join(ans_arr))
    f.close()


DESC = '''
Helper script to record/examine a rollout (record/ render/ playback/ recover) on an environment\n
  > Examine options:\n
    - Record:   Record an execution. (Useful for kinesthetic demonstrations on hardware)\n
    - Render:   Render back the execution. (sim.forward)\n
    - Playback: Playback the rollout action sequence in openloop (sim.step(a))\n
    - Recover:  Playback actions recovered from the observations \n
  > Render options\n
    - either onscreen, or offscreen, or just rollout without rendering.\n
  > Save options:\n
    - save resulting rollouts as RoboHive/Roboset format, and as 2D plots\n
USAGE:\n
    $ python examine_rollout.py --env_name door-v0 \n
    $ python examine_rollout.py --env_name door-v0 --rollout_path my_rollouts.h5 --repeat 10 \n
    $ python logger/examine_logs.py --env_name rpFrankaRobotiqData-v0 --rollout_path teleOp_trace.h5 --rollout_format RoboSet --render offscreen --compress_paths False -c left_cam -c right_cam -c top_cam -c Franka_wrist_cam --plot_paths True
'''

from myosuite.utils.paths_utils import plot as plotnsave_paths
from myosuite.utils import tensor_utils
import gym
import click
import numpy as np
import time
import os

@click.command(help=DESC)
@click.option('-e', '--env_name', type=str, help='environment to load', required=True)
@click.option('-p', '--rollout_path', type=str, help='absolute path of the rollout', default=None)
@click.option('-f', '--rollout_format', type=click.Choice(['RoboHive', 'RoboSet']), help='Data format', default='RoboHive')
@click.option('-m', '--mode', type=click.Choice(['record', 'render', 'playback', 'recover']), help='How to examine rollout', default='playback')
@click.option('-h', '--horizon', type=int, help='Rollout horizon, when mode is record', default=-1)
@click.option('-s', '--seed', type=int, help='seed for generating environment instances', default=123)
@click.option('-n', '--num_repeat', type=int, help='number of repeats for the rollouts', default=1)
@click.option('-r', '--render', type=click.Choice(['onscreen', 'offscreen', 'none']), help='visualize onscreen or offscreen', default='onscreen')
@click.option('-c', '--camera_name', multiple=True, type=str, default=[None,], help=('list of camera names for rendering'))
@click.option('-fs', '--frame_size', type=tuple, default=(424, 240), help=('Camera frame size for rendering'))
@click.option('-o', '--output_dir', type=str, default='./', help=('Directory to save the outputs'))
@click.option('-on', '--output_name', type=str, default=None, help=('The name to save the outputs as'))
@click.option('-sp', '--save_paths', type=bool, default=False, help=('Save the rollout paths'))
@click.option('-cp', '--compress_paths', type=bool, default=True, help=('compress paths. Remove obs and env_info/state keys'))
@click.option('-pp', '--plot_paths', type=bool, default=False, help=('2D-plot of individual paths'))
@click.option('-ea', '--env_args', type=str, default=None, help=('env args. E.g. --env_args "{\'is_hardware\':True}"'))
@click.option('-ns', '--noise_scale', type=float, default=0.0, help=('Noise amplitude in randians}'))
@click.option('-ie', '--include_exteroception', type=bool, default=False, help=('Include exteroception'))
def examine_logs(env_name, rollout_path, rollout_format, mode, horizon, seed, num_repeat, render, camera_name, frame_size, output_dir, output_name, save_paths, compress_paths, plot_paths, env_args, noise_scale, include_exteroception):

    # seed and load environments
    np.random.seed(seed)
    env = gym.make(env_name) if env_args==None else gym.make(env_name, **(eval(env_args)))
    env = env.env
    env.seed(seed)

    # Start a "trace" for recording rollouts
    if rollout_format=='RoboHive':
        from myosuite.logger.grouped_datasets import Trace
    elif rollout_format=='RoboSet':
        from myosuite.logger.roboset_logger import RoboSet_Trace as Trace
    else:
        raise TypeError("unknown rollout_format format")
    trace = Trace("Rollouts")

    # Load old traces as "paths"; none if record
    if mode == 'record':
        assert horizon>0, "Rollout horizon must be specified when recording rollout"
        assert output_name is not None, "Specify the name of the recording"
        if save_paths is False:
            print("Warning: Recording is not being saved. Enable save_paths=True to log the recorded path")
        paths = [None,]*num_repeat # Mark old traces as None
    else:
        assert rollout_path is not None, "Rollout path is required for mode:{} ".format(mode)
        if output_dir == './': # overide the default
            output_dir = os.path.dirname(rollout_path)
        if output_name is None: # default to the rollout name
            rollout_name = os.path.split(rollout_path)[-1]
            output_name, output_type = os.path.splitext(rollout_name)
        paths = Trace.load(trace_path=rollout_path, trace_type=rollout_format)

    # Resolve rendering
    if render == 'onscreen':
        env.mujoco_render_frames = True
    elif render =='offscreen':
        env.mujoco_render_frames = False
    elif render == None:
        env.mujoco_render_frames = False

    # Rollout paths
    for i_loop in range(num_repeat):

        # Rollout path
        print("Starting rollout loop:{}".format(i_loop))
        for path_name, path_data in paths.items():

            # initialize path -----------------------------
            ep_t0 = time.time()
            path_name+='-'+str(i_loop)
            print("Starting {} rollout".format(path_name))
            trace.create_group(path_name)

            # init: reset to starting state
            if path_data and rollout_format=='RoboHive':
                # reset to init state
                if "state" in path_data['env_infos'].keys():
                    path_state = tensor_utils.split_tensor_dict_list(path_data['env_infos']['state'])
                    env.reset(reset_qpos=path_state[0]['qpos'], reset_qvel=path_state[0]['qvel'])
                    env.set_env_state(path_state[0])
                else:
                    env.reset()
            elif path_data and rollout_format=='RoboSet':
                path_data = path_data['data']
                # reset to init state
                reset_qpos = env.init_qpos.copy()
                nq_arm = len(path_data['qp_arm'][0])
                reset_qpos[:nq_arm] = path_data['qp_arm'][0]
                nq_ee = len(path_data['qp_ee'][0])
                reset_qpos[nq_arm:nq_arm+nq_ee] = path_data['qp_ee'][0]
                env.reset(reset_qpos=reset_qpos)
            elif mode=="record":
                env.reset()
            else:
                raise TypeError(f"Unknown rollout_format{rollout_format} / mode:{mode}")
            trace_horizon = horizon if mode=='record' else path_data['time'].shape[0]-1

            # Rollout path --------------------------------
            obs, rwd, done, env_info = env.forward(update_exteroception=include_exteroception)
            ep_rwd = rwd
            for i_step in range(trace_horizon+1):

                # Get step's actions ----------------------

                # Record Execution. Useful for kinesthetic demonstrations on hardware
                if mode=='record':
                    act = env.action_space.sample() # dummy random sample

                # Directly create the scene
                elif mode=='render':
                    # populate actions merely for logging
                    if rollout_format=='RoboSet':
                        act = np.concatenate([path_data['ctrl_arm'][i_step], path_data['ctrl_ee'][i_step]])
                    elif rollout_format=='RoboHive' and "state" in path_data['env_infos'].keys():
                        act = path_data['actions'][i_step]
                    else:
                        raise NotImplementedError("Settings not found")

                # Apply actions in open loop
                elif mode=='playback':
                    if rollout_format=='RoboSet':
                        act = np.concatenate([path_data['ctrl_arm'][i_step], path_data['ctrl_ee'][i_step]])
                    elif rollout_format=='RoboHive':
                        act = path_data['actions'][i_step]

                # Recover actions from states
                elif mode=='recover':
                    # assumes position controls
                    if rollout_format=='RoboSet':
                        act = np.concatenate([path_data['qp_arm'][i_step], path_data['qp_ee'][i_step]])
                    elif rollout_format=='RoboHive':
                        act = path_data['env_infos']['obs_dict']['qp'][i_step]
                    if noise_scale:
                        act = act + env.np_random.uniform(high=noise_scale, low=-noise_scale, size=len(act)).astype(act.dtype)
                    if env.normalize_act:
                        act = env.robot.normalize_actions(controls=act)

                # nan actions for last log entry
                if i_step == trace_horizon:
                    act = np.nan*np.ones(env.action_space.shape)

                # log values at time=t ----------------------------------
                if compress_paths:
                    obs = [] # don't save obs, env_infos has obs_dict
                    if 'state' in env_info.keys(): del env_info['state']  # don't save state, obs_dict has env necessities

                # log: time, obs, act, rwd, info, done
                datum_dict = dict(
                        time=env.time,
                        observations=obs,
                        actions=act.copy(),
                        rewards=rwd,
                        env_infos=env_info,
                        done=done,
                    )
                trace.append_datums(group_key=path_name,dataset_key_val=datum_dict)

                # log: offscreen frames
                if render =='offscreen':
                    for cam in camera_name:
                        curr_frame = env.sim.renderer.render_offscreen(
                            camera_id=cam,
                            width=frame_size[0],
                            height=frame_size[1],
                            device_id=0
                        )
                        # cam_key = cam+'_rgb'
                        trace.append_datum(group_key=path_name,dataset_key=cam, dataset_val=curr_frame)

                # step/forward env using actions from t=>t+1 ----------------------
                if mode=='render' and i_step < trace_horizon:
                    if rollout_format=='RoboSet':
                        env.sim.data.qpos[:nq_arm]= path_data['qp_arm'][i_step+1]
                        env.sim.data.qpos[nq_arm:nq_arm+nq_ee]= path_data['qp_ee'][i_step+1]
                        env.sim.data.qvel[:nq_arm]= path_data['qv_arm'][i_step+1]
                        env.sim.data.qvel[nq_arm:nq_arm+nq_ee]= path_data['qv_ee'][i_step+1]
                        env.sim.data.time = path_data['time'][i_step+1]
                    elif rollout_format=='RoboHive' and "state" in path_data['env_infos'].keys():
                        env.set_env_state(path_state[i_step+1])
                    else:
                        raise NotImplementedError("Settings not found")
                    obs, rwd, done, env_info = env.forward(update_exteroception=include_exteroception)
                    ep_rwd += rwd
                elif i_step < trace_horizon: # incase last step actions (nans) can cause issues in step
                    obs, rwd, done, env_info = env.step(act, update_exteroception=include_exteroception)
                    ep_rwd += rwd

            # save offscreen buffers as video and clear the dataset
            if render == 'offscreen':
                trace.render(output_dir=output_dir, output_format="mp4", groups=[path_name], datasets=camera_name, input_fps=1/env.dt)
                trace.remove_dataset(group_keys=[path_name], dataset_key=camera_name)

            # Finish rollout
            old_stat = f"(Old rollout rewards: {sum(path_data['rewards'][:])})" if path_data else ""
            print(f"Finishing {path_name[:-2]} rollout in {(time.time()-ep_t0):0.4} sec. Total rewards {ep_rwd} "+old_stat)

        # Finish loop
        print("Finished rollout loop:{}".format(i_loop))

    # plot paths ???: Needs upgrade to the new logger
    trace.stack()
    time_stamp = time.strftime("%Y%m%d-%H%M%S")
    # plot paths
    if plot_paths:
        file_name = os.path.join(output_dir, output_name + '{}'.format(time_stamp))
        plotnsave_paths(trace.trace, env=env, fileName_prefix=file_name)

    # Close and save paths
    trace.close()
    if save_paths:
        file_name = os.path.join(output_dir, output_name + '{}_paths.h5'.format(time_stamp))
        trace.save(trace_name=file_name)

if __name__ == '__main__':
    examine_logs()


# YOLOv5 ðŸš€ by Ultralytics, GPL-3.0 license
"""
Download utils
"""

import os
import urllib
import requests
import subprocess
import logging

from pathlib import Path
from zipfile import is_zipfile, ZipFile
from tarfile import is_tarfile
from itertools import repeat
from multiprocessing.pool import ThreadPool

import torch

from .logger import LOGGER

def is_url(url, check=True):
    # Check if string is URL and check if URL exists
    try:
        url = str(url)
        result = urllib.parse.urlparse(url)
        assert all([result.scheme, result.netloc])  # check if is url
        return (urllib.request.urlopen(url).getcode() == 200) if check else True  # check if exists online
    except (AssertionError, urllib.request.HTTPError):
        return False

def gsutil_getsize(url=''):
    # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
    s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')
    return eval(s.split(' ')[0]) if len(s) else 0  # bytes

def url_getsize(url='https://ultralytics.com/images/bus.jpg'):
    # Return downloadable file size in bytes
    response = requests.head(url, allow_redirects=True)
    return int(response.headers.get('content-length', -1))

def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
    # Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes
    file = Path(file)
    assert_msg = f"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}"
    try:  # url1
        LOGGER.info(f'Downloading {url} to {file}...')
        torch.hub.download_url_to_file(url, str(file), progress=LOGGER.level <= logging.INFO)
        assert file.exists() and file.stat().st_size > min_bytes, assert_msg  # check
    except Exception as e:  # url2
        if file.exists():
            file.unlink()  # remove partial downloads
        LOGGER.info(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...')
        os.system(f"curl -# -L '{url2 or url}' -o '{file}' --retry 3 -C -")  # curl download, retry and resume on fail
    finally:
        if not file.exists() or file.stat().st_size < min_bytes:  # check
            if file.exists():
                file.unlink()  # remove partial downloads
            LOGGER.info(f"ERROR: {assert_msg}\n{error_msg}")
        LOGGER.info('')

# def attempt_download(file, repo='ultralytics/yolov5', release='v7.0'):
def attempt_download(file, repo='zjykzj/YOLOv5', release='v1.0'):
    # Attempt file download from GitHub release assets if not found locally. release = 'latest', 'v7.0', etc.
    def github_assets(repository, version='latest'):
        # Return GitHub repo tag (i.e. 'v7.0') and assets (i.e. ['yolov5s.pt', 'yolov5m.pt', ...])
        if version != 'latest':
            version = f'tags/{version}'  # i.e. tags/v7.0
        response = requests.get(f'https://api.github.com/repos/{repository}/releases/{version}').json()  # github api
        return response['tag_name'], [x['name'] for x in response['assets']]  # tag, assets

    file = Path(str(file).strip().replace("'", ''))
    if not file.exists():
        # URL specified
        name = Path(urllib.parse.unquote(str(file))).name  # decode '%2F' to '/' etc.
        if str(file).startswith(('http:/', 'https:/')):  # download
            url = str(file).replace(':/', '://')  # Pathlib turns :// -> :/
            file = name.split('?')[0]  # parse authentication https://url.com/file.txt?auth...
            if Path(file).is_file():
                LOGGER.info(f'Found {url} locally at {file}')  # file already exists
            else:
                safe_download(file=file, url=url, min_bytes=1E5)
            return file

        # GitHub assets
        assets = [f'yolov5{size}{suffix}.pt' for size in 'nsmlx' for suffix in ('', '6', '-cls', '-seg')]  # default
        try:
            tag, assets = github_assets(repo, release)
        except Exception:
            try:
                tag, assets = github_assets(repo)  # latest release
            except Exception:
                try:
                    tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1]
                except Exception:
                    tag = release

        file.parent.mkdir(parents=True, exist_ok=True)  # make parent dir (if required)
        if name in assets:
            url3 = 'https://drive.google.com/drive/folders/1EFQTEUeXWSFww0luse2jB9M1QNZQGwNl'  # backup gdrive mirror
            safe_download(
                file,
                url=f'https://github.com/{repo}/releases/download/{tag}/{name}',
                min_bytes=1E5,
                error_msg=f'{file} missing, try downloading from https://github.com/{repo}/releases/{tag} or {url3}')

    return str(file)

def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')):
    # Unzip a *.zip file to path/, excluding files containing strings in exclude list
    if path is None:
        path = Path(file).parent  # default path
    with ZipFile(file) as zipObj:
        for f in zipObj.namelist():  # list all archived filenames in the zip
            if all(x not in f for x in exclude):
                zipObj.extract(f, path=path)

def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3):
    # Multithreaded file download and unzip function, used in data.yaml for autodownload
    def download_one(url, dir):
        # Download 1 file
        success = True
        if os.path.isfile(url):
            f = Path(url)  # filename
        else:  # does not exist
            f = dir / Path(url).name
            LOGGER.info(f'Downloading {url} to {f}...')
            for i in range(retry + 1):
                if curl:
                    s = 'sS' if threads > 1 else ''  # silent
                    r = os.system(
                        f'curl -# -{s}L "{url}" -o "{f}" --retry 9 -C -')  # curl download with retry, continue
                    success = r == 0
                else:
                    torch.hub.download_url_to_file(url, f, progress=threads == 1)  # torch download
                    success = f.is_file()
                if success:
                    break
                elif i < retry:
                    LOGGER.warning(f'âš ï¸� Download failure, retrying {i + 1}/{retry} {url}...')
                else:
                    LOGGER.warning(f'â�Œ Failed to download {url}...')

        if unzip and success and (f.suffix == '.gz' or is_zipfile(f) or is_tarfile(f)):
            LOGGER.info(f'Unzipping {f}...')
            if is_zipfile(f):
                unzip_file(f, dir)  # unzip
            elif is_tarfile(f):
                os.system(f'tar xf {f} --directory {f.parent}')  # unzip
            elif f.suffix == '.gz':
                os.system(f'tar xfz {f} --directory {f.parent}')  # unzip
            if delete:
                f.unlink()  # remove zip

    dir = Path(dir)
    dir.mkdir(parents=True, exist_ok=True)  # make directory
    if threads > 1:
        pool = ThreadPool(threads)
        pool.imap(lambda x: download_one(*x), zip(url, repeat(dir)))  # multithreaded
        pool.close()
        pool.join()
    else:
        for u in [url] if isinstance(url, (str, Path)) else url:
            download_one(u, dir)


import os
import sys
from glob import glob
from jupyter_packaging import get_data_files
from setuptools import setup, find_packages

long_desc = "{{description}}"

here = os.path.abspath(os.path.dirname(__file__))

setup_args = dict(
    name="{{package-name}}",
    version="{{version}}",
    url="https://github.com/elyra-ai/elyra",
    description="Elyra provides AI Centric extensions to JupyterLab",
    long_description=long_desc,
    author="Elyra Maintainers",
    license="Apache License Version 2.0",
    data_files=get_data_files([{{data - files}}]),
    packages=find_packages(),
    install_requires=[{{install - requires}}],
    include_package_data=True,
    classifiers=(
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
        "Topic :: Scientific/Engineering",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
        "Topic :: Software Development",
        "Topic :: Software Development :: Libraries",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Programming Language :: Python :: 3.11",
    ),
    entry_points={},
)

if __name__ == "__main__":
    setup(**setup_args)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# lda.pyで求めた文書トピック分布を用いてsvm-multi classiferを実装，およびmatplotlibでの図示

from sklearn.datasets import load_digits
from sklearn.cross_validation import train_test_split
from sklearn.svm import LinearSVC
from sklearn.metrics import accuracy_score

# データとラベルを受け取ってsvmを実行
def svm(data, label, train_size):
	# トレーニングデータとテストデータに分割
	data_train, data_test, label_train, label_test = train_test_split(data, label, train_size=train_size)
	# 分類器にパラメータを与える
	estimator = LinearSVC(C=1.0)
	# トレーニングデータで学習する
	estimator.fit(data_train, label_train)
	# テストデータの予測をする
	label_predict = estimator.predict(data_test)

	return label_predict, label_test

# 図示用
def plot_graph(all_error, rates):
	import matplotlib.pylot as plt

	# 各aによってerror rateが変わるかを図示
	plt.plot(rates, all_error[0],'ro',label = "alpha=0.1")
	plt.plot(rates, all_error[1],'b',label = "alpha=1.0")
	plt.plot(rates, all_error[2],label = "alpha=3.0")
	plt.plot(rates, all_error[3],label = "alpha=10.0")

	plt.legend() # 凡例を表示
	plt.xticks(rates,("","0.02","0.05","0.1","0.2"))
	# plt.title("Graph Title")
	plt.xlabel("Training Size")
	plt.ylabel("Error Rate")
	plt.show()
	# トピック次元によって

def main():
	rates = [0.01, 0.02, 0.05, 0.1, 0.2]
	files = ["1",
	"2",
	"3"]
	all_error = []
	# 各データについてsvmを実施する
	for f in files:
		# shelveでオープン
		dic = shelve.open(f)
		data = dic['thetas']
		label = dic['categories']
		error_rates = []
		for r in rates:
			# svmを実行
			label_predict, label_test = svm(data, label, r)
			# Error Rateを求める
			error_rates.append(1.0 - accuracy_score(label_predict, label_test))
		all_error.append(error_rates)
		dic.close()

	# 図示
	result = shelve.open('result')
	result['all_error'] = all_error
	result.close()
	plot_graph(all_error, rates)

def make_graph():
	# 不安だったらここで確認
	result = shelve.open('result')
	all_error = result['all_error']
	result.close()

if __name__ == "__main__":
    # test()
    main()
    make_graph()