'''
deepspeed --num_gpus=1 GPT-2_fine_tune.py
'''

from argparse import ArgumentParser
import os
import pandas as pd
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelWithLMHead

import deepspeed
from deepspeed.ops.adam import DeepSpeedCPUAdam
import wandb

os.environ["TOKENIZERS_PARALLELISM"] = "true"

model_name = "skt/kogpt2-base-v2"
tokenizer = AutoTokenizer.from_pretrained(model_name)
SPECIAL_TOKENS = {
    "bos_token": "<bos>",
    "eos_token": "<eos>",
    "pad_token": "<pad>",
    "sep_token": "<seq>"
    }
tokenizer.add_special_tokens(SPECIAL_TOKENS)

model = AutoModelWithLMHead.from_pretrained(
    model_name
).cuda()

model.resize_token_embeddings(len(tokenizer))

parser = ArgumentParser()
parser.add_argument("--deepspeed_config", type=str, default="ds_config.json")
parser.add_argument("--local_rank", type=int)
parser.add_argument("--epoch", default=50, type=int)
parser.add_argument("--batch_size", default=128, type=int)
parser.add_argument("--sep_token", default=tokenizer.sep_token, type=str)
parser.add_argument("--bos_token", default=tokenizer.bos_token, type=str)
parser.add_argument("--eos_token", default=tokenizer.eos_token, type=str)
args = parser.parse_args()

wandb.init(project="mobot", name=f"mobot-{model_name}")
train_data = pd.read_csv("data/cafe_clear_data_test.tsv", delimiter="\t")
train_data = train_data[:3000]
train_text, train_labels = (
    train_data["text"].values,
    train_data["label"].values,
)

dataset = [
    {"data": t + str(args.bos_token) + l + str(args.eos_token), "label": l}
    for t, l in zip(train_text, train_labels)
]
train_loader = DataLoader(
    dataset,
    batch_size=args.batch_size,
    num_workers=8,
    drop_last=True,
    pin_memory=True,
)

eval_data = pd.read_csv("data/cafe_clear_data_test.tsv", delimiter="\t")
eval_data = eval_data[3000:]

eval_text, eval_labels = (
    eval_data["text"].values,
    eval_data["label"].values,
)

dataset = [
    {"data": t + str(args.bos_token) + l + str(args.eos_token), "label": l }
    for t, l in zip(eval_text, eval_labels)
]
eval_loader = DataLoader(
    dataset,
    batch_size=args.batch_size,
    num_workers=8,
    drop_last=True,
    pin_memory=True,
)

optimizer = DeepSpeedCPUAdam(
    lr=3e-5, weight_decay=3e-7, model_params=model.parameters()
)

engine, optimizer, _, _ = deepspeed.initialize(
    args=args, model=model, optimizer=optimizer
)
for epoch in range(args.epoch):
    model.train()
    for train in tqdm(train_loader):
        optimizer.zero_grad()
        text, label = train["data"], train["label"]
        text_tokens = tokenizer(
            text,
            return_tensors="pt",
            max_length=50,
            truncation=True,
            padding=True,
        )

        input_ids = text_tokens.input_ids.cuda()
        attention_mask = text_tokens.attention_mask.cuda()

        output = engine.forward(
            input_ids=input_ids,
            attention_mask=attention_mask,
            labels=input_ids,
        )

        loss = output.loss
        wandb.log({"loss": loss})
        engine.backward(loss)
        optimizer.step()

    with torch.no_grad():
        model.eval()
        for eval in tqdm(eval_loader):
            eval_text, eval_label = eval["data"], eval["label"]
            eval_text_tokens = tokenizer(
                eval_text,
                return_tensors="pt",
                max_length=50,
                truncation=True,
                padding=True,
            )

            input_ids = eval_text_tokens.input_ids.cuda()
            attention_mask = eval_text_tokens.attention_mask.cuda()

            eval_out = engine.forward(
                input_ids=input_ids,
                attention_mask=attention_mask,
                labels=input_ids,
            )
            wandb.log({"eval_loss": eval_out.loss})
            wandb.log({"epoch": epoch+1})

        torch.save(model.state_dict(), f"model_save/{model_name.replace('/', '-')}.pt")


import logging
from django.views.generic import TemplateView

import uptrace

logger = logging.getLogger(__name__)

class IndexView(TemplateView):
    template_name = "index.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["trace_url"] = uptrace.trace_url()
        return context

class HelloView(TemplateView):
    template_name = "hello.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["trace_url"] = uptrace.trace_url()
        return context

class FailingView(TemplateView):
    template_name = "hello.html"

    def get_context_data(self, **kwargs):
        print(uptrace.trace_url())
        raise ValueError("something went wrong")


from __future__ import print_function
from builtins import map
from pdb import pm

from future.utils import viewitems

from miasm.core.utils import decode_hex
from miasm.analysis.machine import Machine
from miasm.analysis.binary import Container
from miasm.core.asmblock import AsmCFG, AsmConstraint, AsmBlock, \
    AsmBlockBad, AsmConstraintTo, AsmConstraintNext, \
    bbl_simplifier
from miasm.core.graph import DiGraphSimplifier, MatchGraphJoker
from miasm.expression.expression import ExprId
from miasm.core.locationdb import LocationDB

# Initial data: from 'samples/simple_test.bin'
data = decode_hex("5589e583ec10837d08007509c745fc01100000eb73837d08017709c745fc02100000eb64837d08057709c745fc03100000eb55837d080774138b450801c083f80e7509c745fc04100000eb3c8b450801c083f80e7509c745fc05100000eb298b450883e03085c07409c745fc06100000eb16837d08427509c745fc07100000eb07c745fc081000008b45fcc9c3")
loc_db = LocationDB()
cont = Container.from_string(data, loc_db)

# Test Disasm engine
machine = Machine("x86_32")
mdis = machine.dis_engine(cont.bin_stream, loc_db=loc_db)
## Disassembly of one block
first_block = mdis.dis_block(0)
assert len(first_block.lines) == 5
print(first_block)

## Test redisassemble asmcfg
first_block_bis = mdis.dis_block(0)
assert len(first_block.lines) == len(first_block_bis.lines)
print(first_block_bis)

## Disassembly of several block, with cache
asmcfg = mdis.dis_multiblock(0)
assert len(asmcfg) == 17

## Test redisassemble asmcfg
asmcfg = mdis.dis_multiblock(0)
assert len(asmcfg) == 17
## Equality between assembly lines is not yet implemented
assert len(asmcfg.heads()) == 1
assert len(asmcfg.loc_key_to_block(asmcfg.heads()[0]).lines) == len(first_block.lines)

# Test AsmCFG
assert isinstance(asmcfg, AsmCFG)
assert len(asmcfg.pendings) == 0
assert len(asmcfg.nodes()) == 17
assert len(asmcfg.edges2constraint) == len(asmcfg.edges())
assert len(asmcfg.edges()) == 24
assert asmcfg.getby_offset(0x63).lines[0].offset == 0x5f
assert asmcfg.getby_offset(0x69).lines[0].offset == 0x69

## Convert to dot
open("graph.dot", "w").write(asmcfg.dot())

## Modify the structure: link the first and the last block
leaves = asmcfg.leaves()
assert len(leaves) == 1
last_block_loc_key = leaves.pop()

### Remove first_block for the rest of the graph
first_block = asmcfg.loc_key_to_block(asmcfg.heads()[0])
assert len(first_block.bto) == 2
for succ in asmcfg.successors(first_block.loc_key):
    asmcfg.del_edge(first_block.loc_key, succ)

### Modification must be reported from the graph
assert len(first_block.bto) == 0
assert last_block_loc_key in asmcfg.nodes()

### Remove predecessors of last block
for pred in asmcfg.predecessors(last_block_loc_key):
    asmcfg.del_edge(pred, last_block_loc_key)
### Link first and last block
asmcfg.add_edge(first_block.loc_key, last_block_loc_key, AsmConstraint.c_next)
### Only one link between two asmcfg
try:
    asmcfg.add_edge(first_block, last_block_loc_key, AsmConstraint.c_to)
    good = False
except AssertionError:
    good = True
assert good

### Check final state
assert len(first_block.bto) == 1
assert list(first_block.bto)[0].c_t == AsmConstraint.c_next

## Simplify the obtained graph to keep only asmcfg which reach a block
## finishing with RET

def remove_useless_blocks(d_g, graph):
    """Remove leaves without a RET"""
    for leaf_label in graph.leaves():
        block = graph.loc_key_to_block(leaf_label)
        if block.lines[-1].name != "RET":
            graph.del_block(graph.loc_key_to_block(leaf_label))

### Use a graph simplifier to recursively apply the simplification pass
dg = DiGraphSimplifier()
dg.enable_passes([remove_useless_blocks])
asmcfg = dg(asmcfg)

### Only two asmcfg should remain
assert len(asmcfg) == 2
assert first_block.loc_key in asmcfg.nodes()
assert last_block_loc_key in asmcfg.nodes()

## Graph the final output
open("graph2.dot", "w").write(asmcfg.dot())

# Test helper methods
## loc_key_to_block should always be updated
assert asmcfg.loc_key_to_block(first_block.loc_key) == first_block
testlabel = loc_db.get_or_create_name_location("testlabel")
my_block = AsmBlock(loc_db, testlabel)
asmcfg.add_block(my_block)
assert len(asmcfg) == 3
assert asmcfg.loc_key_to_block(first_block.loc_key) == first_block
assert asmcfg.loc_key_to_block(my_block.loc_key) == my_block

## Bad asmcfg
assert len(list(asmcfg.get_bad_blocks())) == 0
assert len(list(asmcfg.get_bad_blocks_predecessors())) == 0
### Add a bad block, not linked
testlabel_bad = loc_db.get_or_create_name_location("testlabel_bad")
my_bad_block = AsmBlockBad(loc_db, testlabel_bad)
asmcfg.add_block(my_bad_block)
assert list(asmcfg.get_bad_blocks()) == [my_bad_block]
assert len(list(asmcfg.get_bad_blocks_predecessors())) == 0
### Link the bad block and update edges
### Indeed, a sub-element has been modified (bto from a block from asmcfg)
my_block.bto.add(AsmConstraintTo(my_bad_block.loc_key))
asmcfg.rebuild_edges()
assert list(asmcfg.get_bad_blocks_predecessors()) == [my_block.loc_key]
### Test strict option
my_block.bto.add(AsmConstraintTo(my_block.loc_key))
asmcfg.rebuild_edges()
assert list(asmcfg.get_bad_blocks_predecessors(strict=False)) == [my_block.loc_key]
assert len(list(asmcfg.get_bad_blocks_predecessors(strict=True))) == 0

## Sanity check
asmcfg.sanity_check()
### Next on itself
testlabel_nextitself = loc_db.get_or_create_name_location("testlabel_nextitself")
my_block_ni = AsmBlock(loc_db, testlabel_nextitself)
my_block_ni.bto.add(AsmConstraintNext(my_block_ni.loc_key))
asmcfg.add_block(my_block_ni)
error_raised = False
try:
    asmcfg.sanity_check()
except RuntimeError:
    error_raised = True
assert error_raised
### Back to a normal state
asmcfg.del_block(my_block_ni)
asmcfg.sanity_check()
### Multiple next on the same node
testlabel_target = loc_db.get_or_create_name_location("testlabel_target")
my_block_target = AsmBlock(loc_db, testlabel_target)
asmcfg.add_block(my_block_target)
testlabel_src1 = loc_db.get_or_create_name_location("testlabel_src1")
testlabel_src2 = loc_db.get_or_create_name_location("testlabel_src2")
my_block_src1 = AsmBlock(loc_db, testlabel_src1)
my_block_src2 = AsmBlock(loc_db, testlabel_src2)
my_block_src1.bto.add(AsmConstraintNext(my_block_target.loc_key))
asmcfg.add_block(my_block_src1)
### OK for now
asmcfg.sanity_check()
### Add a second next from src2 to target (already src1 -> target)
my_block_src2.bto.add(AsmConstraintNext(my_block_target.loc_key))
asmcfg.add_block(my_block_src2)
error_raised = False
try:
    asmcfg.sanity_check()
except RuntimeError:
    error_raised = True
assert error_raised
asmcfg.del_block(my_block_src2)
asmcfg.sanity_check()

## Guess block size
### Initial state
assert not hasattr(first_block, 'size')
assert not hasattr(first_block, 'max_size')
asmcfg.guess_blocks_size(mdis.arch)
assert first_block.size == 39
assert asmcfg.loc_key_to_block(my_block_src1.loc_key).size == 0
assert first_block.max_size == 39
assert asmcfg.loc_key_to_block(my_block_src1.loc_key).max_size == 0

## Check pendings
### Create a pending element
testlabel_pend_src = loc_db.get_or_create_name_location("testlabel_pend_src")
testlabel_pend_dst = loc_db.get_or_create_name_location("testlabel_pend_dst")
my_block_src = AsmBlock(loc_db, testlabel_pend_src)
my_block_dst = AsmBlock(loc_db, testlabel_pend_dst)
my_block_src.bto.add(AsmConstraintTo(my_block_dst.loc_key))
asmcfg.add_block(my_block_src)
### Check resulting state
assert len(asmcfg) == 7
assert len(asmcfg.pendings) == 1
assert my_block_dst.loc_key in asmcfg.pendings
assert len(asmcfg.pendings[my_block_dst.loc_key]) == 1
pending = list(asmcfg.pendings[my_block_dst.loc_key])[0]
assert isinstance(pending, asmcfg.AsmCFGPending)
assert pending.waiter == my_block_src
assert pending.constraint == AsmConstraint.c_to
### Sanity check must fail
error_raised = False
try:
    asmcfg.sanity_check()
except RuntimeError:
    error_raised = True
assert error_raised
### Pending must disappeared when adding expected block
asmcfg.add_block(my_block_dst)
assert len(asmcfg) == 8
assert len(asmcfg.pendings) == 0
asmcfg.sanity_check()

# Test block_merge
data2 = decode_hex("31c0eb0c31c9750c31d2eb0c31ffebf831dbebf031edebfc31f6ebf031e4c3")
cont2 = Container.from_string(data2, loc_db)
mdis = machine.dis_engine(cont2.bin_stream, loc_db=loc_db)
## Elements to merge
asmcfg = mdis.dis_multiblock(0)
## Block alone
asmcfg.add_block(mdis.dis_block(0x1c))
## Bad block
asmcfg.add_block(mdis.dis_block(len(data2)))
## Dump the graph before merging
open("graph3.dot", "w").write(asmcfg.dot())
## Apply merging
asmcfg = bbl_simplifier(asmcfg)
## Dump the graph after merging
open("graph4.dot", "w").write(asmcfg.dot())
## Check the final state
assert len(asmcfg) == 5
assert len(list(asmcfg.get_bad_blocks())) == 1
### Check "special" asmcfg
entry_asmcfg = asmcfg.heads()
bad_block_lbl = next((lbl for lbl in entry_asmcfg
                 if isinstance(asmcfg.loc_key_to_block(lbl), AsmBlockBad)))
entry_asmcfg.remove(bad_block_lbl)
alone_block = next((asmcfg.loc_key_to_block(lbl) for lbl in entry_asmcfg
               if len(asmcfg.successors(lbl)) == 0))
entry_asmcfg.remove(alone_block.loc_key)
assert alone_block.lines[-1].name == "RET"
assert len(alone_block.lines) == 2
### Check resulting function
entry_block = asmcfg.loc_key_to_block(entry_asmcfg.pop())
assert len(entry_block.lines) == 4
assert list(map(str, entry_block.lines)) == ['XOR        EAX, EAX',
                                       'XOR        EBX, EBX',
                                       'XOR        ECX, ECX',
                                       'JNZ        loc_key_27']
assert len(asmcfg.successors(entry_block.loc_key)) == 2
assert len(entry_block.bto) == 2
nextb = asmcfg.loc_key_to_block(next((cons.loc_key for cons in entry_block.bto
                              if cons.c_t == AsmConstraint.c_next)))
tob = asmcfg.loc_key_to_block(next((cons.loc_key for cons in entry_block.bto
                            if cons.c_t == AsmConstraint.c_to)))
assert len(nextb.lines) == 4
assert list(map(str, nextb.lines)) == ['XOR        EDX, EDX',
                                 'XOR        ESI, ESI',
                                 'XOR        EDI, EDI',
                                 'JMP        loc_key_28']
assert asmcfg.successors(nextb.loc_key) == [nextb.loc_key]
assert len(tob.lines) == 2
assert list(map(str, tob.lines)) == ['XOR        EBP, EBP',
                               'JMP        loc_key_27']
assert asmcfg.successors(tob.loc_key) == [tob.loc_key]

# Check split_block
## Without condition for a split, no change
asmcfg_bef = asmcfg.copy()
mdis.apply_splitting(asmcfg)
assert asmcfg_bef == asmcfg
open("graph5.dot", "w").write(asmcfg.dot())
## Create conditions for a block split
inside_firstbbl = loc_db.get_offset_location(4)
tob.bto.add(AsmConstraintTo(inside_firstbbl))
asmcfg.rebuild_edges()
assert len(asmcfg.pendings) == 1
assert inside_firstbbl in asmcfg.pendings
mdis.apply_splitting(asmcfg)
## Check result
assert len(asmcfg) == 6
assert len(asmcfg.pendings) == 0
assert len(entry_block.lines) == 2
assert list(map(str, entry_block.lines)) == ['XOR        EAX, EAX',
                                       'XOR        EBX, EBX']
assert len(asmcfg.successors(entry_block.loc_key)) == 1
lbl_newb = asmcfg.successors(entry_block.loc_key)[0]
newb = asmcfg.loc_key_to_block(lbl_newb)
assert len(newb.lines) == 2
assert list(map(str, newb.lines)) == ['XOR        ECX, ECX',
                                'JNZ        loc_key_27']
preds = asmcfg.predecessors(lbl_newb)
assert len(preds) == 2
assert entry_block.loc_key in preds
assert tob.loc_key in preds
assert asmcfg.edges2constraint[(entry_block.loc_key, lbl_newb)] == AsmConstraint.c_next
assert asmcfg.edges2constraint[(tob.loc_key, lbl_newb)] == AsmConstraint.c_to

# Check double block split
data = decode_hex("74097405b8020000007405b803000000b804000000c3")
cont = Container.from_string(data, loc_db)
mdis = machine.dis_engine(cont.bin_stream, loc_db=loc_db)
asmcfg = mdis.dis_multiblock(0)
## Check resulting disasm
assert len(asmcfg.nodes()) == 6
asmcfg.sanity_check()
## Check graph structure
bbl0 = MatchGraphJoker(name="0")
bbl2 = MatchGraphJoker(name="2")
bbl4 = MatchGraphJoker(name="4")
bbl9 = MatchGraphJoker(name="9")
bblB = MatchGraphJoker(name="B")
bbl10 = MatchGraphJoker(name="10")

matcher = bbl0 >> bbl2 >> bbl4 >> bbl9 >> bblB >> bbl10
matcher += bbl2 >> bbl9 >> bbl10
matcher += bbl0 >> bblB

solutions = list(matcher.match(asmcfg))
assert len(solutions) == 1
solution = solutions.pop()
for jbbl, label in viewitems(solution):
    offset = loc_db.get_location_offset(label)
    assert offset == int(jbbl._name, 16)

loc_key_dum = loc_db.get_or_create_name_location("dummy_loc")
asmcfg.add_node(loc_key_dum)
error_raised = False
try:
    asmcfg.sanity_check()
except RuntimeError:
    error_raised = True
assert error_raised


import os

import matplotlib.pyplot as plt

def plot_result(df, scenario_id):
    plt.figure()
    plt.title(f"Scenario {scenario_id}")
    plt.plot(df.s0, label='not_infected')
    plt.plot(df.s1, label="infected")
    plt.plot(df.s2, label="recovered")
    plt.plot(df.s3, label="dead")
    plt.xlabel("Step")
    plt.ylabel("Count")
    plt.legend()
    plt.savefig(
        os.path.join(
            os.path.join(os.path.dirname(os.path.dirname(__file__)), "data/output"),
            f"Mesa_PopulationInfection_S{scenario_id}" + ".png"
        ),
        dpi=200,
        format="PNG",
    )


import fractions
#Project Euler question 5
#What is the smallest positive number that is evenly divisible by all of the numbers from 1 to 20?

#gets the solution very slowly (brute force)
def pEQuestion5():
    isDivisible = False
    num = 2
    while isDivisible is False:
        for i in range(1,21):
            if num%i==0 and i == 20:
               return num
            num+=1
            print(num)
    return num

#My alt cuts time in half of thee intial solultion
def alt():
    div =1
    isDivisible = False
    while isDivisible is False:
        for i in range(1,21):
            if (20*div)%i != 0:
                break
            else:
                print(div,i)
        div +=1
    return 20*div

#found this soultion at https://www.geeksforgeeks.org/smallest-number-divisible-first-n-numbers/
#This soultion is very fast
def opt():
    div = 1
    for i in range(1,21):
        div = (div*i)/fractions.gcd(div,i)
    return div

print(opt())
#print(alt())
#print(pEQuestion5())


from google.protobuf import message

from recipe_engine import recipe_api, recipe_test_api
from recipe_engine import util as recipe_util

from . import proto_codec

class ProtoOutputPlaceholder(recipe_util.OutputPlaceholder):
  def __init__(self, api, msg_class, codec, add_json_log, name,
               leak_to, decoding_kwargs):
    if codec is proto_codec.BINARY:
      self.raw = api.m.raw_io.output(codec.ext, leak_to=leak_to)
    else:
      self.raw = api.m.raw_io.output_text(codec.ext, leak_to=leak_to)
    self.add_json_log = add_json_log
    self._msg_class = msg_class
    self._codec = codec
    self._decoding_kwargs = decoding_kwargs
    super(ProtoOutputPlaceholder, self).__init__(name=name)

  @property
  def backing_file(self):
    return self.raw.backing_file

  def render(self, test):
    return self.raw.render(test)

  def result(self, presentation, test):
    # This is a bit silly, but we only have the codec information here, and
    # we don't want the user to redundantly provide it in the test.
    if test.enabled and isinstance(test.data, message.Message):
      # We replace the test object with one containing raw bytes for raw_io.
      test = recipe_test_api.PlaceholderTestData(
          data=ProtoApi.encode(test.data, self._codec),
          name=self.name)

    # Save name before self.raw.result() deletes it.
    backing_file = self.backing_file
    raw_data = self.raw.result(presentation, test)
    if raw_data is None:
      if self.add_json_log in (True, 'on_failure'):
        presentation.logs[self.label + ' (read error)'] = [
          'Proto file was missing or unreadable:',
          '  ' + backing_file,
        ]
      return None

    valid = False
    invalid_error = ''
    ret = None
    try:
      ret = ProtoApi.decode(
          raw_data, self._msg_class, self._codec, **self._decoding_kwargs)
      valid = True
    except Exception as ex:  # pragma: no cover
      invalid_error = str(ex)
      jsonErrPrefix = 'Failed to load JSON: '
      if test.enabled and invalid_error.startswith(jsonErrPrefix):
        invalid_error = (
            jsonErrPrefix +
            recipe_util.unify_json_load_err(invalid_error[len(jsonErrPrefix):]))

    if self.add_json_log is True or (
        self.add_json_log == 'on_failure' and presentation.status != 'SUCCESS'):
      if valid:
        jsonpb = ProtoApi.encode(ret, 'JSONPB', indent=2)
        presentation.logs[self.label] = jsonpb.splitlines()
      else:
        presentation.logs[self.label + ' (invalid)'] = raw_data.splitlines()
        presentation.logs[self.label + ' (exception)'] = (
          invalid_error.splitlines())

    return ret

class ProtoApi(recipe_api.RecipeApi):

  @recipe_util.returns_placeholder
  def input(self, proto_msg, codec, **encoding_kwargs):
    """A placeholder which will expand to a file path containing the encoded
    `proto_msg`.

    Example:
       proto_msg = MyMessage(field=10)
       api.step('step name', ['some_cmd', api.proto.input(proto_msg)])
       # some_cmd sees "/path/to/random.pb"

    Args:
      * proto_msg (message.Message) - The message data to encode.
      * codec ('BINARY'|'JSONPB'|'TEXTPB') - The encoder to use.
      * encoding_kwargs - Passed directly to the chosen encoder. See:
        - BINARY: google.protobuf.message.Message.SerializeToString
          * 'deterministic' defaults to True.
        - JSONPB: google.protobuf.json_format.MessageToJson
          * 'preserving_proto_field_name' defaults to True.
          * 'sort_keys' defaults to True.
          * 'indent' defaults to 0.
        - TEXTPB: google.protobuf.text_format.MessageToString

    Returns an InputPlaceholder.
    """
    codec = proto_codec.resolve(codec)
    encoded = self.encode(proto_msg, codec, **encoding_kwargs)
    suffix = '.%s' % (codec.ext,)
    if codec is proto_codec.BINARY:
      return self.m.raw_io.input(encoded, suffix=suffix)
    return self.m.raw_io.input_text(encoded, suffix=suffix)

  @recipe_util.returns_placeholder
  def output(self, msg_class, codec, add_json_log=True, name=None,
             leak_to=None, **decoding_kwargs):
    """A placeholder which expands to a file path and then reads an encoded
    proto back from that location when the step finishes.

    Args:
      * msg_class (protobuf Message subclass) - The message type to decode.
      * codec ('BINARY'|'JSONPB'|'TEXTPB') - The encoder to use.
      * add_json_log (True|False|'on_failure') - Log a copy of the parsed proto
        in JSONPB form to a step link named `name`. If this is 'on_failure',
        only create this log when the step has a non-SUCCESS status.
      * leak_to (Optional[Path]) - This path will be used in place of a random
        temporary file, and the file will not be deleted at the end of the step.
      * decoding_kwargs - Passed directly to the chosen decoder. See:
        - BINARY: google.protobuf.message.Message.Parse
        - JSONPB: google.protobuf.json_format.Parse
          * 'ignore_unknown_fields' defaults to True.
        - TEXTPB: google.protobuf.text_format.Parse
    """
    codec = proto_codec.resolve(codec)
    if not issubclass(msg_class, message.Message): # pragma: no cover
      raise ValueError('msg_class is unexpected type: %r' % (msg_class,))
    if add_json_log not in (True, False, 'on_failure'): # pragma: no cover
      raise ValueError(
          'unexpected value for add_json_log: %r' % (add_json_log,))

    return ProtoOutputPlaceholder(
        self, msg_class, codec, add_json_log, name, leak_to, decoding_kwargs)

  @staticmethod
  def encode(proto_msg, codec, **encoding_kwargs):
    """Encodes a proto message to a string.

    Args:
      * codec ('BINARY'|'JSONPB'|'TEXTPB') - The encoder to use.
      * encoding_kwargs - Passed directly to the chosen encoder. See output
        placeholder for details.

    Returns the encoded proto message.
    """
    if not isinstance(proto_msg, message.Message): # pragma: no cover
      raise ValueError('proto_msg had unexpected type: %s' % (type(proto_msg),))
    return proto_codec.do_enc(codec, proto_msg, **encoding_kwargs)

  @staticmethod
  def decode(data, msg_class, codec, **decoding_kwargs):
    """Decodes a proto message from a string.

    Args:
      * msg_class (protobuf Message subclass) - The message type to decode.
      * codec ('BINARY'|'JSONPB'|'TEXTPB') - The encoder to use.
      * decoding_kwargs - Passed directly to the chosen decoder. See input
        placeholder for details.

    Returns the decoded proto object.
    """
    if not issubclass(msg_class, message.Message): # pragma: no cover
      raise ValueError('msg_class is unexpected type: %r' % (msg_class,))
    return proto_codec.do_dec(data, msg_class, codec, **decoding_kwargs)


# -*- coding: utf-8 -*-
#
#  Copyright (c) 2013-present, Anoop Kunchukuttan
#  All rights reserved.
#
#  This source code is licensed under the MIT license found in the
#  LICENSE file in the root directory of this source tree.
#

#Program to transliterate acronyms from one Latin script to Indic languages
#
# @author Anoop Kunchukuttan
#

from indicnlp.transliterate.unicode_transliterate import UnicodeIndicTransliterator
import string
import random

class LatinToIndicAcronymTransliterator(object):

    LATIN_TO_DEVANAGARI_TRANSTABLE = str.maketrans({
        'a':'ए',
        'b':'बी',
        'c':'सी',
        'd':'डी',
        'e':'ई',
        'f':'एफ',
        'g':'जी',
        'h':'एच',
        'i':'आई',
        'j':'जे',
        'k':'के',
        'l':'एल',
        'm':'एम',
        'n':'एन',
        'o':'ओ',
        'p':'पी',
        'q':'क्यू',
        'r':'आर',
        's':'एस',
        't':'टी',
        'u':'यू',
        'v':'वी',
        'w':'डब्ल्यू',
        'x':'एक्स',
        'y':'वाय',
        'z':'जेड',
    })

    # a_unichr=ord('a')
    # alphabet = [ chr(a_unichr+n) for n in range(26) ]
    LATIN_ALPHABET = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']

    @staticmethod
    def get_transtable():
        return LatinToIndicAcronymTransliterator.LATIN_TO_DEVANAGARI_TRANSTABLE

    @staticmethod
    def transliterate(w,lang):
        return UnicodeIndicTransliterator.transliterate(w.lower().translate(LatinToIndicAcronymTransliterator.LATIN_TO_DEVANAGARI_TRANSTABLE),'hi',lang)

    @staticmethod
    def generate_latin_acronyms(num_acronyms, min_len=2, max_len=6, strategy='random'):
        """
            generate Latin acronyms in lower case
        """

        def sample_acronym(strategy='random'):
            if strategy=='random':
                slen=random.randint(min_len,max_len)
                return ''.join(random.choices(LatinToIndicAcronymTransliterator.LATIN_ALPHABET,k=slen))

        return [ sample_acronym(strategy) for i in range(num_acronyms) ]


import discord
import os
import random
import youtube_module
import pyshorteners as sh
import harvester
import insta_follower
from django.core.validators import URLValidator
from dotenv import load_dotenv
from discord.ext.commands import Bot
from discord.ext import commands

load_dotenv()
TOKEN = os.getenv("DISCORD_TOKEN")
GUILD = os.getenv("DISCORD_GUILD")
validator = URLValidator()
s = sh.Shortener()
client = discord.Client()
img = 'https://cdn.discordapp.com/avatars/743381002382082198/e7bbfd903534e2aecec7748e707c3626.webp?size=1024'

@client.event
async def on_ready():
    print(f"{client.user.name} has connected to Discord!")

@client.event
async def on_member_join(member):
    await member.create_dm()
    await member.dm_channel.send(f"Hi {member.name}, welcome to my Discord server!")

bot = commands.Bot(command_prefix="0")

@bot.command(name="yt")
async def youtube(ctx, url):
    await ctx.message.reply("W A I T . . .", delete_after=5)
    link = youtube_module.get_download_link(url)
    response = "link: {}".format(link)
    await ctx.message.reply(response)

@bot.command(name="cc")
@commands.has_role("admin")
async def create_channel(ctx, channel_name,category_name):
    guild = ctx.guild
    existing_channel = discord.utils.get(guild.channels, name=channel_name)
    if not existing_channel:
        print(f"Creating a new channel: {channel_name}")
        await guild.create_text_channel(channel_name)
        await ctx.message.reply("your channel created successfully", delete_after=5)

@bot.command(name="ig")
async def gain_data(ctx, url):
    await ctx.message.reply("its maybe take a moment...", delete_after=5)
    data = harvester.datas(url)
    await ctx.send(file=discord.File('result.txt'))

@bot.command(name="fn")
async def get_insta(ctx, insta_id):
    await ctx.message.reply("W A I T . . .", delete_after=5)
    numbers = insta_follower.get_follower(insta_id)
    response = "Your follower numbers: {}".format(f'{numbers:,}')
    await ctx.message.reply(response)

@bot.event
async def on_command_error(ctx, error):
    if isinstance(error, commands.errors.CheckFailure):
        await ctx.send("You do not have the required role for this command.")

bot.run(TOKEN)


import torch
import numpy as np

import time

from THHyperIBLT import THHyperIBLT
from fedIBLT import HyperIBLT

def _get_prime(n: int):
    def _is_prime(x):
        for i in range(2, int(np.sqrt(x)) + 1):
            if x % i == 0:
                return False
        return True

    while not _is_prime(n):
        n += 1
    return n

def test_encode_decode(numel):
    device = "cuda" if torch.cuda.is_available() else "cpu"

    NUM_ROW: int = 3
    NUM_COL: int = _get_prime(int(1.25 * numel/NUM_ROW))

    print(f"size of HyperIBLT : num_row = {NUM_ROW}, num_col = {NUM_COL}")

    NUMEL: int = numel
    NUM_BLOCK: int = 20
    MODULO: int = 114514

    tensor = torch.randn(numel, dtype=torch.float32).to(device)

#    index = torch.nonzero(tensor).view(-1)
#    values = tensor[index]
#    index = index.type(torch.int32)

    index = torch.arange(numel, dtype=torch.int32).to(device)
    values = tensor

    encode_table = THHyperIBLT(NUM_ROW, NUM_COL, NUMEL, NUM_BLOCK)

    print("Start of encode")
    start_time = time.perf_counter()
    encode_table.encode(index, values)
    end_time = time.perf_counter()
    print("End of encode")

    print(f"Encode time = {end_time - start_time}")

    results = np.zeros((NUMEL,), dtype=np.float32)
    key_sum = encode_table.key_sum.numpy().astype(np.uint32)
    value_sum = encode_table.value_sum.numpy().astype(np.float32)
    counter = encode_table.counter.numpy().astype(np.uint32)
    hash_bucket = encode_table.hash_bucket.numpy().astype(np.uint32)

    decode_table = HyperIBLT(
      NUM_ROW,
      NUM_COL,
      NUMEL,
      MODULO,
      key_sum,
      value_sum,
      counter,
      hash_bucket
    )

    start_time = time.perf_counter()
    decode_table.decode(results)
    end_time = time.perf_counter()
    print(f"Decode time = {end_time - start_time}")

    print(f"origin tensor = {tensor}")
    print(f"results = {results}")

if __name__ == "__main__":
    test_encode_decode(10_000_000)


from collections import deque

d = deque()

for i in range(int(input())):
    s = input().split()
    if s[0] == 'append':
        d.append(s[1])
    elif s[0] == 'appendleft':
        d.appendleft(s[1])
    elif s[0] == 'pop':
        d.pop()
    elif s[0] == 'popleft':
        d.popleft()
for j in d:
    print(j,end=' ')


from django.urls import path, include
from . import views
from rest_framework import routers

app_name = 'blog'

router = routers.DefaultRouter()
router.register('post', views.PostViewSet)
router.register('category', views.PostCategoryViewSet)
router.register('author', views.AuthorViewSet)

urlpatterns = [
    path('', include(router.urls)),
]


import sys
import time
import subprocess
import progressbar

BAR = progressbar.ProgressBar(max_value=progressbar.UnknownLength, redirect_stdout=False)

def usage():
    message = "Usage: " \
              + sys.argv[0] \
              + " radius_users_file radius_ip radius_port radius_secret" \
              + "radius_ip radius_port radius_secret"

    if len(sys.argv) != 8:
        print(message)
        sys.exit(1)

def get_file_content(file_path):
    content = None

    with open(file_path, "r") as f:
        content = f.read()
        f.close()

    return content

def clear_radius_user_line(user_line):
    cleared_line = []

    for c in user_line:
        if '\t' in c:
            tmp = c.split('\t')
            for d in tmp:
                if len(str(d)) > 0:
                    cleared_line.append(str(d))
        else:
            if len(c) > 0:
                cleared_line.append(str(c))

    return cleared_line

def get_radius_users_from_file(users_file_path):
    users_file = get_file_content(users_file_path)

    if not users_file or len(users_file) <= 0:
        print("Users file is empty.")
        sys.exit(1)

    users = {}
    cpt = 0
    users_temp = users_file.split("\n")
    for line in users_temp:
        if "Cleartext-Password" in line:
            user_line = str(line.strip()).split(" ")
            user_line = clear_radius_user_line(user_line)
            users[str(user_line[0]).strip()] = \
                str(str(user_line[3]).strip()).strip('"')

    return users

def test_server_connectivity(ip, retry):
    tries = 0
    connectivity_ok = False

    while tries <= retry:
        tries +=1
        try:
            output = subprocess.check_output(["/bin/ping", "-W", "3", "-c", "1", str(ip)], timeout=5 )
        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
            time.sleep(1)
            continue
        else:
            if not output or len(str(output)) <= 0 \
              or "Destination Net Unreachable" in str(output) \
              or "No route to host" in str(output) \
              or "0 received" in str(output) :
                time.sleep(1)
                continue
            else:
                connectivity_ok = True
                break

    if not connectivity_ok:
        print("Error : " + ip + " Radius server unreachable.")
        sys.exit(1)

def radtest(users, rad_ip, rad_port, rad_secret, tout, retry):
    '''
    Call radtest as a shell subprocess, verify its return code
    and store the output.

    users: (MAP<str, str>) Map of <user, pwd> which to perform radtest
    rad_ip: (str) IP of the Radius server
    rad_port: (str) Port of the Radius Server
    rad_secret: (str) Secret of the Radius Server (see /ect/freeradius/3.0/clients.conf)
    tout: (int) Timeout of radtest
    retry: (int) Number of retries to perform for each user

    returns a MAP<str, str> of user and output of radtest.
    '''

    test_server_connectivity(rad_ip, 5)

    radtest_output = {}
    found_users = 0

    for user, pwd in users.items():
        found = False
        tries = 0
        while tries <= retry:
            tries += 1
            try:
                output = subprocess.check_output(["/usr/bin/radtest", "-4", str(user), str(pwd), str(rad_ip), str(rad_port), str(rad_secret)], timeout=tout)
            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
                time.sleep(0.07)
                continue
            else:
                radtest_output[str(user)] = str(output)
                found = True
                found_users += 1
                BAR.update(found_users)
                break
        if not found:
                print("User: " + str(user) + " pwd: " + str(pwd) + " radtest returned an error.")
                sys.exit(1)
        time.sleep(0.07)

    return radtest_output

def print_rad_replies_error(out1, out2, ip1, ip2):
    out1  = str(out1).split("\\n")
    out2 = str(str(out2).strip()).split("\\n")

    print("Server " + str(ip1) + " replied:")
    for line in out1:
        print(str(line)) #.strip('\\t'))
    print("Server " + str(ip2) + " replied:")
    for line in out2:
        print(str(line)) #.strip('\\t'))

    sys.exit(1)

def check_rad_replies(radtest_output1, radtest_output2, ip1, ip2):
    '''
    Check if radius replies are the same for the 2 servers.
    '''

    if len(radtest_output1.keys()) != len(radtest_output2.keys()):
        print("Output from server 1 and server 2 arent the same.")
        sys.exit(1)

    error = False
    error_replies = []

    for user, out1 in radtest_output1.items():
        if str(user) not in radtest_output2.keys():
            print("Error for user " + str(user) + ": User not found in server " + str(ip2) + " replies")
            error = True
            continue

        out2 = radtest_output2[str(user)]
        # Verify if outputs are exactly the same
        if out1 == out2:
            continue
        else:
            user_output1 = str(out1).split("\\n")
            user_output2 = str(str(out2).strip()).split("\\n")

            if len(user_output1) != len(user_output2):
                print("Error for user " + str(user) + ": incomplete reply.")
                print_rad_replies_error(out1, out2, ip1, ip2)
                sys.exit(1)

                continue

            for string in user_output1:
                #string = str(string).strip("\\t")
                if str(ip1) in str(string) or str(ip2) in str(string):
                    continue
                else:
                    if str(string).strip() not in user_output2:
                        print("Error for user " + str(user))
                        print("reply " + str(string))
                        print("of server: " + str(ip1) + " not present in the replies of server: " + str(ip2))
                        print("\n")
                        print_rad_replies_error(out1, out2, ip1, ip2)
                        sys.exit(1)

        if error:
            print("Error found for user : " + str(user))
            sys.exit(1)

    if not error:
        print("Checked " + str(len(radtest_output1.keys())) + " entries. OK.")
    else:
        print("Error found.")

def main():
    usage()
    # get all users from the users file and parse them into a user-password array

    users = get_radius_users_from_file(str(sys.argv[1]).strip())
    BAR = progressbar.ProgressBar(max_value=progressbar.UnknownLength, redirect_stdout=False)
    BAR.update(0)
    out1 = radtest(users, str(sys.argv[2]), str(sys.argv[3]), str(sys.argv[4]), 5, 1)
    BAR = progressbar.ProgressBar(max_value=progressbar.UnknownLength, redirect_stdout=False)
    BAR.update(0)
    out2 = radtest(users, str(sys.argv[5]), str(sys.argv[6]), str(sys.argv[7]), 5, 1)
    check_rad_replies(out1, out2, str(sys.argv[2]), str(sys.argv[5]))

if __name__ == "__main__":
    main()


import turtle
from PIL import Image
from turtle import Screen, Turtle
import cv2
polygon = Turtle()
forw = .001

turt = turtle. Turtle(visible=False)
turt.pencolor((0, 0, 100))
turt.pensize(.1)
turt.speed(0)

screen = turtle.Screen()
screen.bgcolor('black')

# turt.begin_fill()
# time.sleep(5)
# turtle.done()

while True:
    turt.forward(forw)
    turt.left(13)
    # turt.left(200)
    forw += .01
    polygon.hideturtle()
    # time.sleep(2)
    ts = screen.getcanvas().postscript(file="w.eps", colormode='color')
    inFile = Image.open("w.eps")
    outFile = "static/1.jpg"
    inFile.save(outFile)


from metric.recall_at_k import RecallAtK
from dataloader.mamo_dataset import MamoDataset
from validator import Validator
from torch.utils.data import DataLoader
from tests.integration.mocks.mock_models import MockAllZeros
from tests.integration.mocks.mock_models import MockNoChange
from tests.integration.mocks.mock_models import MockOpposite
from tests.integration.mocks.mock_models import MockShiftRightByOne
from tests.integration.mocks.mock_loss import MSELoss
import numpy as np

# Packages needed to run test:
# os
# numpy
# torch
# pytest

# Variables
# Mock dataset
input_data = np.array([[1, 0, 0, 0, 0, 0],
                       [0, 1, 0, 0, 0, 0],
                       [0, 0, 1, 0, 0, 0],
                       [0, 0, 0, 1, 0, 0],
                       [0, 0, 0, 0, 1, 0],
                       [0, 0, 0, 0, 0, 1]])
input_data = input_data.astype(float)

# Test demonstrating mock all zeros model to show missing
# metrics or missing objectives
def test_validator_mock_all_zeros_model():
    mock_dataset = MamoDataset(input_data, input_data.copy())
    mock_dataloader = DataLoader(mock_dataset, batch_size=1,
                                 shuffle=False, drop_last=False)
    v_all_zeros = Validator(MockAllZeros(), mock_dataloader,
                            [RecallAtK(1)], None)
    results = v_all_zeros.evaluate()
    assert isinstance(results, tuple)
    assert isinstance(results[0], list)
    assert(results[0][0] == 0)
    assert isinstance(results[1], list)
    assert(results[1] == [])
    v_all_zeros = Validator(MockAllZeros(), mock_dataloader,
                            None, [MSELoss()])
    results = v_all_zeros.evaluate()
    assert isinstance(results, tuple)
    assert isinstance(results[0], list)
    assert(results[0] == [])
    assert isinstance(results[1], list)
    mse = np.mean(input_data)
    assert(round(results[1][0], 2) == round(mse, 2))
    assert(round(v_all_zeros.combine_objectives(results[1]), 2)
           == round(mse, 2))

# Test demonstrating mock no change model
# Recall is 0 as we are recommending already chosen elements
def test_validator_mock_no_change_model():
    mock_dataset = MamoDataset(input_data, input_data.copy())
    mock_dataloader = DataLoader(mock_dataset, batch_size=1,
                                 shuffle=False, drop_last=False)
    v_no_change = Validator(MockNoChange(), mock_dataloader,
                            [RecallAtK(1)], [MSELoss()])
    results = v_no_change.evaluate()
    assert isinstance(results, tuple)
    assert isinstance(results[0], list)
    assert(results[0][0] == 0)
    assert isinstance(results[1], list)
    # Removing chosen elements -so:
    mse = np.mean(input_data)
    assert(round(results[1][0], 2) == round(mse, 2))
    assert(round(v_no_change.combine_objectives(results[1]), 2)
           == round(mse, 2))

# Test demonstrating mock opposite model
def test_validator_mock_opposite_model():
    mock_dataset = MamoDataset(input_data, input_data.copy())
    mock_dataloader = DataLoader(mock_dataset, batch_size=1,
                                 shuffle=False, drop_last=False)
    v_opposite = Validator(MockOpposite(), mock_dataloader,
                           [RecallAtK(1)], [MSELoss()])
    results = v_opposite.evaluate()
    assert isinstance(results, tuple)
    assert isinstance(results[0], list)
    assert(results[0][0] == 0)
    assert isinstance(results[1], list)
    assert(results[1][0] == 1)
    assert(v_opposite.combine_objectives(results[1]) == 1)

# Test demonstrating mock shift right by one model
def test_validator_mock_shift_right_by_one_model():
    mock_dataset = MamoDataset(input_data, np.roll(input_data.copy(),
                               shift=1, axis=1))
    mock_dataloader = DataLoader(mock_dataset, batch_size=1,
                                 shuffle=False, drop_last=False)
    v_shift_right = Validator(MockShiftRightByOne(), mock_dataloader,
                              [RecallAtK(1)], [MSELoss()])
    results = v_shift_right.evaluate()
    assert isinstance(results, tuple)
    assert isinstance(results[0], list)
    assert(results[0][0] == 1)
    assert isinstance(results[1], list)
    assert(results[1][0] == 0)
    assert(v_shift_right.combine_objectives(results[1]) == 0)


import os
import validator
import datetime
time = datetime.datetime.now()

data_path = 'C:/Users/Niki/Documents/Programming Projects/Data/user_record/'
auth_path = 'C:/Users/Niki/Documents/Programming Projects/Data/auth_session/'

def create(accountNumber, first_name, last_name, email, password):

    user_data = str(accountNumber) + ',' + first_name + "," + last_name + "," + email + "," + password + "," + str(0)
    if does_account_number_exist(accountNumber): #returns false if acct # already exists
        return False

    if does_email_exist(email): #returns false if email already exists in existing acct
        print('User already exists')
        return False

    completion_state = False

    try:
        f = open(data_path + str(accountNumber) + ".txt", "x")

    except FileExistsError:
        does_file_contain_data = read(data_path + str(accountNumber) + ".txt")
        if not does_file_contain_data:
            delete(accountNumber)
    else:
        f.write(str(user_data))
        completion_state = True
    finally:
        f.close()
        return completion_state

def delete(accountNumber):
    is_delete_successful = False
    if os.path.exists(data_path + str(accountNumber) + ".txt"):
        try:
            os.remove(data_path + str(accountNumber) + ".txt")
            is_delete_successful = True
        except FileNotFoundError:
            print('User not found')
        finally:
            return is_delete_successful

def read(accountNumber):
    is_valid_account_number = validator.accountNumberValidator(accountNumber) #if validator returns true it gets saved here
    try:
        if is_valid_account_number: #if previous line returned true, then it opens the file that matches the acct #, else does the same without the .txt
            f = open(data_path + str(accountNumber) + ".txt", "r")
        else:
            f = open(data_path + str(accountNumber), "r")
    except FileNotFoundError:
        print('User not found')
    except FileExistsError:
        print("User doesn't exist")
    else:
        return f.readline()
    return False

def does_email_exist(email):
    all_users = os.listdir(data_path)
    for user in all_users:
        user_list = str.split(read(user), ',')
        if email in user_list:
            return True
        else:
            return False

def does_account_number_exist(accountNumber):
    all_users = os.listdir(data_path) # lists all files in folder 'user_record' and assigns to list
    for user in all_users: # checks to see if inside the all_users list there is a file name that matches the accnt number entered by the user
        if user == str(accountNumber) + ".txt":
            return True
        else:
            return False

def auth_user(accountNumber, password):
    if does_account_number_exist(accountNumber): # goes does_account_number_exist function to see if acct # is already in use
        user = str.split(read(accountNumber), ',') # creates list from auth files and then searches for password inside list
        if password == user[4]:
            return user
    else:
        return False

def account_balance(user):
    f = open(data_path + (user[0]) + ".txt", "w")
    user_str = str(user[0]) + ',' + user[1] + "," + user[2] + "," + user[3] + "," + user[4] + "," + str(user[5])
    try:
        f.write(str(user_str))

    finally:
        f.close()

def login_session_stamp(user):
    try:
        f = open(auth_path + (user[1]) + ".txt", "x")
        f.write('User logged in at ' + str(time))
    except FileExistsError:
        os.remove(auth_path + (user[1]) + ".txt",)
        f = open(auth_path + (user[1]) + ".txt", "x")
        f.write('User logged in at ' + str(time))

def logout_stamp_delete(user):
    os.remove(auth_path + (user[1]) + ".txt",)


import pytest

from langchain_community.document_loaders.confluence import ConfluenceLoader

try:
    from atlassian import Confluence  # noqa: F401

    confluence_installed = True
except ImportError:
    confluence_installed = False

@pytest.mark.skipif(not confluence_installed, reason="Atlassian package not installed")
def test_load_single_confluence_page() -> None:
    loader = ConfluenceLoader(url="https://templates.atlassian.net/wiki/")
    docs = loader.load(page_ids=["33189"])

    assert len(docs) == 1
    assert docs[0].page_content is not None
    assert docs[0].metadata["id"] == "33189"
    assert docs[0].metadata["title"] == "An easy intro to using Confluence"
    assert docs[0].metadata["source"] == (
        "https://templates.atlassian.net/wiki/"
        "spaces/RD/pages/33189/An+easy+intro+to+using+Confluence"
    )

@pytest.mark.skipif(not confluence_installed, reason="Atlassian package not installed")
def test_load_full_confluence_space() -> None:
    loader = ConfluenceLoader(url="https://templates.atlassian.net/wiki/")
    docs = loader.load(space_key="RD")

    assert len(docs) == 14
    assert docs[0].page_content is not None

@pytest.mark.skipif(not confluence_installed, reason="Atlassian package not installed")
def test_confluence_pagination() -> None:
    loader = ConfluenceLoader(url="https://templates.atlassian.net/wiki/")
    # this will issue 2 requests; each with a limit of 3 until the max_pages of 5 is met
    docs = loader.load(space_key="RD", limit=3, max_pages=5)

    assert len(docs) == 5
    assert docs[0].page_content is not None

@pytest.mark.skipif(not confluence_installed, reason="Atlassian package not installed")
def test_pass_confluence_kwargs() -> None:
    loader = ConfluenceLoader(
        url="https://templates.atlassian.net/wiki/",
        confluence_kwargs={"verify_ssl": False},
    )

    assert loader.confluence.verify_ssl is False


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/10/25 9:24
# @Author  : Jun
# @Site    :
# @File    : urls.py
# @Software: PyCharm
from django.urls import path
from . import views

app_name = 'user'
urlpatterns = [
    path('loginPage',views.loginPage,name = 'loginPage'),        #登录页面
    path('loginLogic',views.loginLogic,name = 'loginLogic'),
    path('registPage',views.registPage,name = 'registPage'),     #注册页面
    path('registLogic',views.registLogic,name = 'registLogic'),
    path('checkEmail',views.checkEmail,name = 'checkEmail'),       #检测邮箱可用性
    path('forgetPwdPage',views.forgetPwdPage,name = 'forgetPwdPage'),  #忘记密码页面
    path('forgetLogic',views.forgetLogic,name = 'forgetLogic'),     #忘记密码验证
    path('changePwdPage',views.changePwdPage,name = 'changePwdPage'),  #修改密码页面
    path('changePwd',views.changePwd,name = 'changePwd'),     #修改密码
    path('index',views.index,name = 'index'),                 #修改密码
]


import os
import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import math
import matplotlib
#指定默认字体
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['font.family']='sans-serif' #解决负号'-'显示为方块的问题
matplotlib.rcParams['axes.unicode_minus'] = False

class backtest:
    def __init__(self, res_path, source_path, policy_type):
        self.res_path = res_path  # 结果文件位置
        self.hold_num = 20  # 持仓数
        self.adj_turn = 0.001  # 千分之一手续费
        self.st_money = 1000000  # 初始资金
        self.all_money = 1000000  # 现金流
        self.count_days = 0  # 计数当前是第几个交易日
        self.source_path = source_path  # 历史行情数据文件
        self.hold = pd.DataFrame(columns=['kdcode', 'hold'])
        self.policy_type = policy_type

    # 载入历史行情数据
    def load_history(self):
        df = pd.read_csv(self.source_path)
        df = df[['kdcode', 'dt', 'close']]
        return df

    # 查找一个元素在数组里面的位置，如果不存在，则返回-1
    def indexOf(e, a):
        for i in range(0, len(a)):
            if e == a[i]:
                return i
        return -1

    # 获取需要买入的股票代码
    def get_tobuy(self, best_stock, now_stock, max_count):
        count = 0
        toBuy = []
        for i in range(0, len(best_stock)):
            flag = 0
            for j in range(0, len(now_stock)):
                if best_stock[i] == now_stock[j]:
                    flag = 1
                    break;
            if flag == 0:
                toBuy.append(best_stock[i])
                count += 1
                if count >= max_count:
                    return toBuy
        return toBuy

    # 获取需要卖出的股票代码
    def get_tosell(self, now_price, now_stock, max_count):
        df = now_price[now_price['kdcode'].isin(now_stock)]
        #         print(df)
        df = df.sort_values(by=['close'], ascending=True)
        df = df.head(max_count)
        toSell = df['kdcode'].values
        return toSell

    # 模拟交易
    def trading(self, files, now_price):
        now_stock = self.hold['kdcode'].values
        best_stock = self.get_rank(self.res_path + '/' + files)
        best_stock = best_stock['kdcode'].values
        toBuy = best_stock
        toSell = now_stock
        # 现有策略，所有股票等金额
        if self.policy_type == 't3':
            max_count = 10
        if self.policy_type == 't5':
            max_count = 5
        if self.policy_type == 't10':
            max_count = 2
        if self.count_days == 0:
            max_count = 20
        toBuy = self.get_tobuy(best_stock, now_stock, max_count)
        max_count = len(toBuy)
        toSell = self.get_tosell(now_price, now_stock, max_count)
        self.sell(toSell, now_price)
        self.buy(toBuy, now_price)
        return self

    # 对于不在持仓列表的股票  卖出
    def sell(self, toSell, now_price):
        hold = self.hold  # 持仓份额
        for i in range(len(toSell)):
            if now_price[now_price['kdcode'] == toSell[i]].empty:  # 部分股票当日停牌
                code_price = 0.0
            else:
                code_price = 1.0 * now_price[now_price['kdcode'] == toSell[i]]['close'].values
                code_count = 1.0 * hold[hold['kdcode'] == toSell[i]]['hold'].values
                hold.drop(hold.index[hold['kdcode'] == toSell[i]], inplace=True)
            self.all_money += code_price[0] * code_count[0] * (1 - self.adj_turn)
        self.hold = hold
        return self

    # 对于需要持仓的股票  买入
    def buy(self, toBuy, now_price):
        sum_price = self.all_money / len(toBuy)  # 均分到每只股票的金额
        for i in range(len(toBuy)):
            price = 1.0 * now_price[now_price['kdcode'] == toBuy[i]]['close'].values
            hold = sum_price / (price * (1 + self.adj_turn))
            self.hold = self.hold.append({'kdcode': toBuy[i], 'hold': hold}, ignore_index=True)
        self.all_money = 0.0
        return self

    #     # 对于不在持仓列表的股票  卖出
    #     def sell_2(self, toSell, now_price):
    #         for i in range(len(toSell)):
    #             if now_price[now_price['kdcode']==toSell[i]].empty: # 部分股票当日停牌
    #                 code_price = 0.0
    #             else :
    #                 code_price = 1.0*now_price[now_price['kdcode']==toSell[i]]['close'].values
    #             self.all_money += code_price*self.counts*(1+self.adj_turn)

    #         return self

    #     # 对于需要持仓的股票  买入
    #     def buy_2(self, toBuy, now_price):
    #         sum_price = 0.00 # 需要买的股票的总价
    #         for i in range(len(toBuy)):
    #             sum_price += 1.0*now_price[now_price['kdcode']==toBuy[i]]['close'].values
    #         tmp = 0.0
    #         if sum_price:
    #             tmp = self.all_money/sum_price
    #         self.counts = int(tmp*1.0) # 股票份额 下取整
    #         self.all_money -= sum_price*self.counts*(1+self.adj_turn) # 剩余资产-现金流
    #         return self

    # 主函数 遍历所有交易日
    def my_run(self):
        pred_files = os.listdir(self.res_path)  # 获取所有的预测文件
        pred_files.sort()  # 按日期排序
        asset_list = []  # 记录总资产变化
        print("开始交易: ")
        price = self.load_history()
        for files in pred_files:
            today = files.split('.')[0]
            today_price = price[price['dt'] == today]
            self.trading(files, today_price)
            self.count_days += 1
            my_money = self.sum_money(today_price)
            print(my_money)
            asset_list.append(my_money)
        return asset_list

    # 计算当前资产总额--现金+股票
    def sum_money(self, today_price):
        now_stock = self.hold['kdcode'].values  # 获取当前持仓股票
        hold = self.hold
        s_m = self.all_money * 1.0
        for i in range(len(now_stock)):
            if today_price[today_price['kdcode'] == now_stock[i]].empty:  # 部分股票当日停牌
                code_price = 0.0
                s_m += 0
            else:
                code_price = 1.0 * today_price[today_price['kdcode'] == now_stock[i]]['close'].values
                code_count = 1.0 * hold[hold['kdcode'] == now_stock[i]]['hold'].values
                s_m += code_price[0] * code_count[0]
        return s_m

    # 对当日股票进行排序
    def get_rank(self, file_path):
        stock = pd.read_csv(file_path)
        stock = stock.sort_values(by="score", ascending=False)
        stock = stock[0:20].reset_index(drop=True)
        stock = stock[['kdcode', 'dt']]
        return stock

    # 评估函数
    def evaluate(self):
        asset_list = self.my_run()
        print(asset_list)
        print("年化利率： ")
        print(self.evaluate_AnnualizedReturn(asset_list))
        print("夏普率： ")
        print(self.evaluate_SharpeRatio(asset_list))
        print("年化波动： ")
        print(self.evaluate_Volatility(asset_list))
        print("最大回撤: ")
        print(self.evaluate_MaxDrawDown(asset_list))

    # 年化利率
    def evaluate_AnnualizedReturn(self, asset_list):
        test_return = (asset_list[-1] - asset_list[0]) / asset_list[0]
        test_days = len(asset_list) - 1
        ar = (test_return + 1) ** (365 / test_days) - 1
        return ar

    # 夏普率
    def evaluate_SharpeRatio(self, asset_list):
        ar = self.evaluate_AnnualizedReturn(asset_list)
        v = self.evaluate_Volatility(asset_list)
        return ar / v

    # 年化波动
    def evaluate_Volatility(self, asset_list):
        daily_return = [(i - j) / j for i, j in zip(asset_list[1:], asset_list[:-1])]
        daily_return = [i * 365 for i in daily_return]
        erp = sum(daily_return) / len(daily_return)
        v = sum((i - erp) * (i - erp) for i in daily_return)
        v = math.sqrt(v / len(daily_return))
        return v

    # 最大回撤
    def evaluate_MaxDrawDown(self, asset_list):
        min_asset = asset_list.copy()
        for i in range(len(min_asset) - 2, -1, -1):
            min_asset[i] = min(min_asset[i], min_asset[i + 1])
        mdd = max(-(min_asset[i] - asset_list[i]) / asset_list[i] for i in range(len(min_asset)))
        return mdd


from django.urls import path, include
from .views import EventDelete
from . import views

urlpatterns = [
    path('events/', views.eventList, name='events'),
    path('event-detail/<int:pk>', views.eventPage, name='event-detail'),
    path('event-create/', views.createEvent, name='event-create'),
    path('event-delete/<int:pk>', EventDelete.as_view(), name='event-delete'),
    path('event-update/<int:pk>', views.updateEvent, name='event-update'),
    path('delete-comment/<int:pk>', views.deleteComment, name = 'delete-comment')
]


import torch
import torch.nn.functional as F
import torch.nn as nn

def metric_rmse(output,label):
    # this is our MSE loss function
    return torch.sqrt(torch.mean(torch.square(torch.sub(output,label)))) #MSE as loss

# Write a method called `train_one_epoch` that runs one step using the optimizer.
def train_one_epoch(train_loader, model, device, optimizer, log_interval, epoch, batch_size,loss_func):
    model.train()
    losses = []
    counter = []
    metrics=[]

    for i, (img, label) in enumerate(train_loader):
        img, label = img.to(device), label.to(device)

        # ------------------
        # forward pass
        output=model(img)
        # print(label)
        loss=  loss_func(output, label)# loss function goes here

        #zero all gradients
        optimizer.zero_grad()

        #do a backward pass
        loss.backward()

        #update weights
        optimizer.step()
        # print(loss)

        # ------------------

        # Record training loss every log_interval and keep counter of total training images seen
        if (i+1) % log_interval == 0:
            losses.append(loss.item())
            counter.append((i * batch_size) + img.size(0) + epoch * len(train_loader.dataset))
            metrics.append(metric_rmse(nn.Sigmoid()(output),label).item())

    return losses, counter, metrics

# Write a method called `test_one_epoch` that evalutes the trained model on the test dataset. Return the average test loss and the number of samples that the model predicts correctly.
def test_one_epoch(test_loader, model, device,loss_func):
    model.eval()
    test_loss = 0
    counter=0
    mse=0

    with torch.no_grad():
        for i, (img, label) in enumerate(test_loader):
            img, label = img.to(device), label.to(device)

            # ------------------
            output = model(img)
            counter+=1
            test_loss +=  loss_func(output,label).item()# loss function goes here
            mse+=metric_rmse(nn.Sigmoid()(output), label).item()
            # ------------------

    return test_loss/counter, mse/counter


from google.appengine.ext.webapp import template
import webapp2
import os

from google.appengine.ext import ndb
import json

class DemoDayVisitor(ndb.Model):
  email   = ndb.StringProperty(indexed=True, required=True)
  company = ndb.StringProperty(indexed=True)

  def to_dict(self):
    return {
      'email': self.email,
      'company': self.company
    }

class EmailHandler(webapp2.RequestHandler):
  def get(self):
    visitors = DemoDayVisitor.query()
    dumps = map(lambda visitor: visitor.to_dict(), visitors)
    self.response.write(json.dumps({
      'success': True,
      'visitors': dumps
    }))

  def post(self):
    try:
      body = json.loads(self.request.body)
      email = body['email']
      if not email: raise Exception('Email not provided')
      company = body['company'] if 'company' in body else None
      key = ndb.Key('DemoDayVisitor', email)
      visitor = DemoDayVisitor(email=email, company=company, key=key)
      visitor.put()
      self.response.write(json.dumps({
        'success': True,
        'visitor': visitor.to_dict()
      }))
    except:
      self.response.write(json.dumps({
        'success': False
      }))

  def put(self):
    try:
      body = json.loads(self.request.body)
      email = body['email']
      key = ndb.Key('DemoDayVisitor', email)
      visitor = key.get()
      company = body['company'] if 'company' in body else None
      visitor.company = company
      visitor.put()
      self.response.write(json.dumps({
        'success': True,
        'visitor': visitor.to_dict()
      }))
    except:
      self.response.write(json.dumps({
        'success': False
      }))

class DefaultHandler(webapp2.RequestHandler):
  def get(self):
    template_values = {}
    path = os.path.join(os.path.dirname(__file__), 'templates/landing.html')
    self.response.out.write(template.render(path, template_values))

app = webapp2.WSGIApplication([
  ('/api/v1/emails/?', EmailHandler),
  ('.*', DefaultHandler)
])


import argparse
import json
import os
from collections import Counter

import numpy as np
from pandas.io.json import to_json
from sklearn.cluster import DBSCAN

from utils.dataset_utils import get_train_list, get_test_list
from utils.graph_similarity_utils import convert_graph_to_vector, get_feature_vector
from utils.graph_utils import get_removed_backed_graph, get_opcode_sequence
from utils.oep_utils import get_oep_dataset_2, get_preceding_oep

parser = argparse.ArgumentParser()
parser.add_argument('--mode', default="evaluation", type=str)
parser.add_argument('--packer_names', nargs="+", default=["upx"])
parser.add_argument('--file_name', default="accesschk.exe", type=str)
parser.add_argument('--sample_files', nargs="+",
                    default=["AccessEnum.exe", "Cacheset.exe", "ADInsight.exe", "ADExplorer.exe"])
parser.add_argument('--train_path', default="data/train.txt", type=str)
parser.add_argument('--first_k', default=3, type=int)
# Get the arguments
args = parser.parse_args()
data_folder_path = "data"
train_list = get_train_list() + get_test_list()
oep_dictionary_2 = get_oep_dataset_2()

def get_packer_name_and_filename(packer_name_file_name):
    packer_name_of_file, file_name = packer_name_file_name.strip().split("_")[0], "_".join(
        packer_name_file_name.strip().split("_")[1:])
    return packer_name_of_file, file_name

def frequency_feature_vecter_computation(packer_name, packed_files):
    print("Process packer: {}".format(packer_name))
    data = {}
    merged_unique_labels = []
    opcode_sequence = {}
    for packed_file in packed_files:
        packer_name_by_file, file_name = get_packer_name_and_filename(packed_file)
        packed_dot_file = os.path.join(data_folder_path, "asm_cfg", packer_name_by_file,
                                       "{}_model.dot".format(packed_file))
        oep_address = oep_dictionary_2[packed_file]
        print("packed_file = {}, oep_address = {}".format(packed_file, oep_address))
        preceding_oep, msg = get_preceding_oep(packed_dot_file, oep_address)
        if not preceding_oep:
            print("Packer: {}, file_name: {}, error: {}".format(packer_name_by_file, packed_file, msg))
            continue
        packed_graph = get_removed_backed_graph(packer_name_by_file, file_name)
        node_list_packed_file, node_labels_packed_file, original_labels_packed_file, _ = convert_graph_to_vector(
            packed_graph,
            address=preceding_oep)

        data[packed_file] = Counter(list(node_labels_packed_file.values()) + original_labels_packed_file)
        opcode_sequence[packed_file] = get_opcode_sequence(packed_graph, preceding_oep)
        merged_unique_labels = sorted(
            list(set(merged_unique_labels + list(node_labels_packed_file.values()) + original_labels_packed_file)))

    return data, opcode_sequence, merged_unique_labels

def standard_generation(packed_files, opcode_sequence, labels, X, merged_unique_labels, first_k=5):
    standard_feature_vectors = {}
    n_sample_of = {}
    opcode_sequence_of_label = {}

    # for packed_file in packed_files:
    #     _, file_name = get_packer_name_and_filename(packed_file)

    for idx in range(len(packed_files)):
        label = str(labels[idx])
        # print("label = {}".format(label))
        if label == "-1":
            continue
        # print("pass: {}".format(type(label)))
        if not label in opcode_sequence_of_label:
            opcode_sequence_of_label[label] = []

        _, file_name = get_packer_name_and_filename(packed_files[idx])
        opcode_sequence_of_label[label].append(opcode_sequence[packed_files[idx]])

        if label in n_sample_of:
            n_sample_of[label] += 1
        else:
            n_sample_of[label] = 1

        if not (label in standard_feature_vectors):
            standard_feature_vectors[label] = {}

        for idy, label_of_node in enumerate(merged_unique_labels):
            if not (label_of_node in standard_feature_vectors[label]):
                standard_feature_vectors[label][label_of_node] = 0.0
            standard_feature_vectors[label][label_of_node] += X[idx][idy]
    # Check condition opcode sequence in group have to have the same sequence > 0
    for cluster_id, opcode_sequences in opcode_sequence_of_label.items():
        for idx in range(len(opcode_sequences)):
            if opcode_sequences[idx][:first_k] != opcode_sequences[0][:first_k]:
                return standard_feature_vectors, opcode_sequence_of_label, None

    # for cluster_id in opcode_sequence_of_label.keys():
    #     for cluster_other_id in opcode_sequence_of_label.keys():
    #         if cluster_other_id != cluster_id:
    #             if opcode_sequence_of_label[cluster_other_id][0][:first_k] == opcode_sequence_of_label[cluster_id][0][
    #                                                                           :first_k]:
    #                 return standard_feature_vectors, opcode_sequence_of_label, 1
    # only check other groups

    return standard_feature_vectors, opcode_sequence_of_label, n_sample_of

def construct_standard_vector(packer_name, packed_files):
    print("Process packer: {}".format(packer_name))
    data, opcode_sequence, merged_unique_labels = frequency_feature_vecter_computation(packer_name, packed_files)
    print(opcode_sequence)
    X = []
    idx = 0
    packer_name_list = []
    for packed_file in packed_files:
        _, file_name = get_packer_name_and_filename(packed_file)

        try:
            feature_vector = get_feature_vector(data[packed_file], merged_unique_labels)

            print(feature_vector)
            X.append(feature_vector)
            packer_name_list.append(_)
            idx += 1
        except Exception as e:
            pass
    X = np.asarray(X)
    eps_default = 0.3
    while True:
        print("esp = {}".format(eps_default))
        if eps_default <= 0:
            break
        clustering = DBSCAN(eps=eps_default, min_samples=2, metric="cosine").fit(X)
        labels = clustering.labels_
        cluster_by_idx = {}
        for idx, packer_name_iter in enumerate(packer_name_list):
            print("packer name: {}, {}".format(packer_name_iter, labels[idx]))
            if not (labels[idx] in cluster_by_idx):
                cluster_by_idx[labels[idx]] = []
            cluster_by_idx[labels[idx]].append(packer_name_iter)

        not_cross = True

        nb_of_class = 0
        for key in cluster_by_idx.keys():
            if key == "-1" or key == -1:
                continue
            nb_of_class += 1
        for key, value in cluster_by_idx.items():
            if key == "-1" or key == -1:
                continue
            if len(set(value)) != 1:
                not_cross = False
                break
        print("eps = {}, not cross: {}, # of class = {}".format(eps_default, not_cross, nb_of_class))
        standard_feature_vectors, opcode_sequence_of_label, n_sample_of = standard_generation(packed_files,
                                                                                opcode_sequence,
                                                                                labels, X,
                                                                                merged_unique_labels)
        if n_sample_of is None:
            eps_default -= 0.01
        else:
            optimal_standard_feature_vectors = standard_feature_vectors
            optimal_opcode_sequence_of_label = opcode_sequence_of_label
            optimal_n_sample_of = n_sample_of
            break

    assert X.shape[0] == len(packed_files)
    for label in optimal_standard_feature_vectors.keys():
        for idx, label_of_node in enumerate(merged_unique_labels):
            optimal_standard_feature_vectors[label][label_of_node] /= optimal_n_sample_of[label]

    # Save end-of-sequence
    with open("configs/end_of_unpacking_sequence_mixpacker.txt", "a") as f:
        for label, opcode_sequences in optimal_opcode_sequence_of_label.items():
            print("label sequence = {}".format(label))
            for opcode_sequence in opcode_sequences:
                print(opcode_sequence)
            max_length = min([len(seq) for seq in opcode_sequences])
            is_diff = False
            for idx in range(0, max_length):
                opcode = opcode_sequences[0][idx]
                for opcode_sequence in opcode_sequences:
                    if opcode_sequence[idx] != opcode:
                        is_diff = True
                        break
                if is_diff:
                    line = "_".join(opcode_sequences[0][:idx])
                    f.writelines("{},{}\n".format(packer_name, line))
                    break
            if not is_diff:
                line = "_".join(opcode_sequences[0][:max_length])
                f.writelines("{},{}\n".format(packer_name, line))
    return optimal_standard_feature_vectors

def main():
    train_of = {}

    for line in train_list:
        packer_name_of_file, file_name = get_packer_name_and_filename(line.strip())
        if packer_name_of_file != "aspack":
            continue
        if not (packer_name_of_file in train_of):
            train_of[packer_name_of_file] = []
        train_of[packer_name_of_file].append(line.strip())

    packed_all = []
    for packer_name, packed_files in train_of.items():
        packed_all += packed_files
    print(len(packed_all))
    standard_feature_vectors = construct_standard_vector("all", packed_all)
    # print(standard_feature_vectors)
    # with open("configs/standard_feature_vectors_mixed/{}.json".format(packer_name), "w") as outfile:
    #     json.dump(standard_feature_vectors, outfile, indent=4)

if __name__ == '__main__':
    main()