import http.client, json, os, sys, urllib.parse, uuid
from os.path import join
from socket import gaierror
from spreadsheetWrangler import SpreadsheetWrangler

thismodule = sys.modules[__name__]
thismodule.scriptdir = os.path.dirname(os.path.realpath(__file__))
thismodule.parentdir = join(thismodule.scriptdir, "..")

class WordTranslator():
    def __init__(self):
        subscriptionKeysFolder = join(thismodule.parentdir, "subscriptionKeys")
        subscriptionKeyPath = join(subscriptionKeysFolder, "translatorSubscriptionKey.txt")
        self.subscriptionKey = open(subscriptionKeyPath).read()
        self.host = 'api.cognitive.microsofttranslator.com'
        self.path = '/translate?api-version=3.0'

    def translate_words_and_create_spreadsheet(self, inputWords, outputspreadsheet, outputLangCodes, outputLangNames):
        print("I'll print each word when I finish translating it")

        translationsDict = self.generate_translations_dict(inputWords, outputLangCodes)
        SpreadsheetWrangler.write_dict_to_spreadsheet(translationsDict, outputspreadsheet, outputLangCodes)

        print ("I'm finished translating words")

    # returns a dict like {fish: [鱼, Pescado, Cá], kidney beans: [芸豆, Frijoles, Đậu thận]}
    def generate_translations_dict(self, inputWords, outputLangs):
        translations = {}

        for word in inputWords:
            word = word.lstrip().rstrip().capitalize()

            # Empty words get an empty response
            if word is None or word is "":
                translatedWords = []
                for _ in outputLangs:
                    translatedWords.append("")
            else:
                translatedWords = self.get_translated_words(word, outputLangs)

            translations[word] = translatedWords
            print(word)

        return translations

    def get_translated_words(self, inputWord, outputLangs):
        result = self.__get_translations_from_server(inputWord, outputLangs)
        return self.__get_words_from_result(result)

    # gets server translation json for a given word to the given output langs
    def __get_translations_from_server(self, word, outputLangs):
        params = ""
        for lang in outputLangs:
            params += "&to=" + lang.lstrip().rstrip()

        requestBody = [{
            'Text' : word,
        }]

        headers = {
            'Ocp-Apim-Subscription-Key': self.subscriptionKey,
            'Content-type': 'application/json',
            'X-ClientTraceId': str(uuid.uuid4())
        }

        content = json.dumps(requestBody, ensure_ascii=False).encode('utf-8')
        try:
            conn = http.client.HTTPSConnection(self.host)
            conn.request ("POST", self.path + params, content, headers)
            response = conn.getresponse()
            return response.read()
        except gaierror:
            return None

    @staticmethod
    # converts output json into a list of the translated words
    def __get_words_from_result(result):
        if result == None:
            return []

        jsonData = json.loads(result)
        words = []

        for language in jsonData[0]["translations"]:
            words.append(language["text"])

        return words

if __name__ == '__main__':
    inputFolder = join(thismodule.parentdir, "input")
    outputFolder = join(thismodule.parentdir, "output")
    WordTranslator().translate_words_and_create_spreadsheet(
        list(open(join(inputFolder, "words.txt"))),
        join(outputFolder,"translatedWords.xlsx"),
        list(open(join(inputFolder, "toLanguages.txt")))[1:],
        ["Simplified Chinese", "Spanish", "Vietnamese"]) # Yeah it's a bit of a hardcoded cheat, I know


# -*- coding: utf-8 -*-
import json

def main():
    f = open('tang300.v3.json', 'r')
    ps = json.load(f)
    np = []
    for p in ps:
        b = False
        if p[3][0][-1] == u'，':
            t = []
            for i in xrange(len(p[3])/2):
                t.append(p[3][2*i]+p[3][2*i+1])
            p[3] = t
        np.append(p)
    f2 = open('tang300.v4.json', 'w')
    s = json.dumps(np, ensure_ascii=False, indent=2)
    f2.write(s.encode('utf-8'))
    f.close()
    f2.close()

main()


from django.contrib import admin

from goodsapp import models
class TypeInfoAdmin(admin.ModelAdmin):
    list_display = ['id', 'ttitle']

class GoodsInfoAdmin(admin.ModelAdmin):
    list_per_page = 15
    list_display = ['id', 'gtitle', 'gprice', 'gunit', 'gclick', 'gstock', 'gcontent', 'gtype']

admin.site.register(models.TypeInfo, TypeInfoAdmin)
admin.site.register(models.GoodsInfo, GoodsInfoAdmin)


from werkzeug.serving import make_server
from threading import Thread
import base64
import datetime
import collections
from flask import Flask, request
import logging
from requests_oauthlib import OAuth2Session
import requests
from threading import Timer
import time
import webbrowser
import yaml

class FlaskServer:
    def __init__(self, oauth2_server, host="localhost", port=8000):
        self.oauth2_server = oauth2_server
        self.app = Flask(__name__)
        self.srv = make_server(host, port, self.app)
        self.ctx = self.app.app_context()
        self.ctx.push()
        self.setup_routes()

    def setup_routes(self):
        @self.app.route("/callback")
        def callback():
            authorization_code = request.args.get("code")
            response = self.oauth2_server.fetch_token(authorization_code)
            self.oauth2_server.save_token(response)

            shutdown_thread = Thread(target=self.srv.shutdown)
            shutdown_thread.start()
            return "Success!"

    def run(self, authorization_url):
        Timer(1, lambda: webbrowser.open(authorization_url)).start()
        self.srv.serve_forever()

class OAuth2Server:
    def __init__(
        self,
        client_id,
        client_secret,
        authorization_base_url,
        token_url,
        scope,
        redirect_uri="http://localhost:8000/callback",
    ):
        self.server = OAuth2Session(client_id, redirect_uri=redirect_uri, scope=scope)
        self.client_id = client_id
        self.client_secret = client_secret
        self.authorization_base_url = authorization_base_url
        self.token_url = token_url
        self.flask_server = FlaskServer(self)
        self.response = None

    def fetch_token(self, authorization_code):
        return self.server.fetch_token(
            self.token_url,
            client_id=self.client_id,
            client_secret=self.client_secret,
            code=authorization_code,
            include_client_id=True,
        )

    def save_token(self, response):
        self.response = response

    def run(self):
        authorization_url, _ = self.server.authorization_url(
            self.authorization_base_url
        )
        logging.info(f"Please go to {authorization_url} and authorize access.")
        self.flask_server.run(authorization_url)

class CredsFile:
    def __init__(self, path):
        self.path = path

    def read_file(self) -> dict:
        with open(self.path, "r") as stream:
            return yaml.safe_load(stream)

    def get(self, key) -> dict:
        return self.read_file()[key]

    def write(self, entries):
        with open(self.path, "w") as stream:
            # Careful, this rewrites the entire yaml file
            yaml.safe_dump(entries, stream)

    def update(self, new_entry):
        entries = self.read_file()
        updated_entries = self.deep_update(entries, new_entry)
        self.write(updated_entries)
        entries = self.read_file()

    def deep_update(self, d, u):
        # https://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth
        for k, v in u.items():
            if isinstance(v, collections.abc.Mapping):
                d[k] = self.deep_update(d.get(k, {}), v)
            else:
                d[k] = v
        return d

class AccessTokenManager:
    def __init__(self, creds_file):
        self.creds_file = creds_file
        self.creds = None

    def refresh(self, creds_key):
        self.load_creds(creds_key)
        if "access_token" not in self.creds:
            response = self.refresh_with_oauth2()
            self.update_token(response, creds_key)
        elif (
            "expires_at" in self.creds
            and self.token_expired()
            and "refresh_token" in self.creds
        ):
            response = self.refresh_with_refresh_token()
            self.update_token(response, creds_key)

    def token_expired(self):
        return datetime.datetime.today().timestamp() > self.creds["expires_at"]

    def refresh_with_oauth2(self):
        logging.info("Fetching access token using OAuth2")
        server = OAuth2Server(
            self.creds["client_id"],
            self.creds["client_secret"],
            self.creds["authorization_uri"],
            self.creds["token_uri"],
            self.creds.get("scope"),  # Splitwise doesn't API access
        )
        server.run()
        return server.response

    def refresh_with_refresh_token(self):
        logging.info("Using refresh token to fetch fresh access token")
        secret = str(self.creds["client_id"]) + ":" + self.creds["client_secret"]
        header = {"Authorization": "Basic " + self.encode_secret(secret)}
        data = {
            "client_id": self.creds["client_id"],
            "client_secret": self.creds["client_secret"],
            "refresh_token": self.creds["refresh_token"],
            "grant_type": "refresh_token",
        }
        response = requests.post(
            url=self.creds["token_uri"],
            headers=header,
            data=data,
        )
        return response.json()

    @staticmethod
    def encode_secret(secret):
        """Standard protocol for passing bytes over a network to avoid misinterpretation of data
        b64 explanation:
        https://stackoverflow.com/questions/201479/what-is-base-64-encoding-used-for
        Fibit request documentation:
        https://dev.fitbit.com/build/reference/web-api/authorization/client-credentials/
        """
        bytes_secret = secret.encode()
        encoded_bytes_secret = base64.b64encode(bytes_secret)
        encoded_string_secret = encoded_bytes_secret.decode()
        return encoded_string_secret

    def update_token(self, response, creds_key):
        if "expires_in" in response:
            response["expires_at"] = time.time() + response["expires_in"]
        keys_to_update = ["access_token", "refresh_token", "expires_at"]
        new_creds = {
            creds_key: {k: response[k] for k in keys_to_update if k in response}
        }
        self.creds_file.update(new_creds)

    def load_creds(self, creds_key):
        self.creds = self.creds_file.get(creds_key)

if __name__ == "__main__":
    # Creds tests
    print("STRAVA")
    CREDS_KEY = "strava"
    CREDS_PATH = "creds.yml"
    creds_file = CredsFile(CREDS_PATH)
    creds = AccessTokenManager(creds_file)
    creds.refresh(CREDS_KEY)

    print("FITBIT")
    CREDS_KEY = "fitbit"
    CREDS_PATH = "creds.yml"
    creds = AccessTokenManager(creds_file)
    creds.refresh(CREDS_KEY)

    print("SPLITWISE")
    CREDS_KEY = "splitwise"
    CREDS_PATH = "creds.yml"
    creds = AccessTokenManager(creds_file)
    creds.refresh(CREDS_KEY)


import itertools
import json
import os
import re

import numpy as np
import pandas
import pandas as pd
import torch
from pytorch_lightning import LightningDataModule
from pytorch_lightning.utilities.types import TRAIN_DATALOADERS, EVAL_DATALOADERS
from torch.utils.data import DataLoader, Dataset, IterableDataset
from datasets import load_dataset
from typing import List, Optional, Dict
from transformers import PreTrainedTokenizer, AutoTokenizer
import os.path
from datasets import load_dataset
from pathlib import Path
import platform
if platform.system() == "Linux":
    num_workers = 40
else:
    num_workers = 1

class DataModules(LightningDataModule):
    def __init__(self, file_path_dir: str, batch_size: int=32, max_len: int=32):
        super().__init__()
        self.file_path_dir = file_path_dir
        self.batch_size = batch_size
        self.max_len = max_len
        self.save_hyperparameters()

    def setup(self, stage: str) -> None:
        if stage == "fit" or stage is None:
            self.train_set = ToutiaoDataset(self.file_path_dir, "train", select_class=["100", "101"], max_len=self.max_len)
            self.eval_set = ToutiaoDataset(self.file_path_dir, "val", select_class=["100", "101"], max_len=self.max_len)
        if stage == "test" or stage is None:
            self.test_set = ToutiaoDataset(self.file_path_dir, "test", select_class=["100", "101"], max_len=self.max_len)
        if stage == "predict" or stage is None:
            self.predict_set = ToutiaoDataset

    def train_dataloader(self) -> TRAIN_DATALOADERS:
        return DataLoader(self.train_set, batch_size=self.batch_size, shuffle=True, num_workers=num_workers)

    def val_dataloader(self) -> EVAL_DATALOADERS:
        return DataLoader(self.eval_set, batch_size=self.batch_size, shuffle=False, num_workers=num_workers)

    def test_dataloader(self) -> EVAL_DATALOADERS:
        return DataLoader(self.test_set, batch_size=self.batch_size, shuffle=False, num_workers=num_workers)

    def predict_dataloader(self) -> EVAL_DATALOADERS:
        return DataLoader(self.predict_set, batch_size=self.batch_size, shuffle=False, num_workers=4)

class ToutiaoDataset(Dataset):
    def __init__(self, file_path_dir: str, mode: str, select_class: List[str],
                 padding: str="max_length", max_len: int=64, truncation: bool=True, label_map: Dict=None):
        self.file_path_dir = file_path_dir
        self.mode = mode
        self.padding = padding
        self.max_len = max_len
        self.truncation = truncation
        self.data = pd.read_csv(os.path.join(file_path_dir, f"{mode}/corpus-model-filtered-{mode}.csv"), sep="\t", header=0)
        self.select_data = self.data
        # self.select_data = pd.DataFrame(columns=self.data.columns)
        # for label_id in select_class:
        #     tmp = self.data[self.data.label_id == int(label_id)]
        #     self.select_data = pd.concat([self.select_data, tmp])
        # self.select_data = self.select_data.reset_index(drop=True)
        # del self.data
        self.tokenizer = AutoTokenizer.from_pretrained('gpt2-xl')
        # print(self.tokenizer.pad_token_id)
        self.tokenizer.pad_token = self.tokenizer.eos_token
        if label_map is None:
            keys = [str(i) for i in range(100, 117)]
            values = range(0, 17)
            self.label_map = dict(zip(keys, values))
            del keys, values

    def compute_label(self, field):
        label = self.label_map[str(field["label_id"])]
        return np.array(label)

    def convert_str_to_id(self, field):
        text = field["text"]
        output = self.tokenizer(text, text_target=text, padding='max_length', truncation=self.truncation,
                               max_length=self.max_len, add_special_tokens=True)
        output["labels"] = output["labels"][1:]
        return output

    def __len__(self):
        return len(self.select_data)

    def __getitem__(self, item):
        field = self.select_data.loc[item]
        field = field.to_dict()
        feature = self.convert_str_to_id(field)
        # print(feature)
        return (
            np.array(feature["input_ids"]),
            np.array(feature["labels"]),
            np.array(feature["attention_mask"])
        )

class HuggingFaceHugeDatasetLoad(IterableDataset):
    def __init__(self, file_path_dir: str, mode: str, max_len):
        file_list = Path(file_path_dir).glob("*.csv")
        file_list = [file_path.name for file_path in file_list]
        total_len = len(file_list)
        tmp = {"train": [os.path.join(file_path_dir, raw) for raw in file_list[:int(total_len * 0.8)]],
               "validation": [os.path.join(file_path_dir, raw) for raw in file_list[int(total_len * 0.8):int(total_len * 0.9)]],
               "test": [os.path.join(file_path_dir, raw) for raw in file_list[int(total_len * 0.9):]]}
        self.data = load_dataset("csv", data_files=tmp, split=mode, streaming=True)
        self.max_len = max_len

    def __iter__(self):
        for field in self.data:
            tmp = field["input_ids"]
            process = list(map(int, re.sub("'|\[|\]", "", tmp).split(", ")))[:self.max_len]
            yield [np.array(process), np.array(process)[1:], np.ones([self.max_len])]

class PTMDataModules(LightningDataModule):
    def __init__(self, file_path_dir: str, batch_size: int=32, max_len: int=32):
        super().__init__()
        self.file_path_dir = file_path_dir
        self.batch_size = batch_size
        self.max_len = max_len
        self.save_hyperparameters()

    def setup(self, stage: str) -> None:
        if stage == "fit" or stage is None:
            self.train_set = HuggingFaceHugeDatasetLoad(self.file_path_dir, "train", max_len=self.max_len)
            self.eval_set = HuggingFaceHugeDatasetLoad(self.file_path_dir, "validation", max_len=self.max_len)
        if stage == "test" or stage is None:
            self.test_set = HuggingFaceHugeDatasetLoad(self.file_path_dir, "test", max_len=self.max_len)

    def train_dataloader(self) -> TRAIN_DATALOADERS:
        return DataLoader(self.train_set, batch_size=self.batch_size)

    def val_dataloader(self) -> EVAL_DATALOADERS:
        return DataLoader(self.eval_set, batch_size=self.batch_size)

    def test_dataloader(self) -> EVAL_DATALOADERS:
        return DataLoader(self.test_set, batch_size=self.batch_size)

    def predict_dataloader(self) -> EVAL_DATALOADERS:
        return DataLoader(self.predict_set, batch_size=self.batch_size)

#
if __name__ == "__main__":
    # dset = HuggingFaceHugeDatasetLoad(file_path_dir=r"d://workspace//Work//merge_corpus_csv", mode='train', max_len=512)
    # for i in dset:
    #     print(i)
    dm = PTMDataModules(file_path_dir=r"d://workspace//Work//merge_corpus_csv", max_len=32, batch_size=64)
    dm.setup("fit")
    train_loader = dm.train_dataloader()
    for batch, _ in enumerate(train_loader):
        print(_)


"""
This script is based on flake8-markdown from John Franey.
Link to repository: https://github.com/johnfraney/flake8-markdown
"""

import argparse
import glob
import re
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor

if sys.version_info.major == 3 and sys.version_info.minor >= 7:
    SUBPROCESS_ARGS = dict(
        capture_output=True,
        text=True,
    )
else:
    SUBPROCESS_ARGS = dict(
        encoding="ascii",
        universal_newlines=True,
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )

def non_matching_lookahead(pattern):
    return r"(?={})".format(pattern)

def matching_group(pattern):
    return r"({})".format(pattern)

def non_matching_group(pattern):
    return r"(?:{})".format(pattern)

def strip_repl_characters(code):
    """Removes the first four characters from each REPL-style line.

    >>> strip_repl_characters('>>> "banana"') == '"banana"'
    True
    >>> strip_repl_characters('... banana') == 'banana'
    True
    """
    stripped_lines = []
    for line in code.splitlines():
        if line.startswith(">>> ") or line.startswith("... "):
            stripped_lines.append(line[4:])
        else:
            stripped_lines.append(line)
    return "\n".join(stripped_lines)

ONE_OR_MORE_LINES_NOT_GREEDY = r"(?:.*\n)+?"

regex_rule = "".join(
    [
        # Use non-matching group instead of a lookbehind because the code
        # block may have line highlighting hints. See:
        # https://python-markdown.github.io/extensions/fenced_code_blocks/#emphasized-lines
        non_matching_group("^```(python|pycon|py).*$"),
        matching_group(ONE_OR_MORE_LINES_NOT_GREEDY),
        non_matching_lookahead("^```"),
    ]
)

regex = re.compile(regex_rule, re.MULTILINE)

def security_check_markdown_file(markdown_file_path):
    markdown_content = open(markdown_file_path, "r").read()
    code_block_start_lines = []
    for line_no, line in enumerate(markdown_content.splitlines(), start=1):
        # Match python and pycon
        if line.startswith("```py"):
            code_block_start_lines.append(line_no)
    code_block_matches = regex.findall(markdown_content)
    for code_block_match in code_block_matches:
        code_block_type = code_block_match[0]
        match_text = code_block_match[1]
        # pycon lines start with ">>> " or "... ", so strip those characters
        if code_block_type == "pycon":
            match_text = strip_repl_characters(match_text)
        match_text = match_text.lstrip()
        bandit_process = subprocess.run(
            ["bandit", "-"],
            input=match_text,
            **SUBPROCESS_ARGS,
        )
        bandit_output = bandit_process.stdout
        bandit_output = bandit_output.strip()

        bandit_output = "".join(bandit_output)
        bandit_output = bandit_output.replace(r"<stdin>", markdown_file_path)
        print(bandit_output)

def security_check_markdown_glob(markdown_glob):
    files = glob.iglob(markdown_glob, recursive=True)
    with ThreadPoolExecutor() as executor:
        executor.map(security_check_markdown_file, files)

def main(argv=None):
    parser = argparse.ArgumentParser(description="Markdown globs")
    parser.add_argument(
        "globs",
        metavar="glob",
        type=str,
        nargs="+",
        help="a glob of Markdown files to check for vulnerabilities",
    )
    args = parser.parse_args(argv)
    markdown_globs = args.globs
    with ThreadPoolExecutor() as executor:
        executor.map(security_check_markdown_glob, markdown_globs)

main()


import os
import sys
import time
import pickle
import subprocess
import numpy as np
from hyperopt import Trials, STATUS_OK, tpe, hp, fmin

nthreads = 16
opath = "experiments"

r_alpha   = np.arange(0.025, 0.525, 0.025)
r_beta    = np.arange(0.500, 1.025, 0.025)
r_critexp = np.arange(    1,    11,     1)
r_tweight = np.arange(    1,    35,     5)

def evalpnr(space):

  # nextpnr args
  args = ' '.join(map(str, sys.argv[1:]))

  # suppress quiet mode
  args = args.replace('-q','')

  # append optimisation parameters
  cmd = ("%s --placer-heap-alpha %f --placer-heap-beta %f --placer-heap-critexp %i --placer-heap-timingweight %i"
          % (space['alpha'], space['beta'], space['critexp'], space['tweight']))

  # launch process
  process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

  clk = {}
  # parse process log
  while ( process.poll() is None ):
    output = process.stdout.readline().decode()
    if 'Max frequency for clock' in output:
      clkname = output.split()[5].replace('\'','').replace(':','')
      clk['%s' % clkname] = float(output.split()[6])
      continue

  loss = 1000.0 / np.exp( np.sqrt(clk[clkname]))

  print( "space [%s] -> %f(Mhz) loss=%f" % (space, clk[clkname], loss) )

  return{'loss': loss, 'status': STATUS_OK, 'eval_time': time.time(), 'attachments': {'time_module': pickle.dumps(time.time)}}

def main():

  if (sys.argv[1] != "nextpnr-ice40"):
    print("ERROR: only nexpnr-ice40 is supported")
    sys.exit(1)

  space = {
    'alpha'   : hp.quniform ('alpha',     0.010, 0.800, 0.010),
    'beta'    : hp.quniform ('beta',      0.300, 1.000, 0.010),
    'critexp' : hp.randint  ('critexp',   1,     10          ),
    'tweight' : hp.randint  ('tweight',   1,     40          ),
  }

  trials = Trials()

  best = fmin(fn = evalpnr,
              space = space,
              algo = tpe.suggest,
              parallelism = 1,
              max_evals = 100,
              trials = trials)

  print("\n\n")
  print("Best found: ", best)

if __name__ == "__main__":
  main()


from itertools import chain

from flask import render_template
from sqlalchemy import func

from tracker import db
from tracker import tracker
from tracker.form import CVEForm
from tracker.form import GroupForm
from tracker.model import CVE
from tracker.model import CVEGroup
from tracker.model import CVEGroupEntry
from tracker.model import CVEGroupPackage
from tracker.model.cve import cve_id_regex
from tracker.model.cvegroup import vulnerability_group_regex
from tracker.model.enum import status_to_affected
from tracker.user import reporter_required
from tracker.view.error import not_found

@tracker.route('/issue/<regex("{}"):issue>/copy'.format(cve_id_regex[1:-1]), methods=['GET'])
@tracker.route('/cve/<regex("{}"):issue>/copy'.format(cve_id_regex[1:-1]), methods=['GET'])
@tracker.route('/<regex("{}"):issue>/copy'.format(cve_id_regex[1:-1]), methods=['GET'])
@reporter_required
def copy_issue(issue):
    cve = db.get(CVE, id=issue)
    if not cve:
        return not_found()

    form = CVEForm()
    form.cve.data = cve.id
    form.description.data = cve.description
    form.issue_type.data = cve.issue_type
    form.notes.data = cve.notes
    form.reference.data = cve.reference
    form.remote.data = cve.remote.name
    form.severity.data = cve.severity.name

    return render_template('form/cve.html',
                           title='Add CVE',
                           form=form,
                           CVE=CVE,
                           action='/cve/add')

@tracker.route('/group/<regex("{}"):avg>/copy'.format(vulnerability_group_regex[1:-1]), methods=['GET'])
@tracker.route('/avg/<regex("{}"):avg>/copy'.format(vulnerability_group_regex[1:-1]), methods=['GET'])
@tracker.route('/<regex("{}"):avg>/copy'.format(vulnerability_group_regex[1:-1]), methods=['GET'])
@reporter_required
def copy_group(avg):
    group_id = avg.replace('AVG-', '')
    group_data = (db.session.query(CVEGroup, CVE, func.group_concat(CVEGroupPackage.pkgname, ' '))
                  .filter(CVEGroup.id == group_id)
                  .join(CVEGroupEntry, CVEGroup.issues)
                  .join(CVE, CVEGroupEntry.cve)
                  .join(CVEGroupPackage, CVEGroup.packages)
                  .group_by(CVEGroup.id).group_by(CVE.id)
                  .order_by(CVE.id)).all()
    if not group_data:
        return not_found()

    group = group_data[0][0]
    issues = [cve for (group, cve, pkg) in group_data]
    issue_ids = [cve.id for cve in issues]
    pkgnames = set(chain.from_iterable([pkg.split(' ') for (group, cve, pkg) in group_data]))

    form = GroupForm()
    form.advisory_qualified.data = group.advisory_qualified
    form.affected.data = group.affected
    form.bug_ticket.data = group.bug_ticket
    form.cve.data = '\n'.join(issue_ids)
    form.fixed.data = group.fixed
    form.notes.data = group.notes
    form.pkgnames.data = '\n'.join(sorted(pkgnames))
    form.reference.data = group.reference
    form.status.data = status_to_affected(group.status).name

    return render_template('form/group.html',
                           title='Add AVG',
                           form=form,
                           CVEGroup=CVEGroup,
                           action='/avg/add')


from typing import List
class Solution:
    def findDisappearedNumbers(self, nums: List[int]) -> List[int]:
        wp=0
        while wp<len(nums):
            temp=nums[wp]
            if nums[wp]==wp+1 or nums[temp-1]==temp:
                wp+=1
            else:
                nums[wp]=nums[temp-1]
                nums[temp-1]=temp
        ans=[]
        for idx,val in enumerate(nums):
            if val!=idx+1:
                ans.append(idx+1)
        return ans
s=Solution()
nums = [4,3,2,7,8,2,3,1]
print(s.findDisappearedNumbers(nums))


#!/usr/bin/env python
from __future__ import print_function

import os

from taskbase import (
    Task,
    FileType,
    use_file,
)

from pocketfeature.io import (
    backgroundfile,
    featurefile,
    matrixvaluesfile,
)
from pocketfeature.datastructs import PassThroughItems
from pocketfeature.datastructs.metadata import PocketFeatureScoresMatrixMetaData
from pocketfeature.defaults import (
    ALLOWED_SIMILARITY_METHODS,
    ALLOWED_VECTOR_TYPE_PAIRS,
    DEFAULT_BACKGROUND_STATISTICS_FILE,
    DEFAULT_BACKGROUND_NORMALIZATION_FILE,
    DEFAULT_SIMILARITY_METHOD,
    DEFAULT_VECTOR_TYPE_PAIRS,
)

class FeatureFileComparison(Task):
    BACKGROUND_STAT_DEFAULT = DEFAULT_BACKGROUND_STATISTICS_FILE
    BACKGROUND_COEFF_DEFAULT = DEFAULT_BACKGROUND_NORMALIZATION_FILE
    COMPARISON_METHOD_DEFAULT = DEFAULT_SIMILARITY_METHOD
    VECTOR_PAIRS_DEFAULT = DEFAULT_VECTOR_TYPE_PAIRS

    COMPARISON_METHODS = ALLOWED_SIMILARITY_METHODS
    VECTOR_TYPE_PAIRS = ALLOWED_VECTOR_TYPE_PAIRS

    feature_fileA = None
    feature_fileB = None
    statistics = None
    normalizations = None
    allowed_pairs_name = None
    comparison_method_name = None

    def setup(self, params=None, **kwargs):
        params = params or self.params
        defaults = self.defaults(**self.conf)
        self.setup_task(params, defaults=defaults, **kwargs)
        self.setup_params(params, defaults=defaults, **kwargs)
        self.setup_inputs(params, defaults=defaults, **kwargs)

    def load_inputs(self):
        self.load_background = backgroundfile.load(
            stats_file=use_file(self.statistics),
            norms_file=use_file(self.normalizations),
            allowed_pairs=self.allowed_pairs,
            compare_function=self.comparison_method)
        self.featuresA = featurefile.load(use_file(self.feature_fileA))
        self.featuresB = featurefile.load(use_file(self.feature_fileB))

    def execute(self):
        self.run_comparison_matrix()
        self.run_clean_annotate_results()

    def run_comparison_matrix(self):
        vectorsA = self.featuresA
        vectorsB = self.featuresB
        scores = self.background.get_comparison_matrix(vectorsA, vectorsB, matrix_wrapper=PassThroughItems)
        self.original_scores = scores

    def run_clean_annotate_results(self):
        scores = self.original_scores
        fileA_name = getattr(self.feature_fileA, 'name', '<stream>')
        fileB_name = getattr(self.feature_fileB, 'name', '<stream>')
        bg_stats_name = getattr(self.statistics, 'name', '<stream>')
        bg_norms_name = getattr(self.normalizations, 'name', '<stream>')
        scores = scores.round_values(4)
        scores.metadata = self.background.metadata.propagate(kind=PocketFeatureScoresMatrixMetaData,
                                                             FEATURE_FILE_A=fileA_name,
                                                             FEATURE_FILE_B=fileB_name,
                                                             BACKGROUND_STATS_FILE=bg_stats_name,
                                                             BACKGROUND_NORMS_FILE=bg_norms_name)
        self.scores = scores

    def produce_results(self):
        return self.generate_output(matrixvaluesfile.dump,
                                    self.scores,
                                    self.output,
                                    'Writing scores to {}')

    def setup_params(self, params, defaults=None, **kwargs):
        self.apply_setup(params, kwargs, defaults, {
            'comparison_method_name': 'comparison_method',
            'allowed_pairs_name': 'allowed_pairs',
        })
        self.apply_setup(params, kwargs, defaults, (
            'statistics',
            'normalizations',
        ))

        if self.comparison_method_name is not None:
            self.comparison_method = self.COMPARISON_METHODS[self.comparison_method_name]
        else:
            self.comparison_method = None
        if self.allowed_pairs_name is not None:
            self.allowed_pairs = self.VECTOR_TYPE_PAIRS[self.allowed_pairs_name]
        else:
            self.allowed_pairs = None

    def setup_inputs(self, params, defaults=None, **kwargs):
        self.apply_setup(params, kwargs, defaults, (
            'feature_fileA',
            'feature_fileB',
        ))

    @classmethod
    def parser(cls, stdin, stdout, stderr, environ, task_name):
        from argparse import ArgumentParser
        parser = ArgumentParser(prog=task_name,
                                description="""Compute tanimoto matrix for two FEATURE vectors with a background
                                               and score normalizations. If background files are not provided, they
                                               will be checked for in the current directory as well as FEATURE_DIR.
                                               By default the comparison method will be chosen from the background""")
        return parser

    @classmethod
    def arguments(cls, parser, stdin, stdout, stderr, environ, **kwargs):
        cls.input_arguments(parser, stdin, stdout, stderr, environ, **kwargs)
        cls.task_arguments(parser, stdin, stdout, stderr, environ, **kwargs)
        cls.parameter_arguments(parser, stdin, stdout, stderr, environ, **kwargs)

    @classmethod
    def defaults(cls, stdin, stdout, stderr, environ, **kwargs):
        defaults = super(FeatureFileComparison, cls).defaults(stdin, stdout, stderr, environ, **kwargs)
        defaults.update(cls.paremeter_defaults(stdin, stdout, stderr, environ, **kwargs))
        return defaults

    @classmethod
    def parameter_arguments(cls, parser, stdin, stdout, stderr, environ, **kwargs):
        parser.add_argument('-s', '--statistics',
                            metavar='FEATURESTATS',
                            type=FileType.compressed('r'),
                            help='FEATURE file containing standard devations of background [default: %(default)s]')
        parser.add_argument('-n', '--normalizations',
                            metavar='COEFFICIENTS',
                            type=FileType.compressed('r'),
                            help='Map of normalization coefficients for residue type pairs [default: %(default)s]')
        parser.add_argument('--comparison-method',
                            metavar='COMPARISON_METHOD',
                            choices=cls.COMPARISON_METHODS.keys(),
                            help='Scoring method to force (one of %(choices)s) [default: %(default)s]')
        parser.add_argument('--allowed-pairs',
                            metavar='PAIR_SET_NAME',
                            choices=cls.VECTOR_TYPE_PAIRS.keys(),
                            help='Pair selection method force (one of: %(choices)s) [default: %(default)s]')

    @classmethod
    def input_arguments(cls, parser, stdin, stdout, stderr, environ, **kwargs):
        parser.add_argument('feature_fileA',
                            metavar='FEATURE_FILE_A',
                            type=FileType.compressed('r'),
                            help='Path to first FEATURE file')
        parser.add_argument('feature_fileB',
                            metavar='FEATURE_FILE_B',
                            type=FileType.compressed('r'),
                            help='Path to second FEATURE file')

    @classmethod
    def paremeter_defaults(cls, stdin, stdout, stderr, environ, **kwargs):
        background_ff = cls.BACKGROUND_STAT_DEFAULT
        background_coeff = cls.BACKGROUND_COEFF_DEFAULT

        if 'POCKETFEATURE_DIR' in os.environ:
            feature_dir = os.environ.get('POCKETFEATURE_DIR')
            env_bg_ff = os.path.join(feature_dir, background_ff)
            env_bg_coeff = os.path.join(feature_dir, background_coeff)
            if not os.path.exists(background_ff) and os.path.exists(env_bg_ff):
                background_ff = env_bg_ff
            if not os.path.exists(background_coeff) and os.path.exists(env_bg_coeff):
                background_coeff = env_bg_coeff
        elif 'FEATURE_DIR' in os.environ:
            feature_dir = os.environ.get('FEATURE_DIR')
            env_bg_ff = os.path.join(feature_dir, background_ff)
            env_bg_coeff = os.path.join(feature_dir, background_coeff)
            if not os.path.exists(background_ff) and os.path.exists(env_bg_ff):
                background_ff = env_bg_ff
            if not os.path.exists(background_coeff) and os.path.exists(env_bg_coeff):
                background_coeff = env_bg_coeff

        return {
            'statistics': background_ff,
            'normalizations': background_coeff,
            'comparison_method_name': cls.COMPARISON_METHOD_DEFAULT,
            'allowed_pairs_name': cls.VECTOR_PAIRS_DEFAULT,
        }

if __name__ == '__main__':
    import sys
    sys.exit(FeatureFileComparison.run_as_script())


import torch
import matplotlib.pyplot as plt

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def preprocess(obs, env):
    """Performs necessary observation preprocessing."""
    if env in ['CartPole-v1', 'ALE/Pong-v5']:
        return torch.tensor(obs, device=device).float()
    else:
        raise ValueError('Please add necessary observation preprocessing instructions to preprocess() in utils.py.')

def print_pong_episodic_returns():
    j = 1
    for i in range(12):
        file_path = f'data/pong-returns-epoch-{i}.pt'
        data = torch.load(file_path, map_location=torch.device('cpu'))

        returns = data['returns']
        target_update_frequency = data['args']['target_update_frequency']
        train_frequency = data['args']['train_frequency']
        gamma = data['args']['gamma']
        eps_start = data['args']['eps_start']
        eps_end = data['args']['eps_end']
        n_episodes = data['args']['n_episodes']

        x = [x for x,y in returns]
        y = [y for x,y in returns]

        fig = plt.figure()
        ax = fig.add_subplot()
        fig.subplots_adjust(top=0.9)

        # Set titles for the figure and the subplot respectively
        fig.suptitle(f'Epoch {j+1}', fontsize=10, fontweight='bold')
        j+=1
        ax.set_title('Average returns at episode')
        ax.set_xlabel('Episode')
        ax.set_ylabel('Average return')

        plot_x = max(x) - 200
        plot_y = min(y) + 1
        ax.text(plot_x, plot_y, f'target_update_frequency = {target_update_frequency} \n gamma = {gamma} \n eps_start = {eps_start} \n eps_end = {eps_end} \n train_frequency = {train_frequency}',
                bbox={'facecolor': 'orange', 'alpha':0.3, 'pad':5})

        plt.xlim(0, max(x))
        plt.plot(x,y)
        plt.show()

        fig.savefig(f'plots/pong/pong-dqcn-returns-episode-{i}.png', format='png')

        print(data)

def print_cartpole_episodic_returns():
    for i in range(3):
        file_path = f'data/returns-epoch-{i}.pt'
        data = torch.load(file_path, map_location=torch.device('cpu'))

        print(data)
        returns = data
        # target_update_frequency = data['args']['target_update_frequency']
        # train_frequency = data['args']['train_frequency']
        # gamma = data['args']['gamma']
        # n_episodes = data['args']['n_episodes']

        x = [x for (x,y) in returns]
        y = [y for (x,y)in returns]

        print(x)

        fig = plt.figure()
        ax = fig.add_subplot()
        fig.subplots_adjust(top=0.9)

        # Set titles for the figure and the subplot respectively
        fig.suptitle(f'Epoch {i+1}', fontsize=10, fontweight='bold')
        ax.set_title('Average returns at episodes')
        ax.set_xlabel('Episode')
        ax.set_ylabel('Average return')

        # ax.text(250, -1, f'target_update_frequency = {target_update_frequency} \n gamma = {gamma} \n train_frequency = {train_frequency}',
        #         bbox={'facecolor': 'orange', 'alpha':0.3, 'pad':5})

        plt.plot(x,y)
        plt.show()

        fig.savefig(f'plots/cartpole/returns-episode-{i+1}.png', format='png')

        #print(data)

if __name__ == '__main__':
    print_pong_episodic_returns()


import gzip
import os

import numpy as np
import six
from six.moves.urllib import request

from numpy import genfromtxt
from sklearn.cross_validation import train_test_split
from sklearn import preprocessing
import numpy as np
import dateutil.parser
import pdb
import glob
import cPickle as pickle
import shelve
import six
from six.moves.urllib import request

def prepare_data():
    supervised_data = {}
    output_data = []
    new_data_list  = []
    files = glob.glob("/home/deep/development/deep_trading/ib/ftr_csv/*.csv")
    #files = ["/home/deep/development/deep_trading/ib/csv_data/KSCL.csv"]
    for file in files:
        stock_data = genfromtxt(file, delimiter=',', dtype=None, names=True)
        output = None
        daily_gain = None
        #check 1000 can be changed accordingly
        last_n_data = []
        intial_data = stock_data[0]
        for data in stock_data[1:]:
            final_value = data['Low']
            intial_value = intial_data['High']
            #can also add transaction fee
            if (final_value > intial_value):
                output = 1 #should have bought
            elif (final_value < intial_value):
                output = 2 #should have sold
            else:
                output = 0 #should have done nothing
            #percent_gain = (final_value - intial_value) / intial_value
            #converting into 0 to 1
            #output = ((percent_gain - (-1)) / (1 - (-1)) ) * (1 - 0) + 0
            final_date = dateutil.parser.parse(data["DateTime"]).date()
            intial_date = dateutil.parser.parse(intial_data["DateTime"]).date()
            last_n_data.append([intial_data["Low"], intial_data["High"], intial_data["Close"], intial_data["Open"], intial_data["Volume"]])
            if final_date != intial_date:
                opening_price = data["Open"]
                closing_price = intial_data["Close"]
                gain = (opening_price - closing_price) / closing_price
                #converting into 0 to 1 from -1 to 1
                #new_value = ( (old_value - old_min) / (old_max - old_min) ) * (new_max - new_min) + new_min
                daily_gain = ((gain - (-1)) / (1 - (-1)) ) * (1 - 0) + 0
                #todo 1000 can change
            if (daily_gain is not None) and (len(last_n_data) >= 1000):
                last_n_data = last_n_data[-1000:]
                input_vector = make_supervised_vector(last_n_data, daily_gain)
                #list_data = [daily_gain,intial_data["Low"], intial_data["High"], intial_data["Close"], intial_data["Open"], intial_data["Volume"]]
                output_data.append(output)
                #if daily_gain > 0.5:
                    #pdb.set_trace()
                new_data_list.append(input_vector)
            intial_data = data
    stock_data = np.asarray(new_data_list)
    output_data = np.asarray(output_data)
    supervised_data["data"] = stock_data
    supervised_data["target"] = output_data
    return supervised_data

def make_supervised_vector(last_n_data, daily_gain):
    column = len(last_n_data[0])
    standaridized_data = standardization(last_n_data)
    #apply minmax above it to change range from 0 to 1, can be changed
    min_max_scaler = preprocessing.MinMaxScaler()
    standaridized_data = min_max_scaler.fit_transform(standaridized_data)
    vector = standaridized_data[-10:].reshape(10*column,)
    vector = np.append(vector,find_average(standaridized_data[-200:]))
    vector = np.append(vector, find_average(standaridized_data[-1000:]))
    vector = np.append(vector, daily_gain)
    return vector

def standardization(data):
    #Standard standardization with mean
    return preprocessing.scale(data)

def find_average(data):
    return np.mean(data, axis=0)

def load_stock_data():
    if not os.path.exists('stock_5.pkl'):
        dictonary =  prepare_data()
        with open("stock_5.pkl", "wb") as myFile:
            six.moves.cPickle.dump(dictonary, myFile, -1)
    with open('stock_cont.pkl', 'rb') as myFile:
        data = six.moves.cPickle.load(myFile)
    #pdb.set_trace();
    return data


import math
import numpy as np
import tensorflow as tf
from tensorflow.contrib import slim
from PIL import Image
from functools import reduce
import os, cv2

'''
    Be careful about everything you write here because every model use the code like:
    from utils import *
    Give variable names carefully and strictly to avoid namespace conflict
'''

def conv_out_size_same(size, stride):
  return int(math.ceil(float(size) / float(stride)))

# arr should be 4 dimensional
def save_image(arr, name, idx, scale=True, path='./generated/'):
    if scale:
        arr = arr * (255.0 / np.max(arr))
    try:
        for i in range(arr.shape[0]):
            img_to_save = arr[i, :, :, :].astype(np.uint8)
            cv2.imwrite(path + str(idx) + '_' + str(i) + '_' + name, img_to_save)
    except:
        print('Error encountered when saving generated images!')
        return
    print('SAVING GENERATED IMAGES TO: ' + path + name)

def show_all_variables():
    all_variables = tf.trainable_variables()
    slim.model_analyzer.analyze_vars(all_variables, print_info=True)

def get_inception_score(images, sess):
    inps = list(map(lambda img: np.expand_dims(img.astype(np.float32), axis=0), images))
    bs = 100
    preds = list()
    n_batches = int(math.ceil(float(len(inps)) / float(bs)))
    for i in range(n_batches):
        inp = inps[(i * bs):min((i + 1) * bs, len(inps))]
        inp = np.concatenate(inp, 0)
        pred = sess.run(softmax, {'ExpandDims:0': inp})
        preds.append(pred)

# The anime portrait dataset
class animeReader(object):
    def __init__(self, path, low=0, high=5000):
        self.low, self.high = low, high
        fs = os.listdir(path)
        self.files = list(map(lambda x: path + x, fs))
        arrs = list(map(lambda x: cv2.imread(x).astype(np.float32), self.files[low: high]))
        arrs = list(map(lambda x: x.flatten()[None, :], arrs))
        self.data = np.concatenate(arrs, axis=0)
        print('Data successfully loaded --sample number: %d' % (high - low))
        self.counter = 0

    # Randomized output
    def next_batch(self, num):
        index = np.random.randint(low=self.low, high=self.high, size=(1, num))
        x_batch = self.data[index]
        return x_batch[0]

    # Ordinal output
    def ordinal_batch(self, num):
        if self.counter + num < len(self.files):
            end_idx = self.counter + num
            return self.data[self.counter: end_idx, :]
        else:
            start_idx = (self.counter + num) % len(self.files)
            sub1 = self.data[: start_idx, :]
            sub2 = self.data[self.counter:, :]
            return np.concatenate([sub1, sub2], axis=0)

class cifar10Reader(object):
    dataset_dir = 'E:\\datasets\\cifar-10-batches-py\\'

    def __init__(self):
        import pickle
        file_names = ['data_batch_%d' % i for i in range(1, 6)]
        file_names = list(map(lambda x: self.dataset_dir + x, file_names))
        self.dictionaries = list(map(lambda x: pickle.load(open(x, 'rb'), encoding='bytes'), file_names))
        for item in self.dictionaries:
            # The valid value is supposed to be between 0 and 1!!!
            item[b'labels'] = np.array(item[b'labels'])
            item[b'data'] = item[b'data'].astype(np.float32) / 255.0

    def test(self):
        x, y = self.next_batch(64)
        res = x.reshape((64,32,32,3))
        cv2.imwrite('test.png', res[0, :, :, :]*255.0)

    def next_batch(self, num):
        batch_idx = np.random.randint(0, 5)
        image_idx = np.random.randint(0, 10000, size=(num))
        samples = self.dictionaries[batch_idx][b'data'][image_idx]
        labels = self.dictionaries[batch_idx][b'labels'][image_idx]
        # Fucking bloody cifar-10 orders
        for i in range(num):
            res = samples[i, :].reshape((3,32,32)).transpose([1,2,0])
            samples[i, :] = res.reshape((1, 3072))
        return samples, labels

if __name__ == '__main__':
    # Just for tests

    reader = cifar10Reader()
    reader.test()
    # x, y = reader.next_batch(1)
    # print(x, y)
    # print(x.max(), x.min(), x.shape, x.dtype)


import pygame
import math
import random

import settings

from screen.game_screen_class import GameScreen
from character.player import Player
from character.obstacle import StaticObstacle, DynamicObstacle
from character.coin import Coin

class Map(GameScreen):

    """
    Klasa mapy

    player1 - gracz
    obs - przeszkoda
    game_speed - prędkość gry
    baner - element mapy, kierunek wyspa słodowa
    """

    def __init__(self, w, h, selected_player):
        super().__init__(w, h)
        self.font = pygame.font.Font("textures/font.ttf", 16)
        self.baner = pygame.image.load('textures/baner.png').convert_alpha()
        self.baner = pygame.transform.rotozoom(self.baner, 0, 0.30) # powinna być na to funkcja
        # i ogólnie elementy mapy pewnie powinny być w jakiejś liście
        self.baner_rect = self.baner.get_rect(center = (self.w - 150, 300))
        self.game_speed = 1
        self.player1 = Player(350,670, selected_player) # wstępna pozycja
        self.obstacles = []
        self.life= Life(w, h)

        self.game_over = False

        self.score = 0

        # dodawanie serduszek oraz ich skalowanie, myśle że można to gdzies przenieść w dyskretne miejsce żeby oczy nie bolały

        self.scroll = 0

        self.coins = []

    def display_bg(self):
        # wyświetl tło
        #self.background = pygame.transform.scale(self.background, (self.w, self.h))

        i = 0
        while (i < self.tiles):
            self.screen.blit(self.background, (0
                             , self.background.get_height() * - i + self.scroll))
            i += 1

        self.scroll += 5 * self.game_speed

        # resetowanie scrolla
        if abs(self.scroll) > self.background.get_height():
            self.scroll = 0

    # -------------DEBUG------------------------
    def display_debug_rect(self):
        pygame.draw.rect(self.screen, 'green', self.player1.rect, 2)
        for obstacle in self.obstacles:
            pygame.draw.rect(self.screen, 'blue', obstacle.rect, 2)

        """rp = self.player1.get_mask().get_rect()
        pygame.draw.rect(self.screen, 'green', rp, 2)
        for obstacle in self.obstacles:
            ro = obstacle.get_mask().get_rect()
            pygame.draw.rect(self.screen, 'blue', ro, 2)"""
        """print(self.player1.x, self.player1.y)
        olist = self.player1.get_mask().outline()
        pygame.draw.lines(self.screen, 'green' ,1,olist)"""

    def display_score_and_money(self):
        self.calculate_score()
        self.score_txt = self.font.render(f'score: {self.score}', True, 'gold')
        self.score_txt_rect = self.score_txt.get_rect(topleft = (10, 10))
        self.money_txt = self.font.render(f'coins: {self.player1.game_money}', True, 'gold')
        self.money_txt_rect = self.money_txt.get_rect(topleft = (10, 30))

        self.screen.blit(self.score_txt, self.score_txt_rect)
        self.screen.blit(self.money_txt, self.money_txt_rect)

    def increase_speed(self):
        # przyspiesz grę co 3 sekundy -- wartości można zmienić
        if round(pygame.time.get_ticks(), -3) % 3000 == 0:
            self.game_speed *= 1.0005
            self.player1.dx *= 1.0005
        #print(self.game_speed)

    def update_characters(self):
        # update na ekranie pozycję gracza i przeszkód
        # oraz sprawdza kolizję
        # jeżeli wystąpi kolizja to game_over = True
        self.increase_speed()
        self.add_obstacle()
        self.add_coin()

        # ta funkcja zwraca true jezeli gracz straci hp i gra ma sie skonczyc
        self.game_over = self.check_for_obs_collision()

        self.collect_coin()
        self.update_player()
        self.update_obstacles()
        self.update_coins()

    def update_player(self):
        # print(self.check_for_border_collision())
        #self.player1.move_to_initial_pos()
        if self.check_for_border_collision():
            self.player1.move(self.check_for_border_collision())
            self.player1.rect.x = self.player1.x
            self.toggle_player_inivincible()

    def check_for_border_collision(self) -> list[bool, bool]:
        # zwraca kolizję z granicą w postaci dwuelementowej listy
        # w któej idx 0 oznacza kolizję z lewą granicą drogi, a idx 1 oznacza kolizję z prawą granicą drogi
        # jeżeli doda się możliwość przeszkalowania ekranu trzeba też będzie wartości const zmienić na jakieś variable
        if 135 > self.player1.x:
            # kolizja z lewej strony
            return [True, False]
        elif self.player1.rect.right > 465: # or self.player1.rect.left > 405
            # kolizja z prawej strony
            return [False, True]
        # jeżeli brak kolizji
        return [False, False]

    def update_obstacles(self):
        # aktualizuje pozycje przeszkód i wyświetla je na ekranie
        for obstacle in self.obstacles:
            if obstacle:
                obstacle.move(self.game_speed)
                obstacle.rect.y = obstacle.y
                if obstacle.y > self.h + 100:
                    # gdy przeszkoda wyjdzie poza ekran jest usuwana z listy i obiekt też jest usuwany
                    self.obstacles.remove(obstacle)
                    del obstacle

    def update_coins(self):
        for coin in self.coins:
            if coin:
                coin.move(self.game_speed)
                coin.rect.y = coin.y
                if coin.y > self.h + 100:
                    # gdy przeszkoda wyjdzie poza ekran jest usuwana z listy i obiekt też jest usuwany
                    self.coins.remove(coin)
                    del coin

    def check_for_obs_coin_spawn_collision(self, to_be_added) -> bool:
        # sprawdza czy moneta lub przeszkoda do dodania
        # po dodaniu kolidowałaby z już istniejącą
        for obstacle in self.obstacles:
            if to_be_added.rect.y <= obstacle.rect.y + 130 and to_be_added.rect.x == obstacle.rect.x:
                return True
        for coin in self.coins:
            if to_be_added.rect.y <= coin.rect.y - 45 and to_be_added.rect.x == coin.rect.x:
                return True
        return False

    def add_obstacle(self):
        # dodaje przeszkodę we w miarę losowym momencie, nie może być póki co więcej niż 5 na ekranie
        # sprawdza czy w miejscu gdzie ma się pojawić przeszkoda występuje już jakaś inna przeszkoda
        if random.randint(1, 101) % 50 == 0 and len(self.obstacles) < 5:
            if random.randint(1,9) % 8 == 0:
                obs = StaticObstacle()
            else:
                obs = DynamicObstacle()
            if self.check_for_obs_coin_spawn_collision(obs):
                del obs
            else:
                self.obstacles.append(obs)

    def add_coin(self):
        # dodanie monety
        if random.randint(1, 100) % 97 == 0 and len(self.coins) < 4:
            coin = Coin()
            if self.check_for_obs_coin_spawn_collision(coin):
                del coin
            else:
                self.coins.append(coin)

    def collect_coin(self):
        # zbiera monetkę jeżeli auto się z nią zderzy
        for coin in self.coins:
            if pygame.Rect.colliderect(coin.rect, self.player1.rect):
                self.player1.game_money += 1
                self.coins.remove(coin)
                del coin

    """def check_for_obs_collision(self) -> bool:
        # sprawdza czy występuje kolizja gracza z przeszkodą
        for obstacle in self.obstacles:
            if pygame.Rect.colliderect(obstacle.rect, self.player1.rect) and (not self.player1.invincible) and (not self.player1.is_colliding):
                self.player1.blink_invinc_end_time = pygame.time.get_ticks() + 1000
                self.player1.hp -= 1
                self.obstacles.remove(obstacle)
                self.player1.is_colliding = True
                del obstacle
                # zwraca funkcję sprawdzającą hp, która zwraca True jeżeli HP == 0,
                # czyli ta funkcja zwroci True jezeli gracz straci hp - gra ma sie skonczyc
                return self.life.checking_hp(self.player1)
        self.player1.is_colliding = False
        return False"""

    def check_for_obs_collision(self) -> bool:
        # sprawdza czy występuje kolizja gracza z przeszkodą używając maski - pixel perferct collision
        # brak problemów z hitboxami
        player_mask = self.player1.get_mask()
        # p_left_offset = (self.player1.x )
        # p_right_offset =
        col = False
        for obstacle in self.obstacles:
            obstacle_mask = obstacle.get_mask()
            p_left_offset = (self.player1.x - obstacle.rect.topright[0], abs(obstacle.y - round(self.player1.y)))
            p_right_offset = (obstacle.x - self.player1.rect.topright[0], abs(obstacle.y - round(self.player1.y)))
            # p_offset = (abs(self.player1.x - obstacle.x), abs(obstacle.y - round(self.player1.y)))

            if (player_mask.overlap(obstacle_mask, p_left_offset) and p_left_offset[0] < 0) or (player_mask.overlap(obstacle_mask, p_right_offset) and p_right_offset[0] < 0):
                print()
                print(f'overlap: {player_mask.overlap(obstacle_mask, p_left_offset)}, {player_mask.overlap(obstacle_mask, p_right_offset)}')
                col = True

            #if player_mask.overlap(obstacle_mask, p_offset):
            #    col = True

            if col and (not self.player1.invincible) and (not self.player1.is_colliding):
                # print()
                print (f"Left offset: {p_left_offset}, Right offset: {p_right_offset}")
                print (f"Player rectangle pos left: {self.player1.rect.topleft}, Player rectangle pos right: {self.player1.rect.topright}")
                print (f"Obs rectangle pos left: {obstacle.rect.topleft}, Obs rectangle pos right: {obstacle.rect.topright}")
                self.player1.blink_invinc_end_time = pygame.time.get_ticks() + 1000
                self.player1.hp -= 1
                self.obstacles.remove(obstacle)
                self.player1.is_colliding = True
                del obstacle
                # zwraca funkcję sprawdzającą hp, która zwraca True jeżeli HP == 0,
                # czyli ta funkcja zwroci True jezeli gracz straci hp - gra ma sie skonczyc
                return self.life.checking_hp(self.player1)

        # print(col)
        self.player1.is_colliding = False
        return False

    def toggle_player_inivincible(self):
        self.player1.invincible = True if pygame.time.get_ticks() <= self.player1.blink_invinc_end_time + 200 else False

    def display_map_elements(self):
        # wyświetla elementy mapy, takie jak np. baner
        self.screen.blit(self.baner, self.baner_rect)

    def display_player(self):
        # FUNKCJA KTORA MIGA GRACZEM
        # miganie gracza chwilę po tym jak zderzy się z przeszkodą
        # powinien być jeszcze okres invincible
        # czas migania/invcs - jedna sekunda
        # print(self.player1.invincible)
        if pygame.time.get_ticks() <= self.player1.blink_invinc_end_time -800:
            self.player1.display_player(self.player1.blink_image, self.screen)
        elif self.player1.blink_invinc_end_time -600 <= pygame.time.get_ticks() <= self.player1.blink_invinc_end_time -400:
            self.player1.display_player(self.player1.blink_image, self.screen)
        elif self.player1.blink_invinc_end_time -200 <= pygame.time.get_ticks() <= self.player1.blink_invinc_end_time:
            self.player1.display_player(self.player1.blink_image, self.screen)
        else:
            self.player1.display_character(self.screen)

    def display_characters(self):
        # wyświetla charactery
        self.display_player()
        #self.player1.display_character(self.screen)

        # debug:
        self.display_debug_rect()

        for coin in self.coins:
            coin.display_character(self.screen)

        for obstacle in self.obstacles:
            obstacle.display_character(self.screen)

    def calculate_score(self):
        # self.score += int((pygame.time.get_ticks() * self.game_speed**10) // 1000)
        self.score += 1

    def is_game_over(self) -> bool:
        return self.game_over

class Life():

    def __init__(self,w ,h):
        self.w=w
        self.h=h
        # dodawanie serduszek oraz ich skalowanie, myśle że można to gdzies przenieść w dyskretne miejsce żeby oczy nie bolały
        self.hp1 = pygame.image.load('textures/serce.png').convert_alpha()
        self.hp2 = pygame.image.load('textures/serce.png').convert_alpha()
        self.hp3 = pygame.image.load('textures/serce.png').convert_alpha()
        self.hp1_rect = self.hp1.get_rect(center=(self.w + 5, self.h - 670))
        self.hp2_rect = self.hp2.get_rect(center=(self.w + 45, self.h - 670))
        self.hp3_rect = self.hp3.get_rect(center=(self.w + 85, self.h - 670))
        self.hp1 = pygame.transform.rotozoom(self.hp1, 0, 0.15)
        self.hp2 = pygame.transform.rotozoom(self.hp2, 0, 0.15)
        self.hp3 = pygame.transform.rotozoom(self.hp3, 0, 0.15)

    def checking_hp(self, player1) -> bool:
        if (player1.hp == 2):
            # print('2')
            self.hp1 = pygame.image.load('textures/kosa.png').convert_alpha()
            self.hp1_rect = self.hp1.get_rect(center=(self.w - 80, self.h - 750))
            self.hp1 = pygame.transform.rotozoom(self.hp1, 0, 0.40)
            return False
        elif (player1.hp == 1):
            # print('1')
            self.hp2 = pygame.image.load('textures/kosa.png').convert_alpha()
            self.hp2_rect = self.hp2.get_rect(center=(self.w - 40, self.h - 750))
            self.hp2 = pygame.transform.rotozoom(self.hp2, 0, 0.40)
            return False
        elif (player1.hp == 0):
            self.hp3 = pygame.image.load('textures/kosa.png').convert_alpha()
            self.hp3_rect = self.hp3.get_rect(center=(self.w, self.h - 750))
            self.hp3 = pygame.transform.rotozoom(self.hp3, 0, 0.40)
            # print('deat')
            return True

    def show_life(self, screen):
        screen.blit(self.hp1, self.hp1_rect)
        screen.blit(self.hp2, self.hp2_rect)
        screen.blit(self.hp3, self.hp3_rect)


from django.contrib import admin, messages
from django.contrib.admin import AdminSite
# from .views import updateProductionData
from .models import InstalledCmm, DeliveredCmm, NewOrder, WaitingOrderAndInventory, InstalledEachYear
from django.urls import path
from django.http import HttpResponseRedirect
from .productionUpdate import updateProduction

class InsalledCmmAdmin(admin.ModelAdmin):
    fields = ['Year','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
    list_display = ('Year','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec')
    search_fields = ['Year']
    # actions = ['update_data']

    change_list_template = 'op/update_changelist.html'
    # def save_model(self, rquest, obj, form, change):
    #     if 'update_data' in form.changed_data:
    #         messages.add_message(rquest, messages.INFO, 'Car has been sold')
    #     super(InsalledCmmAdmin, self).save_model(rquest, obj, form, change)
    def get_actions(self, request):
        actions = super().get_actions(request)
        if 'delete_selected' in actions:
            del actions['delete_selected']
        return actions

    def get_urls(self):
        urls = super().get_urls()
        my_urls = [
            path('immortal/', self.update_data),

        ]
        return my_urls + urls

    def update_data(self, request):
        message = updateProduction()
        if "全部更新成功" in message:
            self.message_user(request, message)
        else:
            messages.error(request, message)
        return HttpResponseRedirect("../")

    # update_data.short_description='更新数据'

    # def get_actions(self, request):
    #     actions = super().get_actions(request)
    #     if 'delete_selected' in actions:
    #         del actions['delete_selected']
    #     return actions

class WaitingOrderAndInventoryAdmin(admin.ModelAdmin):
    fields = ['YearAndMonth','OrderQty','ShipmentQty','AuctalAssemblyQty','ToBeExcutedTendency','BackLogTendency','NetAvailableTendency','Backlog','ToBeExcutedOrder','NetAvailable']
    list_display = ('YearAndMonth','OrderQty','ShipmentQty','AuctalAssemblyQty','ToBeExcutedTendency','BackLogTendency','NetAvailableTendency','Backlog','ToBeExcutedOrder','NetAvailable')
    search_fields = ['YearAndMonth']
    def get_actions(self, request):
        actions = super().get_actions(request)
        if 'delete_selected' in actions:
            del actions['delete_selected']
        return actions
    change_list_template = 'op/update_changelist.html'
    def get_urls(self):
        urls = super().get_urls()
        my_urls = [
            path('immortal/', self.update_data),

        ]
        return my_urls + urls

    def update_data(self, request):
        message = updateProduction()
        if "全部更新成功" in message:
            self.message_user(request, message)
        else:
            messages.error(request, message)
        return HttpResponseRedirect("../")

class InstalledEachYearAdmin(admin.ModelAdmin):
    fields = ['Year','Global_A','Global_B','Global_C','Global_D','Global_EF','Explorer','Inspector_Pioneer_InspectorP_GlobalP','MH3D_Inspector454_Explorer454','Optive_Vision','Toro_ToroImage','Micro_Plus','Alpha_Apollo','Function_Pluse','Zoo_ZC','Stinger_ll','Global_Mini','Auctual_Build_Qty']
    list_display = ('Year','Global_A','Global_B','Global_C','Global_D','Global_EF','Explorer','Inspector_Pioneer_InspectorP_GlobalP','MH3D_Inspector454_Explorer454','Optive_Vision','Toro_ToroImage','Micro_Plus','Alpha_Apollo','Function_Pluse','Zoo_ZC','Stinger_ll','Global_Mini','Auctual_Build_Qty')
    search_fileds = ['Year']
    def get_actions(self, request):
        actions = super().get_actions(request)
        if 'delete_selected' in actions:
            del actions['delete_selected']
        return actions
    change_list_template = 'op/update_changelist.html'
    def get_urls(self):
        urls = super().get_urls()
        my_urls = [
            path('immortal/', self.update_data),

        ]
        return my_urls + urls

    def update_data(self, request):
        message = updateProduction()
        if "全部更新成功" in message:
            self.message_user(request, message)
        else:
            messages.error(request, message)
        return HttpResponseRedirect("../")

admin.site.register(InstalledCmm, InsalledCmmAdmin)
admin.site.register(DeliveredCmm, InsalledCmmAdmin)
admin.site.register(NewOrder, InsalledCmmAdmin)
admin.site.register(WaitingOrderAndInventory, WaitingOrderAndInventoryAdmin)
admin.site.register(InstalledEachYear, InstalledEachYearAdmin)

admin.site.site_header = "Hexagon"
admin.site.site_title = "管理页面"
admin.site.index_title = "欢迎来到管理页面"

########## duplicated ######
class ProducedAdminSite(AdminSite):
    site_header = 'Installed Admin Site'
    site_title = "Installed Admin Site"
    index_title = 'Installed Admin Site'

produced_admin_site = ProducedAdminSite(name='produced_admin')

produced_admin_site.register(InstalledCmm)
produced_admin_site.register(DeliveredCmm)
produced_admin_site.register(NewOrder)
produced_admin_site.register(WaitingOrderAndInventory)
produced_admin_site.register(InstalledEachYear)


import ast
from glob import glob
import os
from typing import Union
from markdown_it import MarkdownIt
from markdown_it.token import Token
import pytest
import re
from .lib.nb_helper import is_h1, is_markdown, is_python, read_notebook

def check_metadata(notebook, file, expected_kernel):
    metadata = notebook.metadata
    runtime = metadata.kernelspec.display_name

    assert runtime == expected_kernel
    assert metadata.language_info.version.startswith("3.")

    if "colab" in metadata:
        colab_name = metadata["colab"]["name"]
        assert colab_name == file, f"Name in metadata doesn't match filename for {file}."

def check_file(file, expected_kernel="Python [conda env:python-public-policy] *"):
    notebook = read_notebook(file)
    check_metadata(notebook, file, expected_kernel)

notebooks = glob("*.ipynb")
notebooks.sort()
crash_course = os.path.join(os.path.dirname(__file__), "pandas_crash_course.ipynb")
all_notebooks = notebooks + [crash_course]

@pytest.mark.parametrize("notebook", notebooks)
def test_class_notebooks(notebook):
    check_file(notebook)

def test_colab():
    # run in Google Colab
    check_file(crash_course, "Python 3")

@pytest.mark.parametrize("file", all_notebooks)
def test_one_h1(file):
    notebook = read_notebook(file)
    num_h1s = sum(is_h1(cell) for cell in notebook.cells)
    assert num_h1s == 1

@pytest.mark.parametrize("file", all_notebooks)
def test_heading_levels(file):
    notebook = read_notebook(file)
    for cell in notebook.cells:
        meta = cell.metadata
        source = cell.source
        if is_markdown(cell) and "slideshow" in meta and source.startswith("#"):
            # slide with a heading
            slide_type = meta["slideshow"]["slide_type"]
            if slide_type == "slide":
                assert source.startswith(("# ", "## ")), "should be an H1 or H2"
            elif slide_type == "subslide":
                assert source.startswith("###"), "should be H3+"

@pytest.mark.parametrize("file", notebooks)
def test_nested_lists(file):
    """JupyterBook's markdown parser doesn't support nested lists with two spaces."""

    notebook = read_notebook(file)
    for cell in notebook.cells:
        if is_markdown(cell):
            source = cell.source
            for line in source.splitlines():
                match = re.match(r"^( +)(-|\d\.)", line)
                if match:
                    num_spaces = len(match[1])
                    assert (
                        num_spaces % 3 == 0 or num_spaces % 4 == 0
                    ), f"Lists should be indented in multiples of three or four spaces. Text:\n\n{source}\n"

def check_link(token: Token, parent: Token = None):
    if token.type == "link_open":
        href = token.attrGet("href")
        # escaped Jinja2 tags
        if not re.match(r"http|#|%7B%7B", href):
            source = parent.content if parent else token.content
            assert False, f"Link should be absolute. Text:\n\n{source}\n"

@pytest.mark.parametrize("file", notebooks)
def test_links(file):
    """To ensure that links work from the coding environment, ensure all links are absolute."""

    md = MarkdownIt()

    notebook = read_notebook(file)
    for cell in notebook.cells:
        if is_markdown(cell):
            source = cell.source
            tokens = md.parse(source)
            for token in tokens:
                check_link(token)
                if token.type == "inline":
                    for child in token.children:
                        check_link(child, token)

def base_obj(node: ast.Call) -> Union[str, None]:
    """If this is a method call, return the name of the base object it was called on"""
    if isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
        return node.func.value.id
    else:
        return None

class PlotChecker(ast.NodeVisitor):
    def visit_Call(self, node):
        if base_obj(node) == "px":
            args = [kw.arg for kw in node.keywords]
            method = node.func.attr

            if method == "get_trendline_results":
                # `title` not applicable
                return

            assert "title" in args, f"call to `{method}()` missing a `title`"

@pytest.mark.parametrize("file", notebooks)
def test_chart_titles(file):
    """Make sure all charts have titles"""

    notebook = read_notebook(file)
    for cell in notebook.cells:
        tags = cell.metadata.get("tags", [])
        if "skip-plot-check" in tags or not is_python(cell):
            continue

        tree = ast.parse(cell.source)
        checker = PlotChecker()
        checker.visit(tree)

hw_notebooks = glob("hw_*.ipynb")

@pytest.mark.parametrize("file", hw_notebooks)
def test_reminders(file):
    notebook = read_notebook(file)

    assert any(
        "assignments.html" in cell.source for cell in notebook.cells
    ), "Missing assignment submission instructions"

    assert any(
        "participation" in cell.source for cell in notebook.cells
    ), "Missing participation reminder"


from bs4 import BeautifulSoup

def find_title(html_doc):
    soup = BeautifulSoup(html_doc, 'html.parser')
    title_tag = soup.title
    if title_tag:
        return title_tag.text
    else:
        return None

filename = "test1.html"

try:
    with open(filename, 'r') as file:
        html_document = file.read()
        title = find_title(html_document)
        if title:
            print("Знайдений заголовок:", title)
        else:
            print("Заголовок не знайдений.")
except FileNotFoundError:
    print("Файл не знайдений.")


import urllib.request
from bs4 import BeautifulSoup
from lxml import  etree
import re

url = 'https://www.biqukan.com/1_1094/5403177.html'

def get_page(url):
    headers = {'User-Agent':'Mozilla/5.0','Cookie':'UM_distinctid=16b8523768d317-020cad425c2672-e343166-144000-16b8523768e115; bcolor=; font=; size=; fontcolor=; width=; CNZZDATA1260938422=1245222118-1561302037-https%253A%252F%252Fblog.csdn.net%252F%7C1563150333'}
    Request = urllib.request.Request(url,headers=headers)
    response = urllib.request.urlopen(Request)
    html = response.read()
    return html

def parser_page(html):
    soup = BeautifulSoup(html,'lxml',from_encoding='utf-8')
    text = soup.find('div', class_='showtxt')
    cont = text.get_text()
    cont = str(cont).split()

    print(cont)

if __name__ == '__main__':
    html = get_page(url)
    parser_page(html)

爬虫实战/笔趣小说爬取/我自己爬的.py


"""This module is used to test the classes in forger.engine.manager"""

import pytest

from forger.engine.manager import Manager
from forger.engine.pipelines import Pipeline
from tests.conftest import pipeline_samples, pipeline_samples_names

@pytest.fixture()
def manager():
    man = Manager()
    man.Scheduler.pause()
    return man

@pytest.fixture()
def manager_with_pipelines(manager):
    pipelines = []
    for pipeline_sample in pipeline_samples:
        pipelines.append(
            manager.add_pipeline(
                ip=pipeline_sample[1],
                port=pipeline_sample[2],
                topic=pipeline_sample[3],
                frequency=pipeline_sample[4],
                pipeline_name=pipeline_sample[6],
            )
        )
    return manager, pipelines

class TestManager:
    def test_add_pipeline(self, manager_with_pipelines):
        """
        Test the add_pipeline method of the Manager class.
        """
        manager, pipelines = manager_with_pipelines
        for pipeline in pipelines:
            assert isinstance(pipeline, Pipeline)
            assert manager.pipelines[pipeline.pid] == pipeline

    def test_get_names(self, manager_with_pipelines):
        """
        Test the get_names method of the Manager class.
        """
        manager, _ = manager_with_pipelines
        names = manager.get_names()
        assert set(names) == set(pipeline_samples_names)


# this function is to set aqmc parameters and run the method
from initializing import make_hopping, init_trotter, expmk, cluster
from fromscratch import from_scratch
from update import time_wrap, update_G
from measure import correlation

import numpy as np
import cupy as cp
import matplotlib.pyplot as plt
import time

class AQMC:
    def __init__(self, **params):
        self.X_dimension = params["X_dimension"]
        self.Y_dimension = params["Y_dimension"]
        self.N_s = self.X_dimension * self.Y_dimension

        self.periodic_X = params["periodic_X"]
        self.periodic_Y = params["periodic_Y"]
        self.tunneling = params["tunneling"]
        self.U = params["U"]
        self.chemical_potential = params["chemical_potential"] # half filling 0
        self.N_markov = params["N_markov"]
        self.N_sw_measure = params["N_sw_measure"]
        self.N_warm_up = self.N_sw_measure // 5
        self.N_from_scratch = params["N_from_scratch"]
        self.N_qr = params["N_qr"]

        # Set generalized trotter
        self.Beta = params["Beta"]
        self.N_time = params["N_time"]
        self.U_eff =  self.U
        self.directory = params["directory"]
        if self.N_s<32:
            self.BLOCKSIZE = self.N_s
        else:
            self.BLOCKSIZE = 32

    def pin_memory(self):
        memory_pool = cp.cuda.MemoryPool()
        cp.cuda.set_allocator(memory_pool.malloc)
        pinned_memory_pool = cp.cuda.PinnedMemoryPool()
        cp.cuda.set_pinned_memory_allocator(pinned_memory_pool.malloc)

    def run(self):
        # making tunneling Hamiltonian matrix
        H_0 = make_hopping(self.X_dimension, self.Y_dimension, self.periodic_X, self.periodic_Y, self.tunneling)
        H_0_msr = H_0.copy()
        H_0 += np.identity(self.X_dimension * self.Y_dimension) * ((-1) * self.chemical_potential)

        I = cp.array(1)
        sign_U_interact, T_hop, H0_array, lamda, probability, gamma1, gamma2 = init_trotter(self.Beta,self.N_time,self.U_eff,H_0)
        Bk, Bk_inv = expmk(H0_array, T_hop)

        sign_U_interact_m = cp.empty((self.N_markov,self.N_time))
        hs_m = cp.empty((self.N_markov,self.N_time,self.N_s))
        cl_up_m = cp.empty((self.N_markov,self.N_time,self.N_s,self.N_s))
        cl_dn_m = cp.empty((self.N_markov,self.N_time,self.N_s,self.N_s))
        G_up_m = cp.empty((self.N_markov,self.N_s,self.N_s))
        G_dn_m = cp.empty((self.N_markov,self.N_s,self.N_s))
        Bk_m = cp.empty((self.N_markov,self.N_time,self.N_s,self.N_s))
        Bk_inv_m = cp.empty((self.N_markov,self.N_time,self.N_s,self.N_s))
        probability_m = cp.empty((self.N_markov,self.N_time))
        gamma1_m = cp.empty((self.N_markov,self.N_time))
        gamma2_m = cp.empty((self.N_markov,self.N_time))

        hs_cached = self.N_markov * 1000

        map_streams = []
        mark_list = []
        for i in range(self.N_markov):
            mark_list.append(i)
            map_streams.append(cp.cuda.stream.Stream())
            sign_U_interact_m[i] = sign_U_interact
            hs_m[i] = lamda[:,None]*((-1)**cp.random.randint(2,size=(self.N_time,self.N_s)))
            cl_up_m[i],cl_dn_m[i] = cluster(hs_m[i],Bk_m[i],sign_U_interact_m[i])
            G_up_m[i] = from_scratch(cl_up_m[i],self.N_qr)
            G_dn_m[i] = from_scratch(cl_dn_m[i],self.N_qr)
            Bk_m[i] = Bk.copy()
            Bk_inv_m[i] = Bk_inv.copy()
            probability_m[i] = probability.copy()
            gamma1_m[i] = gamma1.copy()
            gamma2_m[i] = gamma2.copy()

        check = cp.zeros((self.N_sw_measure+self.N_warm_up,self.N_time,self.N_markov))
        n_up_dn = cp.zeros(self.N_markov)
        interaction_mar = cp.zeros(self.N_markov)
        sign_mar = cp.zeros(self.N_markov)
        N_measure = cp.zeros(self.N_markov)
        G_up_mar = cp.zeros((self.N_markov,self.N_s,self.N_s))
        G_dn_mar = cp.zeros((self.N_markov,self.N_s,self.N_s))
        szsz_mar = cp.zeros((self.N_markov,self.X_dimension))
        sxsx_mar = cp.zeros((self.N_markov,self.X_dimension))
        rho_mar = cp.zeros((self.N_markov,self.X_dimension))
        sign_partition_accu = cp.array([cp.linalg.slogdet(G_up_m[i] @ G_dn_m[i])[0] for i in range(self.N_markov)],dtype=cp.int32)[:,None]
        HS_list = cp.empty((1, self.N_time, self.N_s)) #keep in mind the size of the array be smaller than the DRAM
        # mark_list, map_streams = self.initialize()
        strt = time.time()
        for msr in range(self.N_sw_measure+self.N_warm_up):
            for l in range(self.N_time):
                for i,stream in zip(mark_list,map_streams):
                    with stream:
                        randomlist = cp.random.random(size=(self.N_s))

                        update_G((1,),(1,),(cp.eye(self.N_s),cp.eye(self.N_s),G_up_m[i],  G_dn_m[i], hs_m[i], randomlist, self.N_s, self.N_time, sign_partition_accu[i], probability_m[i], gamma1_m[i], gamma2_m[i],self.BLOCKSIZE))
                        G_up_m[i], G_dn_m[i], Bk_m[i], Bk_inv_m[i], hs_m[i], cl_up_m[i], cl_dn_m[i], sign_U_interact_m[i],probability_m[i], gamma1_m[i], gamma2_m[i] = time_wrap(G_up_m[i], G_dn_m[i], Bk_m[i], Bk_inv_m[i], hs_m[i], cl_up_m[i], cl_dn_m[i], sign_U_interact_m[i],probability_m[i], gamma1_m[i], gamma2_m[i])
                        if l%self.N_from_scratch == 0:
                            G_up_m[i] = from_scratch(cl_up_m[i],self.N_qr)
                            G_dn_m[i] = from_scratch(cl_dn_m[i],self.N_qr)

                        if msr >= self.N_warm_up and l==self.N_time-1:
                            G_up_ms = cp.eye(self.N_s) - G_up_m[i]
                            G_dn_ms = cp.eye(self.N_s) - G_dn_m[i]
                            G_up_mar[i] += (G_up_ms)*sign_partition_accu[i,0]
                            G_dn_mar[i] += (G_dn_ms)*sign_partition_accu[i,0]
                            n_up_tmp = cp.diag(G_up_ms)
                            n_dn_tmp = cp.diag(G_dn_ms)
                            sz_tmp = (n_up_tmp-n_dn_tmp)/2
                            rho_tmp = (n_up_tmp+n_dn_tmp)/2
                            interaction_mar[i] +=  cp.sum(n_up_tmp * n_dn_tmp * self.U)*sign_partition_accu[i,0]
                            szsz,sxsx,rho = correlation(G_up_ms, G_dn_ms, sz_tmp, rho_tmp, n_up_tmp, n_dn_tmp, self.X_dimension, self.Y_dimension)
                            szsz_mar[i] += szsz*sign_partition_accu[i,0]
                            sxsx_mar[i] += sxsx*sign_partition_accu[i,0]
                            rho_mar[i] += rho*sign_partition_accu[i,0]
                            sign_mar[i] += sign_partition_accu[i,0]
                            N_measure[i] += 1
            HS_list = cp.concatenate([HS_list, hs_m])

            if msr >= self.N_warm_up and (msr+1) % hs_cached == 0:
                HS_list, = self.save_hs(HS_list, self.directory)

        HS_list = self.save_hs(HS_list, self.directory)
        end = time.time()
        N_msr = cp.mean(N_measure,axis=0)
        sign_msr = sign_mar/N_msr

        G_up_msr = cp.array([G_up_mar[i]/(N_msr*sign_msr[i]) for i in range(self.N_markov)])
        G_dn_msr = cp.array([G_dn_mar[i]/(N_msr*sign_msr[i]) for i in range(self.N_markov)])
        interaction_msr = cp.array([interaction_mar[i]/(N_msr*sign_msr[i]) for i in range(self.N_markov)])

        G_msr = G_up_msr+G_dn_msr

        kin = cp.array([cp.trace(cp.array(H_0_msr).dot(G_msr[i]))/self.N_s for i in range(self.N_markov)])
        intr = cp.array([interaction_msr[i]/self.N_s for i in range(self.N_markov)])
        filling = np.trace(cp.mean(G_msr,axis=0))/self.N_s
        mean_onsite_corr = cp.mean(cp.array([interaction_mar[i]/(self.U*cp.mean(cp.diag(G_up_mar[i])*cp.mean(cp.diag(G_dn_mar[i])))) for i in range(self.N_markov)]),axis=0)
        totenrgy = kin + intr
        err = 100*cp.std(totenrgy,axis=0)/cp.abs(cp.mean(totenrgy,axis=0))
        energy_mean = cp.mean(totenrgy,axis=0)

        szsz_msr = cp.array([szsz_mar[i,:]/(N_msr*sign_msr[i]) for i in range(self.N_markov)])
        sxsx_msr = cp.array([sxsx_mar[i,:]/(N_msr*sign_msr[i]) for i in range(self.N_markov)])
        rho_msr = cp.array([rho_mar[i,:]/(N_msr*sign_msr[i]) for i in range(self.N_markov)])

        sign_msr = cp.mean(cp.abs(sign_msr),axis=0)

        measures = {"elpsd_time":end-strt,
                    "kinetic_energy_mean":cp.mean(kin),
                    "interaction_energy_mean":cp.mean(intr),
                    "n_mean":filling,
                    "mean_onsite_corr":mean_onsite_corr,
                    "energy_mean":energy_mean,
                    "err_bar":err,
                    "sign_mean":sign_msr,
                    "markov_data":{
                        "G_up_mar":G_up_mar,
                        "G_dn_mar":G_dn_mar,
                        "N_msr":N_msr,
                        "sign_mar":sign_mar,
                        "interaction_mar":interaction_mar,
                        "correlations":{
                            "szsz_mar":szsz_mar,
                            "sxsx_mar":sxsx_mar,
                            "rho_mar":rho_mar
                        }
                                },
                    "model_params":{
                        "N_sw_measure" : self.N_sw_measure,
                        "N_warm_up" : self.N_sw_measure // 5,
                        "N_from_scratch" : self.N_from_scratch,
                        "N_qr" : self.N_qr,
                        "N_markov" : self.N_markov,
                        "chemical_potential" : self.chemical_potential,
                        "U" : self.U,
                        "tunneling" : self.tunneling,
                        "periodic_Y" : self.periodic_Y,
                        "periodic_X" : self.periodic_X,
                        "X_dimension" : self.X_dimension,
                        "Y_dimension" : self.Y_dimension,
                        "N_s" : self.X_dimension * self.Y_dimension,
                        "N_time" : self.N_time,
                        "Beta" : self.Beta,
                        "directory": self.directory,
                    }
                    }

        print(" time: {}s\n kinetic_energy_mean: {}\n interaction_energy_mean: {}\n n_mean: {}\n mean_onsite_corr: {}\n energy_mean: {}\n err_bar: {}\n sign_mean: {}".format(end - strt,cp.mean(kin),cp.mean(intr),filling,mean_onsite_corr,energy_mean,err,sign_msr))
        plt.plot(cp.asnumpy(szsz_msr[0]))
        return measures,

    def save_hs(self, hs, directory):
        cp.savez_compressed(directory+str(int(time.time()))+".npz", hs = hs)
        return cp.empty((1, self.N_time, self.N_s)) #keep in mind the size of the array be smaller than the DRAM

    def load_hs(self, directory):

        data = cp.load(directory)
        hs = data["hs"][1:]

        return hs

    def calc_var_green(self, directory, paramsjson, paramspjson):
        params = AQMC(**paramsjson)
        paramsp = AQMC(**paramspjson)
        hs_m = self.load_hs(directory)

        H_0 = make_hopping(params.X_dimension, params.Y_dimension, params.periodic_X, params.periodic_Y, params.tunneling)
        H_0 += np.identity(params.X_dimension * params.Y_dimension) * ((-1) * params.chemical_potential)

        sign_U_interact, T_hop, H0_array, _, _, _, _ = init_trotter(params.Beta,params.N_time,params.U_eff,H_0)
        Bk, Bk_inv = expmk(H0_array, T_hop)

        G_up_m = cp.zeros((params.N_s, params.N_s))
        G_dn_m = cp.zeros((params.N_s, params.N_s))

        H_0 = make_hopping(paramsp.X_dimension, paramsp.Y_dimension, paramsp.periodic_X, paramsp.periodic_Y, paramsp.tunneling)
        H_0 += np.identity(paramsp.X_dimension * paramsp.Y_dimension) * ((-1) * paramsp.chemical_potential)

        sign_U_interactp, T_hop, H0_array, _, _, _, _ = init_trotter(paramsp.Beta,paramsp.N_time,paramsp.U_eff,H_0)
        Bkp, Bk_inv = expmk(H0_array, T_hop)

        G_up_p = cp.zeros((paramsp.N_s, paramsp.N_s))
        G_dn_p = cp.zeros((paramsp.N_s, paramsp.N_s))

        pl = 0
        for hs in hs_m:
            cl_up, cl_dn = cluster(hs.copy(),Bk.copy(),sign_U_interact)

            G_up = from_scratch(cl_up, params.N_qr)
            G_dn = from_scratch(cl_dn, params.N_qr)
            print(cp.sum(cl_dn),cp.sum(G_up))
            clp_up, clp_dn = cluster(hs.copy(),Bkp.copy(),sign_U_interactp)

            G_upp = from_scratch(clp_up, paramsp.N_qr)
            G_dnp = from_scratch(clp_dn, paramsp.N_qr)

            signp_up, log_pp_up = cp.linalg.slogdet(G_upp)
            signp_dn, log_pp_dn = cp.linalg.slogdet(G_dnp)
            signp = signp_dn * signp_up
            log_pp = log_pp_up + log_pp_dn

            sign_up, log_p_up = cp.linalg.slogdet(G_up)
            sign_dn, log_p_dn = cp.linalg.slogdet(G_dn)
            sign = sign_dn * sign_up
            log_p = log_p_up + log_p_dn

            G_up_m += G_up * sign
            G_dn_m += G_dn * sign

            G_up_p += G_up * (signp / sign) * cp.exp(log_pp - log_p)
            G_dn_p += G_dn * (signp / sign) * cp.exp(log_pp - log_p)

            print((signp / sign) , log_pp ,log_p,cp.trace(G_dn_p),cp.sum(G_dn_p))
            pl += cp.exp(log_pp - log_p)

            # n_up_tmp = cp.diag(G_up)
            # n_dn_tmp = cp.diag(G_dn)
            # sz_tmp = (n_up_tmp-n_dn_tmp)/2
            # rho_tmp = (n_up_tmp+n_dn_tmp)/2

        return 1-cp.trace(G_up_p)/(pl*self.N_s),1-cp.trace(G_dn_p)/(pl*self.N_s),1-cp.trace(G_up_m)/(len(hs_m)*self.N_s),1-cp.trace(G_dn_m)/(len(hs_m)*self.N_s),
params = {
"N_sw_measure" : 10,
"N_warm_up" : 10 // 5,
"N_from_scratch" : 10,
"N_qr" : 10,
"N_markov" : 2,
"chemical_potential" : 0,
"U" : 4,
"tunneling" : 1,
"periodic_Y" : 1,
"periodic_X" : 1,
"X_dimension" : 16,
"Y_dimension" : 8,
"N_s" : 16*8,
"N_time" : 100,
"Beta" : 4,
"directory":""
}

paramsp = {
"N_sw_measure" : 10,
"N_warm_up" : 10 // 5,
"N_from_scratch" : 10,
"N_qr" : 10,
"N_markov" : 2,
"chemical_potential" : 0.2,
"U" : 4,
"tunneling" : 1,
"periodic_Y" : 1,
"periodic_X" : 1,
"X_dimension" : 16,
"Y_dimension" : 8,
"N_s" : 16*8,
"N_time" : 100,
"Beta" : 4,
"directory":""
}

a = AQMC(**params)


import errno
import sys
import os

from datetime import datetime

from copy import deepcopy

import config as app_config
from pgadmin.utils.route import BaseTestGenerator
from regression.feature_utils.pgadmin_page import PgadminPage
from regression.python_test_utils import test_utils

class BaseFeatureTest(BaseTestGenerator):
    CURRENT_PATH = os.path.dirname(os.path.realpath(__file__))

    def setUp(self):
        self.server = deepcopy(self.server)
        self.server['name'] += ' Feature Tests'
        if app_config.SERVER_MODE and not self.parallel_ui_tests:
            self.skipTest(
                "Currently, config is set to start pgadmin in server mode. "
                "This test doesn't know username and password so doesn't work "
                "in server mode"
            )

        self.page = PgadminPage(self.driver, app_config)
        try:
            if self.parallel_ui_tests:
                self.page.login_to_app(self.server['login_details'])
            test_utils.reset_layout_db()
            self.page.driver.switch_to.default_content()
            self.page.wait_for_app()
            self.page.wait_for_spinner_to_disappear()
            self.page.refresh_page()
            self.page.wait_for_spinner_to_disappear()
            self.before()
        except Exception:
            self._screenshot()
            raise

    def runTest(self):
        # To be implemented by child classes
        pass

    def before(self):
        # To be implemented by child classes
        pass

    def after(self):
        # To be implemented by child classes
        pass

    def tearDown(self):
        python3_failures = hasattr(self, '_outcome') and self.any_step_failed()

        if python3_failures:
            self._screenshot()

        self.after()

    def any_step_failed(self):
        if hasattr(self._outcome, 'errors'):
            for step in self._outcome.errors:
                if step[1] is not None:
                    return True
        return False

    def current_test_failed(self):
        all_failures = self._resultForDoCleanups.errors + \
            self._resultForDoCleanups.failures
        for failure in all_failures:
            if failure[0] == self:
                return True
        return False

    def _screenshot(self):
        screenshots_directory = '{0}/../screenshots'.format(self.CURRENT_PATH)
        screenshots_server_directory = '{0}/{1}'.format(
            screenshots_directory,
            self.server["name"].replace(" ", "_")
        )

        self.ensure_directory_exists(screenshots_directory)
        self.ensure_directory_exists(screenshots_server_directory)

        date = datetime.now().strftime("%Y.%m.%d_%H.%M.%S")
        python_version = sys.version.split(" ", maxsplit=1)[0]

        self.page.driver.save_screenshot(
            '{0}/{1}-{2}-Python-{3}.png'.format(
                screenshots_server_directory,
                self.__class__.__name__,
                date,
                python_version
            )
        )

    def ensure_directory_exists(self, path):
        try:
            os.mkdir(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise