from tools.my_request import MyRequest
from tools.wexcel import *
import os

class InterfaceTest():
    def __init__(self,file):
        self.res = get_data(file, "1")#从excel读取原始数据

#将exlce的类似'a=1,b=2,c=3'格式的，转为字典
    def dic(self,s):
        t=[]
        k=[]
        v=[]
        dic={}
        t=s.split(",")
        for i in t:
            k.append(i.split("=")[0])
            v.append(i.split("=")[1])
        dic=dict(zip(k,v))
        return dic

    def check(self,ck,res):
        fag = 1
        # for k,v in ck.items():
        #     if k in res.keys():
        #         if v==res[k]
        for i in ck.keys():
            if i in res.keys():
                if ck[i]!= str(res[i]):
                    fag=0
        if fag:
            res="校验成功"
        else:
            res="校验失败"
        return res

#将从excel读取的每行内容，提取出url，data，method，check等关键值，并生产字典返回
    def parameter(self):
        dic=[]
        for i in self.res[1:]:
            dic_={}
            dic_=dict(url=i[2],method=i[3],data=self.dic(i[4]),check=self.dic(i[5]))
            dic.append(dic_)
        return dic

#将 parameter类函数返回的字典传入，调用tools包下的MyRequest类，返回请求结果，返回响应和状态结果以list返回
    def my_request(self,data):
        par=data
        res=[]
        for i in par:
            res_=[]
            a=MyRequest(url=i.get("url"),method=i.get("method"),data=i.get("data"))

            res_.append(self.check(i.get("check"),a.response))
            res_.append(str(a.response))
            res.append(res_)
        return res

    def update_excl(self,f):
        req=f
        for i in range(len(req)):
            self.res[i+1][6]=req[i][0]
            self.res[i+1][7] = req[i][1]
        return self.res

#在原始excel目录下，生成结果文件
    def sava_excl(self):

        name1=os.path.basename(file)
        dir=os.path.dirname(file)
        name1_prefix=name1.split(".")[0]
        name=name1_prefix+"_reslut.xlsx"
        exfile=os.path.join(dir,name)
        wexcel(exfile,name1,self.res)

#整合相关请求，只需要调用该类函数即可
    def run(self):
        b = self.my_request(self.parameter())
        self.update_excl(b)
        self.sava_excl()

file = "D:\python_sted\day8\用例模板.xlsx"
cl=InterfaceTest(file)
cl.run()
# b=cl.run(cl.my_request())
# #print(b)
# cl.update_excl(b)
# #print(cl.res)


from calendar import c
import json
from operator import truediv
import random
from web3 import Web3,IPCProvider,middleware
from web3.middleware import geth_poa_middleware
from web3.gas_strategies.time_based import medium_gas_price_strategy
from web3.gas_strategies.rpc import rpc_gas_price_strategy
from eth_keys import keys
import time
from requests.exceptions import ConnectionError
import json
from eth_account import Account
import os
from dotenv import load_dotenv ##pipx install python-dotenv
from decimal import Decimal

load_dotenv()

PRIVATE_KEY = os.getenv("EXTRA_PKEY")
account = Account.from_key(PRIVATE_KEY)
print(f"account.address :               {account.address}")
print(f"\n")

def get(count,fee):
    nonce = web3.eth.get_transaction_count(account.address)
    txn = Random.functions.random(count,0,True).buildTransaction({
        'chainId': web3.eth.chain_id,
        'gasPrice': web3.eth.generate_gas_price() ,
        'from': account.address,
        'nonce': nonce,
        'value': int(fee*1.5*count),
        })

    signed_txn = web3.eth.account.sign_transaction(txn, private_key=PRIVATE_KEY)
    tx_hash = web3.eth.send_raw_transaction(signed_txn.rawTransaction)
    tx_receipt = web3.eth.wait_for_transaction_receipt(tx_hash)
    print(f"requested {count} random sentfee={int(fee*1.5*count)/10**18} fee={int(fee*count)/10**18} totalcost={(tx_receipt.gasUsed*tx_receipt.effectiveGasPrice)/10**18+int(fee*count)/10**18}")
    return tx_receipt

while True:
    try:
        web3 = Web3() ##Web3.HTTPProvider(os.getenv("WEB3_PROVIDER_URI")))
        ##if poa:
        web3.eth.default_account = account.address
        web3.eth.set_gas_price_strategy(rpc_gas_price_strategy)

        print(f"\n")
        print(f"api :                   {web3.api}")
        print(f"clientVersion :         {web3.clientVersion}")
        print(f"\n")

        print(f"chainid :               {web3.eth.chain_id}")
        print(f"gasPrice :              {web3.eth.gasPrice}")
        print(f"generate_gas_price :    {web3.eth.generate_gas_price()}")
        print(f"coinbase :              {web3.eth.coinbase}")
        print(f"hashrate :              {web3.eth.hashrate}")
        print(f"syncing :               {web3.eth.syncing}")
        print(f"accounts :              {web3.eth.accounts}")
        print(f"block_number :          {web3.eth.block_number}")
        ##print(f"protocol_version :      {web3.eth.protocol_version}")
        print(f"mining :                {web3.eth.mining}")
        print(f"\n")

        print(f"net.peer_count :        {web3.net.peer_count}")
        print(f"net.version :           {web3.net.version}")
        print(f"net.listening :         {web3.net.listening}")
        print(f"\n")

        mygasPrice=web3.eth.gasPrice

        mapjsonfilename = 'build/deployments/map.json'
        mapjson = json.load(open(mapjsonfilename))
        RandomAddress = mapjson[str(web3.eth.chain_id)]['Random'][0]

        RandomJSON = json.load(open('build/contracts/Random.json'))
        Random = web3.eth.contract(address=RandomAddress, abi=RandomJSON['abi'])

        print(f"Random Address :        {RandomAddress}")
        print(f"\n")

        Randoms_filter = Random.events.Randoms.createFilter(fromBlock='latest')
        Remaining_filter = Random.events.Remaining.createFilter(fromBlock='latest')
        Reset_filter = Random.events.Reset.createFilter(fromBlock='latest')
        Fee_filter = Random.events.Fee.createFilter(fromBlock='latest')

        fee = Random.functions.fee().call()
        print(f"Random fee :        {fee}")

        while True:
            ##balance = web3.eth.get_balance(web3.eth.default_account)
            ##print(f"Balance :           {Decimal(balance)/10**18}")
            mygasPrice=int(web3.eth.gasPrice*1.1)

            if True: ##(balance > fee*1.1*64):
                startbalance = Decimal(web3.eth.get_balance(web3.eth.default_account))/10**18
                tx_receipt = get(256,fee)
                endbalance = Decimal(web3.eth.get_balance(web3.eth.default_account))/10**18
                deltabalance = endbalance-startbalance
                print(f"balance={endbalance} deltabalance={deltabalance}")

                #print(f"tx.events={tx_receipt}")
                for event in Randoms_filter.get_new_entries():
                    print(f"Randoms:    {event.args.random}")
                    pass
                for event in Remaining_filter.get_new_entries():
                    count = event.args.count
                    print(f"Remaining:  {count}")
                for event in Reset_filter.get_new_entries():
                    count = event.args.count
                    print(f"Reset:  {count}")
                for event in Fee_filter.get_new_entries():
                    fee = event.args.fee
                    print(f"Fee:        {fee}")

            print(f"\n")
            time.sleep(0.1)
    except KeyboardInterrupt:
        raise
    except:
        raise

        time.sleep(5)


import asyncio
import os
import tomli
from pyrogram.client import Client
from caligo.core import database

def load_config(filename: str) -> dict:
    # Check if the file exists and is readable
    if os.path.isfile(filename) and os.access(filename, os.R_OK):
        # Load the configuration from the file in binary mode
        with open(filename, "rb") as config_file:
            config = tomli.load(config_file)
    else:
        # Return an empty dictionary if the file is not found or not readable
        config = {}
    return config

# Define a function to get a value from the configuration or return None
def get_config_value(config: dict, section: str, key: str) -> str:
    # Try to get the value from the configuration dictionary
    value = config.get(section, {}).get(key)
    # Return the value or None if not found or empty
    return value or None

async def create_session() -> None:
    # Load the configuration from config.toml
    config = load_config("config.toml")

    # Get the values for api_id, api_hash, and mongodb_uri from the configuration or return None
    api_id = get_config_value(config, "telegram", "api_id")
    api_hash = get_config_value(config, "telegram", "api_hash")
    mongodb_uri = get_config_value(config, "bot", "db_uri")

    # Prompt the user for input only if the value is None
    if api_id is None:
        api_id = input("Please enter Telegram API ID: ")
    if api_hash is None:
        api_hash = input("Please enter Telegram API HASH: ")
    if mongodb_uri is None:
        mongodb_uri = input("Please enter MongoDB URI: ")

    # Create a Pyrogram client with the given parameters
    client = Client(
        api_id=api_id,
        api_hash=api_hash,
        name="caligo",
        workdir="caligo",
    )

    # Create a MongoDB client with the given URI and connect lazily
    db_client = database.AsyncClient(mongodb_uri, connect=False)

    # Get the CALIGO database from the MongoDB client
    db = db_client.get_database("CALIGO")

    # Set the persistent storage for the Pyrogram client using the database
    client.storage = database.storage.PersistentStorage(db)  # type: ignore

    print("Generating session...")

    # Start and stop the Pyrogram client to generate a session file
    await client.start()
    print("Session generated successfully!")
    await client.stop()

    # Close the MongoDB client connection
    await db_client.close()

if __name__ == "__main__":
    asyncio.run(create_session())


from argparse import ArgumentParser

def run():
    args = ArgumentParser(prog="cwc",description="Character Word Cloud")
    args.add_argument("--text",type=str,help="specify file of words to build the word cloud (default: stdin)")
    args.add_argument("--regex",type=str,help="override the regular expression defining what constitutes a word")
    args.add_argument("--stopwords",type=str,help="specify file of stopwords (containing one word per line) to remove from the given text after parsing")
    args.add_argument("--imagefile",type=str,help="file the completed PNG image should be written to (default: stdout)")
    args.add_argument("--fontfile",type=str,help="path to font file you wish to use (default: DroidSansMono)")
    args.add_argument("--mask",type=str,help="mask to use for the image form")
    args.add_argument("--colormask",type=str,help="color mask to use for the image form")
    args.add_argument("--contour_width",type=int,default=0,help="if greater than 0, draw mask contour (default: 0)")
    args.add_argument("--contour_color",type=str,default="black",help="use given color as mask contour color - accepts any value from PIL.ImageColor.getcolor")
    args.add_argument("--relative_scaling",type=int,default=0,help="scaling of words by frequency (0 - 1)")
    args.add_argument("--margin",type=int,default=2,help="spacing to leave around words")
    args.add_argument("--width",type=int,default=400,help="define output image width")
    args.add_argument("--height",type=int,default=200,help="define output image height")
    args.add_argument("--color",type=str,help="use given color as coloring for the image - accepts any value from PIL.ImageColor.getcolor")
    args.add_argument("--background",type=str,default="black",help="use given color as background color for the image - accepts any value from PIL.ImageColor.getcolor")
    args.add_argument("--collocations",action="store_true",help="add collocations (bigrams) to word cloud (default: add unigrams and bigrams)")
    args.add_argument("--include_numbers",action="store_false",help="include numbers in wordcloud?")
    args.add_argument("--min_word_length",type=int,default=0,help="only include words with more than X letters")
    args.add_argument("--prefer_horizontal",type=float,default=0.9,help="ratio of times to try horizontal fitting as opposed to vertical")
    args.add_argument("--scale",type=float,default=1,help="scaling between computation and drawing")
    args.add_argument("--corlormap",type=str,default="viridis",help="matplotlib colormap name")
    args.add_argument("--mode",type=str,default="RGB",help="use RGB or RGBA for transparent background")
    args.add_argument("--max_words",type=int,default=200,help="maximum number of words")
    args.add_argument("--min_font_size",type=int,default=4,help="smallest font size to use")
    args.add_argument("--max_font_size",type=int,help="maximum font size for the largest word")
    args.add_argument("--font_step",type=int,default=1, help="step size for the font")
    args.add_argument("--random_state",type=int,help="seed for the random number generator")
    args.add_argument("--normalize_plurals",action="store_true",help="whether to remove trailing ‘s’ from words")
    args.add_argument("--repeat",action="store_false", help="whether to repeat words and phrases")
    args.add_argument("--version",help="show program's version number and exit")
    args.parse_args()

if __name__ == '__main__':
    run()


"""Official evaluation script for v1.1 of the SQuAD dataset.
From the SQuAD website: https://rajpurkar.github.io/SQuAD-explorer/
"""

from collections import Counter
import string
import re
import argparse
import json
import sys

def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s))))

def f1_score(prediction, ground_truth):
    prediction_tokens = normalize_answer(prediction).split()
    ground_truth_tokens = normalize_answer(ground_truth).split()
    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
    num_same = sum(common.values())
    if num_same == 0:
        return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(ground_truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1

def exact_match_score(prediction, ground_truth):
    return (normalize_answer(prediction) == normalize_answer(ground_truth))

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)

def evaluate(dataset, predictions):
    f1 = exact_match = total = 0
    for article in dataset:
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                total += 1
                if qa['id'] not in predictions:
                    message = 'Unanswered question ' + qa['id'] + \
                              ' will receive score 0.'
                    print(message, file=sys.stderr)
                    continue
                ground_truths = list(map(lambda x: x['text'], qa['answers']))
                prediction = predictions[qa['id']]
                exact_match += metric_max_over_ground_truths(
                    exact_match_score, prediction, ground_truths)
                f1 += metric_max_over_ground_truths(
                    f1_score, prediction, ground_truths)

    exact_match = 100.0 * exact_match / total
    f1 = 100.0 * f1 / total

    return {'exact_match': exact_match, 'f1': f1}

def unofficial_eval(dataset, predictions):
    """Adapted from evaluate function above. Used during training
    to produce local eval scores with the same formulae used for official eval.
    """
    f1 = exact_match = total = 0
    for example_id, prediction in predictions.items():
        total += 1
        ground_truths = dataset[example_id]['a']
        exact_match += metric_max_over_ground_truths(
            exact_match_score, prediction, ground_truths)
        f1 += metric_max_over_ground_truths(
            f1_score, prediction, ground_truths)

    exact_match = 100.0 * exact_match / total
    f1 = 100.0 * f1 / total

    return {'exact_match': exact_match, 'f1': f1}

def get_pred_text(eval_dict, example_ids, pred_starts, pred_ends, probs, use_official_ids=False):
    """Get text of predicted answers for a number of predicted starts, ends, and the corresponding example_ids.
    If use_official_ids is True, the returned dictionary maps from official UUIDs to predicted (answer text, prob) pairs.
    Otherwise, maps from simple numeric IDs to (answer text, prob) pairs.
    """
    pred_text = {}
    for example_id, pred_start, pred_end, prob in zip(example_ids, pred_starts, pred_ends, probs):
        context = eval_dict[str(example_id)]['c']
        bounds = eval_dict[str(example_id)]['bounds']
        start_idx = bounds[pred_start][0]
        end_idx = bounds[pred_end][1]
        if use_official_ids:
            official_id = eval_dict[str(example_id)]['id']
            pred_text[official_id] = (context[start_idx: end_idx], prob)
        else:
            pred_text[str(example_id)] = (context[start_idx: end_idx], prob)
    return pred_text

if __name__ == '__main__':
    expected_version = '1.1'
    parser = argparse.ArgumentParser(
        description='Evaluation for SQuAD ' + expected_version)
    parser.add_argument('dataset_file', help='Dataset file')
    parser.add_argument('prediction_file', help='Prediction File')
    args = parser.parse_args()
    with open(args.dataset_file) as dataset_file:
        dataset_json = json.load(dataset_file)
        if (dataset_json['version'] != expected_version):
            print('Evaluation expects v-' + expected_version +
                  ', but got dataset with v-' + dataset_json['version'],
                  file=sys.stderr)
        dataset = dataset_json['data']
    with open(args.prediction_file) as prediction_file:
        predictions = json.load(prediction_file)
    print(json.dumps(evaluate(dataset, predictions)))


from PyQt5 import QtWidgets, QtCore
from PyQt5.QtWidgets import QComboBox, QTextEdit

__author__ = 'Pawel'
"""
Class that represents archieve window. It allows you to see every conversation you have
"""
class ArchieveDialog(QtWidgets.QDialog):

    def __init__(self,parent, db):
        super(ArchieveDialog, self).__init__(parent)
        self.conversation = QTextEdit(self)
        self.combo = QComboBox(self)
        self.db = db
        self.initUI()

    def initUI(self):
        self.resize(300,300)
        for key, value in self.db.items():
            self.combo.addItem(key)
        self.conversation.move(0,20)
        self.combo.move(0, 0)
        self.combo.activated[str].connect(self.onActivated)

    def onActivated(self, text):
        self.conversation.clear()
        self.conversation.append(self.db[text])


import numpy as np
import pybullet as p
from constants import *
import scene_builder
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation, writers
import tensorflow as tf
import time
import disect_model
import os
np.random.seed(123456)

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

walls, map_label, bodies = scene_builder.build_scene_full()

STEP_SIZE = PIXEL_FRACTION
ROBOT_RADIUS = PIXEL_FRACTION / 2
NUM_TRIES = 64
MAX_DIST = SCENE_SIZE * np.sqrt(2)
MAX_STEPS = 200
EMBEDDING_SIZE = 512

def pos_to_coords(pos):
    coords = np.floor(pos / PIXEL_FRACTION).astype(np.int)
    return (coords[0], coords[1])

def random_valid_pos(walls, map_label):
    while True:
        pos = scene_builder.random_pos(walls)
        if map_label[pos_to_coords(pos)] == 1:
            return pos

frames = []

def preprocess(img, view):
    return prepocess_image(img), normalize_view(view)

def prepocess_image(img):
    img = img[:,:,:3].astype(np.float) / 255
    img = img * 2 - 1
    return img.reshape([1] + IMG_SHAPE)

def normalize_view(view):
    view = (view - VIEW_MEANS) / VIEW_VARIATION
    return view.reshape((1, VIEW_DIM))

def get_image(pos, rot):
    pos = np.array([pos[0], pos[1], PUPPER_HEIGHT - np.random.random() * HEIGHT_VARIATION])
    pitch = (1 - 2 * np.random.random()) * PITCH_VARIATION
    roll = (1 - 2 * np.random.random()) * ROLL_VARIATION
    rot = np.array([rot, pitch, roll])
    view = scene_builder.pack_viewpoint(pos, rot)
    w, h, rgb, depth, seg = scene_builder.get_image(view)
    return rgb, view

rep_net, map_net, loc_net = disect_model.build_model()

goal = random_valid_pos(walls, map_label)
position = random_valid_pos(walls, map_label)

start = time.time()

embedding = np.zeros([1, EMBEDDING_SIZE])
img, pose = get_image(position, np.random.random() * 2 * np.pi)
pimg, pose = preprocess(img, pose)
embedding = rep_net.predict([pimg, pose, embedding])

est_map = map_net.predict(embedding)[0]
frames.append((pose, pose, goal, est_map, img))

while len(frames) < MAX_STEPS:
    best_score = np.inf
    best_pos = position
    best_rot = 0
    for i in range(NUM_TRIES):
        rot = np.random.random() * 2 * np.pi
        new_pos = STEP_SIZE * np.array([np.cos(rot), np.sin(rot)]) + position
        min_coords = pos_to_coords(new_pos - ROBOT_RADIUS)
        max_coords = pos_to_coords(new_pos + ROBOT_RADIUS)
        prob = np.min(est_map[min_coords[0]:max_coords[0]+1, min_coords[1]:max_coords[1]+1])
        prob = min((prob * 2) ** 2, 1)
        score = np.linalg.norm(goal - new_pos) * prob + MAX_DIST * (1 - prob)
        if score < best_score:
            best_score = score
            best_pos = new_pos
            best_rot = rot
    position = best_pos
    img, pose = get_image(position, best_rot)
    pimg, pose = preprocess(img, pose)
    est_pose = loc_net.predict([pimg, embedding])
    embedding = rep_net.predict([pimg, pose, embedding])
    est_map = map_net.predict(embedding)[0]
    if np.linalg.norm(goal - position) < STEP_SIZE:
        goal = random_valid_pos(walls, map_label)
    frames.append((pose, est_pose, goal, est_map, img))

print(time.time() - start, 'for pathfinding')

def plot_goal(ax, goal):
    ax.scatter(goal[1] * 8, goal[0] * 8, c='g')

def plot_pose(ax, pose, c):
    pose = pose[0]
    pos = (pose[:2] * 4 + 4) * 8
    rot = pose[3:5]
    rot = rot / np.linalg.norm(rot)
    ax.quiver(pos[1], pos[0], rot[1], rot[0], angles='xy')
    ax.scatter(pos[1], pos[0], c=c)

def animate(frame, axs):
    pose, est_pose, goal, est_map, img = frame
    for i in np.ndindex(axs.shape):
        axs[i].clear()
    axs[0, 0].matshow(map_label, extent=(0, MAP_SIZE, MAP_SIZE, 0))
    axs[1, 0].imshow(img, extent=(0, MAP_SIZE, MAP_SIZE, 0))
    axs[0, 1].imshow(est_map > 0.5, extent=(0, MAP_SIZE, MAP_SIZE, 0))
    axs[1, 1].imshow(est_map, extent=(0, MAP_SIZE, MAP_SIZE, 0))
    for i in [(0, 0), (0, 1), (1, 1)]:
        plot_goal(axs[i], goal)
        plot_pose(axs[i], est_pose, 'r')
        plot_pose(axs[i], pose, 'b')

fig, axs = plt.subplots(2, 2)
fig.set_size_inches(6.4, 6.4)
fig.tight_layout()
ani = FuncAnimation(fig, animate, frames=frames, fargs=(axs,), repeat=False)
writer = writers['ffmpeg'](fps=10, bitrate=1800)
ani.save('animation.mp4', writer=writer)


import random
from croe.conf import Conf
import requests
from requests.models import PreparedRequest
from urllib.parse import urlsplit

UA = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"
# http 请求封装函数
def http_req(url, method='get', **kwargs):
    if kwargs.get("disable_normal"):
        # 禁用 URL 规范化处理，仅仅支持GET
        kwargs.pop("disable_normal")
        return req_disable_normal(url, method, **kwargs)

    kwargs.setdefault('verify', False)
    kwargs.setdefault('timeout', (Conf.CONNECT_TIMEOUT, Conf.READ_TIMEOUT))
    kwargs.setdefault('allow_redirects', False)

    headers = kwargs.get("headers", {})
    if headers is None:
        headers = {}

    headers.setdefault("User-Agent", UA)

    random_ip = "10.0.{}.{}".format(random.randint(1, 254), random.randint(1, 254))
    headers.setdefault("X-Real-IP", random_ip)
    headers.setdefault("X-Forwarded-For", random_ip)

    kwargs["headers"] = headers

    proxies = {
        'https': Conf.PROXY_URL,
        'http': Conf.PROXY_URL
    }

    if Conf.PROXY_URL:
        kwargs["proxies"] = proxies

    conn = getattr(requests, method)(url, **kwargs)

    return conn
# 禁用规范化URL处理
# 下面这个函数在开代理的情况下依然是没有用的
def req_disable_normal(url, method='get', **kwargs):
    headers = kwargs.get("headers", {})
    if headers is None:
        headers = {}

    headers.setdefault("User-Agent", UA)

    kwargs["headers"] = headers

    req = requests.Request(method=method, url=url, **kwargs)
    prep = req.prepare()
    prep.url = url

    proxies = {}

    if Conf.PROXY_URL:
        proxies = {
            'http': Conf.PROXY_URL,
            'https': Conf.PROXY_URL
        }

    if "#" in url:
        my_prep = MyPreparedRequest()
        my_prep.prepare(method='GET', url=url)
        my_prep.url = url
        prep = my_prep

    with requests.Session() as session:
        return session.send(prep, verify=False, proxies=proxies, allow_redirects=False,
                            timeout=(Conf.CONNECT_TIMEOUT, Conf.READ_TIMEOUT))
class MyPreparedRequest(PreparedRequest):
    def __init__(self):
        super().__init__()

    @property
    def path_url(self):
        url = []

        p = urlsplit(self.url, allow_fragments=False)

        path = p.path
        if not path:
            path = '/'

        url.append(path)

        query = p.query
        if query:
            url.append('?')
            url.append(query)

        return ''.join(url)


import pygame
import random
import constants as c
from bullet import Bullet
import enemy_spawner

class Enemy2(pygame.sprite.Sprite):
    def __init__(self):
        super(Enemy2, self).__init__()
        self.img_explosion_01 = pygame.image.load('images\explosion_01.png').convert_alpha()
        self.img_explosion_02 = pygame.image.load('images\explosion_02.png').convert_alpha()
        self.img_explosion_03 = pygame.image.load('images\explosion_03.png').convert_alpha()
        self.img_explosion_04 = pygame.image.load('images\explosion_04.png').convert_alpha()
        self.img_explosion_05 = pygame.image.load('images\explosion_05.png').convert_alpha()

        self.anim_explosion = [self.img_explosion_01,
                               self.img_explosion_02,
                               self.img_explosion_03,
                               self.img_explosion_04,
                               self.img_explosion_05]

        self.anim_index = 0
        self.frame_length_max = 8
        self.frame_length = self.frame_length_max
        self.image = pygame.image.load('images\enemy2.png').convert_alpha()
        self.image = pygame.transform.scale(self.image, (int(self.image.get_width() / 9), int(self.image.get_height() / 9)))
        self.is_destroyed = False
        self.is_invincible = False

        self.rect = self.image.get_rect()
        self.rect.x = random.randrange(0, c.display_width - self.rect.width)
        self.rect.y = -self.rect.height
        self.snd_hit = pygame.mixer.Sound('sounds\sound_bullet_hit.ogg')
        self.hp = 3
        self.bullets = pygame.sprite.Group()
        self.bullet_timer_max = 60
        self.bullet_timer = self.bullet_timer_max
        self.states = {'fly_down': 'fly_down',
                       'attack': 'attack'}
        self.state = self.states['fly_down']
        self.init_state = True
        self.score_value = 5
        self.velocity_x = 0
        self.velocity_y = random.randrange(3, 4)

    def update(self):
        self.bullets.update()
        if self.state == 'fly_down':
            self.state_fly_down()
        elif self.state == 'attack':
            self.state_attack()
        self.rect.x += self.velocity_x
        self.rect.y += self.velocity_y

        if self.is_destroyed:
            max_index = len(self.anim_explosion) - 1
            if self.frame_length == 0:
                self.anim_index += 1
                if self.anim_index > max_index:
                    self.kill()
                    enemy_spawner.EnemySpawner().enemies.remove([1])
                else:
                    self.image = self.anim_explosion[self.anim_index]
                    self.frame_length = self.frame_length_max
            else:
                self.frame_length -= 1

    def state_fly_down(self):
        if self.init_state:
            self.init_state = False
        if self.rect.y >= 200:
            self.state = self.states['attack']
            self.init_state = True

    def state_attack(self):
        if self.init_state:
            self.velocity_y = 0
            while self.velocity_x == 0:
                self.velocity_x = random.randrange(-4, 4)
            self.init_state = False
        if self.bullet_timer == 0:
            self.shoot()
            self.bullet_timer = self.bullet_timer_max
        else:
            self.bullet_timer -= 1
        if self.rect.x <= 0:
            self.velocity_x *= -1
        elif self.rect.x + self.rect.width >= c.display_width:
            self.velocity_x *= -1

    def shoot(self):
        new_bullet = Bullet()
        new_bullet.velocity_y = 4
        new_bullet.rect.x = self.rect.x + (self.rect.width // 2)
        new_bullet.rect.y = self.rect.y + self.rect.height
        self.bullets.add(new_bullet)

    def get_hit(self):
        if not self.is_invincible:
            self.snd_hit.play()
            self.hp -=1

            if self.hp <= 0:
                self.is_invincible = True
                self.is_destroyed = True
                self.velocity_x = 0
                self.velocity_y = 0
                self.rect.x = self.rect.x - 30
                self.rect.y = self.rect.y - 44
                self.image = self.anim_explosion[self.anim_index]
        else:
            pass


import json
import requests
import pandas as pd
import boto3

from lxml.html import fromstring
from io import StringIO
from datetime import date

from localities import S3_BUCKET, HISTORICAL_OBS_LOC

def get_today_data():
    '''Scarps the BOM website for the current max, min and rainfall
    '''
    url =  'http://reg.bom.gov.au/nsw/observations/sydney.shtml'
    page = requests.get(url)

    # Getting page data
    tree = fromstring(page.content)
    data_date = tree.xpath('//td[@headers="tSYDNEY-datetime tSYDNEY-station-sydney-observatory-hill"]/text()')[0]
    maxT = float(tree.xpath('//td[@headers="tSYDNEY-hightmp tSYDNEY-station-sydney-observatory-hill"]/text()')[0])
    minT = float(tree.xpath('//td[@headers="tSYDNEY-lowtmp tSYDNEY-station-sydney-observatory-hill"]/text()')[0])
    rain = float(tree.xpath('//td[@headers="tSYDNEY-rainsince9am tSYDNEY-station-sydney-observatory-hill"]/text()')[0])

    # Fixing up the date as it is in day/time format
    # Not using exclusively date.today() as I do want to be careful about
    # timzone problems
    day = int(data_date.split('/')[0])
    today = date.today()
    data_date = today.replace(day=day)

    # Adding it to the historical reading
    recent_obs = pd.DataFrame({'Year':data_date.year,
                               'Month':data_date.month,
                               'Day': data_date.day,
                               'Maximum temperature (°C)':maxT,
                               'Minimum temperature (°C)':minT,
                               'Rainfall (mm)':rain},
                               index=[0])
    return recent_obs

def lambda_handler(event, context):
    # Get most recent observations for this month
    recent_obs = get_today_data()

    # Read historical data from S3
    s3 = boto3.resource('s3')
    historical_obs_loc = s3.Object(bucket_name=S3_BUCKET, key=HISTORICAL_OBS_LOC)
    historical_obs = pd.read_csv(historical_obs_loc.get()['Body'])

    # To keep this data frame up to date, we concatentate it with the most
    # recent month, then drop the earliest rows
    # (assumed more recent data is correct)
    historical_obs = pd.concat([historical_obs, recent_obs])
    historical_obs = historical_obs.drop_duplicates(['Year', 'Month', 'Day'],
                                                    keep='last')

    # Finally upload back to S3
    csv_buffer = StringIO()
    historical_obs.to_csv(csv_buffer, index=False)
    s3.Object(S3_BUCKET, HISTORICAL_OBS_LOC).put(Body=csv_buffer.getvalue())

    return {
        'statusCode': 200,
        'body': json.dumps(recent_obs.to_dict('records'))
    }

if __name__=='__main__':
    lambda_handler(None, None)


from typing import Callable, List, Optional, TypeVar

from reactivex import Observable, abc, compose
from reactivex import operators as ops
from reactivex import typing

_T = TypeVar("_T")

def buffer_with_time_(
    timespan: typing.RelativeTime,
    timeshift: Optional[typing.RelativeTime] = None,
    scheduler: Optional[abc.SchedulerBase] = None,
) -> Callable[[Observable[_T]], Observable[List[_T]]]:
    if not timeshift:
        timeshift = timespan

    return compose(
        ops.window_with_time(timespan, timeshift, scheduler),
        ops.flat_map(ops.to_list()),
    )

__all__ = ["buffer_with_time_"]


from PIL import Image
from io import BytesIO
import urllib.parse

def html_to_img(driver,html_content,id_count):
    '''converts html to image'''
    html_content = urllib.parse.quote(html_content)
    driver.get("data:text/html;charset=utf-8," + html_content)
    window_size = driver.get_window_size()
    max_height,max_width = window_size['height'],window_size['width']
    e = driver.find_element_by_id('c0')

    bboxes = []
    for id in range(id_count):
        e = driver.find_element_by_id('c'+str(id))
        txt = e.text.strip()
        lentext = len(txt)
        loc = e.location
        size_ = e.size
        xmin = loc['x']
        ymin = loc['y']
        xmax = int(size_['width'] + xmin)
        ymax = int(size_['height'] + ymin)
        bboxes.append([lentext,txt,xmin,ymin,xmax,ymax])
        # cv2.rectangle(im,(xmin,ymin),(xmax,ymax),(0,0,255),2)

    png = driver.get_screenshot_as_png()
    im = Image.open(BytesIO(png))
    im = im.crop((0,0, max_width, max_height))

    return im,bboxes


import msprime, pyslim, argparse, random, subprocess
import numpy as np
import re

## This function comes courtesy of Peter Ralph, who directed me to it
## after I posted on the SLiM forum

def remove_mutations(ts, start, end, proportion):
	'''
	This function will return a new tree sequence the same as the input,
	but after removing each non-SLiM mutation within regions specified in lists
	start and end with probability `proportion`, independently. So then, if we
	want to add neutral mutations with rate 1.0e-8 within the regions and 0.7e-8
	outside the regions, we could do
	ts = pyslim.load("my.trees")
	first_mut_ts = msprime.mutate(ts, rate=1e-8)
	mut_ts = remove_mutations(first_mut_ts, start, end, 0.3)

	:param float proportion: The proportion of mutations to remove.
	'''
	new_tables = ts.dump_tables()
	new_tables.mutations.clear()
	mutation_map = [-1 for _ in range(ts.num_mutations)]
	for j, mut in enumerate(ts.mutations()):
	   keep_mutation = True
	for i in range(len(start)):
		left = start[i]
		right = end[i]
		if left == right:
			continue
		assert(left < right)
		if i > 0:
			 assert(end[i - 1] <= left)
		if mut.position >= left and mut.position < right and len(mut.metadata) == 0:
			 keep_mutation = (random.uniform(0, 1) > proportion)
		if keep_mutation:
			mutation_map[j] = new_tables.mutations.num_rows
		if mut.parent < 0:
			 new_parent = -1
		else:
			new_parent = mutation_map[mut.parent]
			new_tables.mutations.add_row(site = mut.site, node = mut.node,
			derived_state = mut.derived_state,
			parent = new_parent,
			metadata = mut.metadata)
	return new_tables.tree_sequence()

class SLiM:
	def __init__(self, configFile):
		regexString = "[-+]?[.]?[\d]+(?:,\d\d\d)*[\.]?\d*(?:[eE][-+]?\d+)?"
		lines = [i.strip().strip(';') for i in open(configFile)]
		elementsRaw = [i.split('(')[-1].strip(')').split(' ') for i in lines if i.startswith('initializeGenomicElement(')]
		self.elementStarts = [int(i[1].split(',')[0]) for i in elementsRaw]
		self.elementEnds = [int(i[2].split(',')[0]) for i in elementsRaw]
#		self.mutRate = float([i for i in lines if i.startswith('initializeMutationRate(')][0].strip().strip(';').strip(')').split('(')[1])
		self.mutRate = float(re.findall(regexString, [i for i in lines if i.startswith('initializeMutationRate(')][0])[0])
		print(self.mutRate)

def main():
	parser = argparse.ArgumentParser(description="PYTHON 3 !!!! This script takes a .trees file from SLiM 3.x and add neutral mutations to the simulated populations. ")
	parser.add_argument("--input","-i",
			required = True,
			dest = "input",
			type =str,
			help = "name the simulation config file")
	parser.add_argument("--trees","-t",
			required = True,
			dest = "trees",
			type =str,
			help = "name the trees file")
#	parser.add_argument("--N",
#			required = True,
#			dest = "N",
#			type = int,
#			help = "The population size that was simulated")
	parser.add_argument("--sample",
			required = True,
			dest = "sample",
			type = int,
			help = "The number of haploid copies to sample from the population")
#	parser.add_argument("--theta",
#			required = True,
#			dest = "theta",
#			type = float,
#			help = "The population effective mutation rate")
	parser.add_argument("-o",
			required = True,
			dest = "output",
			type =str,
			help = "name the output file")
	args = parser.parse_args()

	bgzipPath = '/home/booker/bin/htslib-1.9/'
	tabixPath = '/home/booker/bin/htslib-1.9/'

	config = SLiM(args.input)

	ts = pyslim.load(args.trees).simplify()
	N = ts.get_sample_size()

#	mut_rate = args.theta/(4.*args.N)

	sprinkled = msprime.mutate(ts, rate= config.mutRate, keep=True)

	mutated = remove_mutations(sprinkled, config.elementStarts, config.elementEnds, 0.0)

#	Here we strip out the neutral mutations that were added on top of elements that were already simulated

	haploidSample = np.random.choice(mutated.samples(),args.sample, replace = False)

	ts2 = sprinkled.simplify(samples = haploidSample)

	with open(args.output, "w") as vcf_file:
		ts2.write_vcf(vcf_file, 1)

	bgzip = [bgzipPath + 'bgzip', args.output]
	subprocess.call(bgzip)
	tabix = [tabixPath +'tabix', args.output+'.gz']
	subprocess.call(tabix)

	return
#	for tree in ts2.trees():
#		print("-" * 20)
#		print("tree {}: interval = {}".format(tree.index, tree.interval))
#		print(tree.draw(format="unicode"))

main()


from django.conf.urls import patterns, include, url
from django.views.generic import TemplateView, RedirectView
from django.contrib import admin

admin.autodiscover()

from . import views
from .sitemaps import sitemaps
from widgets.views import BookmarkListView, GalleryHome

urlpatterns = patterns('',
    url(r'^manage-site/', include(admin.site.urls)),
    url(r'^comments/', include('django.contrib.comments.urls')),
    url(r'^posts/', include('blog.urls', namespace='blog')),
)

urlpatterns += patterns('',
    url(r'^$', views.IndexView, name='home'),
    url(r'^bookmarks/$', BookmarkListView.as_view(), name='bookmarklist'),
)

urlpatterns += patterns('',
        url(r'^gallery/$', GalleryHome, name='gallery_home'),
)

urlpatterns += patterns('',
        url(r'^sitemap\.xml$', 'django.contrib.sitemaps.views.sitemap', { 'sitemaps': sitemaps}),
)

urlpatterns += patterns('',
        url(r'^robots\.txt$', TemplateView.as_view(template_name="robots.txt", content_type='text/plain')),
#       url(r'favicon\.ico$', RedirectView.as_view(url='/static/img/favicon.ico')),
)

urlpatterns += patterns('django.contrib.flatpages.views',
        url(r'^(?P<url>.*/)$', 'flatpage'),
)


#!/usr/bin/env python
# coding: utf-8

import util
import numpy as np
import cartopy.crs as ccrs
import absplots as apl

# initialize figure
ll = ccrs.PlateCarree()
proj = ccrs.Stereographic(central_latitude=-10.0, central_longitude=10.0)
fig, ax = apl.subplots_mm(
    figsize=(45, 90), subplot_kw=dict(projection=proj), gridspec_kw=dict(
        left=0, right=0, bottom=0, top=0))
ax.set_rasterization_zorder(2.5)
ax.set_xlim(-7500e3, 7500e3)
ax.set_ylim(-15000e3, 15000e3)

# draw map elements
ax.gridlines(color='0.5', linestyle='-', linewidth=0.1)
ax.coastlines(edgecolor='k', lw=0.25)

# add modelling domain
util.geo.draw_model_domain(ax=ax, extent='alps')

# add record locations
records = ['GRIP', 'EPICA', 'MD01-2444']
colors = ['C1', 'C5', 'C3']
lats = np.array([72.58, -75.1, 37.561])
lons = np.array([-37.64, 123.35, -10.142])
locs = ['uc', 'uc', 'lc']
x, y, _ = ax.projection.transform_points(ll, lons, lats).T
ax.scatter(lons, lats, c=colors, linewidths=0.25, transform=ll)

# add record labels
offset = 6
for i, rec in enumerate(records):
    xy = x[i], y[i]
    loc = locs[i]
    xoffset = ((loc[1] == 'r')-(loc[1] == 'l')) * offset
    yoffset = ((loc[0] == 'u')-(loc[0] == 'l')) * offset
    xytext = xoffset, yoffset
    ha={'r': 'left', 'l': 'right', 'c': 'center'}[loc[1]]
    va={'u': 'bottom', 'l': 'top', 'c': 'center'}[loc[0]]
    ax.annotate(rec, xy=xy, xytext=xytext, ha=ha, va=va, color=colors[i],
                textcoords='offset points', fontweight='bold')

# save
util.com.savefig()


# -*- coding: utf-8 -*-
"""
Created on Sat Jan 28 12:16:28 2023

@author: Nathan Gilmore
"""

import numpy as np
import random
import glob
from astropy.io import fits
from meteostat import Point, Daily, Hourly
from datetime import datetime
import os
import shutil
import time

weather = np.array(['Clear', 'Fair', 'Cloudy', 'Overcast', 'Fog', 'Freezing Fog', 'Light Rain', 'Rain', 'Heavy Rain', 'Freezing Rain', 'Heavy Freezing Rain', 'Sleet', 'Heavy Sleet', 'Light Snowfall', 'Snowfall', 'Heavy Snowfall', 'Rain Shower', 'Heavy Rain Shower', 'Sleet Shower', 'Heavy Sleet Shower', 'Snow Shower', 'Heavy Snow Shower', 'Lightning', 'Hail', 'Thunderstorm', 'Heavy Thunderstorm', 'Storm'])
no_weather_data = 0
out_of_time_range = 0
images_processed = 0
ambiguous_images = 0
clear_images = 0
obscured_images = 0
meteo_weather = 'Null'
coco = 0
fits_dir = r'/beegfs/car/njm/cloud_class/data/fits/'
jpg_dir = r'/beegfs/car/dcampbell/allsky/camera1/jpeg'

def count_files(dir_path):
    return len([f for f in os.listdir(dir_path) if os.path.isfile(os.path.join(dir_path, f))])

#lists all items in the directory
#checks if  they are FILES or DIRECTORIES
#returns the number of FILES

def find_file(filename, root_dir):
    for root, dirs, files in os.walk(root_dir):
        if filename in files:
            return os.path.join(root, filename)

#searches through all files, in all subdirectories, within the parent directory
#checks if the given filename is among the files it has found
#returns the full filepath of that file

def get_weather_data (year, month, day, hour):
        global coco
        global meteo_weather
        #Set time period of image ready for meteostat
        start = datetime(year, month, day, hour)
        end = datetime(year, month, day, hour)

        #Create Point for Bayfordbury, UK foe meteostat
        bayfordbury = Point(51.775, -0.094444)

        #Obtain Image Weather info from Meteostat
        data = Hourly(bayfordbury, start, end) #obtains the data from meteostat for a given date and time
        data = data.fetch() #puts this data into a pandas DataFrame
        clcv=data.iat[0,10] #takes the 0th row, 10th column value (in this case, weather code)
        i = int(clcv)-1
        coco = i+1
        meteo_weather=weather[i] #returns a weather condition as indicated by the weather code

def sort_images (random_file):
        global clear_images
        global obscured_images
        global ambiguous_images
        global coco
        #print(coco)
        #print(meteo_weather)
    #Split filename and extension
        file = random_file.split('.',1)[0] #takes the filename without the ".FIT" extension
        #print(file)
    #Copy .jpg version of clear fits images
        if int(coco) == 1:
            clear_images = clear_images + 1
            #put image into clear folder
            new_dir = r'/beegfs/car/njm/cloud_class/clear_images'
            file_path = find_file(file+'.JPG', jpg_dir) #looks for the image in the jpg directory
            shutil.copy2(file_path,new_dir) #copies the file from "file_path", into "new_dir"
    #Ignore fits images decleared as 'Fair'
        if int(coco) == 2:
            ambiguous_images=ambiguous_images + 1
    #Copy .jpg version of obscured images, ignoring those defined as 'fair'
        if int(coco) == 3:
            obscured_images = obscured_images + 1
            #put image into cloudy folder
            new_dir = r'/beegfs/car/njm/cloud_class/cloudy_images'
            file_path = find_file(file+'.JPG', jpg_dir) #looks for the image in the jpg directory
            shutil.copy2(file_path,new_dir) #copies the file from "file_path", into "new_dir"
        else:
            ambiguous_images=ambiguous_images + 1
        return clear_images, obscured_images, ambiguous_images

#Loop as necessary
file_list = os.listdir(fits_dir)    #creates a list of all of the files in the directory
for unused in range(0, count_files(fits_dir)):     #will loop all of the images in the directory
    dt0 = time.time()
    images_processed = images_processed + 1
    #Get a random FIT Image from FIT Directory,
    random_file = random.choice(file_list)
    #print (random_file)
    dt1 = time.time()
    #print('Time to chose fits:',dt1 - dt0)

    #Extract timestamp from fits header
    with fits.open(fits_dir+'/'+random_file, mode="update") as hdul:
        hdul.info()
        hdul[0].verify('fix') #ensures the HDU data is is consistent and correct and will attempt to fix if not
        hdul[0].header
        datestamp = hdul[0].header['DATE-OBS'] #takes the datestamp of the FITS image
        #print (datestamp)
        #splitting the datestamp into its year, month, day, hour, and minute components
        year = int(datestamp[0:4])
        month = int(datestamp[5:7])
        day = int(datestamp[8:10])
        hour = int(datestamp[11:13])
        minutes = int(datestamp[14:16])

        dt2 = time.time()
        #print('Time to open fits:',dt2 - dt1)

    #Only process images 10mins either side of the hour
    if minutes <= 10 or minutes >= 50:  #good solution to this, i tidied it up to make it a smidgen faster
        if minutes >= 50:
            hour=hour+1
        if hour < 24:  # e.g. if time 23:55:00, it would be rounded to 24:55:00 which is invalid
            try:
                get_weather_data(year,month,day,hour)
                dt3 = time.time()
                #print('Time to get weather data:',dt3 - dt2)

                hdul[0].header.append('WEATHER', meteo_weather)
                sort_images(random_file)

                dt4 = time.time()
                #print('Time to move things',dt4 - dt3)
            except:
                no_weather_data = no_weather_data + 1
    else:
        out_of_time_range = out_of_time_range + 1

#Print Summary
#print ('Number of Images Processed ', images_processed)
#print ('Outside of requisite time-frames ', out_of_time_range)
#print ('Ambiguous images ', ambiguous_images)
#print ('Clear Images ', clear_images)
#print ('Obscured Images ', obscured_images)
#print ('No Weather Data ', no_weather_data)