'''
This script runs noise_tests.test_min_signal_amplitude on a specified chip set
Requires noise_tests library
Requires a .json file containing chip-ids and daisy chain data formatted like
{
    'board': <board-name>,
    'chip_set': [
        [<chip-id>, <io-chain>],
        ...
        ]
}
'''

from __future__ import print_function
import argparse
import logging
from helpers.script_logging import ScriptLogger
import helpers.pathnames as pathnames
import helpers.larpix_scripting as larpix_scripting
import time
import larpix.larpix as larpix
import helpers.noise_tests as noise_tests
from sys import (exit, stdout)
import json
import os

start_time = time.localtime()
default_config = 'physics.json'

parser = argparse.ArgumentParser()
parser.add_argument('-b','--board', default=pathnames.default_board_file(start_time),
                    help='input file containing chipset info (optional, default: '
                    '%(default)s)')
parser.add_argument('-o','--outdir', default=pathnames.default_script_logdir(start_time),
                    help='output directory for log file and data file '
                    '(optional, default: %(default)s)')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--global_threshold_correction', default=0, type=int,
                    help='Amount to increase the global threshold on each chip'
                    ' for sensitivity test'
                    '(optional, default: %(default)s)')
parser.add_argument('--pixel_trim_correction', default=0, type=int,
                    help='Amount to increase the pixel trim on each channel'
                    ' for sensitivity test '
                    '(optional, default: %(default)s)')
parser.add_argument('--max_dac_amp', default=10, type=int,
                    help='(optional, default: %(default)s)')
parser.add_argument('--min_dac_amp', default=0, type=int,
                    help='(optional, default: %(default)s)')
parser.add_argument('--n_pulses', default=10, type=int,
                    help='(optional, default: %(default)s)')
parser.add_argument('-s','--configuration_file', default=None,
                    help='initial chip configuration file to load '
                    'by default will look in %s for individual chip configurations, '
                    'if chip config not found, will load %s, '
                    'if this file does not exist, will load %s and generate new default '
                    '(optional)' % (pathnames.default_config_dir(start_time),
                                    pathnames.default_config_file(start_time),
                                    default_config))
parser.add_argument('-c','--chips', default=None, nargs='+', type=int,
                    help='chips to include in scan '
                    '(optional, default: all chips in chipset file)')
args = parser.parse_args()

infile = args.board
outdir = args.outdir
verbose = args.verbose
global_threshold_correction = args.global_threshold_correction
pixel_trim_correction = args.pixel_trim_correction
max_dac_amp = args.max_dac_amp
min_dac_amp = args.min_dac_amp
n_pulses = args.n_pulses
config_file = args.configuration_file
if config_file is None:
    config_file = pathnames.default_config_dir(start_time)
    default_config = pathnames.make_default_config(start_time, default_config)
chips_to_scan = args.chips

return_code = 0

script_logfile = outdir + '/' + \
    os.path.basename(pathnames.default_script_logfile(start_time))
data_logfile = outdir + '/' + os.path.basename(pathnames.default_data_logfile(start_time))
sl = ScriptLogger(start_time, script_logfile=script_logfile, data_logfile=data_logfile)
log = sl.get_script_log()
log.info('arguments: %s' % str(args))

try:
    controller = larpix.Controller(timeout=0.01)
    # Initial configuration of chips
    board_info = larpix_scripting.load_board(controller, infile)
    log.info('begin initial configuration of chips for board %s' % board_info)
    config_ok, different_registers = larpix_scripting.load_chip_configurations(
        controller, board_info, config_file, silence=True, default_config=default_config)
    if config_ok:
        log.info('initial configuration of chips complete')

    # Run sensitivity test on each chip
    board_results = []
    for chip_idx,chip in enumerate(controller.chips):
        try:
            start_time = time.time()
            chip_id = chip.chip_id
            io_chain = chip.io_chain
            chip_info = (io_chain, chip_id)
            if chips_to_scan is None:
                pass
            else:
                if not chip_id in chips_to_scan:
                    log.info('skipping %d-c%d' % chip_info)
                    board_results += [None]
                    continue

            larpix_scripting.clear_buffer(controller)
            chip_results = noise_tests.test_min_signal_amplitude(\
                controller=controller,chip_idx=chip_idx,
                threshold=chip.config.global_threshold + global_threshold_correction,
                trim=[trim + pixel_trim_correction
                      for trim in chip.config.pixel_trim_thresholds],
                n_pulses=n_pulses,
                min_dac_amp=min_dac_amp,
                max_dac_amp=max_dac_amp,
                reset_cycles=chip.config.reset_cycles)

            board_results += [chip_results]
            larpix_scripting.clear_stored_packets(controller)
            finish_time = time.time()
            if verbose:
                log.debug('%d-c%d sensitivity test took %.2f s' % \
                              (io_chain, chip_id, finish_time - start_time))
        except Exception as error:
            log.exception(error)
            log.error('%d-c%d sensitivity test failed!' % chip_info)
            controller.disable(chip_id=chip_id, io_chain=io_chain)
            return_code = 2
            continue

    log.info('all chips sensitivity check complete')

    # Print sensitivity test results
    for chip_idx,chip in enumerate(controller.chips):
        chip_id = chip.chip_id
        io_chain = chip.io_chain
        if board_results[chip_idx] is None:
            log.info('%s-%d-c%d skipped' % (board_info, io_chain, chip_id))
            continue
        for channel in sorted(board_results[chip_idx].keys()):
            log.info('%s-%d-c%d-ch%d min dac: %d dac, efficiency: %.2f' % \
                         (board_info, io_chain, chip_id, channel,
                          board_results[chip_idx][channel]['min_pulse_dac'],
                          board_results[chip_idx][channel]['eff']))

except Exception as error:
    log.exception(error)
    return_code = 1

exit(return_code)


from __future__ import annotations

import os
import time
import uuid
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Type

import numpy as np
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore

if TYPE_CHECKING:
    import vearch

DEFAULT_TOPN = 4

class Vearch(VectorStore):
    _DEFAULT_TABLE_NAME = "langchain_vearch"
    _DEFAULT_CLUSTER_DB_NAME = "cluster_client_db"
    _DEFAULT_VERSION = 1

    def __init__(
        self,
        embedding_function: Embeddings,
        path_or_url: Optional[str] = None,
        table_name: str = _DEFAULT_TABLE_NAME,
        db_name: str = _DEFAULT_CLUSTER_DB_NAME,
        flag: int = _DEFAULT_VERSION,
        **kwargs: Any,
    ) -> None:
        """Initialize vearch vector store
        flag 1 for cluster,0 for standalone
        """
        try:
            if flag:
                import vearch_cluster
            else:
                import vearch
        except ImportError:
            raise ValueError(
                "Could not import suitable python package. "
                "Please install it with `pip install vearch or vearch_cluster`."
            )

        if flag:
            if path_or_url is None:
                raise ValueError("Please input url of cluster")
            if not db_name:
                db_name = self._DEFAULT_CLUSTER_DB_NAME
                db_name += "_"
                db_name += str(uuid.uuid4()).split("-")[-1]
            self.using_db_name = db_name
            self.url = path_or_url
            self.vearch = vearch_cluster.VearchCluster(path_or_url)

        else:
            if path_or_url is None:
                metadata_path = os.getcwd().replace("\\", "/")
            else:
                metadata_path = path_or_url
            if not os.path.isdir(metadata_path):
                os.makedirs(metadata_path)
            log_path = os.path.join(metadata_path, "log")
            if not os.path.isdir(log_path):
                os.makedirs(log_path)
            self.vearch = vearch.Engine(metadata_path, log_path)
            self.using_metapath = metadata_path
        if not table_name:
            table_name = self._DEFAULT_TABLE_NAME
            table_name += "_"
            table_name += str(uuid.uuid4()).split("-")[-1]
        self.using_table_name = table_name
        self.embedding_func = embedding_function
        self.flag = flag

    @property
    def embeddings(self) -> Optional[Embeddings]:
        return self.embedding_func

    @classmethod
    def from_documents(
        cls: Type[Vearch],
        documents: List[Document],
        embedding: Embeddings,
        path_or_url: Optional[str] = None,
        table_name: str = _DEFAULT_TABLE_NAME,
        db_name: str = _DEFAULT_CLUSTER_DB_NAME,
        flag: int = _DEFAULT_VERSION,
        **kwargs: Any,
    ) -> Vearch:
        """Return Vearch VectorStore"""

        texts = [d.page_content for d in documents]
        metadatas = [d.metadata for d in documents]

        return cls.from_texts(
            texts=texts,
            embedding=embedding,
            metadatas=metadatas,
            path_or_url=path_or_url,
            table_name=table_name,
            db_name=db_name,
            flag=flag,
            **kwargs,
        )

    @classmethod
    def from_texts(
        cls: Type[Vearch],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        path_or_url: Optional[str] = None,
        table_name: str = _DEFAULT_TABLE_NAME,
        db_name: str = _DEFAULT_CLUSTER_DB_NAME,
        flag: int = _DEFAULT_VERSION,
        **kwargs: Any,
    ) -> Vearch:
        """Return Vearch VectorStore"""

        vearch_db = cls(
            embedding_function=embedding,
            embedding=embedding,
            path_or_url=path_or_url,
            db_name=db_name,
            table_name=table_name,
            flag=flag,
        )
        vearch_db.add_texts(texts=texts, metadatas=metadatas)
        return vearch_db

    def _create_table(
        self,
        dim: int = 1024,
        field_list: List[dict] = [
            {"field": "text", "type": "str"},
            {"field": "metadata", "type": "str"},
        ],
    ) -> int:
        """
        Create VectorStore Table
        Args:
            dim:dimension of vector
            fields_list: the field you want to store
        Return:
            code,0 for success,1 for failed
        """

        type_dict = {"int": vearch.dataType.INT, "str": vearch.dataType.STRING}
        engine_info = {
            "index_size": 10000,
            "retrieval_type": "IVFPQ",
            "retrieval_param": {"ncentroids": 2048, "nsubvector": 32},
        }
        fields = [
            vearch.GammaFieldInfo(fi["field"], type_dict[fi["type"]])
            for fi in field_list
        ]
        vector_field = vearch.GammaVectorInfo(
            name="text_embedding",
            type=vearch.dataType.VECTOR,
            is_index=True,
            dimension=dim,
            model_id="",
            store_type="MemoryOnly",
            store_param={"cache_size": 10000},
            has_source=False,
        )
        response_code = self.vearch.create_table(
            engine_info,
            name=self.using_table_name,
            fields=fields,
            vector_field=vector_field,
        )
        return response_code

    def _create_space(
        self,
        dim: int = 1024,
    ) -> int:
        """
        Create VectorStore space
        Args:
            dim:dimension of vector
        Return:
            code,0 failed for ,1 for success
        """
        space_config = {
            "name": self.using_table_name,
            "partition_num": 1,
            "replica_num": 1,
            "engine": {
                "name": "gamma",
                "index_size": 1,
                "retrieval_type": "FLAT",
                "retrieval_param": {
                    "metric_type": "L2",
                },
            },
            "properties": {
                "text": {
                    "type": "string",
                },
                "metadata": {
                    "type": "string",
                },
                "text_embedding": {
                    "type": "vector",
                    "index": True,
                    "dimension": dim,
                    "store_type": "MemoryOnly",
                },
            },
        }
        response_code = self.vearch.create_space(self.using_db_name, space_config)

        return response_code

    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        embeddings = None
        if self.embedding_func is not None:
            embeddings = self.embedding_func.embed_documents(list(texts))
        if embeddings is None:
            raise ValueError("embeddings is None")
        if self.flag:
            dbs_list = self.vearch.list_dbs()
            if self.using_db_name not in dbs_list:
                create_db_code = self.vearch.create_db(self.using_db_name)
                if not create_db_code:
                    raise ValueError("create db failed!!!")
            space_list = self.vearch.list_spaces(self.using_db_name)
            if self.using_table_name not in space_list:
                create_space_code = self._create_space(len(embeddings[0]))
                if not create_space_code:
                    raise ValueError("create space failed!!!")
            docid = []
            if embeddings is not None and metadatas is not None:
                for text, metadata, embed in zip(texts, metadatas, embeddings):
                    profiles: dict[str, Any] = {}
                    profiles["text"] = text
                    profiles["metadata"] = metadata["source"]
                    embed_np = np.array(embed)
                    profiles["text_embedding"] = {
                        "feature": (embed_np / np.linalg.norm(embed_np)).tolist()
                    }
                    insert_res = self.vearch.insert_one(
                        self.using_db_name, self.using_table_name, profiles
                    )
                    if insert_res["status"] == 200:
                        docid.append(insert_res["_id"])
                        continue
                    else:
                        retry_insert = self.vearch.insert_one(
                            self.using_db_name, self.using_table_name, profiles
                        )
                        docid.append(retry_insert["_id"])
                        continue
        else:
            table_path = os.path.join(
                self.using_metapath, self.using_table_name + ".schema"
            )
            if not os.path.exists(table_path):
                dim = len(embeddings[0])
                response_code = self._create_table(dim)
                if response_code:
                    raise ValueError("create table failed!!!")
            if embeddings is not None and metadatas is not None:
                doc_items = []
                for text, metadata, embed in zip(texts, metadatas, embeddings):
                    profiles_v: dict[str, Any] = {}
                    profiles_v["text"] = text
                    profiles_v["metadata"] = metadata["source"]
                    embed_np = np.array(embed)
                    profiles_v["text_embedding"] = embed_np / np.linalg.norm(embed_np)
                    doc_items.append(profiles_v)

                docid = self.vearch.add(doc_items)
                t_time = 0
                while len(docid) != len(embeddings):
                    time.sleep(0.5)
                    if t_time > 6:
                        break
                    t_time += 1
                self.vearch.dump()
        return docid

    def _load(self) -> None:
        """
        load vearch engine for standalone vearch
        """
        self.vearch.load()

    @classmethod
    def load_local(
        cls,
        embedding: Embeddings,
        path_or_url: Optional[str] = None,
        table_name: str = _DEFAULT_TABLE_NAME,
        db_name: str = _DEFAULT_CLUSTER_DB_NAME,
        flag: int = _DEFAULT_VERSION,
        **kwargs: Any,
    ) -> Vearch:
        """Load the local specified table of standalone vearch.
        Returns:
            Success or failure of loading the local specified table
        """
        if not path_or_url:
            raise ValueError("No metadata path!!!")
        if not table_name:
            raise ValueError("No table name!!!")
        table_path = os.path.join(path_or_url, table_name + ".schema")
        if not os.path.exists(table_path):
            raise ValueError("vearch vectorbase table not exist!!!")

        vearch_db = cls(
            embedding_function=embedding,
            path_or_url=path_or_url,
            table_name=table_name,
            db_name=db_name,
            flag=flag,
        )
        vearch_db._load()
        return vearch_db

    def similarity_search(
        self,
        query: str,
        k: int = DEFAULT_TOPN,
        **kwargs: Any,
    ) -> List[Document]:
        """
        Return docs most similar to query.

        """
        if self.embedding_func is None:
            raise ValueError("embedding_func is None!!!")
        embeddings = self.embedding_func.embed_query(query)
        docs = self.similarity_search_by_vector(embeddings, k)
        return docs

    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = DEFAULT_TOPN,
        **kwargs: Any,
    ) -> List[Document]:
        """The most k similar documents and scores of the specified query.
        Args:
            embeddings: embedding vector of the query.
            k: The k most similar documents to the text query.
            min_score: the score of similar documents to the text query
        Returns:
            The k most similar documents to the specified text query.
            0 is dissimilar, 1 is the most similar.
        """
        embed = np.array(embedding)
        if self.flag:
            query_data = {
                "query": {
                    "sum": [
                        {
                            "field": "text_embedding",
                            "feature": (embed / np.linalg.norm(embed)).tolist(),
                        }
                    ],
                },
                "size": k,
                "fields": ["text", "metadata"],
            }
            query_result = self.vearch.search(
                self.using_db_name, self.using_table_name, query_data
            )
            res = query_result["hits"]["hits"]
        else:
            query_data = {
                "vector": [
                    {
                        "field": "text_embedding",
                        "feature": embed / np.linalg.norm(embed),
                    }
                ],
                "fields": [],
                "is_brute_search": 1,
                "retrieval_param": {"metric_type": "InnerProduct", "nprobe": 20},
                "topn": k,
            }
            query_result = self.vearch.search(query_data)
            res = query_result[0]["result_items"]
        docs = []
        for item in res:
            content = ""
            meta_data = {}
            if self.flag:
                item = item["_source"]
            for item_key in item:
                if item_key == "text":
                    content = item[item_key]
                    continue
                if item_key == "metadata":
                    meta_data["source"] = item[item_key]
                    continue
            docs.append(Document(page_content=content, metadata=meta_data))
        return docs

    def similarity_search_with_score(
        self,
        query: str,
        k: int = DEFAULT_TOPN,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """The most k similar documents and scores of the specified query.
        Args:
            embeddings: embedding vector of the query.
            k: The k most similar documents to the text query.
            min_score: the score of similar documents to the text query
        Returns:
            The k most similar documents to the specified text query.
            0 is dissimilar, 1 is the most similar.
        """
        if self.embedding_func is None:
            raise ValueError("embedding_func is None!!!")
        embeddings = self.embedding_func.embed_query(query)
        embed = np.array(embeddings)
        if self.flag:
            query_data = {
                "query": {
                    "sum": [
                        {
                            "field": "text_embedding",
                            "feature": (embed / np.linalg.norm(embed)).tolist(),
                        }
                    ],
                },
                "size": k,
                "fields": ["text_embedding", "text", "metadata"],
            }
            query_result = self.vearch.search(
                self.using_db_name, self.using_table_name, query_data
            )
            res = query_result["hits"]["hits"]
        else:
            query_data = {
                "vector": [
                    {
                        "field": "text_embedding",
                        "feature": embed / np.linalg.norm(embed),
                    }
                ],
                "fields": [],
                "is_brute_search": 1,
                "retrieval_param": {"metric_type": "InnerProduct", "nprobe": 20},
                "topn": k,
            }
            query_result = self.vearch.search(query_data)
            res = query_result[0]["result_items"]
        results: List[Tuple[Document, float]] = []
        for item in res:
            content = ""
            meta_data = {}
            if self.flag:
                score = item["_score"]
                item = item["_source"]
            for item_key in item:
                if item_key == "text":
                    content = item[item_key]
                    continue
                if item_key == "metadata":
                    meta_data["source"] = item[item_key]
                    continue
                if self.flag != 1 and item_key == "score":
                    score = item[item_key]
                    continue
            tmp_res = (Document(page_content=content, metadata=meta_data), score)
            results.append(tmp_res)
        return results

    def _similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        return self.similarity_search_with_score(query, k, **kwargs)

    def delete(
        self,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> Optional[bool]:
        """Delete the documents which have the specified ids.

        Args:
            ids: The ids of the embedding vectors.
            **kwargs: Other keyword arguments that subclasses might use.
        Returns:
            Optional[bool]: True if deletion is successful.
            False otherwise, None if not implemented.
        """

        ret: Optional[bool] = None
        tmp_res = []
        if ids is None or ids.__len__() == 0:
            return ret
        for _id in ids:
            if self.flag:
                ret = self.vearch.delete(self.using_db_name, self.using_table_name, _id)
            else:
                ret = self.vearch.del_doc(_id)
            tmp_res.append(ret)
        ret = all(i == 0 for i in tmp_res)
        return ret

    def get(
        self,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> Dict[str, Document]:
        """Return docs according ids.

        Args:
            ids: The ids of the embedding vectors.
        Returns:
            Documents which satisfy the input conditions.
        """

        results: Dict[str, Document] = {}
        if ids is None or ids.__len__() == 0:
            return results
        if self.flag:
            query_data = {"query": {"ids": ids}}
            docs_detail = self.vearch.mget_by_ids(
                self.using_db_name, self.using_table_name, query_data
            )
            for record in docs_detail:
                if record["found"] is False:
                    continue
                content = ""
                meta_info = {}
                for field in record["_source"]:
                    if field == "text":
                        content = record["_source"][field]
                        continue
                    elif field == "metadata":
                        meta_info["source"] = record["_source"][field]
                        continue
                results[record["_id"]] = Document(
                    page_content=content, metadata=meta_info
                )
        else:
            for id in ids:
                docs_detail = self.vearch.get_doc_by_id(id)
                if docs_detail == {}:
                    continue
                content = ""
                meta_info = {}
                for field in docs_detail:
                    if field == "text":
                        content = docs_detail[field]
                        continue
                    elif field == "metadata":
                        meta_info["source"] = docs_detail[field]
                        continue
                results[docs_detail["_id"]] = Document(
                    page_content=content, metadata=meta_info
                )
        return results


import bmesh
import bpy
import math
import mathutils

ring_count = 16
segments = ring_count * 2
size = 1.0

scene = bpy.context.scene

mesh = bpy.data.meshes.new('SpiralSphere')
spiral_sphere = bpy.data.objects.new('SpiralSphere', mesh)

scene.objects.link(spiral_sphere)
scene.objects.active = spiral_sphere
spiral_sphere.select = True

bm = bmesh.new()

prev_ring = None
ring = []
prev_prev_last = None

for r in range(ring_count):
    ring = []

    for s in range(segments):
        xy_theta = s * 2 * math.pi / segments
        z_theta = (r + (s / segments)) * math.pi / ring_count

        x = math.cos(xy_theta) * math.sin(z_theta)
        y = math.sin(xy_theta) * math.sin(z_theta)
        z = math.cos(z_theta)

        vert = bm.verts.new((x, y, z))
        ring.append(vert)

    if r == 1:
        bm.faces.new([prev_ring[segments - 1], ring[0], prev_ring[0]])
    elif r > 1:
        bm.faces.new([prev_ring[segments - 1], ring[0], prev_ring[0], prev_prev_last])

    for s in range(segments - 1):
        if r == 0:
            if s != 0:
                bm.faces.new([ring[s], ring[s+1], ring[0]])
        else:
            bm.faces.new([ring[s], ring[s+1], prev_ring[s+1], prev_ring[s]])

    if prev_ring is not None:
        prev_prev_last = prev_ring[segments - 1]
    prev_ring = ring

bottom = bm.verts.new((0.0, 0.0, -1.0))

bm.faces.new([bottom, prev_ring[0], prev_prev_last])

for s in range(segments - 1):
    bm.faces.new([bottom, prev_ring[s+1], prev_ring[s]])

bm.verts.ensure_lookup_table()
bm.verts.index_update()

uv_layer = bm.loops.layers.uv.new()
for face in bm.faces:
    safe = False
    for l in range(len(face.loops)):
        loop = face.loops[l]
        x, y, z = loop.vert.co

        u = math.atan2(y, x) / (2 * math.pi)
        if u < 0.0:
            u += 1.0

        v = -math.acos(z) / math.pi

        if l == 0 and u == 0:
            safe = True
        if u == 0.0 and not safe:
            u = 1.0

        loop[uv_layer].uv = mathutils.Vector((u, v))

for v in bm.verts:
    v.co *= size

bm.to_mesh(mesh)
bm.free()


import json
import requests
import csv
url="https://www.mmm920.com/products?coupon=0&keyword=ALL&points=0&redPacket=0"
headers={
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Authorization': 'null',
'Connection': 'keep-alive',
'Content-Length': '134',
'Content-Type': 'application/json',
'Host': 'w.mmm920.com',
'Origin': 'https://www.mmm920.com',
'Referer': 'https://www.mmm920.com/',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-site',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
}
html=requests.post(url,headers=headers)
datas=json.loads(html.text)#把数据解析成json对象
for product in datas['products']:
    print(product['products']['name'])
    t = open('text.csv', 'a+', encoding='utf-8_sig', newline='')
    c = csv.writer(t)
    c.writerow(product['products']['name'])


"""Util that calls Google Search."""
from typing import Any, Dict, List, Optional

from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_core.utils import get_from_dict_or_env

class GoogleSearchAPIWrapper(BaseModel):
    """Wrapper for Google Search API.

    Adapted from: Instructions adapted from https://stackoverflow.com/questions/
    37083058/
    programmatically-searching-google-in-python-using-custom-search

    TODO: DOCS for using it
    1. Install google-api-python-client
    - If you don't already have a Google account, sign up.
    - If you have never created a Google APIs Console project,
    read the Managing Projects page and create a project in the Google API Console.
    - Install the library using pip install google-api-python-client

    2. Enable the Custom Search API
    - Navigate to the APIs & Services→Dashboard panel in Cloud Console.
    - Click Enable APIs and Services.
    - Search for Custom Search API and click on it.
    - Click Enable.
    URL for it: https://console.cloud.google.com/apis/library/customsearch.googleapis
    .com

    3. To create an API key:
    - Navigate to the APIs & Services → Credentials panel in Cloud Console.
    - Select Create credentials, then select API key from the drop-down menu.
    - The API key created dialog box displays your newly created key.
    - You now have an API_KEY

    Alternatively, you can just generate an API key here:
    https://developers.google.com/custom-search/docs/paid_element#api_key

    4. Setup Custom Search Engine so you can search the entire web
    - Create a custom search engine here: https://programmablesearchengine.google.com/.
    - In `What to search` to search, pick the `Search the entire Web` option.
    After search engine is created, you can click on it and find `Search engine ID`
      on the Overview page.

    """

    search_engine: Any  #: :meta private:
    google_api_key: Optional[str] = None
    google_cse_id: Optional[str] = None
    k: int = 10
    siterestrict: bool = False

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid

    def _google_search_results(self, search_term: str, **kwargs: Any) -> List[dict]:
        cse = self.search_engine.cse()
        if self.siterestrict:
            cse = cse.siterestrict()
        res = cse.list(q=search_term, cx=self.google_cse_id, **kwargs).execute()
        return res.get("items", [])

    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        google_api_key = get_from_dict_or_env(
            values, "google_api_key", "GOOGLE_API_KEY"
        )
        values["google_api_key"] = google_api_key

        google_cse_id = get_from_dict_or_env(values, "google_cse_id", "GOOGLE_CSE_ID")
        values["google_cse_id"] = google_cse_id

        try:
            from googleapiclient.discovery import build

        except ImportError:
            raise ImportError(
                "google-api-python-client is not installed. "
                "Please install it with `pip install google-api-python-client"
                ">=2.100.0`"
            )

        service = build("customsearch", "v1", developerKey=google_api_key)
        values["search_engine"] = service

        return values

    def run(self, query: str) -> str:
        """Run query through GoogleSearch and parse result."""
        snippets = []
        results = self._google_search_results(query, num=self.k)
        if len(results) == 0:
            return "No good Google Search Result was found"
        for result in results:
            if "snippet" in result:
                snippets.append(result["snippet"])

        return " ".join(snippets)

    def results(
        self,
        query: str,
        num_results: int,
        search_params: Optional[Dict[str, str]] = None,
    ) -> List[Dict]:
        """Run query through GoogleSearch and return metadata.

        Args:
            query: The query to search for.
            num_results: The number of results to return.
            search_params: Parameters to be passed on search

        Returns:
            A list of dictionaries with the following keys:
                snippet - The description of the result.
                title - The title of the result.
                link - The link to the result.
        """
        metadata_results = []
        results = self._google_search_results(
            query, num=num_results, **(search_params or {})
        )
        if len(results) == 0:
            return [{"Result": "No good Google Search Result was found"}]
        for result in results:
            metadata_result = {
                "title": result["title"],
                "link": result["link"],
            }
            if "snippet" in result:
                metadata_result["snippet"] = result["snippet"]
            metadata_results.append(metadata_result)

        return metadata_results


import cv2
import numpy as np
import keras
import mediapipe as mp
import threading
import time
from pynput.keyboard import Key, Controller
from escenario import Scene
from numpy import asfarray
from win32api import GetSystemMetrics

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
y = 0
x = 0
timeInSecs = 0
gestureSec = 0
lastGesture = ''
lastGesture2 = 0
keepOpen = True
currentScene = Scene()

#   Convertimos los booleanos en una cadena simple para ejemplificar los gestos
def arrayConv(array) -> str:
    resultStr = ''
    for i in range(5):
        if(array[i]):
            resultStr = resultStr + '1'
        else:
            resultStr = resultStr + '0'
    # print (resultStr)
    return resultStr

def secVal(actualGesture: str):      # Validamos que el gesto dure 3 segundos por medio de un contador y cuando pase, ejecutamos la accion
    global lastGesture
    global gestureSec
    global keepOpen
    global currentScene

    if lastGesture != actualGesture:
        lastGesture = actualGesture
        gestureSec = timeInSecs
    else:
        if timeInSecs == (gestureSec + 3):
            currentScene.overlayStuff[0] = 'Reconociendo gesto: '+str(actualGesture)
            gestNUM = currentScene.invokeAction(actualGesture)
            if not gestNUM:
                keepOpen = False
                lastGesture = ''
            elif gestNUM < 10:
                print('Ejecutando gesto '+ str(gestNUM-1))
                currentScene.execAct(gestNUM)
            else:
                currentScene.overlayStuff[0] = 'Este gesto no se acepta'
            gestureSec = 0      # Reiniciamos el contador para que solo se ejecute una vez la accion

def secVal2(actualGesture: int):      # Validamos que el gesto dure 3 segundos por medio de un contador y cuando pase, ejecutamos la accion
    global lastGesture            # Funcion unicamente para equivalencia del metodo con red neuronal
    global gestureSec
    global keepOpen
    global currentScene

    def equiv(n):
        m = 10
        if n == 0:
            m = 1
        elif n == 3:
            m = 2
        elif n == 20:
            m = 3
        elif n == 5:
            m = 4
        elif n == 1:
            m = 5
        ## AQUI Falta gesto 5
        elif n == 10:
            m = 7
        elif n == 23:
            m = 8
        ## AQUI falta gesto 8
        return m

    if lastGesture != actualGesture:
        lastGesture = actualGesture
        gestureSec = timeInSecs
    else:
        if timeInSecs == (gestureSec + 3):
            currentScene.overlayStuff[0] = 'Reconociendo gesto: '+str(actualGesture)
            gestName = actualGesture
            if actualGesture == 0:
                keepOpen = False
                lastGesture2 = 0
            elif equiv(actualGesture) < 10:
                print(gestName)
                currentScene.execAct(1)
            else:
                currentScene.overlayStuff[0] = 'Este gesto no se acepta'
            gestureSec = 0      # Reiniciamos el contador para que solo se ejecute una vez la accion

def secVal3(actualGesture: str):      # Validamos que el gesto dure 3 segundos por medio de un contador y cuando pase, ejecutamos la accion
    global lastGesture                # Funcion unicamente para equivalencia del metodo con red neuronal
    global gestureSec
    global keepOpen
    global currentScene

    def equiv(gesto):
        m = 10
        if gesto == "CincoDedos":
            pass
        elif gesto == "Punio":
            pass

    if lastGesture != actualGesture:
        lastGesture = actualGesture
        gestureSec = timeInSecs
    else:
        if timeInSecs == (gestureSec + 3):
            currentScene.overlayStuff[0] = 'Reconociendo gesto: '+str(actualGesture)

            if lastGesture == "CincoDedos":
                keepOpen = False
                lastGesture = ''
            elif lastGesture == "Punio":
                currentScene.execAct(1)
            else:           #No hay gesto
                pass
            gestureSec = 0      # Reiniciamos el contador para que solo se ejecute una vez la accion

# Esta funcion hace una comparacion por diferencia para saber si un dedo esta abierto o no
def openOrNot(WRIST, FT, FB):
    global y
    global x
    wristVector = np.array([WRIST[0], WRIST[1]])
    fingerTipVector = np.array([x*FT.x, y*FT.y])
    fingerBaseVector = np.array([x*FB.x, y*FB.y])

    # Las siguientes dos lineas operan los vectores para calcular la diferencia
    # entre FT (punta del dedo) hacia FB (base del dedo)
    diffFT = np.linalg.norm(wristVector-fingerTipVector)
    diffFB = np.linalg.norm(wristVector-fingerBaseVector)
    # print ('Distance: {}\n'.format(diffFT - diffFB))

    if diffFT-diffFB>0:
        return True
    else:
        return False

def clock():        # Esta funcion nace como un hilo para llevar un conteo de segundos y un registro
    global timeInSecs       # del ultimo gesto registrado
    global lastGesture
    global keepOpen
    while True:
        print('Segundo {}, gesto {}'.format(timeInSecs, lastGesture))
        timeInSecs = timeInSecs +1
        time.sleep(1)
        if not keepOpen:
            print ('Fin de reconocimiento')
            break

def init(cScene: Scene):

    global x
    global y
    global lastGesture
    global keepOpen
    global currentScene

    currentScene = cScene
    timer = threading.Thread(target=clock)
    timer.start()

    # Obtenemos la entrada de la camara
    cap = cv2.VideoCapture(0)

    # Aqui obtenemos el tamano de X y Y
    x = int (cap.get(3))
    y = int (cap.get(4))
    print('x: {}, y {}'.format(x,y))

    with mp_hands.Hands(
        model_complexity=0,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5) as hands:

        while cap.isOpened() and keepOpen:
            success, image = cap.read()
            cv2.putText(image,
                str(currentScene.overlayStuff[0]),
                (50, 50),
                cv2.FONT_HERSHEY_SIMPLEX,1,
                (0, 255, 255),
                2,
                cv2.LINE_4)
            if not success:
                print("Ignoring empty camera frame.")
                continue

            # To improve performance, optionally mark the image as not writeable to
            # pass by reference.
            image.flags.writeable = False
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            results = hands.process(image)

            # Draw the hand annotations on the image.
            image.flags.writeable = True
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

            if results.multi_hand_landmarks:
                for hand_landmarks in results.multi_hand_landmarks:

                    # Creamos un punto que indique el centro de la palma
                    wristP = hand_landmarks.landmark[mp_hands.HandLandmark.WRIST]
                    midFB = hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_MCP]
                    newCx = int((x*wristP.x + x*midFB.x)/2)
                    newCy = int((y*wristP.y + y*midFB.y)/2)
                    cv2.circle(image, (newCx,newCy), 3, (100,33,200),2)
                    middlePoint = [newCx,newCy]
                    # Creamos un nuevo punto para utilizarlo con el pulgar
                    indFB = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP]
                    thumbFB = hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_CMC]
                    newCx4T = int((x*thumbFB.x + x*1.1*indFB.x)/2)
                    newCy4T = int((y*thumbFB.y + y*.9*indFB.y)/2)
                    # newCx4T = int(((2*x*wristP.x/3) + x*indFB.x)/2)
                    # newCy4T = int(((2*y*wristP.y/3) + y*indFB.y)/2)
                    cv2.circle(image, (newCx4T,newCy4T), 3, (10,150,200),2)
                    thumbPoint = [newCx4T,newCy4T]

                    fingersOpen = []
                    fingersOpen.append(openOrNot(
                        thumbPoint,
                        hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP],
                        hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_MCP]))
                    fingersOpen.append(openOrNot(
                        middlePoint,
                        hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP],
                        hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP]))
                    fingersOpen.append(openOrNot(
                        middlePoint,
                        hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP],
                        hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_MCP]))
                    fingersOpen.append(openOrNot(
                        middlePoint,
                        hand_landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_TIP],
                        hand_landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_MCP]))
                    fingersOpen.append(openOrNot(
                        middlePoint,
                        hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_TIP],
                        hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_MCP]))

                    secVal(arrayConv(fingersOpen))

                    mp_drawing.draw_landmarks(
                    image,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())

            # Si no se detecta ninguna mano, se indica el ultimo gesto reconocido como vacio
            else:
                lastGesture = ''
            cv2.imshow('RGM', image)
            if cv2.waitKey(5) & 0xFF == 13:       # Si presionamos Enter
                keepOpen = False
                break
        cap.release()
        cv2.destroyAllWindows()

def predict(imagen,model):
  result = np.argmax(model.predict(imagen, verbose=0) > 0.5).astype("int32")
  #print(result)
  if result == 0:
    return "CincoDedos"
  elif result == 1:
    return "Punio"
  else: return ""

def ampliarImagen(imagen,width,height):
    return cv2.resize(imagen, (width,height), interpolation = cv2.INTER_AREA)

def recorteImagen(imagen,results,width,height):
    puntos = {'x':[],'y':[]}
    for hand_landmarks in results.multi_hand_landmarks:
        for cont in range(21):
            puntos["x"].append(hand_landmarks.landmark[cont].x)
            puntos["y"].append(hand_landmarks.landmark[cont].y)

    xL = int(width*(min(puntos['x'])))-50
    xR = int(width*(max(puntos['x'])))+50
    yB = int(height*(max(puntos['y'])))+50
    yT = int(height*(min(puntos['y'])))-50
    if xL < 0:
        xL = 0
    if xR > width:
        xR = width
    if yT < 0:
        yT = 0
    if yB > height:
        yB = height
    #print(xL,xR,yT,yB)
    imageCrop = imagen[yT:yB,xL:xR]
    dim = (50, 50)
    # resize image
    try:
        resized = cv2.resize(imageCrop, dim, interpolation = cv2.INTER_AREA)
    except Exception as e:
        print(str(e))

    return resized

def init2(cScene: Scene):

    global x
    global y
    global lastGesture
    global keepOpen
    global currentScene

    currentScene = cScene
    timer = threading.Thread(target=clock)
    timer.start()

    # Carga el modelo
    model = keras.models.load_model("modelo.h5")
    print('Modelo cargado')

    cap = cv2.VideoCapture(0)

    # Aqui obtenemos el tamano de X y Y
    x = int (cap.get(3))
    y = int (cap.get(4))
    #print('x: {}, y {}'.format(x,y))

    with mp_hands.Hands(
        model_complexity=0,
        min_detection_confidence=0.5,
        max_num_hands=1,
        min_tracking_confidence=0.5
        ) as hands:

        while cap.isOpened() and keepOpen:
            ret, frame = cap.read()
            if not ret:
                continue
            height, width, _ = frame.shape
            frame = cv2.flip(frame, 1)
            actualGesture = ""
            results = hands.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
            imagen = frame[0:50,0:50]
            if results.multi_hand_landmarks:
                imagen = recorteImagen(frame,results,width,height)
                numpydata = asfarray(imagen,dtype='float32')
                x = np.expand_dims(numpydata, axis=0)
                actualGesture = predict(x,model)
                secVal3(actualGesture)
                #print(result)
            else:
                lastGesture = ""
                actualGesture = ""
                frame = frame[0:600,0:600]
                imagen = cv2.resize(frame, (200,200), interpolation = cv2.INTER_AREA)
            imagen = ampliarImagen(imagen,200,200)
            cv2.putText(imagen,actualGesture,(20,20),cv2.FONT_HERSHEY_DUPLEX,.7,(51,184,255),2)
            anchoPantalla = GetSystemMetrics(0)
            altoPantalla = GetSystemMetrics(1)
            cv2.imshow("Mano",imagen)
            cv2.moveWindow("Mano", anchoPantalla-200,altoPantalla-300)
            #cv2.imshow('frame', frame)
            if cv2.waitKey(5) & 0xFF == 13:
                keepOpen = False
                break
        cap.release()
        cv2.destroyAllWindows()

if __name__ == '__main__':
    init2(currentScene)


import os
import json
import numpy as np
import pandas as pd
import math
from matplotlib import pyplot as plt
from sklearn.metrics import mean_absolute_error as mae

# 1 is me
def create_errors_of_data(file1, file2):
    with open(file1,'r') as data_file:
        data1 = json.loads(data_file.read())
    with open(file2,'r') as data_file:
        data2 = json.loads(data_file.read())

    size = len(data1)
    # print(size)
    l = []
    for i in range(size):
        di = {}
        train1 = data1[i]['train_config']
        train2 = data2[i]['train_config']

        d = data1[i]['eval_results']
        tgt_langs1 = list(d.keys())
        perfs1 = np.array(list(d.values()), dtype = float)

        d = data2[i]['eval_results']
        tgt_langs2 = list(d.keys())
        perfs2 = np.array(list(d.values()), dtype = float)
        error = np.absolute(perfs1-perfs2)

        di["train_config"] = train1
        d = dict(zip(tgt_langs1, error))
        di["eval_error"] = d
        l.append(di)
    return l

def create_rmse_per_task(file1, file2):
    with open(file1,'r') as data_file:
        data1 = json.loads(data_file.read())
    with open(file2,'r') as data_file:
        data2 = json.loads(data_file.read())

    size = len(data1)
    # print(size)
    l = []
    for i in range(size):
        di = {}
        train1 = data1[i]['train_config']
        train2 = data2[i]['train_config']

        d = data1[i]['eval_results']
        tgt_langs1 = list(d.keys())
        y_pred = np.array(list(d.values()), dtype = float)

        d = data2[i]['eval_results']
        tgt_langs2 = list(d.keys())
        y_test = np.array(list(d.values()), dtype = float)

        # mse = np.square(np.subtract(y_test,y_pred)).mean()
        # rmse = math.sqrt(mse)

        rmse = mae(y_test, y_pred)

        l.append(rmse)
    return np.average(np.array(l))

def create_average_perf_per_lang(file):
    with open(file,'r') as data_file:
        data = json.loads(data_file.read())
    l = []
    size = len(data)

    # p = np.zeros()
    # print(size)
    d = data[0]['eval_error']
    tgt_langs = list(d.keys())
    error = np.array(list(d.values()), dtype = float)
    flag = 0

    for i in range(size):
        di = {}
        train = data[i]['train_config']

        d = data[i]['eval_error']
        tgt_langs1 = list(d.keys())
        if tgt_langs1 != tgt_langs :
            flag = -1
            break
        error1 = np.array(list(d.values()), dtype = float)

        error = (error+error1)

    error = error/size

    if flag == -1:
        print("NOT ALL SAME EVAL LANG")

    return tgt_langs, error

dir1 = "ensem/preds/"
dir2 = "test_gold/"

outdir = "analysis/preds/"
error_per_task = []
for filename in os.listdir(dir1):
    l = {}
    t = {}
    if ".json" in filename:
        file1 = dir1+filename
        file2 = dir2+filename
        l = create_errors_of_data(file1, file2)

        if "surprise" in filename:
            t["filename"] = filename
            t["rmse"] = create_rmse_per_task(file1, file2)
            error_per_task.append(t)

            out = outdir+filename
            with open(out, "w") as outfile:
                json.dump(l, outfile)
        # p = create_average_perf_per_lang(file1)
        # break

out = "analysis/error_per_task.json"
with open(out, "w") as outfile:
    json.dump(error_per_task, outfile)

dir = "analysis/preds/"

error_per_lang = []
for filename in os.listdir(dir):
    l = {}
    if ".json" in filename:
        file = dir+filename
        l["file"] = filename
        lang, err = create_average_perf_per_lang(file)
        d = dict(zip(lang, err))
        l["error_lang"] = d
        error_per_lang.append(l)

out = "analysis/error_per_lang.json"
with open(out, "w") as outfile:
    json.dump(error_per_lang, outfile)

        # p = create_average_perf_per_lang(file1)
        # break


import pytest
from typing import Tuple
import time
from lxml import html
from loguru import logger
from web_tester import WebTester

#site = WebTester("http://covid19-api.exemplartech.com/")
#site = WebTester("http://127.0.0.1:5000", trace=True)
site = None

def remove_identical_nodes(elem1: html.Element, elem2: html.Element) -> bool:

    s1 = html.tostring(elem1)
    s2 = html.tostring(elem2)
    if s1 == s2: return True

    to_remove = []
    n = min(len(elem1), len(elem2))
    for i in range(n):
        is_same = remove_identical_nodes(elem1[i], elem2[i])
        if is_same: to_remove.append(i)

    to_remove.reverse()
    for i in to_remove:
        elem1.remove(elem1[i])
        elem2.remove(elem2[i])

    return False

def get_unique_nodes(content_a: bytes, content_b: bytes) -> Tuple[bytes, bytes]:

    doc_a = html.fromstring(content_a)
    doc_b = html.fromstring(content_b)
    remove_identical_nodes(doc_a, doc_b)
    content_a = html.tostring(doc_a, pretty_print=True)
    content_b = html.tostring(doc_b, pretty_print=True)
    return content_a, content_b

def test_regularization_1():

    if site == None:
        logger.warning("Site connection disabled")
        return

    # this URL trigger the regularization rules
    url = "www.alabamapublichealth.gov/infectiousdiseases/2019-coronavirus.html"

    logger.info("1. get content directly")
    direct_content = site.get(f"https://{url}")

    logger.info("2. get content through raw proxy")
    raw_proxy_content = site.get(f"/proxy-raw/{url}")

    if direct_content != raw_proxy_content:
        pytest.fail("Direct and Raw-Proxy differ ")

    logger.info("3. get content through proxy w/regularization")
    proxy_content = site.get(f"/proxy/{url}")

    content_a, content_b = get_unique_nodes(direct_content, proxy_content)
    logger.info(f"==== before regularization >>>\n{content_a}<<<<\n")
    logger.info(f"==== after regularization >>>\n{content_b}<<<<\n")

    if direct_content == proxy_content:
        pytest.fail("Direct and Proxy are the same ")

    time.sleep(5)

    logger.info("4. get same content again")
    proxy_content_b = site.get(f"/proxy/{url}")

    if proxy_content != proxy_content_b:
        pytest.fail("Proxy and Proxy B are differnt ")


import re
from autoreduce_db.reduction_viewer.models import Experiment, ReductionRun
from django_filters.filters import CharFilter
from django_filters.widgets import RangeWidget
from django_filters import FilterSet, DateFromToRangeFilter
from django.db.models import Q
from django.core.exceptions import ValidationError

def validate_run_number(self):
    """
    Validator method to ensure correct regex in run number field on Search page
    Raises ValidationError if regex does not match
    """
    if "," in self and "-" not in self:
        if not re.match(r'\d+,\d+', self):
            raise ValidationError("Invalid format. There must be a run number before and after the comma.")
    elif "-" in self and "," not in self:
        if not re.match(r'\d+-\d+', self):
            raise ValidationError("Invalid format. There must be a run number before and after the hyphen.")
    elif "-" in self and "," in self:
        if not re.match(r'[0-9, -]+[0-9,]$', self):
            raise ValidationError("Invalid format. Accepted format e.g. 60200-60205, 60210-60215")
    else:
        if not re.match(r'^\d+$', self):
            raise ValidationError("Invalid format. Run number must be numeric.")

# pylint:disable=unused-argument
def filter_run_number(queryset, name, value):
    """
    Method to determine regex used by user in run number field to filter runs
    '-' for a range
    ',' for seperate values
    Or a combination of the two
    """
    if "," in value and "-" not in value:
        list_values = value.split(',')
        query = Q(run_numbers__run_number__in=list_values)
        return queryset.filter(query)
    if "-" in value and "," not in value:
        list_values = value.split('-')
        query = Q(run_numbers__run_number__range=(list_values[0], list_values[1]))
        return queryset.filter(query)
    if "-" in value and "," in value:
        query = Q()
        list_values = value.split(',')
        for pair in list_values:
            seperated_pair = pair.split('-')
            query.add(Q(run_numbers__run_number__range=(seperated_pair[0], seperated_pair[1])), Q.OR)
        return queryset.filter(query)
    query = Q(run_numbers__run_number__exact=value)
    return queryset.filter(query)

class ReductionRunFilter(FilterSet):
    '''Filter model for filtering and querying ReductionRun models. '''
    created = DateFromToRangeFilter(widget=RangeWidget(attrs={
        'type': 'date',
        'placeholder': 'dd-mm-yyyy',
        'label': 'created'
    }))

    run_description = CharFilter(method="filter_run_description")
    run_number = CharFilter(field_name="run_number",
                            method=filter_run_number,
                            label='Run Number',
                            validators=[validate_run_number])

    class Meta:
        model = ReductionRun
        fields = ['run_number', 'instrument', 'run_description', 'created', 'status']

    def __init__(self, *args, run_description_qualifier=None, **kwargs):
        super().__init__(*args, **kwargs)
        # Call empty qs here to prevent all runs showing when search page first loaded
        if self.data == {}:
            self.queryset = self.queryset.none()
        self.run_description_qualifier = run_description_qualifier

    def filter_run_description(self, queryset, name, value):
        """
        Returns filtered queryset based on whether user checked the
        'contains' or 'exact' radio button for run description.
        """
        checkbox = self.run_description_qualifier
        if checkbox == "exact":
            query = Q(run_description__exact=value)
            return queryset.filter(query)
        elif checkbox == "contains":
            query = Q(run_description__contains=value)
            return queryset.filter(query)
        else:
            return queryset.none()

class ExperimentFilter(FilterSet):
    '''Filter model for filtering and querying Experiment models. '''

    class Meta:
        model = Experiment
        fields = ['reference_number']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Call empty qs here to prevent all runs showing when search page first loaded
        if self.data == {}:
            self.queryset = self.queryset.none()


"""Check that the environment variables contain valid boto3 credentials."""
import logging
import os
import boto3
import boto3.session

def check(session):
    client = session.client('s3')
    try:
        response = client.list_buckets()
    except Exception as e:
        logging.exception(e)
        return None
    else:
        return [b['Name'] for b in response['Buckets']]

def check_implicit():
    session = boto3.session.Session()
    buckets = check(session)
    if buckets:
        print('implicit check OK: %r' % buckets)
    else:
        print('implicit check failed')

def check_explicit():
    key_id = os.environ.get('AWS_ACCESS_KEY_ID')
    secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
    if not (key_id and secret_key):
        print('no credentials found in os.environ, skipping explicit check')
        return

    session = boto3.session.Session(aws_access_key_id=key_id, aws_secret_access_key=secret_key)
    buckets = check(session)
    if buckets:
        print('explicit check OK: %r' % buckets)
    else:
        print('explicit check failed')

def main():
    check_implicit()
    check_explicit()

if __name__ == '__main__':
    main()


import copy
from typing import Sequence, Tuple

from bsuite.baselines import base
from bsuite.baselines.utils import replay

import dm_env
from dm_env import specs
import numpy as np
import sonnet as snt
import tensorflow as tf
#tf.config.run_functions_eagerly(True)

from absl import app
from absl import flags
from absl import logging
import dm_memorytasks
import numpy as np
from datetime import datetime
import pandas as pd
import copy
import pygame

FLAGS = flags.FLAGS

_FRAMES_PER_SECOND = 30

flags.DEFINE_list(
    'screen_size', [2048, 1024],
    'Screen width/height in pixels. Scales the environment RGB observations to '
    'fit the screen size.')

flags.DEFINE_string(
    'docker_image_name', None,
    'Name of the Docker image that contains the Memory Tasks. '
    'If None, uses the default dm_memorytask name')

flags.DEFINE_boolean(
    'silent', False,
    'Whether or not the graphical interface should be displayed.')

flags.DEFINE_integer('seed', 123, 'Environment seed.')
flags.DEFINE_string('level_name', 'visible_goal_with_buildings_train',
                    'Name of memory task to run.')
flags.DEFINE_integer('episode_length_seconds', '360', 'Length of episode')

# Tuning parameters
flags.DEFINE_float('learning_rate', .0001, 'Learning rate of the network')
flags.DEFINE_float('epsilon', .1, 'Epsilon of the network')
flags.DEFINE_float('discount', .99, 'Discount of the network')

from typing import Sequence, Tuple
import tree
import trfl
import tensorflow_probability as tfp

tfd = tfp.distributions

# Action structure {'MOVE_BACK_FORWARD': 0, 'STRAFE_LEFT_RIGHT': 0, 'LOOK_LEFT_RIGHT': 0, 'LOOK_DOWN_UP': 0} (ranges from -1, 1)
# Define the 8 possible actions
# Ignore look up and look down since they dont help accomplish the task
_ACTIONS = [
    {'MOVE_BACK_FORWARD': 1},
    {'MOVE_BACK_FORWARD': -1},
    {'STRAFE_LEFT_RIGHT': -1},
    {'STRAFE_LEFT_RIGHT': 1},
    {'LOOK_LEFT_RIGHT': -1},
    {'LOOK_LEFT_RIGHT': 1},
    #{'LOOK_DOWN_UP': -1},
    #{'LOOK_DOWN_UP': 1},
]

# Define default no action
_NO_ACTION = {
    'MOVE_BACK_FORWARD': 0,
    'STRAFE_LEFT_RIGHT': 0,
    'LOOK_LEFT_RIGHT': 0,
    'LOOK_DOWN_UP': 0,
}

class DQN(base.Agent):
  """A simple DQN agent using TF2."""

  def __init__(
      self,
      action_spec: specs.DiscreteArray,
      network: snt.Module,
      batch_size: int,
      discount: float,
      replay_capacity: int,
      min_replay_size: int,
      sgd_period: int,
      target_update_period: int,
      optimizer: snt.Optimizer,
      epsilon: float,
      seed: int = None,
  ):

    # Internalise hyperparameters.
    self._num_actions = action_spec.num_values
    self._discount = discount
    self._batch_size = batch_size
    self._sgd_period = sgd_period
    self._target_update_period = target_update_period
    self._epsilon = epsilon
    self._min_replay_size = min_replay_size

    # Seed the RNG.
    tf.random.set_seed(seed)
    self._rng = np.random.RandomState(seed)

    # Internalise the components (networks, optimizer, replay buffer).
    self._optimizer = optimizer
    self._replay = replay.Replay(capacity=replay_capacity)
    self._online_network = network
    self._target_network = copy.deepcopy(network)
    self._forward = tf.function(network)
    self._total_steps = tf.Variable(0)

  def select_action(self, timestep: dm_env.TimeStep) -> base.Action:
    # Epsilon-greedy policy.
    if self._rng.rand() < self._epsilon:
      return self._rng.randint(self._num_actions)

    observation = tf.convert_to_tensor(timestep.observation['RGB_INTERLEAVED'][None, ...].astype('float32'))
    # Greedy policy, breaking ties uniformly at random.
    q_values = self._forward(observation).numpy()
    #print(np.flatnonzero(q_values == q_values.max()))
    action = self._rng.choice(np.flatnonzero(q_values == q_values.max()))
    return int(action)

  def update(
      self,
      timestep: dm_env.TimeStep,
      action: base.Action,
      new_timestep: dm_env.TimeStep,
  ):
    # Add this transition to replay.
    self._replay.add([
        timestep.observation['RGB_INTERLEAVED'].astype('float32'),
        action,
        new_timestep.reward,
        new_timestep.discount,
        new_timestep.observation['RGB_INTERLEAVED'].astype('float32'),
    ])

    self._total_steps.assign_add(1)
    #print(self._total_steps)
    #print(tf.math.mod(self._total_steps, self._sgd_period).numpy().item())
    if tf.math.mod(self._total_steps, self._sgd_period).numpy().item() != 0:
      #print("NOPE1")
      return

    if self._replay.size < self._min_replay_size:
      #print("NOPE2")
      return

    # Do a batch of SGD.
    transitions = self._replay.sample(self._batch_size)
    self._training_step(transitions)

 # @tf.function
  def _training_step(self, transitions: Sequence[tf.Tensor]) -> tf.Tensor:
    """Does a step of SGD on a batch of transitions."""

    o_tm1, a_tm1, r_t, d_t, o_t = transitions

    r_t = tf.cast(r_t, tf.float32)  # [B]
    d_t = tf.cast(d_t, tf.float32)  # [B]
    o_tm1 = tf.convert_to_tensor(o_tm1)
    o_t = tf.convert_to_tensor(o_t)

    with tf.GradientTape() as tape:
      q_tm1 = self._online_network(o_tm1)  # [B, A]
      q_t = self._target_network(o_t)  # [B, A]

      onehot_actions = tf.one_hot(a_tm1, depth=self._num_actions)  # [B, A]
      qa_tm1 = tf.reduce_sum(q_tm1 * onehot_actions, axis=-1)  # [B]
      qa_t = tf.reduce_max(q_t, axis=-1)  # [B]

      #print(qa_tm1)

      # One-step Q-learning loss.
      target = r_t + d_t * self._discount * qa_t
      td_error = qa_tm1 - target
      loss = 0.5 * tf.reduce_mean(td_error**2)  # []

    # Update the online network via SGD.
    variables = self._online_network.trainable_variables
    gradients = tape.gradient(loss, variables)
    self._optimizer.apply(gradients, variables)

    # Periodically copy online -> target network variables.
    if tf.math.mod(self._total_steps, self._target_update_period).numpy().item() == 0:
      for target, param in zip(self._target_network.trainable_variables,
                              self._online_network.trainable_variables):
        target.assign(param)
    #print('loss', loss)
    return loss

# Default parameters
# def default_agent(obs_spec: specs.Array,
#                   action_spec: specs.DiscreteArray):
#   """Initialize a DQN agent with default parameters."""
#   del obs_spec  # Unused.
#   network = snt.Sequential([
#       snt.Flatten(),
#       snt.nets.MLP([50, 50, action_spec.num_values]),
#   ])
#   optimizer = snt.optimizers.Adam(learning_rate=1e-3)
#   return DQN(
#       action_spec=action_spec,
#       network=network,
#       batch_size=32,
#       discount=0.99,
#       replay_capacity=10000,
#       min_replay_size=100,
#       sgd_period=1,
#       target_update_period=4,
#       optimizer=optimizer,
#       epsilon=0.05,
#       seed=42)

def main(argv):
  if not FLAGS.silent:
    # Init pygame so we can watch the agent
    pygame.init()
    try:
      pygame.mixer.quit()
    except NotImplementedError:
      pass
    pygame.display.set_caption('Memory Tasks DQN Agent')

  env_settings = dm_memorytasks.EnvironmentSettings(
      seed=FLAGS.seed, level_name=FLAGS.level_name, episode_length_seconds=FLAGS.episode_length_seconds)

  with dm_memorytasks.load_from_docker(
      name=FLAGS.docker_image_name, settings=env_settings) as env:

    if not FLAGS.silent:
      # Pygame setup
      screen = pygame.display.set_mode(
        (int(FLAGS.screen_size[0]), int(FLAGS.screen_size[1])))
      rgb_spec = env.observation_spec()['RGB_INTERLEAVED']
      surface = pygame.Surface((rgb_spec.shape[1], rgb_spec.shape[0]))
      clock = pygame.time.Clock()

    # Manually define the action spec
    action_spec = specs.DiscreteArray(
        dtype=int, num_values=len(_ACTIONS), name="action")

    # network = snt.Sequential([
    #   snt.Flatten(),
    #   snt.nets.MLP([50,50, action_spec.num_values]),
    # ])

    # 2
    # network = snt.Sequential([
    #   snt.Conv2D(32, 3, 4),
    #   tf.nn.relu,
    #   snt.Conv2D(64, 2, 2),
    #   snt.Flatten(),
    #   snt.Linear(action_spec.num_values),
    # ])

    network = snt.Sequential([
       snt.Conv2D(32, 3, 4),
      tf.nn.relu,
      snt.Conv2D(32, 2, 2),
      tf.nn.relu,
      snt.Conv2D(64, 2, 2),
      tf.nn.relu,
      snt.Conv2D(64, 2, 2),
      snt.nets.MLP([32,8]),
      snt.Flatten(),
      snt.Linear(action_spec.num_values),
    ])

    # network = snt.Sequential([
    #   snt.Conv2D(32, 3, 4),
    #   tf.nn.max_pool(ksize=3,strides=4, padding="SAME"),
    #   #tf.nn.relu,
    #   #snt.BatchNorm(create_scale=True, create_offset=False, is_tr),
    #   snt.Conv2D(64, 2, 2),
    #   tf.nn.max_pool(ksize=2,strides=2, padding="SAME"),
    #   #tf.nn.relu,
    #   #snt.BatchNorm(create_scale=True, create_offset=False),
    #   snt.Conv2D(64, 3, 1),
    #   tf.nn.max_pool(ksizes=3,strides=1, padding="SAME"),
    #   #tf.nn.relu,
    #   #snt.Dropout(0.5),
    #   #snt.BatchNorm(create_scale=True, create_offset=False),
    #   snt.nets.MLP([32]),
    #   tf.nn.relu,

    #   #snt.LSTM(hidden_size=16),
    #   #snt.LSTM(hidden_size=16),
    #   snt.Flatten(),
    #   snt.Linear(action_spec.num_values),
    # ])

    # Set params of our agent
    optimizer = snt.optimizers.Adam(learning_rate=FLAGS.learning_rate)
    agent = DQN(
      action_spec=action_spec,
      network=network,
      batch_size=64,
      discount=FLAGS.discount,
      replay_capacity=10000,
      min_replay_size=100,
      sgd_period=2,
      target_update_period=6,
      optimizer=optimizer,
      epsilon=FLAGS.epsilon,
      seed=50)
    # Observation: OrderedDict([('Score', Array(shape=(), dtype=dtype('float64'), name='Score')), ('RGB_INTERLEAVED', Array(shape=(72, 96, 3), dtype=dtype('uint8'), name='RGB_INTERLEAVED')), ('AvatarPosition', Array(shape=(3,), dtype=dtype('float32'), name='AvatarPosition'))])

    timestep = env.reset()
    score = 0
    action_tracker = {}
    result_dict = {"timestamp": [], "score": [], "reward":[]}
    while not timestep.last():
      # Returns the index of the traction
      selected_action_idx = agent.select_action(timestep)

      # Track the number of each action taken
      action_ref = selected_action_idx
      if action_ref not in action_tracker:
          action_tracker[action_ref] = 1
      else:
          action_tracker[action_ref] += 1

      #print(action_tracker)

      # Get the key/value of the action by index
      selected_action = _ACTIONS[selected_action_idx]

      # Make a deep copy of the initial array
      action = copy.deepcopy(_NO_ACTION)

      # Get the name of the action and set the value in the array
      selected_action_name = list(selected_action.keys())[0]
      action[selected_action_name] = selected_action[selected_action_name]

      previous_timestep = copy.deepcopy(timestep)
      # Give the actor's action to the envrionment
      timestep = env.step(action)

      agent.update(previous_timestep, selected_action_idx, timestep)

      if not FLAGS.silent:
        # Update the view based on the action
        frame = np.swapaxes(timestep.observation['RGB_INTERLEAVED'], 0, 1)
        pygame.surfarray.blit_array(surface, frame)
        pygame.transform.smoothscale(surface, screen.get_size(), screen)
        pygame.display.update()
        clock.tick(_FRAMES_PER_SECOND)

      if timestep.reward:
        score += timestep.reward
        logging.info('Total score: %1.1f, reward: %1.1f', score,
                     timestep.reward)
        result_dict["timestamp"].append(datetime.now())
        result_dict["score"].append(score)
        result_dict["reward"].append(timestep.reward)

    logging.info('Final score: %1.1f', score)
    logging.info(f'Actions taken (key is the index of _ACTION variable): {action_tracker}')
    result_df = pd.DataFrame(result_dict)
    result_df.to_csv(f"dqn_agent_results_{datetime.now().timestamp()}.csv")

if __name__ == '__main__':
  app.run(main)


import pygame
pygame.init()
gameDisplay = pygame.display.set_mode((800,400))
pygame.display.set_caption("Slither")
blue = (0,0,255)
black = (0,0,0)
white = (255,255,255)
red = (255,0,0)
green = (0,255,0)
lead_x = 400
lead_y = 200
lead_x_change = 0
gameExit = False
while not gameExit:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            gameExit = True
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                lead_x_change =  -25
            if event.key == pygame.K_RIGHT:
                lead_x_change = 25
    lead_x = lead_x + lead_x_change
    gameDisplay.fill(blue)
    pygame.draw.rect(gameDisplay,black,[lead_x,lead_y,25,25])
    pygame.display.update()
pygame.quit()


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from openstack.tests.functional import base

class TestVolume(base.BaseFunctionalTest):
    # Creating a volume backup is incredibly slow.
    TIMEOUT_SCALING_FACTOR = 1.5

    def setUp(self):
        super(TestVolume, self).setUp()
        self.skipTest('Volume functional tests temporarily disabled')
        if not self.user_cloud.has_service('volume'):
            self.skipTest('volume service not supported by cloud')

        if not self.user_cloud.has_service('object-store'):
            self.skipTest('volume backups require swift')

    def test_create_get_delete_volume_backup(self):
        volume = self.user_cloud.create_volume(
            display_name=self.getUniqueString(), size=1
        )
        self.addCleanup(self.user_cloud.delete_volume, volume['id'])

        backup_name_1 = self.getUniqueString()
        backup_desc_1 = self.getUniqueString()
        backup = self.user_cloud.create_volume_backup(
            volume_id=volume['id'],
            name=backup_name_1,
            description=backup_desc_1,
            wait=True,
        )
        self.assertEqual(backup_name_1, backup['name'])

        backup = self.user_cloud.get_volume_backup(backup['id'])
        self.assertEqual("available", backup['status'])
        self.assertEqual(backup_desc_1, backup['description'])

        self.user_cloud.delete_volume_backup(backup['id'], wait=True)
        self.assertIsNone(self.user_cloud.get_volume_backup(backup['id']))

    def test_create_get_delete_volume_backup_from_snapshot(self):
        volume = self.user_cloud.create_volume(size=1)
        snapshot = self.user_cloud.create_volume_snapshot(volume['id'])
        self.addCleanup(self.user_cloud.delete_volume, volume['id'])
        self.addCleanup(
            self.user_cloud.delete_volume_snapshot, snapshot['id'], wait=True
        )

        backup = self.user_cloud.create_volume_backup(
            volume_id=volume['id'], snapshot_id=snapshot['id'], wait=True
        )

        backup = self.user_cloud.get_volume_backup(backup['id'])
        self.assertEqual(backup['snapshot_id'], snapshot['id'])

        self.user_cloud.delete_volume_backup(backup['id'], wait=True)
        self.assertIsNone(self.user_cloud.get_volume_backup(backup['id']))

    def test_create_get_delete_incremental_volume_backup(self):
        volume = self.user_cloud.create_volume(size=1)
        self.addCleanup(self.user_cloud.delete_volume, volume['id'])

        full_backup = self.user_cloud.create_volume_backup(
            volume_id=volume['id'], wait=True
        )
        incr_backup = self.user_cloud.create_volume_backup(
            volume_id=volume['id'], incremental=True, wait=True
        )

        full_backup = self.user_cloud.get_volume_backup(full_backup['id'])
        incr_backup = self.user_cloud.get_volume_backup(incr_backup['id'])
        self.assertEqual(full_backup['has_dependent_backups'], True)
        self.assertEqual(incr_backup['is_incremental'], True)

        self.user_cloud.delete_volume_backup(incr_backup['id'], wait=True)
        self.user_cloud.delete_volume_backup(full_backup['id'], wait=True)
        self.assertIsNone(self.user_cloud.get_volume_backup(full_backup['id']))
        self.assertIsNone(self.user_cloud.get_volume_backup(incr_backup['id']))

    def test_list_volume_backups(self):
        vol1 = self.user_cloud.create_volume(
            display_name=self.getUniqueString(), size=1
        )
        self.addCleanup(self.user_cloud.delete_volume, vol1['id'])

        # We create 2 volumes to create 2 backups. We could have created 2
        # backups from the same volume but taking 2 successive backups seems
        # to be race-condition prone. And I didn't want to use an ugly sleep()
        # here.
        vol2 = self.user_cloud.create_volume(
            display_name=self.getUniqueString(), size=1
        )
        self.addCleanup(self.user_cloud.delete_volume, vol2['id'])

        backup_name_1 = self.getUniqueString()
        backup = self.user_cloud.create_volume_backup(
            volume_id=vol1['id'], name=backup_name_1
        )
        self.addCleanup(self.user_cloud.delete_volume_backup, backup['id'])

        backup = self.user_cloud.create_volume_backup(volume_id=vol2['id'])
        self.addCleanup(self.user_cloud.delete_volume_backup, backup['id'])

        backups = self.user_cloud.list_volume_backups()
        self.assertEqual(2, len(backups))

        backups = self.user_cloud.list_volume_backups(
            search_opts={"name": backup_name_1}
        )
        self.assertEqual(1, len(backups))
        self.assertEqual(backup_name_1, backups[0]['name'])