import os
import pickle
from pathlib import Path
import numpy as np
import cv2
import h5py
import tqdm

def load_pkl(fpath):
    with open(fpath, "rb") as f:
        return pickle.load(f, encoding="latin")

def load_h5py(fpath):
    return h5py.File(fpath, "r")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--root", type=str, required=True, help="path to the PeopleSnapshotData")
    parser.add_argument("--subject", type=str, default="male-3-casual", help="sequence to process")
    args = parser.parse_args()

    dirpath = os.path.join(args.root, args.subject)
    assert os.path.exists(dirpath), f"Cannot open {dirpath}"
    dirpath = Path(dirpath)

    outdir = Path(f"./data/PeopleSnapshot/{args.subject}/")
    os.makedirs(outdir, exist_ok=True)

    # load camera
    camera = load_pkl(dirpath / "camera.pkl")
    K = np.eye(3)
    K[0, 0] = camera["camera_f"][0]
    K[1, 1] = camera["camera_f"][1]
    K[:2, 2] = camera["camera_c"]
    dist_coeffs = camera["camera_k"]

    H, W = camera["height"], camera["width"]
    w2c = np.eye(4)
    w2c[:3, :3] = cv2.Rodrigues(camera["camera_rt"])[0]
    w2c[:3, 3] = camera["camera_t"]

    camera_path = outdir / "cameras.npz"
    np.savez(str(camera_path), **{
        "intrinsic": K,
        "extrinsic": w2c,
        "height": H,
        "width": W,
    })
    print("Write camera to", camera_path)

    # load images
    image_dir = outdir / "images"
    os.makedirs(image_dir, exist_ok=True)

    print("Write images to", image_dir)
    cap = cv2.VideoCapture(str(dirpath / f"{args.subject}.mp4"))
    frame_cnt = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    for i in tqdm.trange(frame_cnt):
        img_path = f"{image_dir}/image_{i:04d}.png"
        ok, frame = cap.read()
        if not ok: break
        frame = cv2.undistort(frame, K, dist_coeffs)
        cv2.imwrite(img_path, frame)

    # load masks
    mask_dir = outdir / "masks"
    os.makedirs(mask_dir, exist_ok=True)

    print("Write mask to", mask_dir)
    masks = np.asarray(load_h5py(dirpath / "masks.hdf5")["masks"]).astype(np.uint8)
    for i, mask in enumerate(tqdm.tqdm(masks)):
        mask_path = f"{mask_dir}/mask_{i:04d}.npy"
        mask = cv2.undistort(mask, K, dist_coeffs)
        np.save(mask_path, mask)

    smpl_params = load_h5py(dirpath / "reconstructed_poses.hdf5")
    smpl_params = {
        "betas": np.asarray(smpl_params["betas"]).astype(np.float32),
        "thetas": np.asarray(smpl_params["pose"]).astype(np.float32),
        "transl": np.asarray(smpl_params["trans"]).astype(np.float32),
    }
    np.savez(str(outdir / "poses.npz"), **smpl_params)


import bpy, bmesh
import os
import mathutils
import math
import imp
from . import objects_organise

from . import modifiers
from . import platforms

imp.reload(modifiers)
imp.reload(platforms)

class op(bpy.types.Operator):
	bl_idname = "fbxbundle.file_export_recent"
	bl_label = "export recent"
	bl_description = "Re-Export recent exported Bundle again."

	@classmethod
	def poll(cls, context):

		if context.space_data.local_view:
			return False

		if bpy.context.scene.FBXBundleSettings.path == "":
			return False

		if len(bpy.context.scene.FBXBundleSettings.recent) <= 0:
			return False

		if len(objects_organise.recent_load_objects()) <= 0:
			return False

		return True

	def execute(self, context):
		export_recent(self)
		return {'FINISHED'}

def export_recent(self):
	objects = objects_organise.recent_load_objects()

	# Warnings
	if len(objects) <= 0:
		raise Exception("")
		self.report({'ERROR_INVALID_INPUT'}, "No previous object selection available" )
		return

	bpy.ops.object.mode_set(mode='OBJECT')

	# Store previous settings
	previous_selection = bpy.context.selected_objects.copy()
	previous_active = bpy.context.view_layer.objects.active
	previous_unit_system = bpy.context.scene.unit_settings.system
	previous_pivot = bpy.context.tool_settings.transform_pivot_point
	previous_cursor = bpy.context.scene.cursor.location.copy()

	bpy.ops.object.select_all(action="DESELECT")
	for obj in objects:
	#	for i in range(len(obj.layers)):
	#		if not obj.layers[i]:
	#			obj.layers[i] = True
		obj.select_set(state = True)

	bpy.context.view_layer.objects.active = objects[-1]
	bpy.ops.fbxbundle.file_export()

	# Restore previous settings
	bpy.context.scene.unit_settings.system = previous_unit_system
	bpy.context.tool_settings.transform_pivot_point = previous_pivot
	bpy.context.scene.cursor.location = previous_cursor
	bpy.context.view_layer.objects.active = previous_active
	bpy.ops.object.select_all(action='DESELECT')
	for obj in previous_selection:
		obj.select_set(state = True)


import serial
import os
import time
#import context  # Ensures paho is in PYTHONPATH
import paho.mqtt.client as MQTT # For MQTT publish/subscribe
import RPi.GPIO as GPIO # For input/output
#from tkinter import *

import pygame.mixer
from sys import exit

import pygame, sys
from pygame.locals import *

slideWinAFlag = False

ser= serial.Serial('/dev/ttyUSB-trellis',9600,timeout=1)
ser.flushInput()

#ser1 = serial.Serial('/dev/ttyUSB-trellis',9600,timeout=1)
# ser1.flushInput()

def printString( str ):
   #"This prints a passed string into this function"
    length = len(str)
    for x in range (length):
        print(x , " is " , (str[x]))
    return;

def convertString(str ):
   #"This take in a string of ASCII chars, converts to Chars and strips last 2 elements"
    stringLen = len(str)
    realLen = stringLen - 2

    tempStr = ""

    for x in range (realLen):
        temp = chr(str[x])
        tempStr = tempStr + temp
    str = tempStr
    return str;
#read from Arduino
# Callback after connecting to MQTT server
def on_connect(self, client, userdata, rc):
  print("Connected!")
  print("Subscribing to ToDevice/All")
  self.subscribe("ToDevice/All")
  print("Subscribing to ToDevice/1")
  self.subscribe("phone/#")
  print("Subscribing to phone")

def redWin_arduino():
    client.publish("phone/trellis/red", 1)
    print(" red <- Recieved Solved")
def redReset_arduino():
    client.publish("phone/trellis/red", 0)
    print(" red <- Recieved Reset")

def redWin_phone():
    ser.write('redSOLVED'.encode())
    print("red -> Sent Solved")
def redReset_phone():
    ser.write('redRESET'.encode())
    print("red -> Sent Reset")

def blueWin_arduino():
    #relay_1_Set_arduino()
    client.publish("phone/trellis/blue", 1)
    print(" blue <- Recieved Solved")
    #client.publish("phone/pots/status", "SOLVED")
def blueWin_phone():
   # ser.write('blueSOLVED'.encode())
    #lient.publish("phone/pots/status", "SOLVED")
    print("blue -> Sent Solved")
def blueReset_arduino():
    client.publish("phone/trellis/blue", 0)
    print(" blue <- Recieved Reset")
    #client.publish("phone/pots/status", "RESET")
def blueReset_phone():
    ser.write('blueRESET'.encode())
    #client.publish("phone/pots/status", "RESET")
    print("blue -> Sent Reset")

# Callback when message is published to MQTT subscribed topic
def on_message(client, userdata, msg):
  # Print the message and topic in which it was published
  print(msg.topic + " " + str(msg.payload))
  # Change the global puzzle state based on message received
  global state
  #temp = int(msg.payload)
  temp = int(msg.payload)

#========================= TRELLIS PUZZLES ========================

  if(msg.topic == "phone/trellis/blue"):
      if(   temp == 1 ):
          blueWin_phone()
      elif( temp == 0 ):
          blueReset_phone()

# ----------------- LOGIC #2 - SEQUENCE -----------------
  if(msg.topic == "phone/trellis/red"):
      if(   temp == 1 ):
          redWin_phone()
      elif( temp == 0 ):
          redReset_phone()

# Callback when client is disconnected from MQTT broker
def on_disconnect(client, userdata, rc):
  client.loop_stop()

# Setup MQTT
client = MQTT.Client()
# Assign callbacks
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
# Connect to MQTT server
client.connect("10.1.10.177", 1883, 60)
# Start the MQTT update loop
client.loop_start()

print("Main Loop Start")

# Main program loop ==============================================================================================================================

while True:

    #print("reading serial ....")

    str_read = ser.readline()
    #print("read String 1")
    newString = convertString(str_read)
    print(newString)

# SERIAL PORT # 1 - Circle Logic and Sequence Logic
    if(newString == "BLUE_WIN"):
        blueWin_arduino()
    elif(newString == "BLUE_RESET"):
        blueReset_arduino()

    elif(newString == "RED_WIN"):
        redWin_arduino()
    elif(newString == "RED_RESET"):
        redReset_arduino()

# SERIAL PORT # 2 - final Puzzle -Phases

    ser.flushInput()
    time.sleep(.5)
# READ FROM SECOND SERIAL PORT
#     str_read2 = ser1.readline()
#     newString1 = convertString(str_read2)
#     print(newString1)
#
#
#     if(newString1 == "PHASE_1"):
#         client.publish("phone/final/phase_1", 1)
#         #plays sound
#     if(newString1 == "PHASE_2"):
#         client.publish("phone/final/phase_2", 1)
#     if(newString1 == "PHASE_3"):
#         client.publish("phone/final/phase_3", 1)
#     if(newString1 == "PHASE_4"):
#         client.publish("phone/final/puzzle", 1)
#     if(newString1 == "FINAL_RESET"):
#         #client.publish("phone/final/puzzle", 0)
#         client.publish("phone/final/phase_1", 0)
#         client.publish("phone/final/phase_2", 0)
#         client.publish("phone/final/phase_3", 0)
#         client.publish("phone/final/start", 0)
#
#
# #    ser.flushInput()
#     ser1.flushInput()
#

 #   time.sleep(.5)


from __future__ import annotations

import copy
import json
import logging
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    List,
    Literal,
    Optional,
    TypedDict,
    Union,
    overload,
)

from langchain_core.callbacks import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain_core.language_models.llms import LLM
from langchain_core.pydantic_v1 import PrivateAttr

if TYPE_CHECKING:
    import openllm

ServerType = Literal["http", "grpc"]

class IdentifyingParams(TypedDict):
    """Parameters for identifying a model as a typed dict."""

    model_name: str
    model_id: Optional[str]
    server_url: Optional[str]
    server_type: Optional[ServerType]
    embedded: bool
    llm_kwargs: Dict[str, Any]

logger = logging.getLogger(__name__)

class OpenLLM(LLM):
    """OpenLLM, supporting both in-process model
    instance and remote OpenLLM servers.

    To use, you should have the openllm library installed:

    .. code-block:: bash

        pip install openllm

    Learn more at: https://github.com/bentoml/openllm

    Example running an LLM model locally managed by OpenLLM:
        .. code-block:: python

            from langchain_community.llms import OpenLLM
            llm = OpenLLM(
                model_name='flan-t5',
                model_id='google/flan-t5-large',
            )
            llm("What is the difference between a duck and a goose?")

    For all available supported models, you can run 'openllm models'.

    If you have a OpenLLM server running, you can also use it remotely:
        .. code-block:: python

            from langchain_community.llms import OpenLLM
            llm = OpenLLM(server_url='http://localhost:3000')
            llm("What is the difference between a duck and a goose?")
    """

    model_name: Optional[str] = None
    """Model name to use. See 'openllm models' for all available models."""
    model_id: Optional[str] = None
    """Model Id to use. If not provided, will use the default model for the model name.
    See 'openllm models' for all available model variants."""
    server_url: Optional[str] = None
    """Optional server URL that currently runs a LLMServer with 'openllm start'."""
    server_type: ServerType = "http"
    """Optional server type. Either 'http' or 'grpc'."""
    embedded: bool = True
    """Initialize this LLM instance in current process by default. Should
    only set to False when using in conjunction with BentoML Service."""
    llm_kwargs: Dict[str, Any]
    """Keyword arguments to be passed to openllm.LLM"""

    _runner: Optional[openllm.LLMRunner] = PrivateAttr(default=None)
    _client: Union[
        openllm.client.HTTPClient, openllm.client.GrpcClient, None
    ] = PrivateAttr(default=None)

    class Config:
        extra = "forbid"

    @overload
    def __init__(
        self,
        model_name: Optional[str] = ...,
        *,
        model_id: Optional[str] = ...,
        embedded: Literal[True, False] = ...,
        **llm_kwargs: Any,
    ) -> None:
        ...

    @overload
    def __init__(
        self,
        *,
        server_url: str = ...,
        server_type: Literal["grpc", "http"] = ...,
        **llm_kwargs: Any,
    ) -> None:
        ...

    def __init__(
        self,
        model_name: Optional[str] = None,
        *,
        model_id: Optional[str] = None,
        server_url: Optional[str] = None,
        server_type: Literal["grpc", "http"] = "http",
        embedded: bool = True,
        **llm_kwargs: Any,
    ):
        try:
            import openllm
        except ImportError as e:
            raise ImportError(
                "Could not import openllm. Make sure to install it with "
                "'pip install openllm.'"
            ) from e

        llm_kwargs = llm_kwargs or {}

        if server_url is not None:
            logger.debug("'server_url' is provided, returning a openllm.Client")
            assert (
                model_id is None and model_name is None
            ), "'server_url' and {'model_id', 'model_name'} are mutually exclusive"
            client_cls = (
                openllm.client.HTTPClient
                if server_type == "http"
                else openllm.client.GrpcClient
            )
            client = client_cls(server_url)

            super().__init__(
                **{
                    "server_url": server_url,
                    "server_type": server_type,
                    "llm_kwargs": llm_kwargs,
                }
            )
            self._runner = None  # type: ignore
            self._client = client
        else:
            assert model_name is not None, "Must provide 'model_name' or 'server_url'"
            # since the LLM are relatively huge, we don't actually want to convert the
            # Runner with embedded when running the server. Instead, we will only set
            # the init_local here so that LangChain users can still use the LLM
            # in-process. Wrt to BentoML users, setting embedded=False is the expected
            # behaviour to invoke the runners remotely.
            # We need to also enable ensure_available to download and setup the model.
            runner = openllm.Runner(
                model_name=model_name,
                model_id=model_id,
                init_local=embedded,
                ensure_available=True,
                **llm_kwargs,
            )
            super().__init__(
                **{
                    "model_name": model_name,
                    "model_id": model_id,
                    "embedded": embedded,
                    "llm_kwargs": llm_kwargs,
                }
            )
            self._client = None  # type: ignore
            self._runner = runner

    @property
    def runner(self) -> openllm.LLMRunner:
        """
        Get the underlying openllm.LLMRunner instance for integration with BentoML.

        Example:
        .. code-block:: python

            llm = OpenLLM(
                model_name='flan-t5',
                model_id='google/flan-t5-large',
                embedded=False,
            )
            tools = load_tools(["serpapi", "llm-math"], llm=llm)
            agent = initialize_agent(
                tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION
            )
            svc = bentoml.Service("langchain-openllm", runners=[llm.runner])

            @svc.api(input=Text(), output=Text())
            def chat(input_text: str):
                return agent.run(input_text)
        """
        if self._runner is None:
            raise ValueError("OpenLLM must be initialized locally with 'model_name'")
        return self._runner

    @property
    def _identifying_params(self) -> IdentifyingParams:
        """Get the identifying parameters."""
        if self._client is not None:
            self.llm_kwargs.update(self._client._config())
            model_name = self._client._metadata()["model_name"]
            model_id = self._client._metadata()["model_id"]
        else:
            if self._runner is None:
                raise ValueError("Runner must be initialized.")
            model_name = self.model_name
            model_id = self.model_id
            try:
                self.llm_kwargs.update(
                    json.loads(self._runner.identifying_params["configuration"])
                )
            except (TypeError, json.JSONDecodeError):
                pass
        return IdentifyingParams(
            server_url=self.server_url,
            server_type=self.server_type,
            embedded=self.embedded,
            llm_kwargs=self.llm_kwargs,
            model_name=model_name,
            model_id=model_id,
        )

    @property
    def _llm_type(self) -> str:
        return "openllm_client" if self._client else "openllm"

    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        try:
            import openllm
        except ImportError as e:
            raise ImportError(
                "Could not import openllm. Make sure to install it with "
                "'pip install openllm'."
            ) from e

        copied = copy.deepcopy(self.llm_kwargs)
        copied.update(kwargs)
        config = openllm.AutoConfig.for_model(
            self._identifying_params["model_name"], **copied
        )
        if self._client:
            res = self._client.generate(
                prompt, **config.model_dump(flatten=True)
            ).responses[0]
        else:
            assert self._runner is not None
            res = self._runner(prompt, **config.model_dump(flatten=True))
        if isinstance(res, dict) and "text" in res:
            return res["text"]
        elif isinstance(res, str):
            return res
        else:
            raise ValueError(
                "Expected result to be a dict with key 'text' or a string. "
                f"Received {res}"
            )

    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        try:
            import openllm
        except ImportError as e:
            raise ImportError(
                "Could not import openllm. Make sure to install it with "
                "'pip install openllm'."
            ) from e

        copied = copy.deepcopy(self.llm_kwargs)
        copied.update(kwargs)
        config = openllm.AutoConfig.for_model(
            self._identifying_params["model_name"], **copied
        )
        if self._client:
            async_client = openllm.client.AsyncHTTPClient(self.server_url)
            res = (
                await async_client.generate(prompt, **config.model_dump(flatten=True))
            ).responses[0]
        else:
            assert self._runner is not None
            (
                prompt,
                generate_kwargs,
                postprocess_kwargs,
            ) = self._runner.llm.sanitize_parameters(prompt, **kwargs)
            generated_result = await self._runner.generate.async_run(
                prompt, **generate_kwargs
            )
            res = self._runner.llm.postprocess_generate(
                prompt, generated_result, **postprocess_kwargs
            )

        if isinstance(res, dict) and "text" in res:
            return res["text"]
        elif isinstance(res, str):
            return res
        else:
            raise ValueError(
                "Expected result to be a dict with key 'text' or a string. "
                f"Received {res}"
            )


import unittest

import discretize
import matplotlib.pyplot as plt
import numpy as np
from pymatsolver import Pardiso as Solver
from scipy.constants import mu_0
from scipy.interpolate import interp1d
from SimPEG import maps
from SimPEG.electromagnetics import analytics
from SimPEG.electromagnetics import time_domain as tdem
from SimPEG.electromagnetics import utils

def halfSpaceProblemAnaDiff(
    meshType,
    srctype="MagDipole",
    sig_half=1e-2,
    rxOffset=50.0,
    bounds=None,
    plotIt=False,
    rxType="MagneticFluxDensityz",
):
    if bounds is None:
        bounds = [1e-5, 1e-3]
    if meshType == "CYL":
        cs, ncx, ncz, npad = 15.0, 30, 10, 15
        hx = [(cs, ncx), (cs, npad, 1.3)]
        hz = [(cs, npad, -1.3), (cs, ncz), (cs, npad, 1.3)]
        mesh = discretize.CylindricalMesh([hx, 1, hz], "00C")

    elif meshType == "TENSOR":
        cs, nc, npad = 20.0, 20, 7
        hx = [(cs, npad, 1.5), (cs, nc), (cs, npad, 1.5)]
        hy = [(cs, npad, 1.5), (cs, nc), (cs, npad, 1.5)]
        hz = [(cs, npad, 1.5), (cs, nc), (cs, npad, 1.5)]
        mesh = discretize.TensorMesh([hx, hy, hz], "CCC")

    active = mesh.cell_centers_z < 0.0
    actMap = maps.InjectActiveCells(mesh, active, np.log(1e-8), nC=mesh.shape_cells[2])
    mapping = maps.ExpMap(mesh) * maps.SurjectVertical1D(mesh) * actMap

    time_steps = [(1e-3, 5), (1e-4, 5), (5e-5, 10), (5e-5, 10), (1e-4, 10)]
    time_mesh = discretize.TensorMesh(
        [
            time_steps,
        ]
    )
    times = time_mesh.nodes_x
    out = utils.VTEMFun(times, 0.00595, 0.006, 100)
    wavefun = interp1d(times, out)
    t0 = 0.006
    waveform = tdem.Src.RawWaveform(off_time=t0, waveform_function=wavefun)

    rx = getattr(tdem.Rx, "Point{}".format(rxType[:-1]))(
        np.array([[rxOffset, 0.0, 0.0]]), np.logspace(-4, -3, 31) + t0, rxType[-1]
    )

    if srctype == "MagDipole":
        src = tdem.Src.MagDipole(
            [rx], waveform=waveform, location=np.array([0, 0.0, 0.0])
        )
    elif srctype == "CircularLoop":
        src = tdem.Src.CircularLoop(
            [rx], waveform=waveform, location=np.array([0.0, 0.0, 0.0]), radius=13.0
        )
    elif srctype == "LineCurrent":
        side = 100.0 * np.sqrt(np.pi)
        loop_path = np.c_[
            [-side / 2, -side / 2, 0],
            [side / 2, -side / 2, 0],
            [side / 2, side / 2, 0],
            [-side / 2, side / 2, 0],
            [-side / 2, -side / 2, 0],
        ].T
        src = tdem.sources.LineCurrent([rx], waveform=waveform, location=loop_path)

    survey = tdem.Survey([src])
    prb = tdem.Simulation3DMagneticFluxDensity(
        mesh, survey=survey, sigmaMap=mapping, time_steps=time_steps
    )
    prb.solver = Solver

    sigma = np.ones(mesh.shape_cells[2]) * 1e-8
    sigma[active] = sig_half
    sigma = np.log(sigma[active])

    if srctype == "MagDipole":
        bz_ana = mu_0 * analytics.hzAnalyticDipoleT(
            rx.locations[0][0] + 1e-3, rx.times - t0, sig_half
        )
    elif srctype == "CircularLoop":
        bz_ana = mu_0 * analytics.hzAnalyticCentLoopT(13, rx.times - t0, sig_half)
    elif srctype == "LineCurrent":
        bz_ana = mu_0 * analytics.hzAnalyticCentLoopT(100.0, rx.times - t0, sig_half)

    bz_calc = prb.dpred(sigma)
    ind = np.logical_and(rx.times - t0 > bounds[0], rx.times - t0 < bounds[1])
    log10diff = np.linalg.norm(
        np.log10(np.abs(bz_calc[ind])) - np.log10(np.abs(bz_ana[ind]))
    ) / np.linalg.norm(np.log10(np.abs(bz_ana[ind])))

    print(
        " |bz_ana| = {ana} |bz_num| = {num} |bz_ana-bz_num| = {diff}".format(
            ana=np.linalg.norm(bz_ana),
            num=np.linalg.norm(bz_calc),
            diff=np.linalg.norm(bz_ana - bz_calc),
        )
    )
    print("Difference: {}".format(log10diff))

    if plotIt is True:
        plt.loglog(
            rx.times[bz_calc > 0] - t0,
            bz_calc[bz_calc > 0],
            "r",
            rx.times[bz_calc < 0] - t0,
            -bz_calc[bz_calc < 0],
            "r--",
        )
        plt.loglog(rx.times - t0, abs(bz_ana), "b*")
        plt.title("sig_half = {:e}".format(sig_half))
        plt.show()

    return log10diff

class TDEM_SimpleSrcTests(unittest.TestCase):
    def test_source(self):
        waveform = tdem.Src.StepOffWaveform()
        assert waveform.eval(0.0) == 1.0

class TDEM_bTests(unittest.TestCase):
    def test_analytic_p0_CYL_1m_MagDipole(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff("CYL", rxOffset=1.0, sig_half=1e0) < 0.01
        )

    def test_analytic_m1_CYL_1m_MagDipole(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff("CYL", rxOffset=1.0, sig_half=1e-1) < 0.01
        )

    def test_analytic_m2_CYL_1m_MagDipole(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff("CYL", rxOffset=1.0, sig_half=1e-2) < 0.01
        )

    def test_analytic_m3_CYL_1m_MagDipole(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff("CYL", rxOffset=1.0, sig_half=1e-3) < 0.01
        )

    def test_analytic_p0_CYL_0m_CircularLoop(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "CYL", srctype="CircularLoop", rxOffset=0.0, sig_half=1e0
            )
            < 0.01
        )

    def test_analytic_m1_CYL_0m_CircularLoop(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "CYL", srctype="CircularLoop", rxOffset=0.0, sig_half=1e-1
            )
            < 0.01
        )

    def test_analytic_m2_CYL_0m_CircularLoop(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "CYL", srctype="CircularLoop", rxOffset=0.0, sig_half=1e-2
            )
            < 0.01
        )

    def test_analytic_m3_CYL_0m_CircularLoop(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "CYL",
                srctype="CircularLoop",
                rxOffset=0.0,
                sig_half=1e-3,
            )
            < 0.01
        )

    def test_analytic_m1_TENSOR_0m_LineCurrent(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "TENSOR",
                srctype="LineCurrent",
                rxOffset=0.0,
                sig_half=1e-1,
            )
            < 0.01
        )

    def test_analytic_m2_TENSOR_0m_LineCurrent(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "TENSOR",
                srctype="LineCurrent",
                rxOffset=0.0,
                sig_half=1e-2,
            )
            < 0.01
        )

    def test_analytic_m3_TENSOR_0m_LineCurrent(self):
        self.assertTrue(
            halfSpaceProblemAnaDiff(
                "TENSOR",
                srctype="LineCurrent",
                rxOffset=0.0,
                sig_half=1e-3,
            )
            < 0.01
        )


from sentence_transformers import SentenceTransformer
from bertopic import BERTopic
from umap import UMAP
import pandas as pd

# from sklearn.feature_extraction.text import CountVectorizer
# vectorizer_model = CountVectorizer(stop_words="english")

# from bertopic.representation import MaximalMarginalRelevance
# representation_model = MaximalMarginalRelevance(diversity=0.2)

# GPU Accelleration
# from cuml.cluster import HDBSCAN
# from cuml.manifold import UMAP
# Create instances of GPU-accelerated UMAP and HDBSCAN
# umap_model = UMAP(n_components=5, n_neighbors=15, min_dist=0.2, metric='cosine')
# hdbscan_model = HDBSCAN(min_samples=10, gen_min_span_tree=True, prediction_data=True)

import plotly.offline as pyo

from helpers.cli_loader import load_bar

FILTER_CAT = "cs"

# Prepare embeddings
with load_bar("Loading local file..."):
    # df = pd.read_json("arxiv-metadata-snapshot-02-23-cs-AI.json", lines=True) # 58k
    df = pd.read_json("arxiv-metadata-oai-snapshot.json", lines=True) # 2.2M
    print("Loaded " + str(len(df)) + " entries.")
    df = df[df["categories"].str.contains(FILTER_CAT)]
    print("Filtered by category " + FILTER_CAT + " to " + str(len(df)) + " entries.")
    docs = df["abstract"].tolist()
    titles = df["title"].tolist()

with load_bar("Generating Topics..."):
    sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
    embeddings = sentence_model.encode(docs, show_progress_bar=False)

    # Train BERTopic
    topic_model = BERTopic(nr_topics="auto").fit(docs, embeddings)

    # Further reduce topics
    # topic_model.reduce_topics(docs, nr_topics=30) # When enabled, disable nr_topics="auto" in topic_model itself!

    # Reduce dimensionality of embeddings, this step is optional but much faster to perform iteratively:
    reduced_embeddings = UMAP(n_neighbors=20, n_components=2, min_dist=0.3, metric='cosine').fit_transform(embeddings)

    topic_model.update_topics(docs, n_gram_range=(1, 3))

    topic_labels = topic_model.generate_topic_labels(nr_words=3,
                                                 topic_prefix=False,
                                                 word_length=10,
                                                 separator=", ")

    topic_model.set_topic_labels(topic_labels)

with load_bar("Generating Visualization..."):

    #topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings).show()
    fig = topic_model.visualize_documents(titles, reduced_embeddings=reduced_embeddings, width=1800, height=1200, hide_annotations=True, custom_labels=True)
    # fig.show()
    # topic_model.visualize_heatmap(titles).show()
    # topic_model.visualize_barchart(titles).show()

    # Save plot as HTML file
    pyo.plot(fig, filename='topic_modelling_arXiv.html')


import datetime as dt
import json
import typing
from pathlib import Path
from sys import getfilesystemencoding

from qgis.gui import (
    QgsFileWidget,
    QgisInterface,
)
from qgis.core import (
    Qgis,
    QgsExpressionContextUtils,
    QgsMapLayer,
    QgsLayerMetadata,
    QgsMapLayerType,
    QgsProject,
)
from PyQt5 import uic
from PyQt5 import QtCore
from PyQt5 import QtGui
from PyQt5 import QtWidgets
from PyQt5.QtPrintSupport import QPrinter

from . import models
from . import utils
from .automation import AutomationButtonsWidget
from .checklist_picker import ChecklistPicker
from .constants import (
    ChecklistItemPropertyColumn,
    ChecklistModelColumn,
    CustomDataRoles,
    DatasetType,
    LayerChooserDataRole,
    QGIS_VARIABLE_PREFIX,
    TabPages,
    ValidationArtifactType,
)
from .report import ReportHandler
from .utils import log_message

# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
UI_DIR = Path(__file__).parents[1] / "ui"
FORM_CLASS, _ = uic.loadUiType(str(UI_DIR / "dataset_qa_workbench_dock.ui"))

class DatasetQaWorkbenchDock(QtWidgets.QDockWidget, FORM_CLASS):
    add_report_to_layer_metadata_pb: QtWidgets.QPushButton
    dataset: typing.Optional[typing.Union[QgsMapLayer, str]]
    iface: QgisInterface
    tab_widget: QtWidgets.QTabWidget
    checklist_checks_tv: models.MyTreeView
    checklist_picker_dlg: ChecklistPicker
    checklist_name_le: QtWidgets.QLineEdit
    checklist_artifacts_le: QtWidgets.QLineEdit
    checklist_types_le: QtWidgets.QLineEdit
    checklist_description_te: QtWidgets.QTextEdit
    choose_checklist_pb: QtWidgets.QPushButton
    clear_checks_pb: QtWidgets.QPushButton
    validate_file_rb: QtWidgets.QRadioButton
    validate_layer_rb: QtWidgets.QRadioButton
    layer_chooser_lv: QtWidgets.QListView
    file_chooser: QgsFileWidget
    selected_checklist: typing.Optional[models.CheckList]
    report_te: QtWidgets.QTextEdit
    save_report_fw: QgsFileWidget
    save_report_pb: QtWidgets.QPushButton
    tab_widget: QtWidgets.QTabWidget
    run_post_validation_pb: QtWidgets.QPushButton
    configure_and_run_post_validation_pb: QtWidgets.QPushButton
    report_handler: typing.Optional[ReportHandler]

    closingPlugin = QtCore.pyqtSignal()

    def __init__(self, iface: QgisInterface, parent=None):
        """Constructor."""
        super(DatasetQaWorkbenchDock, self).__init__(parent)
        # Set up the user interface from Designer through FORM_CLASS.
        # After self.setupUi() you can access any designer object by doing
        # self.<objectname>, and you can use autoconnect slots - see
        # http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html
        # #widgets-and-dialogs-with-auto-connect
        self.setupUi(self)
        self.dataset = None
        self.save_report_pb.setDisabled(True)
        self.iface = iface
        self.checklists = []
        self.selected_checklist = None
        self.tab_widget.currentChanged.connect(self.update_tab_page)
        self.tab_widget.setTabEnabled(TabPages.CHOOSE.value, True)
        self.tab_widget.setTabEnabled(TabPages.VALIDATE.value, False)
        self.tab_widget.setTabEnabled(TabPages.REPORT.value, False)
        self.tab_pages = [self.tab_widget.widget(i.value) for i in TabPages]
        self.choose_checklist_pb.clicked.connect(self.show_checklist_picker)
        self.save_report_fw.fileChanged.connect(self.toggle_save_report_button)
        self.save_report_pb.clicked.connect(self.save_report)
        self.add_report_to_layer_metadata_pb.clicked.connect(
            self.add_report_to_layer_metadata
        )
        self.clear_checks_pb.clicked.connect(self.clear_all_checks)
        self.automate_all_checks_pb.clicked.connect(self.automate_all_checks)
        self.file_chooser.fileChanged.connect(self.selected_file_changed)
        self.validate_layer_rb.toggled.connect(
            self.respond_to_validate_layer_rb_toggled
        )
        self.report_handler = None
        # TODO: It might be necessary to disconnect these when plugin is unloaded
        QgsProject.instance().layersAdded.connect(self.respond_to_layers_added)
        QgsProject.instance().layersRemoved.connect(self.respond_to_layers_removed)

    def respond_to_layers_added(self, layers):
        utils.log_message("layers added")
        if self.selected_checklist is not None:
            current_dataset_type = self.selected_checklist.dataset_type
            model: QtGui.QStandardItemModel = self.layer_chooser_lv.model()
            for layer in layers:
                if utils.match_maplayer_type(layer.type()) == current_dataset_type:
                    item = QtGui.QStandardItem(layer.name())
                    item.setData(
                        layer.id(), LayerChooserDataRole.LAYER_IDENTIFIER.value
                    )
                    model.appendRow([item])

    def respond_to_layers_removed(self, layers: typing.List[str]):
        utils.log_message("layers removed")
        model: QtGui.QStandardItemModel = self.layer_chooser_lv.model()
        if model is not None:
            for row_idx in reversed(range(model.rowCount())):
                index = model.index(row_idx, 0)
                item = model.itemFromIndex(index)
                if item.data() in layers:
                    model.removeRow(row_idx)

    def clear_all_checks(self):
        utils.log_message(f"clear_all_checks_called")
        model = self.checklist_checks_tv.model()
        for row_index in range(model.rowCount()):
            utils.log_message(f"Unchecking row {row_index}...")
            checkbox_index = model.index(row_index, 1)
            model.setData(
                checkbox_index, QtCore.Qt.Unchecked, role=QtCore.Qt.CheckStateRole
            )
            head_index = model.index(row_index, 0)
            notes_index = model.index(
                ChecklistItemPropertyColumn.VALIDATION_NOTES.value, 1, parent=head_index
            )
            model.setData(notes_index, "", role=QtCore.Qt.EditRole)

    def automate_all_checks(self):
        utils.log_message(f"automate_all_checks_called")
        model = self.checklist_checks_tv.model()
        for row_index in range(model.rowCount()):
            item_head_index = model.index(row_index, 0)
            automation_index = model.index(
                ChecklistItemPropertyColumn.AUTOMATION.value, 1, parent=item_head_index
            )
            automation_widget = self.checklist_checks_tv.indexWidget(automation_index)
            if automation_widget:
                automation_widget.automator.perform_automation()

    def update_tab_page(self, index: int):
        if index == TabPages.REPORT.value:
            report = self.update_report()
            current_checklist = self.get_current_checklist()
            if current_checklist is not None and report is not None:
                if current_checklist.report is not None:
                    self.report_handler = ReportHandler(
                        self.iface,
                        report,
                        current_checklist.report.algorithm_id,
                        current_checklist.report.extra_parameters,
                    )

                self.toggle_post_validation_elements(
                    current_checklist.report is not None
                )
            else:
                self.toggle_post_validation_elements(False)

            self.add_report_to_layer_metadata_pb.setEnabled(
                self.validate_layer_rb.isChecked()
            )

    def get_current_checklist(self) -> typing.Optional[models.CheckList]:
        model = self.checklist_checks_tv.model()
        return getattr(model, "checklist", None)

    def update_report(self) -> typing.Optional[typing.Dict]:
        if self.validate_layer_rb.isChecked():
            current_layer_idx = self.layer_chooser_lv.currentIndex()
            layer_model = self.layer_chooser_lv.model()
            layer_id = layer_model.data(
                current_layer_idx, role=LayerChooserDataRole.LAYER_IDENTIFIER.value
            )
            if layer_id is not None:
                project = QgsProject.instance()
                layer = project.mapLayers()[layer_id]
                dataset = layer
            else:
                dataset = None
        else:
            dataset = Path(self.file_chooser.filePath()).name
        report = self.generate_report(dataset)
        if report:
            serialized = utils.serialize_report_to_html(report)
            self.report_te.setDocument(serialized)
        return report

    def toggle_post_validation_elements(self, enabled: bool):
        self.run_post_validation_pb.setEnabled(enabled)
        self.configure_and_run_post_validation_pb.setEnabled(enabled)
        if enabled:
            self.run_post_validation_pb.clicked.connect(
                self.report_handler.handle_report
            )
            self.configure_and_run_post_validation_pb.clicked.connect(
                self.report_handler.configure_and_handle_report
            )
        else:
            try:
                self.run_post_validation_pb.clicked.disconnect()
                self.configure_and_run_post_validation_pb.clicked.disconnect()
            except TypeError:
                # buttons were already disconnected
                pass

    def generate_report(self, dataset: typing.Union[QgsMapLayer, str]):
        checklist_model = self.checklist_checks_tv.model()
        if checklist_model is not None:
            result = get_report_contents(checklist_model, dataset)
        else:
            result = None
        return result

    def toggle_save_report_button(self, current_path: str):
        if current_path:
            self.save_report_pb.setDisabled(False)
        else:
            self.save_report_pb.setDisabled(True)

    def save_report(self):
        output = get_report_path(self.save_report_fw.filePath())
        printer = QPrinter(QPrinter.PrinterResolution)
        printer.setOutputFormat(QPrinter.PdfFormat)
        printer.setPaperSize(QPrinter.A4)
        printer.setOutputFileName(str(output))
        doc: QtGui.QTextDocument = self.report_te.document().clone()
        page_size = printer.pageRect().size()
        doc.setPageSize(QtCore.QSizeF(page_size.width(), page_size.height()))
        doc.print(printer)
        if output.is_file():
            # `doc.print` does not return a value, so this is a hacky way to
            # check if the file was saved correctly
            self.iface.messageBar().pushMessage(
                "Success", "Validation report saved!", level=Qgis.Info
            )
        else:
            self.iface.messageBar().pushMessage(
                "Error",
                f"Could not save validation report to: {output}",
                level=Qgis.Critical,
            )

    def add_report_to_layer_metadata(self):
        report = self.generate_report(dataset=self.dataset)
        add_report_to_layer(report, self.dataset)
        self.iface.messageBar().pushMessage(
            "Success", "Validation report added to layer metadata!", level=Qgis.Info
        )

    def selected_layer_changed(
        self, current: QtCore.QModelIndex, previous: QtCore.QModelIndex
    ):
        # NOTE: This method does not get called when list item is deselected
        self.dataset = self._get_current_layer()
        if self.dataset:
            self.load_checklist_steps(current, previous)

    def load_checklist_steps(
        self, current: QtCore.QModelIndex, previous: QtCore.QModelIndex
    ):
        utils.log_message(
            f"inside load_checklist_steps selected_checklist: {self.selected_checklist}"
        )
        utils.log_message(
            f"selected_checklist checks: {self.selected_checklist.checks}"
        )
        for head_check in self.selected_checklist.checks:
            head_check: models.ChecklistItemHead
            utils.log_message(
                f"check {head_check.name} description: "
                f"{head_check.check_properties[ChecklistItemPropertyColumn.DESCRIPTION.value]}"
            )
        checklist_checks_model = models.CheckListItemsModel(self.selected_checklist)
        self.checklist_checks_tv.setModel(checklist_checks_model)
        self.checklist_checks_tv.resized.connect(self.force_model_update)
        self.checklist_checks_tv.setTextElideMode(QtCore.Qt.ElideNone)
        self.checklist_checks_tv.setWordWrap(True)
        self.checklist_checks_tv.setAlternatingRowColors(True)
        self.add_automation_widgets()
        header = self.checklist_checks_tv.header()
        header.setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeToContents)
        delegate = models.ChecklistItemsModelDelegate(self.checklist_checks_tv)
        self.checklist_checks_tv.setItemDelegate(delegate)

    def add_automation_widgets(self):
        model = self.checklist_checks_tv.model()
        for head_row in range(model.rowCount()):
            head_index = model.index(head_row, 0)
            item_head: models.ChecklistItemHead = head_index.internalPointer().ref
            if item_head.automation.algorithm_id is not None:
                automation_index = model.index(
                    ChecklistItemPropertyColumn.AUTOMATION.value, 1, head_index
                )
                automation_widget = AutomationButtonsWidget(
                    checklist_item_head_index=head_index, dataset=self.dataset
                )
                self.checklist_checks_tv.setIndexWidget(
                    automation_index, automation_widget
                )

    def force_model_update(self):
        model = self.checklist_checks_tv.model()
        for row in range(model.rowCount()):
            parent = model.index(row, 0)
            properties = (
                ChecklistItemPropertyColumn.DESCRIPTION.value,
                ChecklistItemPropertyColumn.GUIDE.value,
                ChecklistItemPropertyColumn.VALIDATION_NOTES.value,
            )
            for property_row in properties:
                property_index = model.index(property_row, 1, parent)
                model.dataChanged.emit(property_index, property_index)

    def selected_file_changed(self, raw_path: str):
        utils.log_message(f"selected_file_changed raw_path: {raw_path}")
        if raw_path and self.validate_file_rb.isChecked():
            self.toggle_other_pages(True)
            self.dataset = Path(raw_path).expanduser().resolve()
            self.load_checklist_steps(None, None)
        elif not raw_path and self.validate_file_rb.isChecked():
            self.toggle_other_pages(False)
            self.dataset = None

    def toggle_other_pages(self, enabled: bool):
        self.tab_widget.setTabEnabled(TabPages.VALIDATE.value, enabled)
        self.tab_widget.setTabEnabled(TabPages.REPORT.value, enabled)

    def respond_to_validate_layer_rb_toggled(self, checked: bool):
        if checked:
            self.layer_chooser_lv.setEnabled(True)
            self.file_chooser.setEnabled(False)
            try:
                current_layer_selection: QtCore.QItemSelection = (
                    self.layer_chooser_lv.selectionModel().selection()
                )
            except AttributeError:
                pass
            else:
                self.selected_layer_selection_changed(current_layer_selection, None)
        else:
            self.layer_chooser_lv.setEnabled(False)
            self.file_chooser.setEnabled(True)
            self.selected_file_changed(self.file_chooser.filePath())

    def selected_layer_selection_changed(
        self, selected: QtCore.QItemSelection, deselected: QtCore.QItemSelection
    ):
        if len(selected.indexes()) > 0 and self.validate_layer_rb.isChecked():
            self.toggle_other_pages(True)
            # TODO: check if self.dataset is None
            self.dataset = self._get_current_layer()
        elif len(selected.indexes()) == 0 and self.validate_layer_rb.isChecked():
            self.toggle_other_pages(False)
            self.dataset = None

    def _get_current_layer(self) -> typing.Optional[QgsMapLayer]:
        current_layer_idx = self.layer_chooser_lv.currentIndex()
        utils.log_message(f"current_layer_idx: {current_layer_idx}")
        result = None
        if current_layer_idx != QtCore.QModelIndex():
            layer_model = self.layer_chooser_lv.model()
            layer_id = layer_model.data(
                current_layer_idx, role=LayerChooserDataRole.LAYER_IDENTIFIER.value
            )
            utils.log_message(f"layer_id: {layer_id}")
            layer_model = self.layer_chooser_lv.model()
            project = QgsProject.instance()
            try:
                result = project.mapLayers()[layer_id]
            except KeyError:
                utils.log_message(f"Unable to load layer {layer_id}")
        return result

    def show_checklist_picker(self):
        self.checklist_picker_dlg = ChecklistPicker(self.iface)
        self.checklist_picker_dlg.button_box.accepted.connect(self.load_checklist)
        self.checklist_picker_dlg.setModal(True)
        self.checklist_picker_dlg.show()
        self.checklist_picker_dlg.exec_()

    def load_checklist(self):
        selected_indexes = self.checklist_picker_dlg.checklists_tv.selectedIndexes()
        if any(selected_indexes):
            self.selected_checklist = self.get_selected_checklist(selected_indexes[0])
            log_message(f"the selected checklist is: {self.selected_checklist.name}")
            self.reset_loaded_checklist()
            self.load_checklist_elements(self.selected_checklist)
        else:
            log_message("no checklist was selected")

    def get_selected_checklist(self, index: QtCore.QModelIndex) -> models.CheckList:
        model = index.model()
        identifier_item = model.item(index.row(), ChecklistModelColumn.IDENTIFIER.value)
        result = identifier_item.data(
            role=CustomDataRoles.CHECKLIST_DOWNLOADER_IDENTIFIER.value
        )
        return result

    def load_checklist_elements(self, checklist: models.CheckList):
        self.checklist_name_le.setEnabled(True)
        self.checklist_artifacts_le.setEnabled(True)
        self.checklist_types_le.setEnabled(True)
        self.checklist_description_te.setEnabled(True)
        self.checklist_name_le.setText(checklist.name)
        self.checklist_artifacts_le.setText(checklist.validation_artifact_type.value)
        self.checklist_types_le.setText(checklist.dataset_type.value)
        self.checklist_description_te.setText(checklist.description)

        enable_layer_chooser = False
        enable_file_chooser = False
        if checklist.dataset_type in (DatasetType.VECTOR, DatasetType.RASTER):
            if checklist.validation_artifact_type == ValidationArtifactType.DATASET:
                enable_layer_chooser = True
            elif checklist.validation_artifact_type == ValidationArtifactType.STYLE:
                enable_layer_chooser = True
                enable_file_chooser = True
            else:
                enable_file_chooser = True
        else:
            enable_file_chooser = True
        if enable_file_chooser:
            self.validate_file_rb.setEnabled(True)
            self.validate_file_rb.setChecked(True)
            self.file_chooser.setEnabled(True)
            self.selected_file_changed(self.file_chooser.filePath())
        if enable_layer_chooser:
            self.validate_layer_rb.setEnabled(True)
            self.validate_layer_rb.setChecked(True)
            self.layer_chooser_lv.setEnabled(True)
            model = get_list_view_layers(checklist.dataset_type)
            self.layer_chooser_lv.setModel(model)
            selection_model: QtCore.QItemSelectionModel = (
                self.layer_chooser_lv.selectionModel()
            )
            log_message(f"selection_model: {selection_model}")
            selection_model.currentChanged.connect(self.selected_layer_changed)
            selection_model.selectionChanged.connect(
                self.selected_layer_selection_changed
            )

    def enable_dataset_chooser_elements(
        self,
        dataset_type: DatasetType,
        validation_artifact_type: ValidationArtifactType,
    ):
        enable_layer_chooser = False
        enable_file_chooser = False
        if dataset_type in (DatasetType.VECTOR, DatasetType.RASTER):
            if validation_artifact_type == ValidationArtifactType.DATASET:
                enable_layer_chooser = True
            elif validation_artifact_type == ValidationArtifactType.STYLE:
                enable_layer_chooser = True
                enable_file_chooser = True
            else:
                enable_file_chooser = True
        else:
            enable_file_chooser = True
        if enable_file_chooser:
            self.validate_file_rb.setEnabled(True)
            self.validate_file_rb.setChecked(True)
            self.file_chooser.setEnabled(True)
        if enable_layer_chooser:
            self.validate_layer_rb.setEnabled(True)
            self.validate_layer_rb.setChecked(True)
            self.layer_chooser_lv.setEnabled(True)
            model = get_list_view_layers(dataset_type)
            self.layer_chooser_lv.setModel(model)
            selection_model: QtCore.QItemSelectionModel = (
                self.layer_chooser_lv.selectionModel()
            )
            log_message(f"selection_model: {selection_model}")
            selection_model.currentChanged.connect(self.selected_layer_changed)
            selection_model.selectionChanged.connect(
                self.selected_layer_selection_changed
            )

    def reset_loaded_checklist(self):
        self.tab_widget.setTabEnabled(TabPages.VALIDATE.value, False)
        self.tab_widget.setTabEnabled(TabPages.REPORT.value, False)

        self.checklist_name_le.setEnabled(False)
        self.checklist_artifacts_le.setEnabled(False)
        self.checklist_types_le.setEnabled(False)
        self.checklist_description_te.setEnabled(False)
        self.validate_layer_rb.setEnabled(False)
        self.validate_file_rb.setEnabled(False)
        self.file_chooser.setEnabled(False)
        self.layer_chooser_lv.setEnabled(False)

        self.checklist_name_le.clear()
        self.checklist_artifacts_le.clear()
        self.checklist_types_le.clear()
        self.checklist_description_te.clear()
        self.validate_layer_rb.setChecked(False)
        self.validate_file_rb.setChecked(False)
        # TODO: clear loaded layers list view
        # TODO: clear file chooser

    def closeEvent(self, event: QtGui.QCloseEvent) -> None:
        self.closingPlugin.emit()
        event.accept()

def get_list_view_layers(dataset_type: DatasetType) -> QtGui.QStandardItemModel:
    project = QgsProject.instance()
    result = QtGui.QStandardItemModel()
    for index, (id_, layer) in enumerate(project.mapLayers().items()):
        if utils.match_maplayer_type(layer.type()) == dataset_type:
            item = QtGui.QStandardItem(layer.name())
            item.setData(id_, LayerChooserDataRole.LAYER_IDENTIFIER.value)
            result.appendRow([item])
            log_message(
                f"retrieving layer id from the "
                f"item: {item.data(LayerChooserDataRole.LAYER_IDENTIFIER.value)}"
            )
    return result

def get_legal_layers(dataset_type: models.DatasetType) -> typing.Dict[str, QgsMapLayer]:
    project = QgsProject.instance()
    result = {}
    for id_, layer in project.mapLayers().items():
        if utils.match_maplayer_type(layer.type()) == dataset_type:
            result[id_] = layer
    return result

def get_report_contents(
    checklist_items: models.CheckListItemsModel, dataset: typing.Union[QgsMapLayer, str]
) -> typing.Dict:
    if isinstance(dataset, QgsMapLayer):
        name = dataset.name()
    else:
        name = dataset
    result = {
        "name": "Validation report",
        "validator": utils.get_qgis_variable(
            f"{QGIS_VARIABLE_PREFIX}_validator", "user_full_name"
        ),
        "generated": dt.datetime.now(dt.timezone.utc).isoformat(),
        "dataset": name,
        "dataset_is_valid": checklist_items.result,
        "checklist": checklist_items.checklist.name,
        "dataset_type": checklist_items.checklist.dataset_type.value,
        "artifact_type": checklist_items.checklist.validation_artifact_type.value,
        "description": checklist_items.checklist.description,
        "checks": [],
    }
    for row in range(checklist_items.rowCount()):
        name_idx = checklist_items.index(row, 0)
        node = name_idx.internalPointer()
        checklist_head: models.ChecklistItemHead = node.ref
        description_prop: models.ChecklistItemProperty = (
            checklist_head.check_properties[
                ChecklistItemPropertyColumn.DESCRIPTION.value
            ]
        )
        notes_prop: models.ChecklistItemProperty = checklist_head.check_properties[
            ChecklistItemPropertyColumn.VALIDATION_NOTES.value
        ]
        check = {
            "name": checklist_head.name,
            "validated": True
            if checklist_head.validated == QtCore.Qt.Checked
            else False,
            "description": checklist_head.description,
            "notes": checklist_head.validation_notes,
        }
        result["checks"].append(check)
    return result

def serialize_report(report: typing.Dict) -> str:
    return json.dumps(report, indent=2)

def add_report_to_layer(report: typing.Dict, layer: QgsMapLayer):
    history_msg = (
        f'{report["generated"]} - Validation report: '
        f'{"Valid" if report["dataset_is_valid"] else "Invalid"}'
    )
    abstract_msg = utils.serialize_report_to_plain_text(report)
    metadata: QgsLayerMetadata = layer.metadata()
    history: typing.List = metadata.history()
    history.append(history_msg)
    abstract: str = metadata.abstract()
    abstract = "\n\n---\n\n".join((abstract, abstract_msg))
    metadata.setAbstract(abstract)
    metadata.setHistory(history)
    layer.setMetadata(metadata)

def get_report_path(raw_path: str) -> Path:
    result = Path(raw_path).expanduser().resolve()
    if result.suffix.lower() != ".pdf":
        result = result.parent / f"{result.name}.pdf"
    return result


import argparse
import time
import math
import os
import sys
import torch
from data_utils import get_lm_corpus
from utils.exp_utils import get_logger
import numpy as np

parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model')
parser.add_argument('--data', type=str, default='../data/enwik8',
                    help='location of the data corpus')
parser.add_argument('--dataset', type=str, default='enwik8',
                    choices=['wt103', 'lm1b', 'enwik8', 'text8'],
                    help='dataset name')
parser.add_argument('--split', type=str, default='onnx',
                    choices=['all', 'valid', 'test','onnx'],
                    help='which split to evaluate')
parser.add_argument('--batch_size', type=int, default=1,
                    help='batch size')
parser.add_argument('--tgt_len', type=int, default=128,
                    help='number of tokens to predict')
parser.add_argument('--ext_len', type=int, default=0,
                    help='length of the extended context')
parser.add_argument('--mem_len', type=int, default=0,
                    help='length of the retained previous heads')
parser.add_argument('--clamp_len', type=int, default=-1,
                    help='max positional embedding index')
parser.add_argument('--pre_data_save_path', type=str, default='./bin_data',
                    help='location of the bin data')
parser.add_argument('--pre_target_save_path', type=str, default='./bin_target',
                    help='location of the bin data')

args = parser.parse_args()

assert args.ext_len >= 0, 'extended context length must be non-negative'

if not os.path.exists(args.pre_data_save_path):
    os.makedirs(args.pre_data_save_path)
if not os.path.exists(args.pre_target_save_path):
    os.makedirs(args.pre_target_save_path)

# Load dataset
corpus = get_lm_corpus(args.data, args.dataset)
ntokens = len(corpus.vocab)

valid_iter = corpus.get_iterator('valid', args.batch_size, args.tgt_len,
                                 device='cpu', ext_len=args.ext_len)
test_iter = corpus.get_iterator('test', args.batch_size, args.tgt_len,
                                device='cpu', ext_len=args.ext_len)

f_info_file = open("bin_file.info", "wt")

for idx, (data, target, seq_len) in enumerate(valid_iter):
    if idx < valid_iter.n_batch-1:
        data_seq = np.asarray(data, dtype=np.int64)
        target_seq = np.asarray(target, dtype=np.int64)
        data_bin_file_path = os.path.join(args.pre_data_save_path, 'data_' + str(idx) + ".bin")
        target_bin_file_path = os.path.join(args.pre_target_save_path, 'data_' + str(idx) + ".bin")
        data_seq.tofile(data_bin_file_path)
        target_seq.tofile(target_bin_file_path)
        f_info_file.write(str(idx) + ' ' + args.pre_data_save_path + '/data_' + str(idx) + ".bin" + '\n')
        f_info_file.write(str(idx) + ' ' + args.pre_target_save_path + '/data_' + str(idx) + ".bin" + '\n')
        print('\rhave done {} batches'.format(str(idx+1)), end='')
    else:
        break
print('\nCompleted!')
f_info_file.close()


"""
running environment:
    Windows:
        `SYSTEM = Windows`
    Baidu macbook pro:
        conda activate defect
        `SYSTEM = Mac`
"""

import typing
import json
import numpy as np
from collections import Counter

from classifiers import model_builder
from processors import smote
from datasets.ck_nasa_loaders import get_dataset, load_train_test_data

SYSTEM = 'Windows'

def experiments(config: typing.Dict) -> typing.Dict:
    """
    experiment wrapper with configuration as parameter
    :param config: configuration dict loaded from json file
    :return: dict evaluation metrics of each experiment,
        format: {$DATA_SOURCE_NAME: {$DATA_SET_NAME: {$SPLIT: {$METHOD_NAME: {$METRIC_NAME: $METRIC_VALUE}}}}}
        example: {'CK': {'ant1': {'10%': {'lr': {'precision': .333, 'recall': .333, 'f-score': .333}}}}}
    """
    models = [model_builder(model_name.strip()) for model_name in config['model'].split(',')]
    metrics_dict = {}
    training_rates = ('10%', '20%', '30%')
    for datasource in config['data_sources']:
        datasets = get_dataset(config['data_root'][SYSTEM], datasource)
        mean_dict = [{} for _ in range(len(training_rates))]
        for dataset in datasets:
            trains, tests = load_train_test_data(config['data_root'][SYSTEM], datasource, dataset)
            for split in range(len(training_rates)):
                trainXY, testXY = trains[split], tests[split]
                trainX, trainY = trainXY[:, :-1], np.array(
                    [1 if y >0 else 0 for y in trainXY[:, -1]])      # 1/-1 label to 1/0 label
                testX, testY = testXY[:, :-1], np.array([1 if y > 0 else 0 for y in testXY[:, -1]])
                print('Original dataset shape %s' % Counter(trainY))
                trainX, trainY = smote(trainX, trainY)
                print('Resampled dataset shape %s' % Counter(trainY))
                for model in models:
                    metrics = model(trainX, trainY, testX, testY)
                    metrics_dict.setdefault(datasource, {}).setdefault(dataset, {}) \
                        .setdefault(training_rates[split], {})[str(model).split(' ')[1]] = metrics
                    for (key, val) in metrics.items():
                        mean_dict[split].setdefault(str(model).split(' ')[1], {})[key] \
                            = mean_dict[split].setdefault(str(model).split(' ')[1], {}).setdefault(key, 0.0) + val
        for split in range(len(training_rates)):
            for model in models:
                for (key, val) in mean_dict[split][str(model).split(' ')[1]].items():
                    mean_dict[split][str(model).split(' ')[1]][key] = val / len(datasets)
        metrics_dict[datasource]['mean'] = {training_rates[idx]: mean_dict[idx] for idx in range(len(training_rates))}
    return metrics_dict

if __name__ == '__main__':
    with open('configs/ck_nasa_config.json', 'r') as f:
        config = json.load(f)
    results = experiments(config)
    for datasource in config['data_sources']:
        print('{}: {}'.format(datasource, results[datasource]['mean']['10%']))
    with open('results.json', 'w') as f:
        json.dump(results, f)


"""
Created by Constantin Philippenko, 1st September 2022.
"""
import argparse

import numpy as np
from matplotlib.lines import Line2D

from src.SGD import SeriesOfSGD, SGDArtemis, SGDCompressed, SGDVanilla
from src.federated_learning.Client import check_clients, Client
from src.main import plot_eigen_values
from src.utilities.PlotUtils import plot_only_avg, setup_plot_with_SGD
from src.utilities.Utilities import create_folder_if_not_existing

DIM = 100
POWER_COV = 1
R_SIGMA=0

EIGENVALUES = None
USE_ORTHO_MATRIX = True
DO_LOGISTIC_REGRESSION = False

HETEROGENEITY = "wstar"

NB_RUNS = 5

step_size = lambda it, r2, omega, K: 1 / (2*r2)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--nb_clients",
        type=int,
        help="Number of clients",
        required=True,
    )
    parser.add_argument(
        "--stochastic",
        type=str,
        help="Stochastic or full batch",
        required=True,
        default=True,
    )
    parser.add_argument(
        "--batch_size",
        type=int,
        help="Batch size when stochastic.",
        required=False,
        default=32,
    )
    parser.add_argument(
        "--use_ortho_matrix",
        type=str,
        help="Use an orthogonal matrix.",
        required=False,
        default=True,
    )
    parser.add_argument(
        "--noiseless",
        type=str,
        help="Use noise or not",
        required=False,
        default=False,
    )
    args = parser.parse_args()
    nb_clients = args.nb_clients
    batch_size = args.batch_size
    stochastic = True if args.stochastic == "True" else False
    noiseless = True if args.noiseless == "True" else False
    use_ortho_matrix = True if args.use_ortho_matrix == "True" else False

    legend_line = [Line2D([0], [0], color="black", lw=2, label='w.o. control variate.'),
                    Line2D([0], [0], linestyle="--", color="black", lw=2, label='w. control variate.')]

    # if stochastic:
    size_dataset = 10 ** 8
    nb_epoch = 1

    lower_sigma = 0 if noiseless else None

    np.random.seed(10)
    clients = [Client(i, DIM, size_dataset // nb_clients, POWER_COV, nb_clients, USE_ORTHO_MATRIX, HETEROGENEITY,
                      lower_sigma=lower_sigma) for i in range(nb_clients)]

    sgd_series = SeriesOfSGD()
    for run_id in range(NB_RUNS):
        check_clients(clients, HETEROGENEITY)
        synthetic_dataset = clients[0].dataset
        synthetic_dataset.power_cov = POWER_COV

        hash_string = synthetic_dataset.string_for_hash(nb_runs=NB_RUNS,stochastic=stochastic, batch_size=batch_size,
                                                        noiseless=noiseless)

        labels = ["no compr.", "1-quantiz.", "sparsif.", "sketching",
                  r"rand-$h$", "partial part."]

        w_star = np.mean([client.dataset.w_star for client in clients], axis=0)
        vanilla_sgd = SGDVanilla(clients, step_size, nb_epoch=nb_epoch, sto=stochastic, batch_size=batch_size,
                                 start_averaging=0, reg=0)
        sgd_nocompr = vanilla_sgd.gradient_descent(label=labels[0])
        all_sgd = [sgd_nocompr]

        my_compressors = [synthetic_dataset.quantizator, synthetic_dataset.sparsificator, synthetic_dataset.sketcher,
                          synthetic_dataset.rand1, synthetic_dataset.all_or_nothinger]

        for i in range(len(my_compressors)):
            compressor = my_compressors[i]
            print("Compressor: {0}".format(compressor.get_name()))
            all_sgd.append(
                SGDCompressed(clients, step_size, compressor, nb_epoch=nb_epoch, sto=stochastic,
                              batch_size=batch_size, start_averaging=0, reg=0).gradient_descent(label=labels[i + 1]))
            all_sgd.append(
                SGDArtemis(clients, step_size, compressor, nb_epoch=nb_epoch, sto=stochastic,
                           batch_size=batch_size, start_averaging=0, reg=0).gradient_descent(label=labels[i + 1] + "-art"))

        optimal_loss = 0
        print("Optimal loss:", optimal_loss)
        sgd_series.append(all_sgd)
        create_folder_if_not_existing("pickle")
        sgd_series.save("pickle/C{0}-{1}".format(nb_clients, hash_string))

        plot_only_avg(sgd_series, optimal_loss=optimal_loss,
                      hash_string="C{0}-{1}-artemis".format(nb_clients, hash_string),
                      custom_legend=legend_line, with_artemis=True, stochastic=stochastic)

        for client in clients:
            client.regenerate_dataset()

    setup_plot_with_SGD(sgd_series, optimal_loss=optimal_loss,
                        hash_string="C{0}-{1}-artemis_both".format(nb_clients, hash_string),
                        custom_legend=legend_line, with_artemis=True)

    plot_eigen_values(sgd_series, hash_string="C{0}-{1}-artemis".format(nb_clients, hash_string),
                      custom_legend=legend_line)


from flask import Blueprint
from flask import request
from flask import session
from flask import redirect, url_for
from flask import render_template, flash
from flask_login import login_required, current_user
from werkzeug.utils import secure_filename
from werkzeug.datastructures import FileStorage
import os
from .models import Item, Wish, Watchlist, Bid, User, Image, BookCategory, BookCondition, AutographStatus
from . import db
from auction.forms import BookCreationForm, BidForm
from sqlalchemy import func, desc
from datetime import datetime
bp = Blueprint('item', __name__, url_prefix='/items')

@bp.route('/<id>')
def show(id):
    item = Item.query.filter_by(id=id).first()  # grab item query data by id
    images = Image.query.filter_by(item_id=id).all()
    # this is the user who has listed the item, need to pull their username
    user = User.query.filter_by(id=item.user_id).first()
    bids = Bid.query.filter_by(item_id=item.id).order_by(
        Bid.bid_datetime.desc()).limit(10).all()  # grabs top ten most recent bids
    # formats into a nice datetime that can be used to show item listing date; this is optional
    formatted_datetime = item.item_datetime.strftime("%d/%m/%Y, %H:%M:%S")
    bidders = db.session.query(User).join(Bid, User.id == Bid.user_id, isouter=True).join(
        Item, id == Bid.item_id, isouter=True).filter_by(id=Bid.item_id).order_by(Bid.bid_datetime.desc()).group_by(User.id).limit(10).all()

    bid_form = BidForm()
    return render_template("item/show.html", item=item, images=images,
                           user=user, bids=bids, formatted_datetime=formatted_datetime,
                           bidders=bidders, bid_form=bid_form)

@bp.route('/<item_id>/bid', methods=["GET", "POST"])
@login_required
def bid(item_id):
    form = BidForm()
    book = Item.query.filter(Item.id == item_id).first()

    if form.validate_on_submit():
        # make sure seller cannot bid on his own book
        if current_user.id == book.user_id:
            flash("You cannot bid on your own book!", "table-warning")
            return redirect(url_for('item.show', id=item_id))

        # get the highest bid
        bids = book.bids
        if len(bids) != 0:
            max_bid_price = bids[0].bid_price
        else:
            max_bid_price = book.starting_bid

        for b in bids:
            if b.bid_price > max_bid_price:
                max_bid_price = b.bid_price
        # validate the bid price
        if form.bid_price.data > max_bid_price:

            bid = Bid(bid_price=form.bid_price.data, item_id=item_id,
                      user_id=current_user.id, bid_datetime=datetime.now())
            db.session.add(bid)
            db.session.commit()
        else:  # erro message less than current bid price
            flash(
                f"Please enter a bid that is greater than ${max_bid_price}", "table-warning")
    else: # handle unexpected user inputs
        if form.bid_price.errors[0] == "Not a valid float value":
            flash("Please enter a valid bid price!", "table-danger")

    return redirect(url_for('item.show', id=item_id))

@bp.route('/<item_id>/watch')
@login_required
def watch(item_id):
    book = Item.query.filter(Item.id == item_id).first()
    if book.item_status == "Closed":
        flash("Cannot add a closed auction to watchlist", "table-danger")
        return redirect(url_for('item.show', id=item_id))
    watchlist = Watchlist.query.filter(
        Watchlist.user_id == current_user.id).first()
    if watchlist == None:
        new_watchlist = Watchlist(user_id=current_user.id)
        db.session.add(new_watchlist)
        db.session.commit()
        watchlist = Watchlist.query.filter(
            Watchlist.user_id == current_user.id).first()
    # user can only add the book to watchlist once
    new_wish = Wish(watchlist_id=watchlist.id,
                    item_id=book.id)
    repeated = False
    for wish in watchlist.wishes:
        if wish.item_id == new_wish.item_id:
            flash("Item already exists in the watchlist!","table-danger")
            repeated = True
            break
    if repeated==False:
        db.session.add(new_wish)
        db.session.commit()
        flash("Item successfully added to watchlist!", "table-success")
    return redirect(url_for('item.show', id=item_id))

@bp.route('/<item_id>/close')
@login_required
def close(item_id):
    book = Item.query.filter(Item.id == item_id).first()
    if book.item_status == "Closed":
        flash("Auction is already closed!", "table-warning")
        return redirect(url_for('item.show', id=item_id))
    if current_user.id != book.user_id:  # if someone's trying to hack by editing the url
        flash("You cannot close someone else's auction!", "table-danger")
        return redirect(url_for('item.show', id=item_id))
    book.item_status = "Closed"
    db.session.commit()
    flash("Auction successfully closed!", "table-success")
    return redirect(url_for('item.show', id=item_id))

# add the types of files allowed as a set
ALLOWED_FILE = {'png', 'jpg', 'JPG', 'PNG'}

@bp.route('/create', methods=['GET', 'POST'])
@login_required
def create():

    # print('Method type: ', request.method)
    form = BookCreationForm()
    if(form.validate_on_submit()):
        # extract data from the form
        listing_title = form.listing_title.data
        book_title = form.book_title.data
        book_category = BookCategory[form.category.data]
        autograph_status = AutographStatus[form.autograph_status.data]
        book_condition = BookCondition[form.book_condition.data]
        author_name = form.author.data
        book_ISBN = form.isbn.data
        listing_description = form.description.data
        starting_bid = form.starting_bid.data
        creator_id = current_user.id

        valid = True
        # ensure the listing_title is unique
        if not Item.query.filter(Item.listing_title == listing_title).first() == None:
            flash("The listing title already exists!", "listing_title_repeated")
            # ensure uploaded files are all images
            valid = False

        # check images' file types
        images = request.files.getlist(form.images.name)
        for image in images:
            filename = image.filename
            if not filename.split('.')[1] in ALLOWED_FILE:
                flash('Only supports png, jpg, JPG, PNG', "file_type")
                valid = False
                break
        # invalid input detected
        if valid == False:
            return render_template('item/create.html', form=form)

        # create a new book
        new_book = Item(listing_title=listing_title,
                        book_title=book_title,
                        book_category=book_category,
                        autograph_status=autograph_status,
                        book_condition=book_condition,
                        author_name=author_name,
                        book_ISBN=book_ISBN,
                        listing_description=listing_description,
                        starting_bid=starting_bid,
                        user_id=creator_id)
        db.session.add(new_book)
        db.session.commit()
        # link images to the new book
        BASE_PATH = os.path.dirname(__file__)
        for image in images:
            filename = image.filename
            upload_path = os.path.join(
                BASE_PATH + '/static/images', secure_filename(filename))

            db_upload_path = '/static/images/' + secure_filename(filename)
            image.save(upload_path)

            img_model = Image(image=db_upload_path, item_id=new_book.id)
            db.session.add(img_model)
            db.session.commit()

        return redirect(url_for('item.show', id=new_book.id))
    return render_template('item/create.html', form=form)


import numpy as np
import streamlit as st
from PIL import Image, ImageOps
import tensorflow
from sklearn.cluster import MiniBatchKMeans
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import cv2
from lime import lime_image
from skimage.segmentation import mark_boundaries

def color_compress_KNN(image, KNN=4):
    import warnings; warnings.simplefilter('ignore')
    Kmeans_images = []
    rescaled_image = np.array(image) / 255.0 # use 0...1 scale
    preprocessed_image = rescaled_image.reshape(150 * 150, 3)
    kmeans = MiniBatchKMeans(KNN)
    kmeans.fit(preprocessed_image)
    new_colors = kmeans.cluster_centers_[kmeans.predict(preprocessed_image)]
    color_compressed_image = new_colors.reshape(np.array(image).shape)
    #Changed to predict in Warren's Model
    reshaped_cci = cv2.resize(color_compressed_image, (75,75)).reshape(1,75,75,3)
    return reshaped_cci

def import_and_predict(image_data, model):

        img_output = color_compress_KNN(image_data)
        prediction = model.predict(img_output)[0]

        return prediction, img_output

def image_expl(model,image_array):
    '''
    Takes a model, single image array, name string of the model, and the true label of the image.
    Uses LIME to get an image explainer and plots the image and explaination.
    '''

    explained_image, ax = plt.subplots()
    #instantiate explainer
    explainer = lime_image.LimeImageExplainer(random_state=1)
    #explain an image caste to double bc updated keras version, pass predictor
    explanation = explainer.explain_instance(image_array[0].astype('double'), model.predict)
    #show the image
    ax = plt.imshow(image_array[0])
    #get the mask for the prediction
    temp, mask = explanation.get_image_and_mask(model.predict(image_array.reshape((1,75,75,3))).argmax(axis=1)[0], positive_only=False, hide_rest=False)
    #show image and mask together
    ax = plt.title("Best Explaination of What Our Model is Seeing.")
    ax = plt.imshow(mark_boundaries(temp, mask))
    plt.axis("off")
    return explained_image

class_names=['mountain', 'street', 'glacier', 'buildings', 'sea', 'forest']
def pick_a_graph(model, image, color_coder, graph_type="bar"):
    if graph_type == "bar":
        fig_bar, ax = plt.subplots()
        ax = plt.bar([0,1,2,3,4,5], model.predict(image)[0])
        for i in color_coder:
            if color_coder[i] == model.predict(image)[0].max():
                ax[i].set_color("r")
                Best_guess = mpatches.Patch(color='red', label=round(color_coder[i],2))
            else:
                ax[i].set_color("b")
        plt.legend(handles=[Best_guess])
        plt.title("Model's Confidence as a Bar Plot")
        plt.ylabel("Model Confidence")
        plt.xlabel("Classes")
        ax = plt.xticks(ticks=[0,1,2,3,4,5],labels=[x.title() for x in class_names], size=10)
        return fig_bar
    elif graph_type == "donut":
        fig_pie, ax = plt.subplots(subplot_kw=dict(aspect="equal"))
        recipe = color_coder
        data = color_coder.values()
        wedges, texts = plt.pie(color_coder.values(), wedgeprops=dict(width=0.5), startangle=-40)
        bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=0.72)
        kw = dict(arrowprops=dict(arrowstyle="-"), bbox=bbox_props, zorder=0, va="center")

        ax.legend(wedges, [x.title() for x in class_names], title="Classes")
        for i, p in enumerate(wedges):
            ang = (p.theta2 - p.theta1)/2. + p.theta1
            y = np.sin(np.deg2rad(ang))
            x = np.cos(np.deg2rad(ang))
            horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))]
            connectionstyle = "angle,angleA=0,angleB={}".format(ang)
            kw["arrowprops"].update({"connectionstyle": connectionstyle})
            ax.annotate(round(color_coder[i],2), xy=(x, y), xytext=(1.35*np.sign(x), 1.4*y),
                        horizontalalignment=horizontalalignment, **kw)
        ax.set_title("Model's Confidence as a Pie Chart")
        return fig_pie
    elif graph_type == "lollipop":
        # creating an empty chart
        fig_lollipop, ax = plt.subplots()
        # plotting using plt.stem
        ax = plt.stem([0,1,2,3,4,5], model.predict(image)[0]
                  ,use_line_collection=True, basefmt=' ')
        plt.title("Model's Confidence as a Lollipop Graph")
        plt.ylabel("Model Confidence")
        plt.xlabel("Classes")
        ax = plt.xticks(ticks=[0,1,2,3,4,5],labels=[x.title() for x in class_names])
        return fig_lollipop
    elif graph_type == "scatter":
        fig_scatter= plt.figure()
        ax1=fig_scatter.add_subplot(111)
        X=[0,1,2,3,4,5]
        y=model.predict(image)[0]
        for i in color_coder:
            if color_coder[i] == y.max():
                ax1.scatter([i], y[i], c ='r', marker="*", s=750)
                ax1.scatter(np.delete(X,i), np.delete(y,i))
                Best_guess = mpatches.Patch(color='red', label=round(color_coder[i],2))
        plt.legend(handles=[Best_guess])
        plt.title("Model's Confidence as a Scatter Plot")
        plt.ylabel("Model Confidence")
        plt.xlabel("Classes")
        ax1= plt.xticks(ticks=X,labels=[x.title() for x in class_names])
        return fig_scatter

# Start of the website
# Loading in the Model
model = tensorflow.keras.models.load_model("Models/3rd-color_comp_model.h5")
page_image=("Sample_Pictures/Icon.jpg")
st.set_page_config(page_title="Image Classification",page_icon=page_image,layout="wide")
# Creating a centered layout
_, center_,_ = st.columns([1,2,1])
with center_:
    st.markdown("""
             # Intel Nature Image Predictor
             """
             )
    st.markdown(" ## **Data found on Kaggle**:")
    st.caption("https://www.kaggle.com/puneet6060/intel-image-classification")

col1, col2, col3, col4, col5, col6 = st.columns(6)
with col1:
    st.image("Sample_Pictures/Buildings_Image.jpg", caption="Building Example", width= 250)
with col2:
    st.image("Sample_Pictures/Forest_Image.jpg", caption="Forest Example", width= 250)
with col3:
    st.image("Sample_Pictures/Glacier_Image.jpg", caption="Glacier Example", width= 250)
with col4:
    st.image("Sample_Pictures/Mountain_Image.jpg", caption="Mountain Example", width= 250)
with col5:
    st.image("Sample_Pictures/Sea_Image.jpg", caption="Sea Example", width= 250)
with col6:
    st.image("Sample_Pictures/Street_Image.jpg", caption="Street Example", width= 250)

st.markdown("## This image classifier web app will predict which of 6 classes your uploaded Image should be.")
st.markdown("## Today we will be showing you how this Image performs in our model.")

file = st.file_uploader("Please upload an image file", type=["jpg","png"])
#
if file is None:
    st.text("Please select an Image from the Kaggle Dataset to be classified")
else:
    image = Image.open(file)
    prediction, new_image = import_and_predict(image, model)
    explained_image=image_expl(model, new_image)
    color_coder=dict(enumerate(prediction,0))

    fig_bar, ax = plt.subplots()
    plt.title("Model's Probablity for Class of Given Image")
    ax=plt.bar([0,1,2,3,4,5], prediction, edgecolor='black')
    for i in color_coder:
        if color_coder[i] == prediction.max():
            ax[i].set_color("r")
            Best_guess = mpatches.Patch(color='red', label=round(color_coder[i],2))
        else:
            ax[i].set_color("b")
    plt.legend(handles=[Best_guess])
    ax=plt.xticks(ticks=[0,1,2,3,4,5],labels=['mountain', 'street', 'glacier', 'buildings', 'sea', 'forest'])
    cola , colb, colc = st.columns([1,1,1])
    with cola:
        st.image(image,caption="Original Image", width=500)
    with colb:
        st.image(new_image,caption="Color Compressed Image", width=500)
    with colc:
        choice = st.selectbox("Graph Selector", ["Bar", "Donut", "Lollipop", "Scatter"])
        result_fig=pick_a_graph(model, new_image, color_coder, choice.lower())
        st.pyplot(result_fig)

# Lime required Section
    _,final,_ =st.columns([1,2,1])
    with final:
        if st.button('How did the Model do that!?!',  help="Click Me to Find out!"):
            st.pyplot(explained_image)


import asyncio
from asyncio import Future

import reactivex
from reactivex import Observable
from reactivex import operators as ops
from reactivex.scheduler.eventloop import AsyncIOScheduler

def to_async_iterable():
    def _to_async_iterable(source: Observable):
        class AIterable:
            def __aiter__(self):
                class AIterator:
                    def __init__(self):
                        self.notifications = []
                        self.future = Future()

                        source.pipe(ops.materialize()).subscribe(self.on_next)

                    def feeder(self):
                        if not self.notifications or self.future.done():
                            return

                        notification = self.notifications.pop(0)
                        dispatch = {
                            "N": lambda: self.future.set_result(notification.value),
                            "E": lambda: self.future.set_exception(
                                notification.exception
                            ),
                            "C": lambda: self.future.set_exception(StopAsyncIteration),
                        }

                        dispatch[notification.kind]()

                    def on_next(self, notification):
                        self.notifications.append(notification)
                        self.feeder()

                    async def __anext__(self):
                        self.feeder()

                        value = await self.future
                        self.future = Future()
                        return value

                return AIterator()

        return AIterable()

    return _to_async_iterable

async def go(loop):
    scheduler = AsyncIOScheduler(loop)

    ai = reactivex.range(0, 10, scheduler=scheduler).pipe(to_async_iterable())
    async for x in ai:
        print("got %s" % x)

def main():
    loop = asyncio.get_event_loop()
    loop.run_until_complete(go(loop))

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
# @Time    : 2019/8/7 17:09
# @Author  : GUO Huimin
# @Email   : guohuimin2619@foxmail.com
# @FileName: dicom-main.py

from pathlib import Path
import SimpleITK as stk
import matplotlib.pyplot as plt
import numpy as np

plt.ion()

class readDICOM():
    def __init__(self, ImagePath):
        '''

        :param ImagePath: string, single DICOM image path.
        '''
        self.ImagePath = ImagePath

    def readDICOM(self):
        reader = stk.ImageSeriesReader()
        dicom_names = reader.GetGDCMSeriesFileNames(self.ImagePath)
        reader.SetFileNames(dicom_names)
        image = reader.Execute()
        # [z, y, x]
        image_numpy = stk.GetArrayFromImage(image)
        origin = image.GetOrigin()  # x, y, z
        spacing = image.GetSpacing()  # x, y, z
        # [z,y,x]
        image_origin = np.array(list(reversed(origin)))
        # [z,y,x]
        image_spacing = np.array(list(reversed(spacing)))
        return image_numpy, image_origin, image_spacing

    def showDICOM(self):
        reader = stk.ImageSeriesReader()
        dicom_names = reader.GetGDCMSeriesFileNames(self.ImagePath)
        reader.SetFileNames(dicom_names)
        image = reader.Execute()
        # [z, y, x]
        image_numpy = stk.GetArrayFromImage(image)
        print(f"The shape of the current DICOM image is {image_numpy.shape}")
        if len(image_numpy.shape) == 3:
            slice_size = image_numpy.shape[0]
            for slice in range(slice_size):
                plt.imshow(image_numpy[slice, :, :], cmap='gray')
                plt.pause(pow(0.1, 8))
                plt.ioff()
                plt.show()
        if len(image_numpy.shape) == 4:
            Image_tmp = image_numpy.squeeze()
            self.showDICOM(Image_tmp)

if __name__ == '__main__':
    ImagePath = str(Path('D:/DICOM'))
    readDICOM = readDICOM(ImagePath)
    image_numpy, image_origin, image_spacing = readDICOM.readDICOM()
    readDICOM.showDICOM()


from pyexpat import model
from django.shortcuts import get_object_or_404, render, redirect
from django.http import HttpResponse
from django.views.generic import ListView, CreateView, UpdateView, DetailView
from django.db.models import Func, F
from django.core.paginator import Paginator

from books.models import BookItem, Publisher
from clothes.models import ClothesItem
from electronics.models import ElectronicItem, Laptop, MobilePhone, Tivi
from shoes.models import ShoesItem

# Create your views here.

class HomeView(ListView):
    context_object_name = 'bookItems'
    queryset = BookItem.objects.filter(status=True)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['electronicsItems'] = ElectronicItem.objects.filter(
            status=True)
        context['shoesItems'] = ShoesItem.objects.filter(status=True)
        context['ShoesItems'] = ShoesItem.objects.filter(status=True)
        return context

    template_name = 'home.html'
class CategoryElectronicView(ListView):
    context_object_name = 'electronicItems'

    def get_queryset(self):
        electronicItems = ElectronicItem.objects.filter(status=True)
        _manufacturer = self.request.GET.get('manufacturer')
        _price = self.request.GET.get('price')
        _sort = self.request.GET.get('sort')
        _page = self.request.GET.get('page')
        if _manufacturer:
            electronicItems = electronicItems.filter(
                electronic__manufacturer=_manufacturer)
        if _price:
            if _price == 'duoi5tr':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__lt=5000000)
            elif _price == '5trden10tr':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=5000000, priceBought__lt=10000000)
            elif _price == '10trden15tr':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=10000000, priceBought__lt=15000000)
            elif _price == '15trden25tr':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=15000000, priceBought__lt=25000000)
            elif _price == 'tren25tr':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=25000000)
        if _sort:
            if _sort == 'low-to-high':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('priceBought')
            elif _sort == 'high-to-low':
                electronicItems = electronicItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('-priceBought')

        paginator = Paginator(electronicItems, 8)
        electronicItems = paginator.get_page(_page)

        return electronicItems

    template_name = 'category_electronic.html'

class CategoryElectronicDetailView(DetailView):
    model = ElectronicItem

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['sameItems'] = ElectronicItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?')
        print(ElectronicItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?'))
        return context

    template_name = 'electronic_detail.html'

class CategoryBookView(ListView):
    context_object_name = 'bookItems'

    def get_queryset(self):
        bookItems = BookItem.objects.filter(status=True)
        _publisher = self.request.GET.get('publisher')
        _price = self.request.GET.get('price')
        _sort = self.request.GET.get('sort')
        _page = self.request.GET.get('page')
        if _publisher:
            bookItems = bookItems.filter(
                book__publisher__name=_publisher)
        if _price:
            if _price == 'duoi100':
                bookItems = bookItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__lt=100000)
            elif _price == '100den200':
                bookItems = bookItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=100000, priceBought__lte=200000)
            elif _price == 'tren200':
                bookItems = bookItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gt=200000)
        if _sort:
            if _sort == 'low-to-high':
                bookItems = bookItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('priceBought')
            elif _sort == 'high-to-low':
                bookItems = bookItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('-priceBought')

        paginator = Paginator(bookItems, 8)
        bookItems = paginator.get_page(_page)
        return bookItems

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['publishers'] = Publisher.objects.filter(status=True)
        return context
    template_name = 'category_book.html'

class CategoryBookDetailView(DetailView):
    model = BookItem

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['sameBookItems'] = BookItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?')
        return context
    template_name = 'book_detail.html'

class CategoryShoesView(ListView):
    context_object_name = 'ShoesItems'

    def get_queryset(self):
        ShoesItems = ShoesItem.objects.filter(status=True)

        _style = self.request.GET.get('style')
        _price = self.request.GET.get('price')
        _sort = self.request.GET.get('sort')
        _page = self.request.GET.get('page')

        if _style:
            ShoesItems = ShoesItems.filter(shoes__style=_style)

        if _price:
            if _price == 'duoi100000':
                ShoesItems = ShoesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__lt=100000)
            elif _price == '100000den200000':
                ShoesItems = ShoesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=100000, priceBought__lt=200000)
            elif _price == 'tren200000':
                ShoesItems = ShoesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('priceBought')
        if _sort:
            if _sort == 'low-to-high':
                ShoesItems = ShoesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('priceBought')
            elif _sort == 'high-to-low':
                ShoesItems = ShoesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('-priceBought')

        paginator = Paginator(ShoesItems, 8)
        ShoesItems = paginator.get_page(_page)

        return ShoesItems

    template_name = 'category_Shoes.html'

class CategoryShoesDetailView(DetailView):
    model = ShoesItem

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['sameItems'] = ShoesItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?')
        print(ShoesItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?'))
        return context

    template_name = 'Shoes_detail.html'

class CategoryClothesView(ListView):
    context_object_name = 'clothesItems'

    def get_queryset(self):
        clothesItems = ClothesItem.objects.filter(status=True)
        _price = self.request.GET.get('price')
        _sort = self.request.GET.get('sort')
        _page = self.request.GET.get('page')

        if _price:
            if _price == 'duoi100000':
                clothesItems = clothesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__lt=100000)
            elif _price == '100000den200000':
                clothesItems = clothesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=100000, priceBought__lt=200000)
            elif _price == 'tren200000':
                clothesItems = clothesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).filter(priceBought__gte=200000)

        if _sort:
            if _sort == 'low-to-high':
                clothesItems = clothesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('priceBought')
            elif _sort == 'high-to-low':
                clothesItems = clothesItems.annotate(priceBought=F(
                    'price')*((100-F('discount'))/100)).order_by('-priceBought')

        paginator = Paginator(clothesItems, 8)
        clothesItems = paginator.get_page(_page)

        return clothesItems

    template_name = 'category_clothes.html'

class CategoryClothesDetailView(DetailView):
    model = ClothesItem

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['sameItems'] = ClothesItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?')
        print(ClothesItem.objects.filter(
            status=True).exclude(barCode=self.kwargs['pk']).order_by('?'))
        return context

    template_name = 'clothes_detail.html'