import logging
import os
import pathlib
import shutil
import subprocess
import sys
from dataclasses import dataclass
from enum import Enum
from pathlib import Path, PosixPath
from typing import Dict, List, Optional, Tuple
from unittest import mock
from unittest.mock import MagicMock, patch
from uuid import uuid4

import pytest
from _pytest.capture import CaptureFixture
from azureml._restclient.constants import RunStatus
from azureml.core import RunConfiguration, ScriptRunConfig, Workspace
from azureml.data.azure_storage_datastore import AzureBlobDatastore
from azureml.data.dataset_consumption_config import DatasetConsumptionConfig
from azureml.train.hyperdrive import HyperDriveConfig

import health.azure.himl as himl
from health.azure.azure_util import (EXPERIMENT_RUN_SEPARATOR, WORKSPACE_CONFIG_JSON, get_most_recent_run,
                                     get_workspace, is_running_in_azure_ml)
from health.azure.datasets import DatasetConfig, _input_dataset_key, _output_dataset_key, get_datastore
from testazure.test_data.make_tests import render_environment_yaml, render_test_script
from testazure.util import DEFAULT_DATASTORE, change_working_directory, check_config_json, repository_root

INEXPENSIVE_TESTING_CLUSTER_NAME = "lite-testing-ds2"
EXPECTED_QUEUED = "This command will be run in AzureML:"
GITHUB_SHIBBOLETH = "GITHUB_RUN_ID"  # https://docs.github.com/en/actions/reference/environment-variables

logger = logging.getLogger('test.health.azure')
logger.setLevel(logging.DEBUG)

# region Small fast local unit tests

@pytest.mark.fast
def test_submit_to_azure_if_needed_returns_immediately() -> None:
    """
    Test that himl.submit_to_azure_if_needed can be called, and returns immediately.
    """
    with mock.patch("sys.argv", ["", "--azureml"]):
        with pytest.raises(Exception) as ex:
            himl.submit_to_azure_if_needed(
                aml_workspace=None,
                workspace_config_file=None,
                entry_script=Path(__file__),
                compute_cluster_name="foo",
                snapshot_root_directory=Path(__file__).parent)
        # N.B. This assert may fail when run locally since we may find a workspace_config_file through the call to
        # _find_file(CONDA_ENVIRONMENT_FILE) in submit_to_azure_if_needed
        if _is_running_in_github_pipeline():
            assert "No workspace config file given, nor can we find one" in str(ex)
    with mock.patch("sys.argv", [""]):
        result = himl.submit_to_azure_if_needed(
            entry_script=Path(__file__),
            compute_cluster_name="foo",
            conda_environment_file=Path("env.yml"))
        assert isinstance(result, himl.AzureRunInfo)
        assert not result.is_running_in_azure_ml
        assert result.run is None

def _is_running_in_github_pipeline() -> bool:
    """
    :return: Is the test running in a pipeline/action on GitHub, i.e. not locally?
    """
    return GITHUB_SHIBBOLETH in os.environ

@pytest.mark.fast
@patch("health.azure.himl.Run")
def test_write_run_recovery_file(mock_run: mock.MagicMock) -> None:
    # recovery file does not exist:
    mock_run.id = uuid4().hex
    mock_run.experiment.name = uuid4().hex
    expected_run_recovery_id = mock_run.experiment.name + EXPERIMENT_RUN_SEPARATOR + mock_run.id
    recovery_file = Path(himl.RUN_RECOVERY_FILE)
    if recovery_file.exists():
        recovery_file.unlink()
    himl._write_run_recovery_file(mock_run)
    recovery_file_text = Path(himl.RUN_RECOVERY_FILE).read_text()
    assert expected_run_recovery_id == recovery_file_text
    # recovery file exists from above:
    mock_run.id = uuid4().hex
    mock_run.experiment.name = uuid4().hex
    himl._write_run_recovery_file(mock_run)
    recovery_file_text = Path(himl.RUN_RECOVERY_FILE).read_text()
    assert expected_run_recovery_id != recovery_file_text

@pytest.mark.fast
@patch("azureml.data.OutputFileDatasetConfig")
@patch("health.azure.himl.DatasetConsumptionConfig")
@patch("health.azure.himl.Workspace")
@patch("health.azure.himl.DatasetConfig")
def test_to_datasets(
        mock_dataset_config: mock.MagicMock,
        mock_workspace: mock.MagicMock,
        mock_dataset_consumption_config: mock.MagicMock,
        mock_output_file_dataset_config: mock.MagicMock) -> None:
    def to_input_dataset(workspace: Workspace, dataset_index: int, ) -> DatasetConsumptionConfig:
        return mock_dataset_consumption_config

    def to_output_dataset(workspace: Workspace, dataset_index: int, ) -> DatasetConsumptionConfig:
        return mock_output_file_dataset_config

    mock_dataset_consumption_config.name = "A Consumption Config"
    mock_output_file_dataset_config.name = "An Output File Dataset Config"
    mock_dataset_config.to_input_dataset = to_input_dataset
    mock_dataset_config.to_output_dataset = to_output_dataset
    cleaned_input_datasets = [mock_dataset_config]
    # TODO: There's something wrong here. We feed 2 datasets in, but only get one out. When creating the dictionary,
    # we should check if the names are unique. In practice, this is unlikely to happen because the names are
    # INPUT_0, INPUT_1, etc
    cleaned_output_datasets = [mock_dataset_config, mock_dataset_config]
    inputs, outputs = himl.convert_himl_to_azureml_datasets(
        cleaned_input_datasets=cleaned_input_datasets,
        cleaned_output_datasets=cleaned_output_datasets,
        workspace=mock_workspace)
    assert len(inputs) == 1
    assert len(outputs) == 1
    assert inputs[mock_dataset_consumption_config.name] == mock_dataset_consumption_config
    assert outputs[mock_output_file_dataset_config.name] == mock_output_file_dataset_config

@pytest.mark.fast
@patch("health.azure.himl.register_environment")
@patch("health.azure.himl.create_python_environment")
@patch("health.azure.himl.Workspace")
def test_create_run_configuration_fails(
        mock_workspace: mock.MagicMock,
        _: mock.MagicMock,
        __: mock.MagicMock,
) -> None:
    existing_compute_target = "this_does_exist"
    mock_workspace.compute_targets = {existing_compute_target: 123}
    with pytest.raises(ValueError) as e:
        himl.create_run_configuration(
            compute_cluster_name="b",
            workspace=mock_workspace)
    assert "One of the two arguments 'aml_environment_name' or 'conda_environment_file' must be given." == str(e.value)
    with pytest.raises(ValueError) as e:
        himl.create_run_configuration(
            conda_environment_file=Path(__file__),
            compute_cluster_name="b",
            workspace=mock_workspace)
    assert "Could not find the compute target b in the AzureML workspace" in str(e.value)
    assert existing_compute_target in str(e.value)

@pytest.mark.fast
@patch("health.azure.himl.DockerConfiguration")
@patch("health.azure.datasets.DatasetConfig.to_output_dataset")
@patch("health.azure.datasets.DatasetConfig.to_input_dataset")
@patch("health.azure.himl.Environment.get")
@patch("health.azure.himl.Workspace")
def test_create_run_configuration(
        mock_workspace: mock.MagicMock,
        mock_environment_get: mock.MagicMock,
        mock_to_input_dataset: mock.MagicMock,
        mock_to_output_dataset: mock.MagicMock,
        mock_docker_configuration: mock.MagicMock,
) -> None:
    existing_compute_target = "this_does_exist"
    mock_env_name = "Mock Env"
    mock_environment_get.return_value = mock_env_name
    mock_workspace.compute_targets = {existing_compute_target: 123}
    aml_input_dataset = MagicMock()
    aml_input_dataset.name = "dataset_in"
    aml_output_dataset = MagicMock()
    aml_output_dataset.name = "dataset_out"
    mock_to_input_dataset.return_value = aml_input_dataset
    mock_to_output_dataset.return_value = aml_output_dataset
    run_config = himl.create_run_configuration(
        workspace=mock_workspace,
        compute_cluster_name=existing_compute_target,
        aml_environment_name="foo",
        num_nodes=10,
        max_run_duration="1h",
        input_datasets=[DatasetConfig(name="input1")],
        output_datasets=[DatasetConfig(name="output1")],
        docker_shm_size="2g"
    )
    assert isinstance(run_config, RunConfiguration)
    assert run_config.target == existing_compute_target
    assert run_config.environment == mock_env_name
    assert run_config.node_count == 10
    assert run_config.mpi.node_count == 10
    assert run_config.max_run_duration_seconds == 60 * 60
    assert run_config.data == {"dataset_in": aml_input_dataset}
    assert run_config.output_data == {"dataset_out": aml_output_dataset}
    mock_docker_configuration.assert_called_once()
    run_config = himl.create_run_configuration(
        workspace=mock_workspace,
        compute_cluster_name=existing_compute_target,
        aml_environment_name="foo",
    )
    assert run_config.max_run_duration_seconds is None
    assert run_config.mpi.node_count == 1
    assert not run_config.data
    assert not run_config.output_data

@pytest.mark.fast
def test_invalid_entry_script(tmp_path: Path) -> None:
    snapshot_dir = tmp_path / uuid4().hex
    snapshot_dir.mkdir(exist_ok=False)
    ok_entry_script = snapshot_dir / "entry_script.py"
    ok_entry_script.write_text("print('hello world')\n")

    run_config = himl.create_script_run(
        entry_script=ok_entry_script,
        snapshot_root_directory=snapshot_dir,
        script_params=[])
    assert run_config.script == str(ok_entry_script.relative_to(snapshot_dir))

    problem_entry_script_dir = tmp_path / uuid4().hex
    problem_entry_script_dir.mkdir(exist_ok=False)
    problem_entry_script = problem_entry_script_dir / "entry_script.py"
    problem_entry_script.write_text("print('hello world')\n")

    with pytest.raises(ValueError) as e:
        himl.create_script_run(
            entry_script=problem_entry_script,
            snapshot_root_directory=snapshot_dir,
            script_params=[])
    assert "entry script must be inside of the snapshot root directory" in str(e)

    with mock.patch("sys.argv", ["foo"]):
        script_run = himl.create_script_run()
        assert script_run.source_directory == str(Path.cwd())
        assert script_run.script == "foo"
        assert script_run.arguments == []

    # Entry scripts where the path is not absolute should be left unchanged
    script_run = himl.create_script_run(entry_script="some_string", script_params=["--foo"])
    assert script_run.script == "some_string"
    assert script_run.arguments == ["--foo"]

@pytest.mark.fast
def test_get_script_params() -> None:
    expected_params = ["a string"]
    assert expected_params == himl._get_script_params(expected_params)
    with mock.patch("sys.argv", ["", "a string", "--azureml"]):
        assert expected_params == himl._get_script_params()
    with mock.patch("sys.argv", ["", "a string"]):
        assert expected_params == himl._get_script_params()

@pytest.mark.fast
@patch("health.azure.azure_util.is_running_in_azure_ml")
def test_get_workspace_no_config(
        mock_is_running_in_azure: mock.MagicMock,
        tmp_path: Path) -> None:
    """
    Test if the workspace config path setting is ignored if a workspace is already given, and there is no config.json
    file in the current directory.
    """
    mock_is_running_in_azure.return_value = False
    with change_working_directory(tmp_path):
        with pytest.raises(ValueError) as ex:
            with mock.patch("sys.argv", ["", "--azureml"]):
                himl.submit_to_azure_if_needed(compute_cluster_name="foo")
        assert "No workspace config file given" in str(ex)

@pytest.mark.fast
@patch("health.azure.himl.Run")
@patch("health.azure.himl.Workspace")
@patch("health.azure.himl._generate_azure_datasets")
@patch("health.azure.himl.RUN_CONTEXT")
def test_submit_to_azure_if_needed_azure_return(
        mock_run_context: mock.MagicMock,
        mock_generate_azure_datasets: mock.MagicMock,
        mock_workspace: mock.MagicMock,
        mock_run: mock.MagicMock) -> None:
    """
    When running in AzureML, the call to submit_to_azure_if_needed should return immediately, without trying to
    submit a new job.
    """
    # The presence of the "experiment" flag is the trigger to recognize an AzureML run.
    mock_run_context.experiment = mock.MagicMock(workspace=mock_workspace)
    # This import needs to be local, after mocking the RUN_CONTEXT
    import health
    assert is_running_in_azure_ml(health.azure.himl.RUN_CONTEXT)
    expected_run_info = himl.AzureRunInfo(
        run=mock_run,
        input_datasets=[],
        output_datasets=[],
        is_running_in_azure_ml=True,
        output_folder=Path.cwd(),
        logs_folder=Path.cwd())
    mock_generate_azure_datasets.return_value = expected_run_info
    with mock.patch("sys.argv", ["", "--azureml"]):
        run_info = himl.submit_to_azure_if_needed(
            aml_workspace=mock_workspace,
            entry_script=Path(__file__),
            compute_cluster_name="foo",
            conda_environment_file=Path("env.yml"))
    assert run_info == expected_run_info

@pytest.mark.fast
@patch("health.azure.himl.DatasetConfig")
@patch("health.azure.himl.RUN_CONTEXT")
def test_generate_azure_datasets(
        mock_run_context: mock.MagicMock,
        mock_dataset_config: mock.MagicMock) -> None:
    mock_run_context.input_datasets = {}
    mock_run_context.output_datasets = {}
    for i in range(4):
        mock_run_context.input_datasets[_input_dataset_key(i)] = f"input_{i}"
        mock_run_context.output_datasets[_output_dataset_key(i)] = f"output_{i}"
    run_info = himl._generate_azure_datasets(
        cleaned_input_datasets=[mock_dataset_config] * 2,
        cleaned_output_datasets=[mock_dataset_config] * 3)
    assert run_info.is_running_in_azure_ml
    assert len(run_info.input_datasets) == 2
    assert len(run_info.output_datasets) == 3
    for i, d in enumerate(run_info.input_datasets):
        assert isinstance(d, Path)
        assert str(d) == f"input_{i}"
    for i, d in enumerate(run_info.output_datasets):
        assert isinstance(d, Path)
        assert str(d) == f"output_{i}"

@pytest.mark.fast
def test_append_to_amlignore(tmp_path: Path) -> None:
    # If there is no .amlignore file before the test, there should be none afterwards
    amlignore_path = tmp_path / Path(uuid4().hex)
    with himl.append_to_amlignore(
            amlignore=amlignore_path,
            lines_to_append=["1st line", "2nd line"]):
        amlignore_text = amlignore_path.read_text()
    assert "1st line\n2nd line" == amlignore_text
    assert not amlignore_path.exists()

    # If there is no .amlignore file before the test, and there are no lines to append, then there should be no
    # .amlignore file during the test
    amlignore_path = tmp_path / Path(uuid4().hex)
    with himl.append_to_amlignore(
            amlignore=amlignore_path,
            lines_to_append=[]):
        amlignore_exists_during_test = amlignore_path.exists()
    assert not amlignore_exists_during_test
    assert not amlignore_path.exists()

    # If there is an empty .amlignore file before the test, it should be there afterwards
    amlignore_path = tmp_path / Path(uuid4().hex)
    amlignore_path.touch()
    with himl.append_to_amlignore(
            amlignore=amlignore_path,
            lines_to_append=["1st line", "2nd line"]):
        amlignore_text = amlignore_path.read_text()
    assert "1st line\n2nd line" == amlignore_text
    assert amlignore_path.exists()
    assert amlignore_path.read_text() == ""

    # If there is a .amlignore file before the test, it should be identical afterwards
    amlignore_path = tmp_path / Path(uuid4().hex)
    amlignore_path.write_text("0th line")
    with himl.append_to_amlignore(
            amlignore=amlignore_path,
            lines_to_append=["1st line", "2nd line"]):
        amlignore_text = amlignore_path.read_text()
    assert "0th line\n1st line\n2nd line" == amlignore_text
    amlignore_text = amlignore_path.read_text()
    assert "0th line" == amlignore_text

class TestTagOption(Enum):
    TAGS = 1  # Set the tags parameter
    NO_TAGS = 2  # Set the tags parameter to None
    ARGS = 3  # script_run_config is a ScriptRunConfig with arguments
    HYPER = 4  # script_run_config is a HyperDriveConfig with arguments

@pytest.mark.fast
@pytest.mark.parametrize("wait_for_completion", [True, False])
@pytest.mark.parametrize("set_tags", [
    TestTagOption.TAGS, TestTagOption.NO_TAGS, TestTagOption.ARGS, TestTagOption.HYPER])
@patch("health.azure.himl.Run")
@patch("health.azure.himl.Experiment")
@patch("health.azure.himl.Workspace")
def test_submit_run(mock_workspace: mock.MagicMock,
                    mock_experiment: mock.MagicMock,
                    mock_run: mock.MagicMock,
                    wait_for_completion: bool,
                    set_tags: TestTagOption,
                    capsys: CaptureFixture
                    ) -> None:
    mock_experiment.return_value.submit.return_value = mock_run
    mock_run.get_status.return_value = RunStatus.COMPLETED
    mock_run.status = RunStatus.COMPLETED
    mock_run.get_children.return_value = []
    mock_tags = {'tag1': '1', 'tag2': '2'}
    mock_arguments = ["--arg1", "--message=\\'Hello World :-)\\'"]
    # Pretend to be a ScriptRunConfig
    mock_script_run_config = mock.MagicMock(spec=[ScriptRunConfig])
    mock_script_run_config.arguments = None
    tags: Optional[Dict[str, str]] = None
    if set_tags == TestTagOption.TAGS:
        tags = mock_tags
    else:
        if set_tags == TestTagOption.ARGS:
            mock_script_run_config.arguments = mock_arguments
        elif set_tags == TestTagOption.HYPER:
            # Pretend to be a HyperDriveConfig
            mock_script_run_config = mock.MagicMock(spec=[HyperDriveConfig])
            mock_script_run_config.run_config = MagicMock()
            mock_script_run_config.run_config.arguments = mock_arguments
    an_experiment_name = "an experiment"
    _ = himl.submit_run(
        workspace=mock_workspace,
        experiment_name=an_experiment_name,
        script_run_config=mock_script_run_config,
        tags=tags,
        wait_for_completion=wait_for_completion,
        wait_for_completion_show_output=True,
    )
    out, err = capsys.readouterr()
    assert not err
    assert "Successfully queued run" in out
    assert "Experiment name and run ID are available" in out
    assert "Experiment URL" in out
    assert "Run URL" in out
    if set_tags == TestTagOption.TAGS:
        mock_run.set_tags.assert_called_once_with(mock_tags)
    elif set_tags == TestTagOption.NO_TAGS:
        mock_run.set_tags.assert_called_once_with(None)
    else:
        mock_run.set_tags.assert_called_once_with({"commandline_args": " ".join(mock_arguments)})
    if wait_for_completion:
        assert "Waiting for the completion of the AzureML run" in out
        assert "AzureML completed" in out
        mock_run.get_status.return_value = RunStatus.UNAPPROVED
        mock_run.status = RunStatus.UNAPPROVED
        with pytest.raises(ValueError) as e:
            _ = himl.submit_run(
                workspace=mock_workspace,
                experiment_name=an_experiment_name,
                script_run_config=mock_script_run_config,
                wait_for_completion=wait_for_completion,
                wait_for_completion_show_output=True,
            )
        error_msg = str(e.value)
        out, err = capsys.readouterr()
        assert "runs failed" in error_msg
        assert "AzureML completed" not in out

@pytest.mark.fast
def test_str_to_path(tmp_path: Path) -> None:
    assert himl._str_to_path(tmp_path) == tmp_path
    assert himl._str_to_path(str(tmp_path)) == tmp_path

# endregion Small fast local unit tests

# region Elevate to AzureML unit tests

here = pathlib.Path(__file__).parent.resolve()

class RunTarget(Enum):
    LOCAL = 1
    AZUREML = 2

def spawn_and_monitor_subprocess(process: str, args: List[str],
                                 cwd: Path, env: Dict[str, str]) -> Tuple[int, List[str]]:
    """
    Helper function to spawn and monitor subprocesses.
    :param process: The name or path of the process to spawn.
    :param args: The args to the process.
    :param cwd: Working directory.
    :param env: The environment variables for the process (default is the environment variables of the parent).
    :return: Return code after the process has finished, and the list of lines that were written to stdout by the
    subprocess.
    """
    p = subprocess.Popen(
        [process] + args,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        cwd=cwd,
        env=env)

    # Read and print all the lines that are printed by the subprocess
    stdout_lines = [line.decode('UTF-8').strip() for line in p.stdout]  # type: ignore

    logging.info("~~~~~~~~~~~~~~")
    logging.info("\n".join(stdout_lines))
    logging.info("~~~~~~~~~~~~~~")

    return p.wait(), stdout_lines

def render_and_run_test_script(path: Path,
                               run_target: RunTarget,
                               extra_options: Dict[str, str],
                               extra_args: List[str],
                               expected_pass: bool,
                               suppress_config_creation: bool = False) -> str:
    """
    Prepare test scripts, submit them, and return response.

    :param path: Where to build the test scripts.
    :param run_target: Where to run the script.
    :param extra_options: Extra options for template rendering.
    :param extra_args: Extra command line arguments for calling script.
    :param expected_pass: Whether this call to subprocess is expected to be successful.
    :param suppress_config_creation: (Optional, defaults to False) do not create a config.json file if none exists
    :return: Either response from spawn_and_monitor_subprocess or run output if in AzureML.
    """
    # target hi-ml-azure package version, if specified in an environment variable.
    version = ""
    run_requirements = False

    himl_wheel_filename = os.getenv('HIML_AZURE_WHEEL_FILENAME', '')
    himl_test_pypi_version = os.getenv('HIML_AZURE_TEST_PYPI_VERSION', '')
    himl_pypi_version = os.getenv('HIML_AZURE_PYPI_VERSION', '')

    if not himl_wheel_filename:
        # If testing locally, can build the package into the "dist" folder and use that.
        dist_folder = Path.cwd().joinpath('dist')
        whls = sorted(list(dist_folder.glob('*.whl')))
        if len(whls) > 0:
            last_whl = whls[-1]
            himl_wheel_filename = str(last_whl)

    if himl_wheel_filename:
        # Testing against a private wheel.
        himl_wheel_filename_full_path = str(Path(himl_wheel_filename).resolve())
        extra_options['private_pip_wheel_path'] = f'Path("{himl_wheel_filename_full_path}")'
        print(f"Added private_pip_wheel_path: {himl_wheel_filename_full_path} option")
    elif himl_test_pypi_version:
        # Testing against test.pypi, add this as the pip_extra_index_url, and set the version.
        extra_options['pip_extra_index_url'] = "https://test.pypi.org/simple/"
        version = himl_test_pypi_version
        print(f"Added test.pypi: {himl_test_pypi_version} option")
    elif himl_pypi_version:
        # Testing against pypi, set the version.
        version = himl_pypi_version
        print(f"Added pypi: {himl_pypi_version} option")
    else:
        # No packages found, so copy the src folder as a fallback
        src_path = repository_root() / "hi-ml-azure" / "src"
        if src_path.is_dir():
            shutil.copytree(src=src_path / 'health', dst=path / 'health')
            run_requirements = True
            print("Copied 'src' folder.")

    environment_yaml_path = path / "environment.yml"
    render_environment_yaml(environment_yaml_path, version, run_requirements)

    entry_script_path = path / "test_script.py"
    workspace_config_file_arg = "None" if suppress_config_creation else "WORKSPACE_CONFIG_JSON"
    render_test_script(entry_script_path, extra_options, INEXPENSIVE_TESTING_CLUSTER_NAME, environment_yaml_path,
                       workspace_config_file_arg=workspace_config_file_arg)

    score_args = [str(entry_script_path)]
    if run_target == RunTarget.AZUREML:
        score_args.append("--azureml")
    score_args.extend(extra_args)

    env = dict(os.environ.items())

    def spawn() -> Tuple[int, List[str]]:
        code, stdout = spawn_and_monitor_subprocess(
            process=sys.executable,
            args=score_args,
            cwd=path,
            env=env)
        return code, stdout

    if suppress_config_creation:
        code, stdout = spawn()
    else:
        with check_config_json(path):
            code, stdout = spawn()
    assert code == 0 if expected_pass else 1, f"Expected the script to {'pass' if expected_pass else 'fail'}, but " \
                                              f"got a return code {code}"
    captured = "\n".join(stdout)

    if run_target == RunTarget.LOCAL or not expected_pass:
        assert EXPECTED_QUEUED not in captured
        return captured
    else:
        assert EXPECTED_QUEUED in captured
        with check_config_json(path):
            workspace = get_workspace(aml_workspace=None, workspace_config_path=path / WORKSPACE_CONFIG_JSON)

        run = get_most_recent_run(run_recovery_file=path / himl.RUN_RECOVERY_FILE,
                                  workspace=workspace)
        assert run.status == "Completed"
        log_root = path / "logs"
        log_root.mkdir(exist_ok=False)
        run.get_all_logs(destination=log_root)
        driver_log = log_root / "azureml-logs" / "70_driver_log.txt"
        log_text = driver_log.read_text()
        return log_text

@pytest.mark.parametrize("run_target", [RunTarget.LOCAL, RunTarget.AZUREML])
def test_invoking_hello_world_no_config(run_target: RunTarget, tmp_path: Path) -> None:
    """
    Test invoking rendered 'simple' / 'hello_world_template.txt' when there is no config file in the current working
    directory. This should pass fine for local runs, but fail when trying to submit to AzureML.
    :param run_target: Where to run the script.
    :param tmp_path: PyTest test fixture for temporary path.
    """
    message_guid = uuid4().hex
    extra_options = {
        'workspace_config_file': 'None',
        'args': 'parser.add_argument("-m", "--message", type=str, required=True, help="The message to print out")',
        'body': 'print(f"The message was: {args.message}")'
    }
    extra_args = [f"--message={message_guid}"]
    expected_output = f"The message was: {message_guid}"
    if run_target == RunTarget.LOCAL:
        output = render_and_run_test_script(tmp_path, run_target, extra_options, extra_args,
                                            expected_pass=True,
                                            suppress_config_creation=True)
        assert expected_output in output
    else:
        response = render_and_run_test_script(tmp_path, run_target, extra_options, extra_args,
                                              expected_pass=False,
                                              suppress_config_creation=True)
        assert "No workspace config file given" in response

@pytest.mark.parametrize("run_target", [RunTarget.LOCAL, RunTarget.AZUREML])
@pytest.mark.parametrize("use_package", [True, False])
def test_invoking_hello_world_config(run_target: RunTarget, use_package: bool, tmp_path: Path) -> None:
    """
    Test that invoking hello_world.py elevates itself to AzureML with config.json.
    Test against either the local src folder or a package. If running locally, ensure that there
    are no whl's in the dist folder, or that will be used.
    :param run_target: Local execution if True, else in AzureML.
    :param use_package: True to test against package, False to test against copy of src folder.
    :param tmp_path: PyTest test fixture for temporary path.
    """
    if not use_package and \
            not os.getenv('HIML_AZURE_WHEEL_FILENAME', '') and \
            not os.getenv('HIML_AZURE_TEST_PYPI_VERSION', '') and \
            not os.getenv('HIML_AZURE_PYPI_VERSION', ''):
        # Running locally, no need to duplicate this test.
        return

    message_guid = uuid4().hex
    extra_options = {
        'args': 'parser.add_argument("-m", "--message", type=str, required=True, help="The message to print out")',
        'body': 'print(f"The message was: {args.message}")'
    }
    extra_args = [f"--message={message_guid}"]
    if use_package:
        output = render_and_run_test_script(tmp_path, run_target, extra_options, extra_args, True)
    else:
        with mock.patch.dict(os.environ, {"HIML_AZURE_WHEEL_FILENAME": '',
                                          "HIML_AZURE_TEST_PYPI_VERSION": '',
                                          "HIML_AZURE_PYPI_VERSION": ''}):
            output = render_and_run_test_script(tmp_path, run_target, extra_options, extra_args, True)
    expected_output = f"The message was: {message_guid}"
    assert expected_output in output

@patch("health.azure.himl.submit_to_azure_if_needed")
def test_calling_script_directly(mock_submit_to_azure_if_needed: mock.MagicMock) -> None:
    with mock.patch("sys.argv", ["",
                                 "--workspace_config_file", "1",
                                 "--compute_cluster_name", "2",
                                 "--snapshot_root_directory", "3",
                                 "--entry_script", "4",
                                 "--conda_environment_file", "5"]):
        himl.main()
    assert mock_submit_to_azure_if_needed.call_args[1]["workspace_config_file"] == PosixPath("1")
    assert mock_submit_to_azure_if_needed.call_args[1]["compute_cluster_name"] == "2"
    assert mock_submit_to_azure_if_needed.call_args[1]["snapshot_root_directory"] == PosixPath("3")
    assert mock_submit_to_azure_if_needed.call_args[1]["entry_script"] == PosixPath("4")
    assert mock_submit_to_azure_if_needed.call_args[1]["conda_environment_file"] == PosixPath("5")

def test_invoking_hello_world_no_private_pip_fails(tmp_path: Path) -> None:
    """
    Test that invoking rendered 'simple' / 'hello_world_template.txt' raises a FileNotFoundError on
    invalid private_pip_wheel_path.
    :param tmp_path: PyTest test fixture for temporary path.
    """
    extra_options: Dict[str, str] = {}
    extra_args: List[str] = []
    with mock.patch.dict(os.environ, {"HIML_AZURE_WHEEL_FILENAME": 'not_a_known_file.whl'}):
        output = render_and_run_test_script(tmp_path, RunTarget.AZUREML, extra_options, extra_args, False)
    error_message_begin = "FileNotFoundError: Cannot add add_private_pip_wheel:"
    error_message_end = "not_a_known_file.whl, it is not a file."

    assert error_message_begin in output
    assert error_message_end in output

@pytest.mark.parametrize("run_target", [RunTarget.LOCAL, RunTarget.AZUREML])
def test_invoking_hello_world_env_var(run_target: RunTarget, tmp_path: Path) -> None:
    """
    Test that invoking rendered 'simple' / 'hello_world_template.txt' elevates itself to AzureML with config.json,
    and that environment variables are passed through.
    :param run_target: Where to run the script.
    :param tmp_path: PyTest test fixture for temporary path.
    """
    message_guid = uuid4().hex
    extra_options: Dict[str, str] = {
        'environment_variables': f"{{'message_guid': '{message_guid}'}}",
        'body': 'print(f"The message_guid env var was: {os.getenv(\'message_guid\')}")'
    }
    extra_args: List[str] = []
    output = render_and_run_test_script(tmp_path, run_target, extra_options, extra_args, True)
    expected_output = f"The message_guid env var was: {message_guid}"
    assert expected_output in output

def _create_test_file_in_blobstore(datastore: AzureBlobDatastore,
                                   filename: str, location: str, tmp_path: Path) -> str:
    # Create a dummy folder.
    dummy_data_folder = tmp_path / "dummy_data"
    dummy_data_folder.mkdir()

    # Create a dummy text file.
    dummy_txt_file = dummy_data_folder / filename
    message_guid = uuid4().hex
    dummy_txt_file.write_text(f"some test data: {message_guid}")

    # Upload dummy text file to blob storage
    datastore.upload_files(
        [str(dummy_txt_file.resolve())],
        relative_root=str(dummy_data_folder),
        target_path=location,
        overwrite=True,
        show_progress=True)

    dummy_txt_file_contents = dummy_txt_file.read_text()

    # Discard dummies
    dummy_txt_file.unlink()
    dummy_data_folder.rmdir()

    return dummy_txt_file_contents

@dataclass
class TestInputDataset:
    # Test file name. This will be populated with test data and uploaded to blob storage.
    filename: str
    # Name of container for this dataset in blob storage.
    blob_name: str
    # Local folder for this dataset when running locally.
    folder_name: Path
    # Contents of test file.
    contents: str = ""

@dataclass
class TestOutputDataset:
    # Name of container for this dataset in blob storage.
    blob_name: str
    # Local folder for this dataset when running locally or when testing after running in Azure.
    folder_name: Path

@pytest.mark.parametrize("run_target", [RunTarget.LOCAL, RunTarget.AZUREML])
def test_invoking_hello_world_datasets(run_target: RunTarget, tmp_path: Path) -> None:
    """
    Test that invoking rendered 'simple' / 'hello_world_template.txt' elevates itself to AzureML with config.json,
    and that datasets are mounted in all combinations.
    :param run_target: Where to run the script.
    :param tmp_path: PyTest test fixture for temporary path.
    """
    input_count = 4
    input_datasets = [TestInputDataset(
        filename=f"{uuid4().hex}.txt",
        blob_name=f"himl_dataset_test_input{i}",
        folder_name=tmp_path / f"local_dataset_test_input{i}")
        for i in range(0, input_count)]
    output_count = 3
    output_datasets = [TestOutputDataset(
        blob_name=f"himl_dataset_test_output{i}",
        folder_name=tmp_path / f"local_dataset_test_output{i}")
        for i in range(0, output_count)]

    # Get default datastore
    with check_config_json(tmp_path):
        workspace = get_workspace(aml_workspace=None,
                                  workspace_config_path=tmp_path / WORKSPACE_CONFIG_JSON)
        datastore: AzureBlobDatastore = get_datastore(workspace=workspace,
                                                      datastore_name=DEFAULT_DATASTORE)

    # Create dummy txt files, one for each item in input_datasets.
    for input_dataset in input_datasets:
        input_dataset.contents = _create_test_file_in_blobstore(
            datastore=datastore,
            filename=input_dataset.filename,
            location=input_dataset.blob_name,
            tmp_path=tmp_path)

        if run_target == RunTarget.LOCAL:
            # For running locally, download the test files from blobstore
            downloaded = datastore.download(
                target_path=input_dataset.folder_name,
                prefix=f"{input_dataset.blob_name}/{input_dataset.filename}",
                overwrite=True,
                show_progress=True)
            assert downloaded == 1

            # Check that the input file is downloaded
            downloaded_dummy_txt_file = input_dataset.folder_name / input_dataset.blob_name / input_dataset.filename
            # Check it has expected contents
            assert input_dataset.contents == downloaded_dummy_txt_file.read_text()

    if run_target == RunTarget.LOCAL:
        for output_dataset in output_datasets:
            output_blob_folder = output_dataset.folder_name / output_dataset.blob_name
            output_blob_folder.mkdir(parents=True)
    else:
        # Check that these files are not already in the output folders.
        for input_dataset in input_datasets:
            for output_dataset in output_datasets:
                downloaded = datastore.download(
                    target_path=str(output_dataset.folder_name),
                    prefix=f"{output_dataset.blob_name}/{input_dataset.filename}",
                    overwrite=True,
                    show_progress=True)
                assert downloaded == 0

    # Format input_datasets for use in script.
    input_file_names = [
        f'("{input_dataset.filename}", "{input_dataset.blob_name}", Path("{str(input_dataset.folder_name)}"))'
        for input_dataset in input_datasets]
    script_input_datasets = ',\n        '.join(input_file_names)

    # Format output_datasets for use in script.
    output_file_names = [
        f'("{output_dataset.blob_name}", Path("{str(output_dataset.folder_name)}"))'
        for output_dataset in output_datasets]
    script_output_datasets = ',\n        '.join(output_file_names)

    extra_options: Dict[str, str] = {
        'prequel': """
    target_folder = "foo"
        """,
        'ignored_folders': '[".config", ".mypy_cache", "hello_world_output"]',
        'default_datastore': f'"{DEFAULT_DATASTORE}"',
        'input_datasets': f"""[
            "{input_datasets[0].blob_name}",
            DatasetConfig(name="{input_datasets[1].blob_name}", datastore="{DEFAULT_DATASTORE}"),
            DatasetConfig(name="{input_datasets[2].blob_name}", datastore="{DEFAULT_DATASTORE}",
                          target_folder=target_folder),
            DatasetConfig(name="{input_datasets[3].blob_name}", datastore="{DEFAULT_DATASTORE}",
                          use_mounting=True),
        ]""",
        'output_datasets': f"""[
            "{output_datasets[0].blob_name}",
            DatasetConfig(name="{output_datasets[1].blob_name}", datastore="{DEFAULT_DATASTORE}"),
            DatasetConfig(name="{output_datasets[2].blob_name}", datastore="{DEFAULT_DATASTORE}",
                          use_mounting=False),
        ]""",
        'body': f"""
    input_datasets = [
        {script_input_datasets}
    ]
    output_datasets = [
        {script_output_datasets}
    ]
    for i, (filename, input_blob_name, input_folder_name) in enumerate(input_datasets):
        input_folder = run_info.input_datasets[i] or input_folder_name / input_blob_name
        for j, (output_blob_name, output_folder_name) in enumerate(output_datasets):
            output_folder = run_info.output_datasets[j] or output_folder_name / output_blob_name
            file = input_folder / filename
            shutil.copy(file, output_folder)
            print(f"Copied file: {{file.name}} from {{input_blob_name}} to {{output_blob_name}}")
        """
    }
    extra_args: List[str] = []
    output = render_and_run_test_script(tmp_path, run_target, extra_options, extra_args, True)

    for input_dataset in input_datasets:
        for output_dataset in output_datasets:
            expected_output = \
                f"Copied file: {input_dataset.filename} from {input_dataset.blob_name} to {output_dataset.blob_name}"
            assert expected_output in output

            if run_target == RunTarget.AZUREML:
                # If test ran in Azure, need to download the outputs to check them.
                downloaded = datastore.download(
                    target_path=str(output_dataset.folder_name),
                    prefix=f"{output_dataset.blob_name}/{input_dataset.filename}",
                    overwrite=True,
                    show_progress=True)
                assert downloaded == 1

            output_dummy_txt_file = output_dataset.folder_name / output_dataset.blob_name / input_dataset.filename
            assert input_dataset.contents == output_dummy_txt_file.read_text()

# endregion Elevate to AzureML unit tests


from time import sleep

from pyecore.resources import ResourceSet, URI
from pyecore.utils import DynamicEPackage, alias
from pyecore.resources.resource import HttpURI
from app.helpers.xmlresource import XMLResource
from pyecore.notification import EObserver
import uuid
from app.helpers.StringURI import StringURI
from io import BytesIO

class EnergySystemHandler:
    """Class to handle (load, read, and update) an ESDL Energy System"""

    def __init__(self, name=None):
        # create a resourceSet that hold the contents of the esdl.ecore model and the instances we use/create
        self.rset = ResourceSet()

        # Assign files with the .esdl extension to the XMLResource instead of default XMI
        self.rset.resource_factory['esdl'] = lambda uri: XMLResource(uri)

        # Read the esdl.ecore from the tmp folder
        esdl_model_resource = self.rset.get_resource(URI('tmp/esdl/esdl.ecore'))

        esdl_model = esdl_model_resource.contents[0]
        # print('Namespace: {}'.format(esdl_model.nsURI))
        self.rset.metamodel_registry[esdl_model.nsURI] = esdl_model

        # Create a dynamic model from the loaded esdl.ecore model, which we can use to build Energy Systems
        self.esdl = DynamicEPackage(esdl_model)

        # fix python buildin 'from' that is also used in ProfileElement as attribute
        # use 'start' instead of 'from' when using a ProfileElement
        alias('start', self.esdl.ProfileElement.findEStructuralFeature('from'))

        # have a nice __repr__ for some ESDL classes when printing ESDL objects (includes all Assets and EnergyAssets)
        self.esdl.Item.python_class.__repr__ = lambda x: '{}: ({})'.format(x.name, EnergySystemHandler.attr_to_dict(x))
        self.esdl.Carrier.python_class.__repr__ = lambda x: '{}: ({})'.format(x.name, EnergySystemHandler.attr_to_dict(x))
        self.esdl.Geometry.python_class.__repr__ = lambda x: '{}: ({})'.format(x.name, EnergySystemHandler.attr_to_dict(x))
        self.esdl.QuantityAndUnitType.python_class.__repr__ = lambda x: '{}: ({})'.format(x.id, EnergySystemHandler.attr_to_dict(x))
        self.esdl.QuantityAndUnitReference.python_class.__repr__ = lambda x: '{}: ({})'.format('QuantityAndUnitReference', EnergySystemHandler.attr_to_dict(x))
        self.esdl.KPI.python_class.__repr__ = lambda x: '{}: ({})'.format(x.name, EnergySystemHandler.attr_to_dict(x))
        self.esdl.ProfileElement.python_class.__repr__ = lambda x: 'ProfileElement ({})'.format(EnergySystemHandler.attr_to_dict(x))

        if name:
            self.name = name
            self.load_energy_system(name)

    # Creates a dict of all the attributes of an ESDL object
    @staticmethod
    def attr_to_dict(esdl_object):
        d = dict()
        d['esdlType'] = esdl_object.eClass.name
        for attr in dir(esdl_object):
            attr_value = esdl_object.eGet(attr)
            if attr_value is not None:
                d[attr] = attr_value
        return d

    # Creates a uuid: useful for generating unique IDs
    @staticmethod
    def generate_uuid():
        return str(uuid.uuid4())

    def load_energy_system(self, name):
        # load the ESDL file
        self.resource = self.rset.get_resource(URI(name))
        self.es = self.resource.contents[0]

    # Add Energy System Information
    def add_energy_system_information(self):
        esi = self.esdl.EnergySystemInformation(id='energy_system_information')
        self.es.energySystemInformation = esi

    def add_data_source(self, name, description):
        data_source = self.esdl.DataSource(id='data_source', name=name, description=description)
        self.es.dataSource = data_source

    # Add energy system information to the energy system if it is not there yet
    # Energy System information can be used to globally define the quantity and units of this system,
    # instead of defining them manually per KPI in each area: this fosters reuse (but is not necessary)
    def get_quantity_and_units(self):
        q_and_u = None

        if self.get_by_id('energy_system_information') is None:
            self.add_energy_system_information()

            q_and_u = self.esdl.QuantityAndUnits(id='quantity_and_units')
            self.es.energySystemInformation.quantityAndUnits = q_and_u

        else:
            q_and_u = self.get_by_id('quantity_and_units')

        return q_and_u

    # Add Measures object to Energy System
    def add_measures(self):
        # Create new Measures object
        measures = self.esdl.Measures(id='measures')
        self.es.instance[0].area.measures = measures

    # Append measure to Measures object
    def append_measure(self, measure):
        self.es.instance[0].area.measures.measure.append(measure)

    # Append asset measure to Measures object
    def append_asset_to_measure(self, measure, asset):
        measure.asset.append(asset)
        self.es.instance[0].area.measures.measure.append(measure)

    # Add KPIs object to Energy System
    def add_kpis(self):
        # create new KPIs object
        kpis = self.esdl.KPIs(id='kpis', description='KPIs')
        self.es.instance[0].area.KPIs = kpis

    # Create new KPI object
    def create_kpi(self, kpi_type, kpi_id, name, q_and_u):
        return getattr(self.esdl, kpi_type)(
            id=kpi_id,
            name=name,
            quantityAndUnit=q_and_u
        )

    # Add KPI to KPIs object
    def add_kpi(self, kpi):
        self.es.instance[0].area.KPIs.kpi.append(kpi)

    # Get a list of assets of a specific ESDL type in the main instance's area
    # def get_assets_of_type(self, esdl_type):
    #     assets = []
    #
    #     for current_asset in self.es.instance[0].area.asset:
    #         if isinstance(current_asset, esdl_type):
    #             assets.append(current_asset)
    #     return assets

    # Get a list of assets of a specific ESDL type in the specified area or asset
    def get_assets_of_type(self, area, esdl_type):
        assets = []

        for current_asset in area.asset:
            if isinstance(current_asset, esdl_type):
                assets.append(current_asset)

        return assets

    # Get a list of assets of a specific ESDL type in the specified area or asset
    # filtered by a specified attribute-value combination
    def get_assets_of_type_and_attribute_value(self, area, esdl_type, attribute, value):
        assets = []

        for current_asset in area.asset:
            if isinstance(current_asset, esdl_type):
                if str(getattr(current_asset, attribute)) == value:
                    assets.append(current_asset)
        return assets

    # Get a list of potentials of a specific ESDL type in the main instance's area
    def get_potentials_of_type(self, esdl_type):
        potentials = []

        for current_potential in self.es.instance[0].area.potential:
            if isinstance(current_potential, esdl_type):
                potentials.append(current_potential)
        return potentials

    # returns a generator of all assets or potentials of a specific type. Not only the ones defined in the main Instance's Area
    # e.g. QuantityAndUnits can be defined in the KPI of an Area or in the EnergySystemInformation object
    # this function returns all of them at once
    def get_all_instances_of_type(self, esdl_type):
        return esdl_type.allInstances()

    # Using this function you can query for objects by ID
    # After loading an ESDL-file, all objects that have an ID defines are stored in resource.uuid_dict automatically
    # Note: If you add things later to the resource, it won't be added automatically to this dictionary though.
    # Use get_by_id_slow() for that
    def get_by_id(self, id):
        if id in self.resource.uuid_dict:
            return self.resource.uuid_dict[id]
        else:
            return None

    # This function iterates over all the contents of the Energy System and is much slower than get_by_id()
    def get_by_id_slow(self, id):
        for child in self.es.eAllContents():
            if hasattr(child, 'id'):
                if child.id == id:
                    return child

    # create a readable list of the attributes of an ESDL class
    def get_asset_attribute(self, esdl_type, attribute):
        asset_data = []

        for current_asset in self.es.instance[0].area.asset:

            if isinstance(current_asset, esdl_type):
                asset_data.append({
                    'name': current_asset.name,  # name
                    'attribute': {
                        'key': attribute,
                        'value': getattr(current_asset, attribute)
                    }
                })

        return asset_data

    # create a readable list of the attributes of an ESDL class
    # (scoped to a specific area)
    def get_asset_attribute(self, area, esdl_type, attribute):
        asset_data = []

        for current_asset in area.asset:

            if isinstance(current_asset, esdl_type):
                asset_data.append({
                    'name': current_asset.name,  # name
                    'attribute': {
                        'key': attribute,
                        'value': getattr(current_asset, attribute)
                    }
                })

        return asset_data

    # returns a specific KPI by id, see also get_by_id for a faster method
    def get_kpi_by_id(self, id):
        for kpi in self.es.instance[0].area.KPIs.kpi:
            if kpi.id == id:
                return kpi

    # returns a specific KPI by name
    def get_kpi_by_name(self, name):
        for kpi in self.es.instance[0].area.KPIs.kpi:
            if kpi.name == name:
                return kpi

    # save the resource
    def save(self, filename):
        uri = URI(filename)
        fileresource = self.rset.create_resource(uri)
        # add the current energy system
        fileresource.append(self.es)
        # save the resource
        fileresource.save()
        # return the resource
        return fileresource

    # get the energy system as a XML String
    # does not change the 'active' resource
    # so save() will still save as a file
    def get_as_string(self):
        # to use strings as resources, we simulate a string as being a file
        uri = StringURI('tmp/anyname.esdl')
        # create the string resource
        stringresource = self.rset.create_resource(uri)
        # add the current energy system
        stringresource.append(self.es)
        # save the resource
        stringresource.save()
        # remove the temporary resource in the resource set
        self.rset.remove_resource(stringresource)
        # return the string
        return uri.getvalue()

    def get_as_stream(self):
        # to use strings as resources, we simulate a string as being a file
        uri = StringURI('tmp/anyname.esdl')
        # create the string resource
        stringresource = self.rset.create_resource(uri)
        # add the current energy system
        stringresource.append(self.es)
        # save the resource
        stringresource.save()
        # remove the temporary resource in the resource set
        self.rset.remove_resource(stringresource)
        # return the string
        return uri.get_stream()

    # load an EnergySystem from a string (using UTF-8 encoding)
    def load_from_string(self, string):
        uri = StringURI('loadfromstring', string)
        # this overrides the current loaded resource
        self.resource = self.rset.create_resource(uri)
        self.resource.load()
        self.es = self.resource.contents[0]
        return self.es

class PrintNotification(EObserver):
    def __init__(self, notifier=None):
        super().__init__(notifier=notifier)

    def notifyChanged(self, notification):
        print('Notification: {}'.format(notification))


from django.test import TestCase
from django_filters import OrderingFilter

from ...filtersets.CategoryFilterSet import CategoryFilterSet
from ...models.Category import Category

class CategoryFilterSetTest(TestCase):
    databases = {'master', }

    @classmethod
    def setUpTestData(cls):
        cls.tested_class = CategoryFilterSet

        Category.objects.create(**{
            'title': 'q' * 50,
        })

        Category.objects.create(**{
            'title': 'w' * 50,
        })

        Category.objects.create(**{
            'title': 'e' * 50,
        })

    def test_Should_IncludeDefiniteFilters(self):
        expected_filters = [
            'order_by',
        ]
        real_filters = list(self.tested_class.get_filters())

        self.assertEqual(expected_filters, real_filters)

    def test_Should_SpecificTypeForEachFilter(self):
        expected_filters = {
            'order_by': OrderingFilter,
        }
        real_filters = {
            key: value.__class__
            for key, value in self.tested_class.get_filters().items()
        }

        self.assertEqual(expected_filters, real_filters)

    def test_When_OrderByTitle_Should_ReturnOrderedData(self):
        data = {
            'order_by': 'title',
        }
        base_queryset = Category.objects.all()

        filtered_data = self.tested_class(data=data, queryset=base_queryset)

        expected_queryset = list(base_queryset.order_by('title'))
        real_queryset = list(filtered_data.qs)

        self.assertEqual(expected_queryset, real_queryset)

    def test_When_OrderByTitleDesc_Should_ReturnOrderedDescData(self):
        data = {
            'order_by': '-title',
        }
        base_queryset = Category.objects.all()

        filtered_data = self.tested_class(data=data, queryset=base_queryset)

        expected_queryset = list(base_queryset.order_by('-title'))
        real_queryset = list(filtered_data.qs)

        self.assertEqual(expected_queryset, real_queryset)


import sys
from base64 import b64encode
from threading import Thread
import time
import asyncio
import io
import avro.protocol
from avro.io import DatumWriter
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory

from collections import namedtuple

from avro.schema import Parse as schema_parse
from avro.protocol import Parse as protocol_parse

class ETPClientProtocol(WebSocketClientProtocol):
    def __init__(self):
        self.messageMaps = {}
        self.messageHeaderSchema = schema_parse(
            open("Schemas/Energistics/Datatypes/MessageHeader.avsc").read()
        )
        self.protocolsMessage = {}

        schema_path = "etp.avpr"

        protocol = protocol_parse(open(schema_path).read())

        for type in protocol.types:
            if "protocol" in type.other_props:
                protocol_no = int(type.other_props["protocol"])
                message_type = int(type.other_props["messageType"])
                if not protocol_no in self.protocolsMessage:
                    self.protocolsMessage[protocol_no] = {}
                self.protocolsMessage[protocol_no][message_type] = type
                self.messageMaps[type.namespace + "." + type.name] = type

    def onConnect(self, response):
        print("Connected to Server: {}".format(response.peer))

    async def onOpen(self):
        print("WebSocket connection open.")

        self.factory.messageMaps = self.messageMaps
        self.factory.messageHeaderSchema = self.messageHeaderSchema
        self.protocolsMessage = self.protocolsMessage
        self.send_etp_message(
            self.create_request_session_message(),
            "Energistics.Protocol.Core.RequestSession",
        )
        await asyncio.sleep(2)

    def send_etp_message(self, message, messageType):
        bytes_writer = io.BytesIO()
        encoder = avro.io.BinaryEncoder(bytes_writer)

        # need to send based on the message type -> cache this messageType from dictionary
        header = {
            "protocol": 0,
            "messageType": 1,
            "correlationId": 0,
            "messageId": 1,
            "messageFlags": 0,
        }

        # write header
        writer = avro.io.DatumWriter(self.factory.messageHeaderSchema)
        writer.write(header, encoder)

        # write body
        writer = avro.io.DatumWriter(self.factory.messageMaps[messageType])
        writer.write(message, encoder)

        self.sendMessage(bytes_writer.getvalue(), True)

    def onClose(self, wasClean, code, reason):
        print("WebSocket connection closed: {}".format(reason))

    def onMessage(self, payload, isBinary):
        if isBinary:
            print("Binary message received: {0} bytes".format(len(payload)))
            header, message = self.decode(payload)
            print("header: {}".format(header))
            print("message: {}".format(message))
        else:
            print("Text message received: {0}".format(payload.decode("utf8")))

    def decode(self, payload):
        bytes_reader = io.BytesIO(payload)
        decoder = avro.io.BinaryDecoder(bytes_reader)

        # read header
        reader = avro.io.DatumReader(self.messageHeaderSchema)
        header = reader.read(decoder)

        message_header = namedtuple("MessageHeader", list(header.keys()))(
            *list(header.values())
        )

        reader = avro.io.DatumReader(
            self.protocolsMessage[message_header.protocol][message_header.messageType]
        )
        body = reader.read(decoder)

        message_body = namedtuple("MessageBody", list(body.keys()))(
            *list(body.values())
        )

        return message_header, message_body

    def create_request_session_message(self):
        requestSession = {
            "applicationName": "",
            "applicationVersion": "",
            "requestedProtocols": [
                {
                    "protocol": 1,
                    "protocolVersion": {
                        "major": 1,
                        "minor": 1,
                        "revision": 0,
                        "patch": 0,
                    },
                    "role": "producer",
                    "protocolCapabilities": {},
                },
                {
                    "protocol": 3,
                    "protocolVersion": {
                        "major": 1,
                        "minor": 1,
                        "revision": 0,
                        "patch": 0,
                    },
                    "role": "store",
                    "protocolCapabilities": {},
                },
                {
                    "protocol": 4,
                    "protocolVersion": {
                        "major": 1,
                        "minor": 1,
                        "revision": 0,
                        "patch": 0,
                    },
                    "role": "store",
                    "protocolCapabilities": {},
                },
                {
                    "protocol": 5,
                    "protocolVersion": {
                        "major": 1,
                        "minor": 1,
                        "revision": 0,
                        "patch": 0,
                    },
                    "role": "store",
                    "protocolCapabilities": {},
                },
                {
                    "protocol": 6,
                    "protocolVersion": {
                        "major": 1,
                        "minor": 1,
                        "revision": 0,
                        "patch": 0,
                    },
                    "role": "store",
                    "protocolCapabilities": {},
                },
            ],
            "supportedObjects": [],
        }

        return requestSession

if __name__ == "__main__":
    headers = {
        "Authorization": "Basic {}".format(
            b64encode(b"ilab.user:n@6C5rN!").decode("utf-8")
        )
    }

    factory = WebSocketClientFactory(
        "wss://witsmlstudio.pds.software/staging/api/etp",
        headers=headers,
        protocols=["energistics-tp"],
    )
    factory.protocol = ETPClientProtocol
    # self.factory.on_etp_message = self.on_etp_message

    loop = asyncio.get_event_loop()
    coro = loop.create_connection(factory, "witsmlstudio.pds.software", 80)
    loop.run_until_complete(coro)
    loop.run_forever()
    loop.close()


import json
import time
import unittest

from selenium.webdriver import ActionChains
from selenium import webdriver
import HTMLTestRunner

class Testunitest(unittest.TestCase):

    def setUp(self):
        self.driver = webdriver.Chrome()
        self.driver.maximize_window()
        self.driver.get("http://192.168.2.104:5003/home/index?menuid=JINHAI")

        # 首先清除浏览器已有的cookies
        self.driver.delete_all_cookies()
        # 读取cookie
        with open('cookie.txt', 'r',encoding='utf-8') as cookief:
            cookieslist = json.load(cookief)
            # 方法1 将expiry类型变为int,这样就不会过期
            for cookie in cookieslist:
                if isinstance(cookie.get('expiry'), float):
                    cookie['expiry'] = int(cookie['expiry'])
                self.driver.add_cookie(cookie)
        time.sleep(10)
        self.driver.get("http://192.168.2.104:5003/home/index?menuid=JINHAI")
        self.driver.refresh()

        driver = self.driver
        driver.implicitly_wait(10)

        # 点击模块展开按钮
        elem2 = self.driver.find_element_by_id('longnows_app_flexible')
        ActionChains(self.driver).click(elem2).perform()

        # 点击进出口管理
        time.sleep(5)
        elem3 = self.driver.find_element_by_xpath('//*[@id="nav-left"]/li[3]/a/span')
        ActionChains(self.driver).click(elem3).perform()
        time.sleep(3)

        # 点击进口管理
        elem4 = self.driver.find_element_by_xpath('//*[@id="nav-left"]/li[3]/dl/dd[2]/a/span')
        ActionChains(self.driver).click(elem4).perform()
        time.sleep(5)

        # 点击进口台账模块，进入进口台账页面
        elem5 = self.driver.find_element_by_xpath('//*[@id="nav-left"]/li[3]/dl/dd[2]/dl/dd[3]/a/cite')
        ActionChains(self.driver).click(elem5).perform()
        time.sleep(8)

    def test_case1(self):
        driver = self.driver
        driver.implicitly_wait(10)

        # ====================================================表头==============================================
        # 在进口台账页面点击新增按钮,这边用到iframe
        elem6 = driver.find_element_by_xpath('//*[@id="longnows_body"]/div[2]/div[2]/div[2]/iframe')
        driver.switch_to.frame(elem6)
        # 点击新增按钮
        elem7 = driver.find_element_by_xpath('/html/body/div[2]/div[1]/div[1]/div/button[1]')
        ActionChains(driver).click(elem7).perform()
        time.sleep(5)
        # 获取当前打开的所有窗口的句柄
        num1 = driver.window_handles
        print(num1)
        driver.switch_to.window(num1[0])

        # 在进口台账的新增页面，选择进境关别下拉数据
        elem8 = driver.find_element_by_xpath('//*[@id="longnows_tab_container"]/div[3]/iframe')
        print(elem8)
        # elem8=WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH,".//iframe[contains(@src,'/Dec/CopAct/Modify?Type=3&IEFlag=I')]")))
        driver.switch_to.frame(elem8)
        # ============
        # 点击进境关口下拉
        elem9 = driver.find_element_by_xpath('//*[@id="head"]/div[3]/div[2]/span/span/a')
        ActionChains(driver).click(elem9).perform()
        time.sleep(1)
        # 选择进境关闭
        elem10 = driver.find_element_by_xpath('/html/body/div[16]/div/div[1]')
        ActionChains(driver).click(elem10).perform()
        time.sleep(1)

        # ============
        # 点击入境口岸下拉
        elem11 = driver.find_element_by_xpath('//*[@id="head"]/div[3]/div[4]/span/span/a')
        ActionChains(driver).click(elem11).perform()
        time.sleep(1)
        # 选择入境口岸
        elem12 = driver.find_element_by_xpath('/html/body/div[17]/div/div[1]')
        ActionChains(driver).click(elem12).perform()
        time.sleep(1)

        # ============
        # 点击申报地海关下拉
        elem13 = driver.find_element_by_xpath('//*[@id="head"]/div[2]/div[6]/span/span/a')
        ActionChains(driver).click(elem13).perform()
        time.sleep(1)
        # 选择入境口岸
        elem14 = driver.find_element_by_xpath('/html/body/div[15]/div/div[1]')
        ActionChains(driver).click(elem14).perform()
        time.sleep(1)

        # ============
        # 点击报关行单位
        elem15 = driver.find_element_by_xpath('//*[@id="head"]/div[7]/div[2]/span/span/a')
        ActionChains(driver).click(elem15).perform()
        time.sleep(1)
        # 选择入境口岸
        elem16 = driver.find_element_by_xpath('/html/body/div[23]/div/div[1]')
        ActionChains(driver).click(elem16).perform()
        time.sleep(1)

        # ============
        # 点击监管方式
        elem17 = driver.find_element_by_xpath('//*[@id="head"]/div[9]/div[2]/span/span/a')
        ActionChains(driver).click(elem17).perform()
        time.sleep(1)
        # 选择监管方式
        elem18 = driver.find_element_by_xpath('/html/body/div[25]/div/div[1]')
        ActionChains(driver).click(elem18).perform()
        time.sleep(1)

        # ============
        # 点击起运国
        elem19 = driver.find_element_by_xpath('//*[@id="head"]/div[10]/div[2]/span/span/a')
        ActionChains(driver).click(elem19).perform()
        time.sleep(1)
        # 选择起运国
        elem20 = driver.find_element_by_xpath('/html/body/div[29]/div/div[1]')
        ActionChains(driver).click(elem20).perform()
        time.sleep(1)

        # ============
        # 点击境内目的地
        elem21 = driver.find_element_by_xpath('//*[@id="head"]/div[11]/div[2]/span/span/a')
        ActionChains(driver).click(elem21).perform()
        time.sleep(1)
        # 选择境内目的地
        elem22 = driver.find_element_by_xpath('/html/body/div[35]/div/div[1]')
        ActionChains(driver).click(elem22).perform()
        time.sleep(1)

        # ============
        # 点击特殊关确认
        elem23 = driver.find_element_by_xpath('//*[@id="head"]/div[12]/div[2]/span/span/a')
        ActionChains(driver).click(elem23).perform()
        time.sleep(1)
        # 选择特殊关系确认
        elem24 = driver.find_element_by_xpath('/html/body/div[39]/div/div[1]')
        ActionChains(driver).click(elem24).perform()
        time.sleep(1)

        # ============
        # 点击成交方式
        elem25 = driver.find_element_by_xpath('//*[@id="head"]/div[9]/div[4]/span/span/a')
        ActionChains(driver).click(elem25).perform()
        time.sleep(1)
        # 选择成交方式
        elem26 = driver.find_element_by_xpath('/html/body/div[27]/div/div[1]')
        ActionChains(driver).click(elem26).perform()
        time.sleep(1)

        # ============
        # 点击经停港
        elem27 = driver.find_element_by_xpath('//*[@id="head"]/div[10]/div[4]/span/span/a')
        ActionChains(driver).click(elem27).perform()
        time.sleep(1)
        # 选择经停港
        elem28 = driver.find_element_by_xpath('/html/body/div[30]/div/div[1]')
        ActionChains(driver).click(elem28).perform()
        time.sleep(1)

        # ============
        # 点击贸易国别（地区）
        elem29 = driver.find_element_by_xpath('//*[@id="head"]/div[11]/div[4]/span/span/a')
        ActionChains(driver).click(elem29).perform()
        time.sleep(1)
        # 选择贸易国别（地区）
        elem30 = driver.find_element_by_xpath('/html/body/div[33]/div/div[1]')
        ActionChains(driver).click(elem30).perform()
        time.sleep(1)

        # ============
        # 点击价格影响确认
        elem31 = driver.find_element_by_xpath('//*[@id="head"]/div[12]/div[4]/span/span/a')
        ActionChains(driver).click(elem31).perform()
        time.sleep(1)
        # 选择价格影响确认
        elem32 = driver.find_element_by_xpath('/html/body/div[40]/div/div[1]')
        ActionChains(driver).click(elem32).perform()
        time.sleep(1)

        # ============
        # 点击运输方式
        elem33 = driver.find_element_by_xpath('//*[@id="head"]/div[9]/div[6]/span/span/a')
        ActionChains(driver).click(elem33).perform()
        time.sleep(1)
        # 选择运输方式
        elem34 = driver.find_element_by_xpath('/html/body/div[28]/div/div[1]')
        ActionChains(driver).click(elem34).perform()
        time.sleep(1)

        # ============
        # 点击支付特殊权使确认
        elem35 = driver.find_element_by_xpath('//*[@id="head"]/div[12]/div[6]/span/span/a')
        ActionChains(driver).click(elem35).perform()
        time.sleep(1)
        # 选择支付特殊权使确认
        elem36 = driver.find_element_by_xpath('/html/body/div[41]/div/div[1]')
        ActionChains(driver).click(elem36).perform()
        time.sleep(1)

        # ============
        # 点击供应商
        elem37 = driver.find_element_by_xpath('//*[@id="head"]/div[8]/div[2]/span/span/a')
        ActionChains(driver).click(elem37).perform()
        time.sleep(1)
        # 选择供应商
        elem38 = driver.find_element_by_xpath('/html/body/div[24]/div/div[1]')
        ActionChains(driver).click(elem38).perform()
        time.sleep(1)

        # ============
        # 点击货代
        elem39 = driver.find_element_by_xpath('//*[@id="head"]/div[7]/div[4]/span/span/a')
        ActionChains(driver).click(elem39).perform()
        time.sleep(1)
        # 选择货代
        elem40 = driver.find_element_by_xpath('/html/body/div[18]/div/div[1]')
        ActionChains(driver).click(elem40).perform()
        time.sleep(1)

        # ============
        # 点击表头保存按钮
        elem41=driver.find_element_by_xpath('/html/body/div[1]/button[2]')
        elem41 = driver.find_element_by_xpath('/html/body/div[1]/button[contains(@lay-event,"headSave")]')
        ActionChains(driver).click(elem41).perform()
        time.sleep(1)

        # ===========================================表体======================================================

        # 点击表头新增按钮
        elem42 = driver.find_element_by_id('listInit')
        ActionChains(driver).click(elem42).perform()
        time.sleep(1)

        # =============
        # 点击商品料号下拉
        elem43 = driver.find_element_by_xpath('//*[@id="act_list_form"]/div[2]/div[2]/span/span/a')
        ActionChains(driver).click(elem43).perform()
        time.sleep(1)
        # 选择商品料号
        elem44 = driver.find_element_by_xpath('/html/body/div[11]/div/div[1]')
        ActionChains(driver).click(elem44).perform()
        time.sleep(1)

        # 设置申报数量
        elem45 = driver.find_element_by_id('Qty')
        elem45.clear()
        elem45.send_keys(10)
        time.sleep(1)

        # 设置申报单价
        elem46 = driver.find_element_by_id('DecPrice')
        elem46.clear()
        elem46.send_keys(6)
        time.sleep(1)

        # 设置法定第一数量
        elem47 = driver.find_element_by_id('Qty1')
        elem47.clear()
        elem47.send_keys(10)
        time.sleep(1)

        # 设置净重
        elem48 = driver.find_element_by_id('NetWt').send_keys(10)
        time.sleep(1)

        # 设置原产国
        # 点击下拉按钮
        elem49 = driver.find_element_by_xpath('//*[@id="act_list_form"]/div[9]/div[2]/span/span/a')
        ActionChains(driver).click(elem49).perform()
        time.sleep(1)
        # 选择下拉
        elem50 = driver.find_element_by_xpath('/html/body/div[58]/div/div')
        ActionChains(driver).click(elem50).perform()
        time.sleep(1)

        # 点击保存按钮
        elem51 = driver.find_element_by_xpath('/html/body/div[2]/div[1]/div/div[2]/button[contains(@lay-event,"listSave")]')
        ActionChains(driver).click(elem51).perform()
        time.sleep(1)

    def test_edit(self):
        driver = self.driver
        driver.implicitly_wait(10)

        # ====================================================表头==============================================
        # 在进口台账页面点击编辑按钮,这边用到iframe
        e_elem1 = driver.find_element_by_xpath('//*[@id="longnows_body"]/div[2]/div[2]/div[2]/iframe')
        driver.switch_to.frame(e_elem1)
        # 点击编辑按钮
        e_elem2 = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[4]/div[2]/table/tbody/tr/td/div/button[1]/i')
        ActionChains(driver).click(e_elem2).perform()
        driver.implicitly_wait(10)
        # 获取当前打开的所有窗口的句柄
        num11 = driver.window_handles
        print(num11)
        driver.switch_to.window(num11[0])
        # 进入编辑页面的iframe
        e_elem3= driver.find_element_by_xpath('/html/body/div[1]/div[2]/div[2]/div[2]/div[3]/iframe')
        print(e_elem3)
        driver.switch_to.frame(e_elem3)
        # 修改出口日期
        e_elem4 = driver.find_element_by_xpath('//*[@id="head"]/div[3]/div[4]/span/span/a')
        ActionChains(driver).click(e_elem4).perform()
        time.sleep(2)
        e_elem5=driver.find_element_by_xpath('/html/body/div[17]/div/div[3]')
        ActionChains(driver).click(e_elem5).perform()

        #点击保存按钮
        e_elem6 = driver.find_element_by_xpath('/html/body/div[2]/div[1]/div/div[2]/button[contains(@lay-event,"listSave")]')

    def tearDown(self):
        print('Test Over')

    # if __name__=='__main__':
    #     unittest.main()

if __name__=='__main__':
    testunit = unittest.TestSuite()
    testunit.addTest(Testunitest('test_case1'))
    # 获取当前时间
    testtime = time.strftime('%Y%m%d_%H%M%S_')
    # 报告地址及报告名称
    file_path = 'D:\\PythonProject\\Testunitreport\\report.html'
    file_result = open(file_path, 'wb')
    runner = HTMLTestRunner.HTMLTestRunner(
        stream = file_result,
        title = 'Test Report',
        description = 'Test Result'
    )
    # 自动进行测试
    runner.run(testunit)
    file_result.close()


from funcy import lmap, partial, silent
from odd_models import DataEntity, DataEntityType, DataSet
from odd_models.models import DataTransformer
from oddrn_generator import PostgresqlGenerator

from ..models import Table
from .columns import map_column
from .metadata import get_table_metadata
from .utils import has_vector_column

def map_view(generator: PostgresqlGenerator, view: Table):
    generator.set_oddrn_paths(
        **{"schemas": view.table_schema, "views": view.table_name}
    )
    map_view_column = partial(map_column, generator=generator, path="views")

    # If view contains vector column we consider it as a vector store, otherwise - an ordinary view
    data_entity_type = (
        DataEntityType.VECTOR_STORE
        if has_vector_column(view.columns)
        else DataEntityType.VIEW
    )

    return DataEntity(
        oddrn=generator.get_oddrn_by_path("views"),
        name=view.table_name,
        type=data_entity_type,
        owner=view.table_owner,
        description=view.description,
        metadata=[get_table_metadata(entity=view)],
        dataset=DataSet(
            rows_number=silent(int)(view.table_rows),
            field_list=lmap(map_view_column, view.columns),
        ),
        data_transformer=DataTransformer(
            sql=view.view_definition, inputs=[], outputs=[]
        ),
    )


from pydantic import BaseSettings, root_validator

class Settings(BaseSettings):

    google_client_id: str | None = None
    google_authorize_url: str | None = None
    google_redirect_uri: str | None = None
    google_scope: str = 'openid email'

    github_client_id: str | None = None
    github_authorize_url: str | None = None
    github_redirect_uri: str | None = None
    github_scope: str = 'openid email'

    @root_validator
    def check_google_params(cls, values):
        three_values = [
            values.get('google_client_id'),
            values.get('google_authorize_url'),
            values.get('google_redirect_uri')
        ]
        count = len([v for v in three_values if v])

        if count not in (0, 3):
            raise ValueError(
                'google_client_id, google_authorize_url and google_redirect_uri'
                ' should be either all absent or all present'
            )

        return values

    @root_validator
    def check_github_params(cls, values):
        three_values = [
            values.get('github_client_id'),
            values.get('github_authorize_url'),
            values.get('github_redirect_uri')
        ]
        count = len([v for v in three_values if v])

        if count not in (0, 3):
            raise ValueError(
                'github_client_id, github_authorize_url and github_redirect_uri'
                ' should be either all absent or all present'
            )

        return values

    class Config:
        env_prefix = 'PAPERMERGE__AUTH__'

def get_settings():
    return Settings()


import os
from PyPDF2 import PdfFileWriter, PdfFileReader

# Splits a single page pdf into several pages.
def pdf_splitter(path, parts):
    # file name.
    fname = os.path.splitext(os.path.basename(path))[0]
    pdf_writer = PdfFileWriter()
    page = get_base_page(path)
    dim = get_dim(page)
    # print(dim)
    upper_right = dim["upperRight"]
    scale = upper_right[1] / parts
    pages = []

    # Generates pages.
    for part in range(0, parts):
        page = get_base_page(path)
        page.cropBox.upperLeft = (0, part * scale)
        page.cropBox.lowerLeft = (0, (part + 1) * scale)
        pages.append(page)

    # Adds the pages to the new pdf in the right order.
    for split_page in range(len(pages)):
        pdf_writer.addPage(pages.pop())

    # Writes new pdf
    output_filename = '{}_{}.pdf'.format(fname, "split")
    with open(output_filename, 'wb') as out:
        pdf_writer.write(out)

# returns first page.
def get_base_page(path):
    pdf = PdfFileReader(path)
    return pdf.getPage(0)

# returns page dimensions.
def get_dim(page):
    return {
        "lowerLeft": page.cropBox.getLowerLeft(),
        "lowerRight" : page.cropBox.getLowerRight(),
        "upperLeft" : page.cropBox.getUpperLeft(),
        "upperRight" : page.cropBox.getUpperRight()
    }


#-----
# Description   : Example script to use cpu backprojection code
# Date          : Nov 2020
# Author        : Berkan Lafci
# E-mail        : lafciberkan@gmail.com
#-----

#%% Start logging
import logging
logging.basicConfig(filename='exampleCpuBP.log', filemode='w', level=logging.INFO)
logging.info('  Script      "exampleCpuBP"      : exampleCpuBP.py')

#%% Import libraries
import os
import numpy as np
import matplotlib.pyplot as plt

# stop writing __pycache__ files
import sys
sys.dont_write_bytecode = True

from pyoat import *

#%% Data paths (defined by users)
folderPath          = 'data/rawData'
scanName            = 'mouseCrossSection'

#%% read signal file
oaData              = oaReader(folderPath=folderPath, scanName=scanName, averaging=False)
sigMat              = oaData.sigMat

#%% Initialize cpuBP object
bp                  = cpuBP()

#%% Reconstruction parameters (defined by user)
bp.speedOfSound     = 1535              # change SoS based on water temperature (default: 1480)
bp.fieldOfView      = 0.024             # FOV to reconstruct (default: 0.03)
bp.pixelNumber      = 512               # increase this number for higher resolution (default: 128)
bp.cupType          = 'ring'    		# ring, multisegment, virtualRing (default: ringCup)
bp.reconType        = 'full'            # full, direct or derivative (default: full)
bp.delayInSamples   = 64                # reception delay in samples
bp.wavelengths      = [800]             # wavelengths used in acquisition
bp.lowCutOff        = 0.1e6             # low cutoff for bandpass (default: 0.1e6)
bp.highCutOff       = 6e6               # high cutoff for bandpass (default: 6e6)

#%% Reconstruction
imageRecon          = bp.reconBP(sigMat)

# #%% Visualize reconstructed images (optional)
# plt.figure()
# plt.imshow(imageRecon[:,:,0,0], cmap='gray')
# plt.show()

#%% Save reconstructed images (optional)

# save as png
imageReconRotated = np.rot90(imageRecon,1)
pngPath = 'data/pngImages'
saveImagePng(reconObject=bp, pngPath=pngPath, saveName='BP_'+scanName, imageRecon=imageReconRotated)

# save as mat
imageReconRotated = np.rot90(imageRecon,1)
matPath = 'data/matImages'
saveImageMat(reconObject=bp, matPath=matPath, saveName='BP_'+scanName, imageRecon=imageReconRotated)


# -*- coding: utf-8 -*-
import openpyxl
"""DATA_OF_2017={
    'First_Number' : 749, #First_Number = 우선선발 신청한 인원 수
    'Second_Number' : 448, #Second_Number = 일반선발A 신청한 인원 수
    'Third_Number' : 1, #Third_Number = 일반선발B 신청한 인원 수
    'First_Input' : 82900, #First_Input = 우선선발의 총 용량(가중치가 포함 됨)
    'Second_Input' : 403514, #Second_Input = 일반선발A의 총 용량(가중치가 포함 됨)
    'Third_Input' : 3843, #Thrid_Input = 일반선발B의 총 용량(가중치가 포함 됨)
    'First_Capacity' : 150000, #First_Capacity = 250000(공고용량) * 0.6
    'Second_Capacity' : 75000, #Second_Capacity = 250000(공고용량) * 0.3
    'Third_Capacity' : 25000, #Third_Capacity = 250000(공고용량) * 0.1
    'Upper_Limit' : 110, #2017년 SMP 평균 가격 81.125  REC 평균 가격 103,473 (육지기준)
    'Average' : 100, # 1REC = 1000kw (가중치는 이미 계산 됨) then Upper_Limit = 110,205 (SMP가 연 평균이므로 110으로 생각)
}
"""
DATA_OF_2017 = {
    'First_Number' : 1548, #First_Number = 우선선발 신청한 인원 수
    'Second_Number' : 711,  #Second_Number = 일반선발A 신청한 인원 수
    'Third_Number' : 1, #Third_Number = 일반선발B 신청한 인원 수
    'First_Input' : 171985, #First_Input = 우선선발의 총 용량(가중치가 포함 됨)
    'Second_Input' : 575856, #Second_Input = 일반선발A의 총 용량(가중치가 포함 됨)
    'Third_Input' : 3844, #Thrid_Input = 일반선발B의 총 용량(가중치가 포함 됨)
    'First_Capacity' : 150000, #First_Capacity = 250000(공고용량) * 0.6
    'Second_Capacity' : 75000, #Second_Capacity = 250000(공고용량) * 0.3
    'Third_Capacity' : 25000, #Third_Capacity = 250000(공고용량) * 0.1
    'Upper_Limit' : 110, #2017년 SMP 평균 가격 81.125  REC 평균 가격 103,473 (육지기준) then Upper_Limit = 110,205 (SMP가 연 평균이므로 110으로 생각)
    'Average' : 103, # 1REC = 1000kw (가중치는 이미 계산 됨) then  Average = 103.473 (SMP가 연 평균이므로 103으로 생각) 우선선발, 일반선발A, 일반선발B 모두 똑같다고 가정
}

First_Average=float(DATA_OF_2017['First_Input'])/float(DATA_OF_2017['First_Number'])
Second_Average=float(DATA_OF_2017['Second_Input'])/float(DATA_OF_2017['Second_Number'])
Third_Average=float(DATA_OF_2017['Third_Input'])/float(DATA_OF_2017['Third_Number'])

Standard_Deviation_List =[5,10,20,30]  #표준편차  = 다양하게 계산 할 예정

wb=openpyxl.load_workbook("연습.xlsx")
ws=wb.active
ws["B1"],ws["B14"],ws["B27"],ws["B40"]="알파 퍼센트(공정성)","알파 퍼센트(공정성)","알파 퍼센트(공정성)","알파 퍼센트(공정성)"
ws["A1"],ws["A14"],ws["A27"],ws["A40"]="표준편차","표준편차","표준편차","표준편차"
ws["A2"],ws["A15"],ws["A28"],ws["A41"]="5","10","20","30"
ws["C1"],ws["C14"],ws["C27"],ws["C40"]="효율성","효율성","효율성","효율성"
ws["D1"],ws["D14"],ws["D27"],ws["D40"]="Jain Index","Jain Index","Jain Index","Jain Index"
ws["E1"],ws["E14"],ws["E27"],ws["E40"]="예정","예정","예정","예정"

for i in range(2,12):
    ws.cell(row=i,column=2).value=0.02*(i-1)
for i in range(2,12):
    ws.cell(row=i+13,column=2).value=0.02*(i-1)
for i in range(2,12):
    ws.cell(row=i+26,column=2).value=0.02*(i-1)
for i in range(2,12):
    ws.cell(row=i+39,column=2).value=0.02*(i-1)

in_advance_List=[0.02,0.04,0.06,0.08,0.10,0.12,0.14,0.16,0.18,0.20]

efficiency=0
Jain_fairness_index=0


"""Implementation of a Swiss-system tournament."""

import psycopg2

WIN = 1
LOSS = 2
TIE = 3
BYE = 4

def connect():
    """Connect to the PostgreSQL tournament database.

    :returns: tournament database connection
    :rtype: psycopg2.connection

    """
    return psycopg2.connect("dbname=tournament")

def run_query(query, query_args=(), query_type='SELECT'):
    """Run a query against the tournament database.

    The query result will depend on the query type, although the result will
    always be contained in a dict or None. The scope of this project is small
    enough that opening and closing connections per query is acceptable.

    param str query: query string to run
    param tuple query_args: query args to pass to execute
    param query_type: query type to run (SELECT | UPDATE | DELETE | INSERT)
    :returns: query result; result type depends on query type
    :rtype: dict | None

    """
    query_type = query_type.upper()

    with connect() as connection:
        with connection.cursor() as cursor:
            cursor.execute(query, query_args)

            if query_type == 'SELECT':
                result = cursor.fetchall()
            elif query_type == 'INSERT' and 'RETURNING' in query:
                try:
                    # Requires RETURNING be used
                    result, = cursor.fetchone()
                except psycopg2.ProgrammingError:
                    result = None
            elif query_type in ('UPDATE', 'DELETE', 'INSERT'):
                result = cursor.rowcount
            else:
                raise ValueError(
                    "Query type %s is not supported." % query_type)

    return {'result': result}

def delete_all_from_table(table):
    """Delete all rows from a table.

    :param str table: name of the table to delete all rows from
    :returns: count of rows deleted
    :rtype: int

    """
    query = "DELETE FROM %s;" % table
    deleted = run_query(query, query_type='DELETE')
    return deleted['result']

def delete_matches():
    """Delete all the match records from the database.

    :returns: count of rows deleted from match table
    :rtype: int

    """
    return delete_all_from_table('match')

def delete_players():
    """Delete all the player records from the database.

    :returns: count of rows deleted from player table
    :rtype: int

    """
    return delete_all_from_table('player')

def count_players():
    """Count of all players currently registered.

    :returns: count of all registered players
    :rtype: int

    """
    query = "SELECT count(id) FROM player;"
    players = run_query(query)
    return players['result'][0][0]

def register_player(name):
    """Add a player to the tournament database.

    :param str name: name of player to register
    :returns: id of the registered player
    :rtype: int

    """
    query = "INSERT INTO player (name) VALUES (%s) RETURNING id;"
    inserted = run_query(query, query_args=(name,), query_type='INSERT')
    return inserted['result']

def register_tournament(name, players):
    """Add a tournament to the tournament database.

    :param str name: name of tournament to register
    :param int players: number of tournament entrants
    :returns: id of the registered tournament
    :rtype: int

    """
    query = ("INSERT INTO tournament (name, players) "
             "VALUES (%s, %s) "
             "RETURNING id;")
    inserted = run_query(
        query, query_args=(name, players), query_type='INSERT')
    return inserted['result']

def register_player_in_tournament(player, tournament):
    """Register a player in a tournament as an entrant.

    :param int player: id of the player to register
    :param int tournament: id of the tournament to register player in
    :returns: rowcount of the player inserted, 0 | 1
    :rtype: int

    """
    query = ("INSERT INTO entrant (player_id, tournament_id) "
             "VALUES (%s, %s);")
    inserted = run_query(
        query, query_args=(player, tournament), query_type='INSERT')
    return inserted['result']

def player_standings():
    """Get a list of the players and their win records, sorted by wins.

    :returns: list of players and win records
    :rtype: list

    Return format:
        [(15, 'Bruno Walton', 2, 0), (16, "Boots O'Neal", 1, 0), ...]

    """
    query = ("SELECT id, name, wins, matches "
             "FROM player "
             "ORDER BY wins DESC;")
    standings = run_query(query)
    return standings['result']

def player_opponents(player, tournament):
    """Get a list of the played opponents for a player in a tournament.

    :param int player: id of the player to get played opponents for
    :param int tournament: id of the tournament
    :returns: list of opponents that specified player has played
    :rtype: list

    """
    subquery = ("(SELECT id "
                "FROM match "
                "WHERE player_id = %s "
                "AND tournament_id = %s)")
    query = ("SELECT player_id "
             "FROM match "
             "WHERE id IN (%s) "
             "AND player_id != %s;") % (subquery, '%s')
    opponents = run_query(
        query, query_args=(player, tournament, player))
    return [result[0] for result in opponents['result']]

def player_opponents_match_wins(player, tournament):
    """Get a sum of the played opponents match wins.

    :param int player: id of the player
    :param int tournament: id of the tournament
    :returns: sum of opponent wins for a specified player
    :rtype: int

    """
    opponents = player_opponents(player, tournament)

    opponents_count = len(opponents)
    if opponents_count == 0:
        return 0
    elif opponents_count == 1:
        placeholder = "%s"
    else:
        placeholder = ("%s, " * opponents_count)[:-2]

    query = ("SELECT sum(wins) "
             "FROM player "
             "WHERE id IN (%s);") % placeholder
    opponents_match_wins = run_query(
        query, query_args=([o for o in opponents]))

    return opponents_match_wins['result'][0][0]

def player_standings_by_tournament(tournament):
    """Get a list of the players and their win records by tournament.

    :param int tournament: id of tournament to get standings for
    :returns: list of players and win records
    :rtype: list

    Return format:
        [(15, 'Bruno Walton', 2, 0), (16, "Boots O'Neal", 1, 0), ...]

    """
    query = ("SELECT id, name, wins, matches "
             "FROM player p, entrant e "
             "WHERE p.id = e.player_id "
             "AND tournament_id = %s "
             "ORDER BY wins DESC;")
    standings = run_query(query, query_args=(tournament,))
    return standings['result']

def player_has_received_bye(player, tournament):
    """Get whether or not a player has received a bye for a tournament.

    :param int player: id of the player to check
    :param int tournament: id of tournament to check
    :returns: whether a player received a bye in a tournament; True | False
    :rtype: boolean

    """
    query = ("SELECT bye "
             "FROM entrant "
             "WHERE player_id = %s "
             "AND tournament_id = %s;")
    bye = run_query(query, query_args=(player, tournament,))

    return bye['result'][0][0]

def update_match_wins(player):
    """Update match wins in standings for a player.

    :param int player: id of the winning player
    :returns: count of the rows updated; 0 | 1
    :rtype: int

    """
    query = ("UPDATE player "
             "SET wins = wins + 1 "
             "WHERE id = %s;")
    updated = run_query(query, query_args=(player,), query_type='UPDATE')
    return updated['result']

def update_matches_played(players):
    """Update matches played in standings for a player or players.

    :param list players: id of the winning player(s)
    :returns: count of the row(s) updated
    :rtype: int

    """
    players_count = len(players)
    if players_count == 1:
        placeholder = "%s"
    else:
        placeholder = ("%s, " * players_count)[:-2]

    query = ("UPDATE player "
             "SET matches = matches + 1 "
             "WHERE id IN (%s);") % placeholder
    updated = run_query(
        query, query_args=tuple(p for p in players), query_type='UPDATE')

    return updated['result']

def report_match(winner, loser, tournament, tie=False):
    """Report the outcome of a single match between two players.

    Both win-lose and tie matches are reported by report_match. In the event
    of a tie both players are considered winners.

    :param int winner: id of the winner
    :param int loser: id of the loser
    :param int tournament: id of the tournament the match was played in
    :param bool tie: whether match was a tie; True | False
    :returns: id of the reported match
    :rtype: int

    """
    # Add winner or tie
    result_type = TIE if tie else WIN
    query = ("INSERT INTO match "
             "(player_id, tournament_id, result_id) "
             "VALUES (%s, %s, %s) "
             "RETURNING id;")
    inserted = run_query(
        query,
        query_args=(winner, tournament, result_type), query_type='INSERT')

    match_id = inserted['result']

    # Add loser or tie
    result_type = TIE if tie else LOSS
    query = ("INSERT INTO match "
             "(id, player_id, tournament_id, result_id) "
             "VALUES (%s, %s, %s, %s);")
    run_query(
        query,
        query_args=(match_id, loser, tournament, result_type),
        query_type='INSERT')

    # Update matches played and wins
    update_matches_played([winner, loser])
    update_match_wins(winner)

    # Ties count as wins
    if tie:
        update_match_wins(loser)

    return match_id

def report_match_bye(player, tournament):
    """Report a bye for a player in a tournament.

    :param int player: id of the player to report a bye for
    :param int tournament: id of the tournament to report the bye in
    :returns: rowcount of the player updated; 0 | 1
    :rtype: int

    """
    query = ("UPDATE entrant "
             "SET bye = TRUE "
             "WHERE player_id = %s "
             "AND tournament_id = %s;")
    updated = run_query(
        query, query_args=(player, tournament), query_type='UPDATE')

    # Add bye match to match table
    query = ("INSERT INTO match "
             "(player_id, tournament_id, result_id) "
             "VALUES (%s, %s, %s);")
    run_query(
        query, query_args=(player, tournament, BYE), query_type='INSERT')

    # Update matches played and wins
    update_matches_played([player])
    update_match_wins(player)

    return updated['result']

def rank_by_opponent_match_wins(standings, tournament):
    """Rank like players using opponent match wins.

    Players that have equal match wins should be ranked according to the
    strength of the opponents beaten.

    :param list standings: standings for a tournament
    :param int tournament: standings for a tournament
    :returns: standings sorted by opponent match wins
    :rtype: list

    """
    def omw_sort(player1, player2):
        if player1[2] == player2[2]:
            player1_omw = player_opponents_match_wins(player1[0], tournament)
            player2_omw = player_opponents_match_wins(player2[0], tournament)
            if player1_omw > player2_omw:
                return -1
        return 1

    return sorted(standings, cmp=omw_sort)

def swiss_pairings(tournament):
    """Pair players for the next round in a swiss-style tournament.

    Players are paired with an opponent with a equal or nearly-equal win
    record. A player cannot play the same opponent twice.

    :param int tournament: id of the tournament to pair the next round for
    :returns: tuples containing pairings in the format --
        (id1, name1, id2, name2)
    :rtype: list

    """
    standings = player_standings_by_tournament(tournament)
    standings = rank_by_opponent_match_wins(standings, tournament)
    standings = [standing[:2] for standing in standings]

    # Player receives a bye if odd number of players
    if len(standings) % 2 != 0:
        for standing in reversed(standings):
            player, _ = standing
            if not player_has_received_bye(player, tournament):
                report_match_bye(player, tournament)
                standings.pop(standings.index(standing))
                break

    players, opponents = [], []
    while standings:
        # Add the player we are going to find a match for
        player = standings.pop(0)
        players.append(player)

        opponents_played = player_opponents(player[0], tournament)
        # Iterate over the remaining opponents (standings after pop)
        for index, opponent in enumerate(iter(standings)):
            opponent_id, _ = opponent
            # Never play the same opponent twice
            if opponent_id not in opponents_played:
                # Match found; move to next player in standings
                opponents.append(standings.pop(index))
                break

    # Pair players with their respective opponents (determined above)
    pairings = []
    for matchup in zip(players, opponents):
        player, opponent = matchup
        pairings.append(player + opponent)

    return pairings


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import numpy as np
import matplotlib.pyplot as plt
import xarray as xr

eps_r    = 0.18
q_0      = 1.42

ds1 = xr.open_dataset("data_netcdf/txE_resonant_multiscale.nc")
ds2 = xr.open_dataset("data_netcdf/txE_resonant_woelectronscale.nc")
ds3 = xr.open_dataset("data_netcdf/txE_offresonant_multiscale.nc")
ds4 = xr.open_dataset("data_netcdf/txE_offresonant_woelectronscale.nc")
ds_list = [ds1, ds2, ds3, ds4]

# In[ ]:

fig=plt.figure(figsize=(3.4,2.5),dpi=600) # figsize=(幅,高さ(inch)),dpi(dots per inch)
plt.style.use('../nature_style.txt')

###
label = [r"Resonant $v=2v_\mathrm{te}$ (Multi-scale)",
         r"Resonant $v=2v_\mathrm{te}$ (w/o electron-scale)",
         r"Off-resonant $v=2.5v_\mathrm{te}$ (Multi-scale)",
         r"Off-resonant $v=2.5v_\mathrm{te}$ (w/o electron-scale)"]
color = ["tab:red", "k", "b", "tab:green"]
linestyle = ["solid", "dashed", "solid", "dashed"]
ax=fig.add_subplot(211)
ax.axes.xaxis.set_ticklabels([])
ax.set_ylabel("Poloidal electric \n field "+r"$E_y$ ($T_\mathrm{i}/eR_0$)")
for i in range(4):
    ds = ds_list[i]
    y = ds.y
    j0Ey = ds.j0Ey
    ax.plot((y-y[0])*q_0/eps_r,j0Ey,label=label[i],c=color[i],linestyle=linestyle[i],lw=1)
ax.axhline(0,c="k",lw=0.5)
ax.set_xlim(0,1450)
ax.set_ylim(-20,20)
ax.set_yticks([-15,0,15])
fig.legend(loc="lower left",ncol=1,bbox_to_anchor=(0.15,0.88),labelspacing=0)
ax=fig.add_subplot(212)
ax.set_xlabel(r"Toroidal path length $y q / \epsilon_r$ ($\rho_\mathrm{ti}$)")
ax.set_ylabel("Radial\n displacement\n" + r"$x$ ($\rho_\mathrm{ti}$)")
for i in range(4):
    ds = ds_list[i]
    y = ds.y
    x = ds.x
    ax.plot((y-y[0])*q_0/eps_r,x-x[0],label=label[i],c=color[i],linestyle=linestyle[i],lw=1)
ax.axhline(0,c="k",lw=0.5)
ax.set_xlim(0,1450)
ax.set_ylim(-15,50)
ax.set_yticks([0,20,40])

fig.text(0.045,0.885,"a",color="k", fontfamily="sans-serif", fontweight="bold", fontsize=8)
fig.text(0.045,0.460,"b",color="k", fontfamily="sans-serif", fontweight="bold", fontsize=8)

plt.savefig("fig2.jpg",dpi=600)
plt.savefig('fig2.svg',bbox_inches="tight")
import subprocess
subprocess.call('inkscape fig2.svg -M fig2.emf', shell=True)
subprocess.call('inkscape fig2.svg -E fig2.eps', shell=True)
subprocess.call('inkscape fig2.svg -A fig2.pdf', shell=True)
plt.show()

# In[ ]:

# In[ ]:


from django import forms
from .models import AdminProfile
from accounts.models import Account

class AdminForm(forms.ModelForm):
    class Meta:
        model=Account
        fields=('first_name','last_name','phone_number')

    def __init__(self,*args,**kwargs):
        super(AdminForm,self).__init__(*args,**kwargs)
        for field in self.fields:
            self.fields[field].widget.attrs['class']='form-control'

class AdminProfileForm(forms.ModelForm):
    profile_picture = forms.ImageField(required=False, error_messages = {'invalid':("Image files only")}, widget=forms.FileInput)
    class Meta:
        model=AdminProfile
        fields=('address_line_1','address_line_2','city','state','country','profile_picture')

    def __init__(self,*args,**kwargs):
        super(AdminProfileForm,self).__init__(*args,**kwargs)
        for field in self.fields:
            self.fields[field].widget.attrs['class']='form-control'