import datetime
import os
import shutil
import tempfile
from unittest import mock

import pytest
import pytz

from testifycompat import assert_equal
from testifycompat import assert_in
from testifycompat import run
from testifycompat import setup
from testifycompat import teardown
from testifycompat import TestCase
from tests.assertions import assert_raises
from tron.config import config_parse
from tron.config import config_utils
from tron.config import ConfigError
from tron.config import schedule_parse
from tron.config import schema
from tron.config.config_parse import build_format_string_validator
from tron.config.config_parse import CLEANUP_ACTION_NAME
from tron.config.config_parse import valid_cleanup_action_name
from tron.config.config_parse import valid_config
from tron.config.config_parse import valid_job
from tron.config.config_parse import valid_node_pool
from tron.config.config_parse import valid_output_stream_dir
from tron.config.config_parse import validate_fragment
from tron.config.config_utils import NullConfigContext
from tron.config.schedule_parse import ConfigDailyScheduler
from tron.config.schema import ConfigNodeAffinity
from tron.config.schema import MASTER_NAMESPACE

BASE_CONFIG = dict(
    ssh_options=dict(agent=False, identities=["tests/test_id_rsa"]),
    time_zone="EST",
    output_stream_dir="/tmp",
    nodes=[dict(name="node0", hostname="node0"), dict(name="node1", hostname="node1"),],
    node_pools=[dict(name="NodePool", nodes=["node0", "node1"])],
)

def make_ssh_options():
    return schema.ConfigSSHOptions(
        agent=False,
        identities=("tests/test_id_rsa",),
        known_hosts_file=None,
        connect_timeout=30,
        idle_connection_timeout=3600,
        jitter_min_load=4,
        jitter_max_delay=20,
        jitter_load_factor=1,
    )

def make_mock_schedule():
    return ConfigDailyScheduler(days=set(), hour=0, minute=0, second=0, original="00:00:00 ", jitter=None,)

def make_command_context():
    return {
        "python": "/usr/bin/python",
        "batch_dir": "/tron/batch/test/foo",
    }

def make_nodes():
    return {
        "node0": schema.ConfigNode(name="node0", username="foo", hostname="node0", port=22,),
        "node1": schema.ConfigNode(name="node1", username="foo", hostname="node1", port=22,),
    }

def make_node_pools():
    return {
        "NodePool": schema.ConfigNodePool(nodes=("node0", "node1"), name="NodePool",),
    }

def make_mesos_options():
    return schema.ConfigMesos(
        master_address=None,
        master_port=5050,
        secret_file=None,
        role="*",
        principal="tron",
        enabled=False,
        default_volumes=(),
        dockercfg_location=None,
        offer_timeout=300,
    )

def make_k8s_options():
    return schema.ConfigKubernetes(enabled=False, default_volumes=())

def make_action(**kwargs):
    kwargs.setdefault("name", "action"),
    kwargs.setdefault("command", "command")
    kwargs.setdefault("executor", schema.ExecutorTypes.ssh.value)
    kwargs.setdefault("requires", ())
    kwargs.setdefault("expected_runtime", datetime.timedelta(1))
    return schema.ConfigAction(**kwargs)

def make_cleanup_action(**kwargs):
    kwargs.setdefault("name", "cleanup"),
    kwargs.setdefault("command", "command")
    kwargs.setdefault("executor", schema.ExecutorTypes.ssh.value)
    kwargs.setdefault("expected_runtime", datetime.timedelta(1))
    return schema.ConfigCleanupAction(**kwargs)

def make_job(**kwargs):
    kwargs.setdefault("namespace", "MASTER")
    kwargs.setdefault("name", f"{kwargs['namespace']}.job_name")
    kwargs.setdefault("node", "node0")
    kwargs.setdefault("enabled", True)
    kwargs.setdefault("monitoring", {})
    kwargs.setdefault(
        "schedule",
        schedule_parse.ConfigDailyScheduler(
            days=set(), hour=16, minute=30, second=0, original="16:30:00 ", jitter=None,
        ),
    )
    kwargs.setdefault("actions", {"action": make_action()})
    kwargs.setdefault("queueing", True)
    kwargs.setdefault("run_limit", 50)
    kwargs.setdefault("all_nodes", False)
    kwargs.setdefault("cleanup_action", make_cleanup_action())
    kwargs.setdefault("max_runtime")
    kwargs.setdefault("allow_overlap", False)
    kwargs.setdefault("time_zone", None)
    kwargs.setdefault("expected_runtime", datetime.timedelta(0, 3600))
    kwargs.setdefault("use_k8s", False)
    return schema.ConfigJob(**kwargs)

def make_master_jobs():
    return {
        "MASTER.test_job0": make_job(
            name="MASTER.test_job0", schedule=make_mock_schedule(), expected_runtime=datetime.timedelta(1),
        ),
        "MASTER.test_job1": make_job(
            name="MASTER.test_job1",
            schedule=schedule_parse.ConfigDailyScheduler(
                days={1, 3, 5}, hour=0, minute=30, second=0, original="00:30:00 MWF", jitter=None,
            ),
            actions={
                "action": make_action(requires=("action1",), expected_runtime=datetime.timedelta(0, 7200),),
                "action1": make_action(name="action1", expected_runtime=datetime.timedelta(0, 7200),),
            },
            time_zone=pytz.timezone("Pacific/Auckland"),
            expected_runtime=datetime.timedelta(1),
            cleanup_action=None,
            allow_overlap=True,
        ),
        "MASTER.test_job2": make_job(
            name="MASTER.test_job2",
            node="node1",
            actions={"action2_0": make_action(name="action2_0", command="test_command2.0",),},
            time_zone=pytz.timezone("Pacific/Auckland"),
            expected_runtime=datetime.timedelta(1),
            cleanup_action=None,
        ),
        "MASTER.test_job_actions_dict": make_job(
            name="MASTER.test_job_actions_dict",
            node="node1",
            schedule=make_mock_schedule(),
            actions={
                "action": make_action(),
                "action1": make_action(name="action1"),
                "action2": make_action(name="action2", requires=("action", "action1"), node="node0",),
            },
            cleanup_action=None,
            expected_runtime=datetime.timedelta(1),
        ),
        "MASTER.test_job4": make_job(
            name="MASTER.test_job4",
            node="NodePool",
            schedule=schedule_parse.ConfigDailyScheduler(
                original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None,
            ),
            all_nodes=True,
            enabled=False,
            cleanup_action=None,
            expected_runtime=datetime.timedelta(1),
        ),
        "MASTER.test_job_mesos": make_job(
            name="MASTER.test_job_mesos",
            node="NodePool",
            schedule=schedule_parse.ConfigDailyScheduler(
                original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None,
            ),
            actions={
                "action_mesos": make_action(
                    name="action_mesos",
                    command="test_command_mesos",
                    executor=schema.ExecutorTypes.mesos.value,
                    cpus=0.1,
                    mem=100,
                    disk=600,
                    docker_image="container:latest",
                ),
            },
            cleanup_action=None,
            expected_runtime=datetime.timedelta(1),
        ),
        "MASTER.test_job_k8s": make_job(
            name="MASTER.test_job_k8s",
            node="NodePool",
            schedule=schedule_parse.ConfigDailyScheduler(
                original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None,
            ),
            actions={
                "action_k8s": make_action(
                    name="action_k8s",
                    command="test_command_k8s",
                    executor=schema.ExecutorTypes.kubernetes.value,
                    cpus=0.1,
                    mem=100,
                    disk=600,
                    cap_add=["KILL"],
                    cap_drop=["CHOWN", "KILL"],
                    docker_image="container:latest",
                    secret_env=dict(
                        TEST_SECRET=schema.ConfigSecretSource(secret_name="tron-secret-test-secret--1", key="secret_1")
                    ),
                    secret_volumes=(
                        schema.ConfigSecretVolume(
                            secret_volume_name="abc",
                            secret_name="secret1",
                            container_path="/b/c",
                            default_mode="0644",
                            items=(schema.ConfigSecretVolumeItem(key="secret1", path="abcd", mode="777"),),
                        ),
                    ),
                    node_selectors={"yelp.com/pool": "default"},
                    node_affinities=(ConfigNodeAffinity(key="instance_type", operator="In", value=("a1.1xlarge",),),),
                ),
            },
            cleanup_action=None,
            expected_runtime=datetime.timedelta(1),
        ),
    }

def make_tron_config(
    action_runner=None,
    output_stream_dir="/tmp",
    command_context=None,
    ssh_options=None,
    time_zone=pytz.timezone("EST"),
    state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE,
    nodes=None,
    node_pools=None,
    jobs=None,
    mesos_options=None,
    k8s_options=None,
):
    return schema.TronConfig(
        action_runner=action_runner or {},
        output_stream_dir=output_stream_dir,
        command_context=command_context or dict(batch_dir="/tron/batch/test/foo", python="/usr/bin/python"),
        ssh_options=ssh_options or make_ssh_options(),
        time_zone=time_zone,
        state_persistence=state_persistence,
        nodes=nodes or make_nodes(),
        node_pools=node_pools or make_node_pools(),
        jobs=jobs or make_master_jobs(),
        mesos_options=mesos_options or make_mesos_options(),
        k8s_options=k8s_options or make_k8s_options(),
    )

def make_named_tron_config(jobs=None):
    return schema.NamedTronConfig(jobs=jobs or make_master_jobs())

class ConfigTestCase(TestCase):
    JOBS_CONFIG = dict(
        jobs=[
            dict(
                name="test_job0",
                node="node0",
                schedule="daily 00:00:00",
                actions=[dict(name="action", command="command")],
                cleanup_action=dict(command="command"),
            ),
            dict(
                name="test_job1",
                node="node0",
                schedule="daily 00:30:00 MWF",
                allow_overlap=True,
                time_zone="Pacific/Auckland",
                actions=[
                    dict(name="action", command="command", requires=["action1"], expected_runtime="2h",),
                    dict(name="action1", command="command", expected_runtime="2h",),
                ],
            ),
            dict(
                name="test_job2",
                node="node1",
                schedule="daily 16:30:00",
                expected_runtime="1d",
                time_zone="Pacific/Auckland",
                actions=[dict(name="action2_0", command="test_command2.0")],
            ),
            dict(
                name="test_job_actions_dict",
                node="node1",
                schedule="daily 00:00:00 ",
                actions=dict(
                    action=dict(command="command"),
                    action1=dict(command="command"),
                    action2=dict(node="node0", command="command", requires=["action", "action1"],),
                ),
            ),
            dict(
                name="test_job4",
                node="NodePool",
                all_nodes=True,
                schedule="daily",
                enabled=False,
                actions=[dict(name="action", command="command")],
            ),
            dict(
                name="test_job_mesos",
                node="NodePool",
                schedule="daily",
                actions=[
                    dict(
                        name="action_mesos",
                        executor="mesos",
                        command="test_command_mesos",
                        cpus=0.1,
                        mem=100,
                        disk=600,
                        docker_image="container:latest",
                    ),
                ],
            ),
            dict(
                name="test_job_k8s",
                node="NodePool",
                schedule="daily",
                actions=[
                    dict(
                        name="action_k8s",
                        executor="kubernetes",
                        command="test_command_k8s",
                        cpus=0.1,
                        mem=100,
                        disk=600,
                        docker_image="container:latest",
                        secret_env=dict(TEST_SECRET=dict(secret_name="tron-secret-test-secret--1", key="secret_1")),
                        secret_volumes=[
                            dict(
                                secret_volume_name="abc",
                                secret_name="secret1",
                                container_path="/b/c",
                                default_mode="0644",
                                items=[dict(key="secret1", path="abcd", mode="777"),],
                            ),
                        ],
                        cap_add=["KILL"],
                        cap_drop=["CHOWN", "KILL"],
                        node_selectors={"yelp.com/pool": "default"},
                        node_affinities=[{"key": "instance_type", "operator": "In", "value": ["a1.1xlarge"]}],
                    ),
                ],
            ),
        ],
    )

    config = dict(
        command_context=dict(batch_dir="/tron/batch/test/foo", python="/usr/bin/python",), **BASE_CONFIG, **JOBS_CONFIG,
    )

    @mock.patch.dict("tron.config.config_parse.ValidateNode.defaults")
    def test_attributes(self):
        config_parse.ValidateNode.defaults["username"] = "foo"
        expected = make_tron_config()

        test_config = valid_config(self.config)

        assert test_config.command_context == expected.command_context
        assert test_config.ssh_options == expected.ssh_options
        assert test_config.mesos_options == expected.mesos_options
        assert test_config.time_zone == expected.time_zone
        assert test_config.nodes == expected.nodes
        assert test_config.node_pools == expected.node_pools
        assert test_config.k8s_options == expected.k8s_options
        for key in ["0", "1", "2", "_actions_dict", "4", "_mesos"]:
            job_name = f"MASTER.test_job{key}"
            assert job_name in test_config.jobs, f"{job_name} in test_config.jobs"
            assert job_name in expected.jobs, f"{job_name} in test_config.jobs"
            assert test_config.jobs[job_name] == expected.jobs[job_name]
        assert test_config == expected

    def test_empty_node_test(self):
        valid_config(dict(nodes=None))

class TestNamedConfig(TestCase):
    config = ConfigTestCase.JOBS_CONFIG

    def test_attributes(self):
        expected = make_named_tron_config(
            jobs={
                "test_job": make_job(
                    name="test_job",
                    namespace="test_namespace",
                    schedule=make_mock_schedule(),
                    expected_runtime=datetime.timedelta(1),
                ),
            },
        )
        test_config = validate_fragment(
            "test_namespace",
            dict(
                jobs=[
                    dict(
                        name="test_job",
                        namespace="test_namespace",
                        node="node0",
                        schedule="daily 00:00:00 ",
                        actions=[dict(name="action", command="command")],
                        cleanup_action=dict(command="command"),
                    ),
                ],
            ),
        )
        assert_equal(test_config, expected)

    def test_attributes_with_master_context(self):
        expected = make_named_tron_config(
            jobs={
                "test_namespace.test_job": make_job(
                    name="test_namespace.test_job",
                    namespace="test_namespace",
                    schedule=make_mock_schedule(),
                    expected_runtime=datetime.timedelta(1),
                ),
            },
        )
        master_config = dict(
            nodes=[dict(name="node0", hostname="node0",),], node_pools=[dict(name="nodepool0", nodes=["node0"],),],
        )
        test_config = validate_fragment(
            "test_namespace",
            dict(
                jobs=[
                    dict(
                        name="test_job",
                        namespace="test_namespace",
                        node="node0",
                        schedule="daily 00:00:00",
                        actions=[dict(name="action", command="command")],
                        cleanup_action=dict(command="command"),
                    ),
                ],
            ),
            master_config=master_config,
        )
        assert_equal(test_config, expected)

    def test_invalid_job_node_with_master_context(self):
        master_config = dict(nodes=[dict(name="node0", hostname="node0",),],)
        test_config = dict(
            jobs=[
                dict(
                    name="test_job",
                    namespace="test_namespace",
                    node="node1",
                    schedule="daily 00:30:00 ",
                    actions=[dict(name="action", command="command")],
                    cleanup_action=dict(command="command"),
                ),
            ],
        )
        expected_message = "Unknown node name node1 at test_namespace.NamedConfigFragment.jobs.Job.test_job.node"
        exception = assert_raises(ConfigError, validate_fragment, "test_namespace", test_config, master_config,)
        assert_in(expected_message, str(exception))

    def test_invalid_action_node_with_master_context(self):
        master_config = dict(
            nodes=[dict(name="node0", hostname="node0",),], node_pools=[dict(name="nodepool0", nodes=["node0"],),],
        )
        test_config = dict(
            jobs=[
                dict(
                    name="test_job",
                    namespace="test_namespace",
                    node="node0",
                    schedule="daily 00:30:00 ",
                    actions=[dict(name="action", node="nodepool1", command="command")],
                    cleanup_action=dict(command="command"),
                ),
            ],
        )
        expected_message = "Unknown node name nodepool1 at test_namespace.NamedConfigFragment.jobs.Job.test_job.actions.Action.action.node"

        exception = assert_raises(ConfigError, validate_fragment, "test_namespace", test_config, master_config,)
        assert_in(expected_message, str(exception))

class TestJobConfig(TestCase):
    def test_no_actions(self):
        test_config = dict(jobs=[dict(name="test_job0", node="node0", schedule="daily 00:30:00 "),], **BASE_CONFIG,)

        expected_message = "Job test_job0 is missing options: actions"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_message, str(exception))

    def test_empty_actions(self):
        test_config = dict(
            jobs=[dict(name="test_job0", node="node0", schedule="daily 00:30:00 ", actions=None,),], **BASE_CONFIG,
        )

        expected_message = "Value at config.jobs.Job.test_job0.actions"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_message, str(exception))

    def test_dupe_names(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd"), dict(name="action", command="cmd"),],
                ),
            ],
            **BASE_CONFIG,
        )

        expected = "Duplicate name action at config.jobs.Job.test_job0.actions"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected, str(exception))

    def test_bad_requires(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd")],
                ),
                dict(
                    name="test_job1",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action1", command="cmd", requires=["action"],),],
                ),
            ],
            **BASE_CONFIG,
        )

        expected_message = "jobs.MASTER.test_job1.action1 has a dependency " '"action" that is not in the same job!'
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_message, str(exception))

    def test_circular_dependency(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[
                        dict(name="action1", command="cmd", requires=["action2"],),
                        dict(name="action2", command="cmd", requires=["action1"],),
                    ],
                ),
            ],
            **BASE_CONFIG,
        )

        expect = "Circular dependency in job.MASTER.test_job0: action1 -> action2"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expect, str(exception))

    def test_circular_dependency_multiaction(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[
                        dict(name="action1", command="cmd", requires=["action2"],),
                        dict(name="action2", command="cmd", requires=["action3"],),
                        dict(name="action3", command="cmd", requires=["action4"],),
                        dict(name="action4", command="cmd", requires=["action5"],),
                        dict(name="action5", command="cmd", requires=["action3"],),
                    ],
                ),
            ],
            **BASE_CONFIG,
        )

        expect = "Circular dependency in job.MASTER.test_job0: action3 -> action4 -> action5"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expect, str(exception))

    def test_config_cleanup_name_collision(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[dict(name=CLEANUP_ACTION_NAME, command="cmd"),],
                ),
            ],
            **BASE_CONFIG,
        )
        expected_message = "config.jobs.Job.test_job0.actions.Action.cleanup.name"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_message, str(exception))

    def test_config_cleanup_action_name(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd"),],
                    cleanup_action=dict(name="gerald", command="cmd"),
                ),
            ],
            **BASE_CONFIG,
        )

        expected_msg = "Cleanup actions cannot have custom names"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_msg, str(exception))

    def test_config_cleanup_requires(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd"),],
                    cleanup_action=dict(command="cmd", requires=["action"]),
                ),
            ],
            **BASE_CONFIG,
        )

        expected_msg = "Unknown keys in CleanupAction : requires"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_equal(expected_msg, str(exception))

    def test_validate_job_no_actions(self):
        job_config = dict(name="job_name", node="localhost", schedule="daily 00:30:00", actions=[],)
        config_context = config_utils.ConfigContext("config", ["localhost"], None, None,)
        expected_msg = "Required non-empty list at config.Job.job_name.actions"
        exception = assert_raises(ConfigError, valid_job, job_config, config_context,)
        assert_in(expected_msg, str(exception))

class TestValidSecretSource(TestCase):
    def test_missing_secret_name(self):
        secret_env = dict(key="no_secret_name")

        with pytest.raises(ConfigError) as missing_exc:
            config_parse.valid_secret_source(secret_env, NullConfigContext)

        assert "missing options: secret" in str(missing_exc.value)

    def test_validate_job_extra_secret_env(self):
        secret_env = dict(
            secret_name="tron-secret-k8s-name-no--secret--name", key="no_secret_name", extra_key="unknown",
        )
        with pytest.raises(ConfigError) as missing_exc:
            config_parse.valid_secret_source(secret_env, NullConfigContext)

        assert "Unknown keys in SecretSource : extra_key" in str(missing_exc.value)

    def test_valid_job_secret_env_success(self):
        secret_env = dict(secret_name="tron-secret-k8s-name-no--secret--name", key="no_secret_name",)

        expected_env = schema.ConfigSecretSource(**secret_env)

        built_env = config_parse.valid_secret_source(secret_env, NullConfigContext)
        assert built_env == expected_env

class TestNodeConfig(TestCase):
    def test_validate_node_pool(self):
        config_node_pool = valid_node_pool(dict(name="theName", nodes=["node1", "node2"]),)
        assert_equal(config_node_pool.name, "theName")
        assert_equal(len(config_node_pool.nodes), 2)

    def test_overlap_node_and_node_pools(self):
        tron_config = dict(
            nodes=[dict(name="sameName", hostname="localhost"),],
            node_pools=[dict(name="sameName", nodes=["sameNode"]),],
        )
        expected_msg = "Node and NodePool names must be unique sameName"
        exception = assert_raises(ConfigError, valid_config, tron_config)
        assert_in(expected_msg, str(exception))

    def test_invalid_node_name(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="unknown_node",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd")],
                ),
            ],
            **BASE_CONFIG,
        )

        expected_msg = "Unknown node name unknown_node at config.jobs.Job.test_job0.node"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_equal(expected_msg, str(exception))

    def test_invalid_nested_node_pools(self):
        test_config = dict(
            nodes=[dict(name="node0", hostname="node0"), dict(name="node1", hostname="node1"),],
            node_pools=[dict(name="pool0", nodes=["node1"]), dict(name="pool1", nodes=["node0", "pool0"]),],
            jobs=[
                dict(
                    name="test_job0",
                    node="pool1",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd")],
                ),
            ],
        )

        expected_msg = "NodePool pool1 contains other NodePools: pool0"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_msg, str(exception))

    def test_invalid_node_pool_config(self):
        test_config = dict(
            nodes=[dict(name="node0", hostname="node0"), dict(name="node1", hostname="node1"),],
            node_pools=[dict(name="pool0", hostname=["node1"]), dict(name="pool1", nodes=["node0", "pool0"]),],
            jobs=[
                dict(
                    name="test_job0",
                    node="pool1",
                    schedule="daily 00:30:00",
                    actions=[dict(name="action", command="cmd")],
                ),
            ],
        )

        expected_msg = "NodePool pool0 is missing options"
        exception = assert_raises(ConfigError, valid_config, test_config,)
        assert_in(expected_msg, str(exception))

    def test_invalid_named_update(self):
        test_config = dict(bozray=None)
        expected_message = "Unknown keys in NamedConfigFragment : bozray"
        exception = assert_raises(ConfigError, validate_fragment, "foo", test_config,)
        assert_in(expected_message, str(exception))

class TestValidateJobs(TestCase):
    def test_valid_jobs_success(self):
        test_config = dict(
            jobs=[
                dict(
                    name="test_job0",
                    node="node0",
                    schedule="daily",
                    expected_runtime="20m",
                    actions=[
                        dict(name="action", command="command", expected_runtime="20m",),
                        dict(
                            name="action_mesos",
                            command="command",
                            executor="mesos",
                            cpus=4,
                            mem=300,
                            disk=600,
                            constraints=[dict(attribute="pool", operator="LIKE", value="default",),],
                            docker_image="my_container:latest",
                            docker_parameters=[dict(key="label", value="labelA"), dict(key="label", value="labelB"),],
                            env=dict(USER="batch"),
                            extra_volumes=[dict(container_path="/tmp", host_path="/home/tmp", mode="RO",),],
                        ),
                        dict(
                            name="test_trigger_attrs",
                            command="foo",
                            triggered_by=["foo.bar"],
                            trigger_downstreams=True,
                        ),
                    ],
                    cleanup_action=dict(command="command"),
                ),
            ],
            **BASE_CONFIG,
        )

        expected_jobs = {
            "MASTER.test_job0": make_job(
                name="MASTER.test_job0",
                schedule=make_mock_schedule(),
                actions={
                    "action": make_action(expected_runtime=datetime.timedelta(0, 1200),),
                    "action_mesos": make_action(
                        name="action_mesos",
                        executor=schema.ExecutorTypes.mesos.value,
                        cpus=4.0,
                        mem=300.0,
                        disk=600.0,
                        constraints=(schema.ConfigConstraint(attribute="pool", operator="LIKE", value="default",),),
                        docker_image="my_container:latest",
                        docker_parameters=(
                            schema.ConfigParameter(key="label", value="labelA",),
                            schema.ConfigParameter(key="label", value="labelB",),
                        ),
                        env={"USER": "batch"},
                        extra_volumes=(
                            schema.ConfigVolume(
                                container_path="/tmp", host_path="/home/tmp", mode=schema.VolumeModes.RO.value,
                            ),
                        ),
                        expected_runtime=datetime.timedelta(hours=24),
                    ),
                    "test_trigger_attrs": make_action(
                        name="test_trigger_attrs", command="foo", triggered_by=("foo.bar",), trigger_downstreams=True,
                    ),
                },
                expected_runtime=datetime.timedelta(0, 1200),
            ),
        }

        context = config_utils.ConfigContext("config", ["node0"], None, MASTER_NAMESPACE,)
        config_parse.validate_jobs(test_config, context)
        assert expected_jobs == test_config["jobs"]

class TestValidMesosAction(TestCase):
    def test_missing_docker_image(self):
        config = dict(name="test_missing", command="echo hello", executor="mesos", cpus=0.2, mem=150, disk=450,)
        with pytest.raises(ConfigError):
            config_parse.valid_action(config, NullConfigContext)

    def test_cleanup_missing_docker_image(self):
        config = dict(command="echo hello", executor="mesos", cpus=0.2, mem=150, disk=450,)
        with pytest.raises(ConfigError):
            config_parse.valid_action(config, NullConfigContext)

class TestValidCleanupActionName(TestCase):
    def test_valid_cleanup_action_name_pass(self):
        name = valid_cleanup_action_name(CLEANUP_ACTION_NAME, None)
        assert_equal(CLEANUP_ACTION_NAME, name)

    def test_valid_cleanup_action_name_fail(self):
        assert_raises(
            ConfigError, valid_cleanup_action_name, "other", NullConfigContext,
        )

class TestValidOutputStreamDir(TestCase):
    @setup
    def setup_dir(self):
        self.dir = tempfile.mkdtemp()

    @teardown
    def teardown_dir(self):
        shutil.rmtree(self.dir)

    def test_valid_dir(self):
        path = valid_output_stream_dir(self.dir, NullConfigContext)
        assert_equal(self.dir, path)

    def test_missing_dir(self):
        exception = assert_raises(ConfigError, valid_output_stream_dir, "bogus-dir", NullConfigContext,)
        assert_in("is not a directory", str(exception))

    # TODO: docker tests run as root so everything is writeable
    # def test_no_ro_dir(self):
    #     os.chmod(self.dir, stat.S_IRUSR)
    #     exception = assert_raises(
    #         ConfigError,
    #         valid_output_stream_dir, self.dir, NullConfigContext,
    #     )
    #     assert_in("is not writable", str(exception))

    def test_missing_with_partial_context(self):
        dir = "/bogus/path/does/not/exist"
        context = config_utils.PartialConfigContext("path", "MASTER")
        path = config_parse.valid_output_stream_dir(dir, context)
        assert_equal(path, dir)

class TestBuildFormatStringValidator(TestCase):
    @setup
    def setup_keys(self):
        self.context = dict.fromkeys(["one", "seven", "stars"])
        self.validator = build_format_string_validator(self.context)

    def test_validator_passes(self):
        template = "The {one} thing I {seven} is {stars}"
        assert self.validator(template, NullConfigContext)

    def test_validator_unknown_variable_error(self):
        template = "The {one} thing I {seven} is {unknown}"
        exception = assert_raises(ConfigError, self.validator, template, NullConfigContext,)
        assert_in("Unknown context variable", str(exception))

    def test_validator_passes_with_context(self):
        template = "The {one} thing I {seven} is {mars}"
        context = config_utils.ConfigContext(None, None, {"mars": "ok"}, None,)
        assert self.validator(template, context) == template

    def test_validator_valid_string_without_no_percent_escape(self):
        template = "The {one} {seven} thing is {mars} --year %Y"
        context = config_utils.ConfigContext(path=None, nodes=None, command_context={"mars": "ok"}, namespace=None,)
        assert self.validator(template, context)

class TestValidateConfigMapping(TestCase):
    config = dict(**BASE_CONFIG, command_context=dict(some_var="The string"))

    def test_validate_config_mapping_missing_master(self):
        config_mapping = {"other": mock.Mock()}
        seq = config_parse.validate_config_mapping(config_mapping)
        exception = assert_raises(ConfigError, list, seq)
        assert_in("requires a MASTER namespace", str(exception))

    def test_validate_config_mapping(self):
        master_config = self.config
        other_config = TestNamedConfig.config
        config_mapping = {
            "other": other_config,
            MASTER_NAMESPACE: master_config,
        }
        result = list(config_parse.validate_config_mapping(config_mapping))
        assert_equal(len(result), 2)
        assert_equal(result[0][0], MASTER_NAMESPACE)
        assert_equal(result[1][0], "other")

class TestConfigContainer(TestCase):
    config = BASE_CONFIG

    @setup
    def setup_container(self):
        other_config = TestNamedConfig.config
        self.config_mapping = {
            MASTER_NAMESPACE: valid_config(self.config),
            "other": validate_fragment("other", other_config),
        }
        self.container = config_parse.ConfigContainer(self.config_mapping)

    def test_create(self):
        config_mapping = {
            MASTER_NAMESPACE: self.config,
            "other": TestNamedConfig.config,
        }

        container = config_parse.ConfigContainer.create(config_mapping)
        assert_equal(set(container.configs.keys()), {"MASTER", "other"})

    def test_create_missing_master(self):
        config_mapping = {"other": mock.Mock()}
        assert_raises(
            ConfigError, config_parse.ConfigContainer.create, config_mapping,
        )

    def test_get_job_names(self):
        job_names = self.container.get_job_names()
        expected = [
            "test_job1",
            "test_job0",
            "test_job_actions_dict",
            "test_job2",
            "test_job4",
            "test_job_mesos",
            "test_job_k8s",
        ]
        assert_equal(set(job_names), set(expected))

    def test_get_jobs(self):
        expected = [
            "test_job1",
            "test_job0",
            "test_job_actions_dict",
            "test_job2",
            "test_job4",
            "test_job_mesos",
            "test_job_k8s",
        ]
        assert_equal(set(expected), set(self.container.get_jobs().keys()))

    def test_get_node_names(self):
        node_names = self.container.get_node_names()
        expected = {"node0", "node1", "NodePool"}
        assert_equal(node_names, expected)

class TestValidateSSHOptions(TestCase):
    @setup
    def setup_context(self):
        self.context = config_utils.NullConfigContext
        self.config = {"agent": True, "identities": []}

    @mock.patch.dict("tron.config.config_parse.os.environ")
    def test_post_validation_failed(self):
        if "SSH_AUTH_SOCK" in os.environ:
            del os.environ["SSH_AUTH_SOCK"]
        assert_raises(
            ConfigError, config_parse.valid_ssh_options.validate, self.config, self.context,
        )

    @mock.patch.dict("tron.config.config_parse.os.environ")
    def test_post_validation_success(self):
        os.environ["SSH_AUTH_SOCK"] = "something"
        config = config_parse.valid_ssh_options.validate(self.config, self.context,)
        assert_equal(config.agent, True)

class TestValidateIdentityFile(TestCase):
    @setup
    def setup_context(self):
        self.context = config_utils.NullConfigContext
        self.private_file = tempfile.NamedTemporaryFile()

    def test_valid_identity_file_missing_private_key(self):
        exception = assert_raises(ConfigError, config_parse.valid_identity_file, "/file/not/exist", self.context,)
        assert_in("Private key file", str(exception))

    def test_valid_identity_files_missing_public_key(self):
        filename = self.private_file.name
        exception = assert_raises(ConfigError, config_parse.valid_identity_file, filename, self.context,)
        assert_in("Public key file", str(exception))

    def test_valid_identity_files_valid(self):
        filename = self.private_file.name
        fh_private = open(filename + ".pub", "w")
        try:
            config = config_parse.valid_identity_file(filename, self.context)
        finally:
            fh_private.close()
            os.unlink(fh_private.name)
        assert_equal(config, filename)

    def test_valid_identity_files_missing_with_partial_context(self):
        path = "/bogus/file/does/not/exist"
        context = config_utils.PartialConfigContext("path", "MASTER")
        file_path = config_parse.valid_identity_file(path, context)
        assert_equal(path, file_path)

class TestValidKnownHostsFile(TestCase):
    @setup
    def setup_context(self):
        self.context = config_utils.NullConfigContext
        self.known_hosts_file = tempfile.NamedTemporaryFile()

    def test_valid_known_hosts_file_exists(self):
        filename = config_parse.valid_known_hosts_file(self.known_hosts_file.name, self.context,)
        assert_equal(filename, self.known_hosts_file.name)

    def test_valid_known_hosts_file_missing(self):
        exception = assert_raises(ConfigError, config_parse.valid_known_hosts_file, "/bogus/path", self.context,)
        assert_in("Known hosts file /bogus/path", str(exception))

    def test_valid_known_hosts_file_missing_partial_context(self):
        context = config_utils.PartialConfigContext
        expected = "/bogus/does/not/exist"
        filename = config_parse.valid_known_hosts_file(expected, context,)
        assert_equal(filename, expected)

class TestValidateVolume(TestCase):
    @setup
    def setup_context(self):
        self.context = config_utils.NullConfigContext

    def test_missing_container_path(self):
        config = {
            "container_path_typo": "/nail/srv",
            "host_path": "/tmp",
            "mode": "RO",
        }
        assert_raises(
            ConfigError, config_parse.valid_volume.validate, config, self.context,
        )

    def test_missing_host_path(self):
        config = {
            "container_path": "/nail/srv",
            "hostPath": "/tmp",
            "mode": "RO",
        }
        assert_raises(
            ConfigError, config_parse.valid_volume.validate, config, self.context,
        )

    def test_invalid_mode(self):
        config = {
            "container_path": "/nail/srv",
            "host_path": "/tmp",
            "mode": "RA",
        }
        assert_raises(
            ConfigError, config_parse.valid_volume.validate, config, self.context,
        )

    def test_valid(self):
        config = {
            "container_path": "/nail/srv",
            "host_path": "/tmp",
            "mode": schema.VolumeModes.RO.value,
        }
        assert_equal(
            schema.ConfigVolume(**config), config_parse.valid_volume.validate(config, self.context),
        )

    def test_mesos_default_volumes(self):
        mesos_options = {"master_address": "mesos_master"}
        mesos_options["default_volumes"] = [
            {"container_path": "/nail/srv", "host_path": "/tmp", "mode": "RO",},
            {"container_path": "/nail/srv", "host_path": "/tmp", "mode": "invalid",},
        ]

        with pytest.raises(ConfigError):
            config_parse.valid_mesos_options.validate(mesos_options, self.context)

        # After we fix the error, expect error to go away.
        mesos_options["default_volumes"][1]["mode"] = "RW"
        assert config_parse.valid_mesos_options.validate(mesos_options, self.context,)

    def test_k8s_default_volumes(self):
        k8s_options = {"kubeconfig_path": "some_path"}
        k8s_options["default_volumes"] = [
            {"container_path": "/nail/srv", "host_path": "/tmp", "mode": "RO",},
            {"container_path": "/nail/srv", "host_path": "/tmp", "mode": "invalid",},
        ]

        with pytest.raises(ConfigError):
            config_parse.valid_kubernetes_options.validate(k8s_options, self.context)

        # After we fix the error, expect error to go away.
        k8s_options["default_volumes"][1]["mode"] = "RW"
        assert config_parse.valid_kubernetes_options.validate(k8s_options, self.context,)

class TestValidPermissionMode:
    @pytest.mark.parametrize(
        ("permission", "normalized"),
        [("777", "777"), ("0", "0"), ("0000", "0000"), ("0123", "0123"), (0, "0"), (7777, "7777")],
    )
    def test_valid_permissions(self, permission, normalized):
        result = config_parse.valid_permission_mode(permission, NullConfigContext)
        assert result == normalized

    @pytest.mark.parametrize("permission", ["778", "é", -1, "", {}, [], ()])
    def test_invalid_permissions(self, permission):
        with pytest.raises(ConfigError):
            config_parse.valid_permission_mode(permission, NullConfigContext)

class TestValidSecretVolumeItem:
    @pytest.mark.parametrize(
        "config",
        [
            {"path": "abc"},
            {"key": "abc",},
            {"key": "abc", "path": "abc", "extra_key": None,},
            {"key": "abc", "path": "abc", "mode": "a"},
        ],
    )
    def test_invalid(self, config):
        with pytest.raises(ConfigError):
            config_parse.valid_secret_volume_item(config, NullConfigContext)

    @pytest.mark.parametrize(
        "config", [{"key": "abc", "path": "abc"}, {"key": "abc", "path": "abc", "mode": "777"}],
    )
    def test_valid_job_secret_volume_success(self, config):
        config_parse.valid_secret_volume_item(config, NullConfigContext)

class TestValidSecretVolume:
    @pytest.mark.parametrize(
        "config",
        [
            dict(
                secret_volume_name="abc",
                secret_name="secret1",
                container_path="/b/c",
                default_mode="0644",
                items=[dict(key="secret1", path="abcd", mode="7778"),],
            ),
            dict(
                secret_volume_name="abc",
                container_path="/b/c",
                default_mode="0644",
                items=[dict(key="secret1", path="abcd", mode="7777"),],
            ),
            dict(secret_volume_name="abc", secret_name="secret1", container_path=123,),
            dict(
                secret_volume_name="abc",
                secret_name="secret1",
                container_path="/b/c",
                items=[dict(key="secret1", path="abcd", mode="7777"), dict(key="secret1", path="abcde", mode="7777")],
            ),
        ],
    )
    def test_invalid(self, config):
        with pytest.raises(ConfigError):
            config_parse.valid_secret_volume(config, NullConfigContext)

    def test_wrong_item_key(self):
        config = dict(
            secret_volume_name="abc",
            secret_name="secret1",
            container_path="/b/c",
            items=[dict(key="secret2", path="abc"),],
        )
        with pytest.raises(ConfigError):
            config_parse.valid_secret_volume(config, NullConfigContext)

    @pytest.mark.parametrize(
        "config",
        [
            dict(
                secret_volume_name="abc",
                secret_name="secret1",
                container_path="/b/c",
                default_mode="0644",
                items=[dict(key="secret1", path="abc"),],
            ),
            dict(secret_volume_name="abc", secret_name="secret1", container_path="/b/c", items=[],),
            dict(secret_volume_name="abc", secret_name="secret1", container_path="/b/c",),
        ],
    )
    def test_valid(self, config):
        config_parse.valid_secret_volume(config, NullConfigContext)

class TestValidMasterAddress:
    @pytest.fixture
    def context(self):
        return config_utils.NullConfigContext

    @pytest.mark.parametrize(
        "url", ["http://blah.com", "http://blah.com/", "blah.com", "blah.com/",],
    )
    def test_valid(self, url, context):
        normalized = "http://blah.com"
        result = config_parse.valid_master_address(url, context)
        assert result == normalized

    @pytest.mark.parametrize(
        "url", ["https://blah.com", "http://blah.com/something", "blah.com/other", "http://", "blah.com?a=1",],
    )
    def test_invalid(self, url, context):
        with pytest.raises(ConfigError):
            config_parse.valid_master_address(url, context)

if __name__ == "__main__":
    run()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('controller_setup', '0028_auto_20160207_1101'),
    ]

    operations = [
        migrations.AlterField(
            model_name='linkusertoplant',
            name='comment',
            field=models.CharField(default=b'', help_text=b"Optional. Will appear on the 'control' page", max_length=200, blank=True),
        ),
        migrations.AlterField(
            model_name='linkusertoplant',
            name='distant_project_path',
            field=models.CharField(default=b'/home/pi/modbus_controller', help_text=b'Absolute path on the local controller to the project folder', max_length=200),
        ),
        migrations.AlterField(
            model_name='table',
            name='callback',
            field=models.IntegerField(default=0, choices=[(1, b'two 16bits to form a 32bits, for data_format=>HH'), (0, b'length one answer, ex (xxx,)'), (2, b'concatenate strings'), (3, b'swapped version of 2*16bits->1*32bits, make sure your data_format is correct, so either unsigned >HH or signed >hh')]),
        ),
        migrations.AlterField(
            model_name='table',
            name='data_format',
            field=models.CharField(default=b'', help_text=b'Use for example >H, must be a precise value that must work for modbus_tk. For more information and example see https://docs.python.org/2/library/struct.html#format-characters', max_length=10),
        ),
    ]


#!/usr/bin/env python

import webapp2
import json
import time
import logging
import uuid

from google.appengine.api import channel
from google.appengine.api import memcache
from google.appengine.api import users

#from google.appengine.ext import ndb

MC_LISTENING_CLIENTS="listening_clients_%s"

def add_listener(lid, room):
    mc_client = memcache.Client()
    while True:
        listeners = mc_client.gets(MC_LISTENING_CLIENTS % room)
        if listeners is None:
            memcache.set(MC_LISTENING_CLIENTS % room, {lid: time.time()})
            break

        keys_to_del = []
        for k, l in listeners.items():
            if l < time.time() - 3600:
                keys_to_del.append(k)
        for k in keys_to_del:
            del listeners[k]

        listeners[lid] = time.time()

        if mc_client.cas(MC_LISTENING_CLIENTS % room, listeners):
            break

    return channel.create_channel(lid + room)

def broadcast(from_lid, room, msg):
    listeners = memcache.get(MC_LISTENING_CLIENTS % room)
    if not listeners:
        return
    for lid, t in listeners.items():
        if t < time.time() - 3600:
            continue
        if lid == from_lid:
            continue

        logging.info("sending message to %s" % lid)
        send_message(from_lid, lid, room, msg)

def send_message(from_, to, room, message):
    channel.send_message(to + room, json.dumps({
        'message': message,
        'from': from_,
        'to': to,
    }))

class MessageHandler(webapp2.RequestHandler):
    def post(self, to=None):
        user_id = self.request.cookies.get('uid')

        if user_id:
            logging.info('Send message');
            if to:
                send_message(user_id, to, self.request.get('room', 'default'), json.loads(self.request.get('message', '{}')))
            else:
                broadcast(user_id, self.request.get('room', 'default'), json.loads(self.request.get('message', '{}')))

class TokenHandler(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'application/json'
        user_id = self.request.cookies.get('uid')

        if not user_id:
            user_id = str(uuid.uuid4())
            self.response.set_cookie('uid', user_id)

        self.response.out.write(json.dumps({
            'token': add_listener(user_id, self.request.get('room', 'default'))
        }))

app = webapp2.WSGIApplication([
    # not found api error
    (r'/api/get-token', TokenHandler),
    (r'/api/message', MessageHandler),
])


import logging
# __name__によってファイルを参照して出力される。
logger = logging.getLogger(__name__)
# loggerの閾値(DEBUG以上のerrorを取得可能となる。)
logger.setLevel(logging.DEBUG)
# handlerの定義

# ターミナル上にエラーを出力
s_handler = logging.StreamHandler()
# ファイル上にエラーを出力(文字化け予防のため、utf-8を指定)
f_handler = logging.FileHandler('./logging/sample_2.log', encoding='utf-8')

# handlerのログレベル定義
s_handler.setLevel(logging.DEBUG)
f_handler.setLevel(logging.ERROR)

# Formatter(出力形式を定義)
s_formatter = logging.Formatter('%(name)s-%(levelname)s-%(message)s')
f_formatter = logging.Formatter(
    '%(asctime)s-%(name)s-%(levelname)s-%(message)s'
)
# handlerにformatterを設定
s_handler.setFormatter(s_formatter)
f_handler.setFormatter(f_formatter)

# loggerにhandlerをセット(loggerはログの管理者みたいなもの)
logger.addHandler(s_handler)
logger.addHandler(f_handler)

logger.debug('デバッグログ')
logger.info('情報ログ')
logger.warning('警告ログ')
logger.error('エラーログ')
logger.critical('重大エラーログ')

# 例外処理の出力(0では割り切れない)
a = 10
b = 0
try:
    c = a/b
except Exception as e:
    # exc-infoを用いることで、エラーがどのファイルのどの位置にあるのかがわかる。
    logger.error(e, exc_info=True)


# !/usr/bin/env python3
import asyncio
import asyncio.subprocess
import json
import logging
import os
import re
import shutil
from typing import Union

from requests import post

from utils.abema import abema_download
from utils.bgm_nfo import episode_nfo, save_nfo, subject_name, subject_nfo
from utils.bot import async_send_messsge, bot_register
from utils.download import download
from utils.global_vars import bot, client, config, queue, sql
from utils.msg_events import ani_chat_detecting, nc_chat_detecting, nc_group_detecting
from utils.schedule_orm import run_schedule, set_schedule
from utils.queue_api import start_api, stop_api

logging.getLogger().setLevel(logging.INFO)
logging.basicConfig(
    format="[%(levelname)s]%(asctime)s: %(message)s",
    handlers=[
        logging.FileHandler("data/run.log", encoding="UTF-8"),
        logging.StreamHandler(),
    ],
)

async def down_worker(name):
    """
    0 -> url: 下载链接 (必须) str
    1 -> season_name: 番剧名称 (必须) str
    2 -> file_type: 文件类型后缀 (必须) str
    3 -> volume: 剧集集数 (必须) int
    4 -> platform: 平台名称 (必须) str
    5 -> bgm_id: BGM ID (必须) str
    6 -> tmdb_d: TMDB ID (可选) <tv|movie/id> str
    7 -> chat_msg_id: 视频频道消息 ID (可选) int"""
    while True:
        queue_item: list = await queue.get()
        url: str = queue_item[0]
        season_name: str = queue_item[1]
        file_type: str = queue_item[2]
        volume: Union[int, float] = queue_item[3]
        platform: str = queue_item[4]
        bgm_id: str = queue_item[5]
        # 如果没有获得到 TMDB，则为 None
        tmdb_d, chat_msg_id = None, None
        if len(queue_item) > 6:
            tmdb_d: str = queue_item[6]
        if len(queue_item) > 7:
            chat_msg_id: int = queue_item[7]
        if platform == "Abema":
            up_folder_name = season_name
        else: # 除 abema 其他平台
            # 处理获取到的剧集名称并去除季度信息
            season = re.search(r"第(.*)季", season_name)
            if season: season_name = season_name.replace(season.group(0), "").strip()
            # 使用 BGM ID 获取番剧日语名称将其设置为 GD 目的目录并去除季度信息
            up_folder_name = sql.inquiry_name_ja(bgm_id)
            if not up_folder_name:
                up_folder_name = subject_name(bgm_id)
                season = re.search(r"Season(.*)", up_folder_name)
                if season: up_folder_name = up_folder_name.replace(season.group(0), "").strip()
                sql.insert_data(bgm_id, tmdb_d, season_name, up_folder_name)
            else:
                sql.update_time(bgm_id)
        if isinstance(volume, int):
            _volume = f"{volume:02d}"
            episode_type: int = 0
        elif isinstance(volume, float):
            _volume = volume
            episode_type: int = 1
        # 拼接视频名称以及临时下载目录名称
        video_name = f"{season_name} - S01E{_volume} - {platform}"
        worker_path = f"{config['save_path']}{video_name}"
        # 创建临时下载目录
        if not os.path.exists(worker_path):
            os.mkdir(worker_path)
        # 创建 NFO 文件 先判断 GD 是否存在该目录 如果存在则不创建
        try:
            proc = await asyncio.create_subprocess_exec(
                "rclone", "lsjson", f"{config['rclone_config_name']}:NC-Raws", "--dirs-only",
                stdout=asyncio.subprocess.PIPE)
            output = await proc.communicate()
            dirs_list = json.loads(output[0].decode("utf-8"))
            dirs = [dirs["Name"] for dirs in dirs_list if up_folder_name in dirs["Name"]]
            # 生成 NFO 文件
            subject_data = subject_nfo(bgm_id, tmdb_d) if not dirs else None
            episode_data = episode_nfo(bgm_id, volume, episode_type)
            save_nfo(worker_path, video_name, subject_data, episode_data)
        except Exception as e:
            logging.error(f"[video_name: {video_name}] - 生成 NFO 数据出错: {e}")
            pass
        # 下载视频文件并上传到 GD
        try:
            logging.info(f"[video_name: {video_name}] - 开始下载")
            try:
                if platform == "Abema":
                    abema_download(url, bgm_id)
                else:
                    download(url, f"{worker_path}/{video_name}.{file_type}")
            except Exception as e:
                if chat_msg_id:
                    logging.warning(f"[video_name: {video_name}] - 下载失败，正在尝试从频道下载: {e}")
                    await bot.send_message(config["notice_chat"],
                        f"\\[#报告] `{bgm_id}`\n - {video_name}\n\nNC Drive 下载失败，正在尝试从频道下载", parse_mode="Markdown")
                    chat_msg = await client.get_messages(config["nc_chat_id"], ids=chat_msg_id)
                    loop = asyncio.get_event_loop()
                    task = loop.create_task(client.download_media(chat_msg, f"{video_name}/{video_name}.{file_type}"))
                    await asyncio.wait_for(task, timeout=3600)
                else: raise e
            logging.info(f"[video_name: {video_name}] - 开始上传")
            proc = await asyncio.create_subprocess_exec(
                "rclone", "move", worker_path,
               f"{config['rclone_config_name']}:NC-Raws/{up_folder_name}/",
                "--transfers", "12", stdout=asyncio.subprocess.DEVNULL)
            await proc.wait()
            if proc.returncode == 0:
                await async_send_messsge(bot, bgm_id, video_name)
                if config["finish_url"]:
                    post(config["finish_url"])
                logging.info(f"[video_name: {video_name}] - 已下载并上传成功")
        except Exception as e:
            logging.error(f"[video_name: {video_name}] - 下载或上传失败: {e}")
            await bot.send_message(config["notice_chat"],
                f"\\[#出错啦] `{bgm_id}`\n - {video_name} 下载或上传失败: \n\n`{e}`", parse_mode="Markdown")
        finally:
            shutil.rmtree(worker_path)
            queue.task_done()

if __name__ == "__main__":
    sql.create_season_db()
    sql.create_subscribe_db()
    sql.create_abema_db()
    set_schedule()
    bot_register(bot)
    client.start()
    client.add_event_handler(nc_chat_detecting)
    client.add_event_handler(ani_chat_detecting)
    client.add_event_handler(nc_group_detecting)
    tasks = []
    try:
        loop = asyncio.get_event_loop()
        tasks.append(loop.create_task(bot.polling(non_stop=True)))
        tasks.append(loop.create_task(run_schedule()))
        for i in range(config["max_num"]):
            tasks.append(loop.create_task(down_worker(f"down_worker-{i}")))
        start_api()
        print("已启动 (按 Ctrl+C 停止)")
        client.run_until_disconnected()
    finally:
        for task in tasks:
            task.cancel()
        client.disconnect()
        sql.close()
        stop_api()
        print("已停止")


# -*- coding: utf-8 -*-

"""
Fórmula de Newton-Cotes
"""
from scipy.integrate import quad
import numpy as np

def l(x, i, n, nodos):
    arr = []
    for j in range(i):
        arr.append((x - nodos[j])/(nodos[i]-nodos[j]))
    for j in range(i+1, n+1):
        arr.append((x - nodos[j])/(nodos[i]-nodos[j]))
    return np.prod(np.array(arr))

def newton_cotes( f, a, b, n ):
    nodos = []
    h = (b-a)/n
    for i in range(n+1):
        nodos.append(a + i*h)

    coef = []
    for i in range(n+1):
        coef.append(quad(l, a, b, args=(i, n, nodos))[0])

    valores = []
    for i in range(n+1):
        valores.append(coef[i]*f(nodos[i]))

    return [coef, sum(valores)]

"""
Programa principal
"""
f = lambda x: np.log(x)
a = 1
b = 2
n = 5

result = newton_cotes(f, a,b,n)

print("\nEl valor aproximado es:",result[1])


import json
import os
import sys

from loguru import logger as log
from datetime import datetime
from collections import OrderedDict

from api.tesouro import BuySell
from api.utils import fake_file

class DataProcess:

    @classmethod
    def open_file(cls, file):
        with open(file, 'rt') as f:
            file = json.loads(f.read())
        return file

    @staticmethod
    def load_last_day(file):
        """
        if the file doesn't exist, create it using the fake_file variable
        """
        try:
            log.info('Loading the last day...')
            file = DataProcess.open_file(file=file)

            if file.get('xlsLastUpdated'):
                return file
        except BaseException as err:
            log.error(f'The file was not found or is not valid: {file}')
            try:
                log.info('Creating a new file...')
                with open('last_day.json', 'w') as f:
                    f.write(json.dumps(fake_file, indent=2))
            except:
                log.error(f'Error: {err}')
                log.error(f'Occurred an error while creating the file')
            finally:
                log.info('Loading the last day...')
                file = DataProcess.open_file(file=file)

                if file.get('xlsLastUpdated'):
                    return file

    @staticmethod
    def get_current_day():
        log.info('Getting the current day...')
        bs = BuySell()
        buy, sell, last_update = bs.get_price_rate()
        return buy, sell, last_update

    @staticmethod
    def format_current_day(last_day, buy, sell, last_update):
        log.info('Formatting the current day...')
        global bid, ask, puc, puv
        new_data = {}

        for ld in last_day:
            primary_key = ld
            if primary_key != 'xlsLastUpdated':

                try:
                    # Securities to invest
                    bid = buy[primary_key]['rate']
                    ask = sell[primary_key]['rate']
                    puc = buy[primary_key]['price']
                    puv = sell[primary_key]['price']
                except:
                    # Securities to redeem
                    bid = 0
                    ask = sell[primary_key]['rate']
                    puc = 0
                    puv = sell[primary_key]['price']
                finally:
                    new_data[primary_key] = {
                        'date': last_update,
                        'bid': bid,
                        'ask': ask,
                        'puc': puc,
                        'puv': puv,
                        'txm': (bid + ask) / 2,
                        'pum': (puc + puv) / 2
                    }

        return new_data

    @staticmethod
    def calcute_variation(current_day, last_day):
        """
        Calculate the variation between the current day and the last day
        """
        log.info('Calculating the variation...')
        variation = {}
        for key in current_day:
            if key != 'xlsLastUpdated':
                if current_day[key]['bid'] > 0:  # only buy bonds
                    variation[key] = {
                        'year': key.split('\t')[1].split('-')[0],
                        'rate': current_day[key]['bid'],
                        'rate_diff': round(
                            100 * (current_day[key]['bid'] - last_day[key]['bid']) / last_day[key]['bid'], 1),
                        'price': current_day[key]['puc'],
                        'price_diff': round(
                            100 * (current_day[key]['puc'] - last_day[key]['puc']) / last_day[key]['puc'], 1),
                    }
        return variation

    @staticmethod
    def aggregate_data(variation):
        """
        Aggregate the data to a single list aggregated by name of the bond.
        """
        log.info('Aggregating the data...')
        buy_bonds = []
        for key, value in variation.items():
            buy_bonds.append({'name': key[:-11], 'value': value})

        od = OrderedDict()

        for bond in buy_bonds:
            od.setdefault(bond['name'], list()).append(
                {'year': bond['value']['year'], 'rate': bond['value']['rate'], 'rate_diff': bond['value']['rate_diff'],
                 'price': bond['value']['price']})

        agg_data = [{k: v.pop() if len(v) == 1 else v} for k, v in od.items()]
        return agg_data


from django.contrib import admin
from greatdebate.apps.decisionMakers.models import DecisionMaker, DecisionMakerResponse

class DecisionMakerAdmin(admin.ModelAdmin):
  fields = ('name','title')
  list_display = ('name','title')
admin.site.register(DecisionMaker,DecisionMakerAdmin)

class DecisionMakerResponseAdmin(admin.ModelAdmin):
  fields = ('decision_maker', 'campaign', 'response_url')
  list_display = ('decision_maker', 'response_url')
admin.site.register(DecisionMakerResponse, DecisionMakerResponseAdmin)


# coding: utf8
# flux download

import datetime
import os
import re
import requests
import sys
import time
import unicodedata
import urllib

import config

from lxml import etree
from time import sleep
from urllib.request import urlopen

from classes.flux.flux import flux
from classes.flux.flux_manager import flux_manager
from classes.show.show import show
from classes.show.show_manager import show_manager

class flux_download():

	"""download flux from url"""

	def __init__(self):
		self.conf = config.get_conf()

	def run(self, flux_id = None):

		if flux_id == None:
			try:
				flux_id = int(input('Please insert the id of the show you want to download : '))
			except ValueError:
				print('Please insert an integer as show id')
				self.run()

		self.flux_mng = flux_manager()
		self.flux = self.flux_mng.get_by_id(flux_id)

		self.show_list = []
		XMLLoading = self.load_xml()

		if( XMLLoading):

			try :
				download_list = self.download_show_list()
				report = 'Download report for ' + self.flux.name + ': ' + download_list
				return [True, True, report, True]
			except Exception as e:
				print(e)
		else:
			return [True, False, 'Le flux n\'est pas accessible', True]

	def load_xml(self):

		try:
			ret = requests.head(self.flux.url)
		except requests.exceptions.ConnectionError as e:
			return False

		tree = etree.parse(self.flux.url)
		self.xml_root = tree.getroot()

		for item in self.xml_root.iter('item'):

			show_data = []
			if self.flux.keywords != None and len(self.flux.keywords) != 0:
				'''
				k = ''
				for keyword in self.flux.keywords:
					k = k + ' ' + keyword
				'''

				itemTitle = item.find('title').text
				# DebugKeyWordCheck = any(keyword in item.find('title').text for keyword in self.flux.keywords)
				if any(keyword in item.find('title').text for keyword in self.flux.keywords):
					show_data = self.get_show_data(item)
					new_show = show()
					new_show.load(show_data)

					self.show_list.append(new_show)
			else:
				show_data = self.get_show_data(item)
				new_show = show()
				new_show.load(show_data)

				self.show_list.append(new_show)

		self.getOldestShowTimestamp()

		return True

	def get_show_data(self, xml_item):

			show_data =  [
				None,
				xml_item.find('podcastRF:magnetothequeID', self.flux.namespaces).text,
				self.flux.id,
				xml_item.find('title').text,
				xml_item.find('guid').text,
				xml_item.find('pubDate').text,
				time.mktime(time.strptime(xml_item.find('pubDate').text, '%a, %d %b %Y %H:%M:%S ' + xml_item.find('pubDate').text[-5:])),
				None,
				'remote',
			]

			return show_data

	def getOldestShowTimestamp(self):

		if not self.show_list:
			pass
		else:
			latestShow = self.show_list[-1]
			self.oldestShowTimestamp = time.mktime(time.strptime(latestShow.diffusion_date, '%a, %d %b %Y %H:%M:%S ' + latestShow.diffusion_date[-5:]))

	def download_show_list(self):

		if not self.show_list:

			k = ''
			for keyword in self.flux.keywords:
				k = k + ' ' + keyword
			download_report = '\n\tNo new podcast matching keywords (' + k + ' ) for ' + self.flux.name + '.'

		else:

			self.show_mng = show_manager(self.flux.id)
			show_dict = self.show_mng.get_all_by_remote_id({'diffusion_timestamp': self.oldestShowTimestamp})

			download_report = ''

			showsDownloadList = []
			for show in self.show_list:
				if show.remote_id in show_dict and show_dict[show.remote_id].status == 'downloaded':
					pass
				else:
					showsDownloadList.append(show)

			self.fileCounter = 1
			self.filesCount = len(showsDownloadList)

			if not showsDownloadList:
				download_report += '\n\tNo new podcast available for ' + self.flux.name + '.'
			else:
				for show in showsDownloadList:

					# Dates
					timestamp = time.mktime(time.strptime(show.diffusion_date, '%a, %d %b %Y %H:%M:%S ' + show.diffusion_date[-5:]))
					show.diffusion_date = datetime.datetime.fromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S' + show.diffusion_date[-5:])
					show_diff_date = datetime.datetime.fromtimestamp(int(timestamp)).strftime('%Y-%m-%d')

					# Filname & path
					show_filename = show_diff_date + '_' + self.get_valid_filename(self.flux.name + '_' + show.title)
					if len(show_filename) > 65:
						show_filename = show_filename[:65]
					show_filename += '.mp3'
					show_path = self.conf['download_dir'] + '/' + self.flux.name + '/' + show_filename

					show.status = 'error'

					try:

						remote_file_size = self.download_file(show, show_path)

						if remote_file_size == os.stat(show_path).st_size:
							show.status = 'downloaded'

					except Exception as e:
						print(e)

					show.download_date = '{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
					self.show_mng.save([show.__dict__])
					self.fileCounter = self.fileCounter+1

					download_report += '\n\t' + show_filename + ' - ' + show.status

		return download_report

	def download_file(self, show, show_path):

		url = show.url

		file_name = show_path
		u = urlopen(url)
		f = open(file_name, 'wb')
		meta = u.info()

		file_size = int(meta.get_all("Content-Length")[0])
		baseText = 'Downloading ' + self.flux.name[:8] + '... ' + str(self.fileCounter) + '/' + str(self.filesCount) + ': ' + show.diffusion_date[0:10] + ' ' + show.title[:27] + '. ' + str(file_size / 1000000)[:5] + ' Mo'

		file_size_dl = 0
		block_sz = 1000000

		while True:

			try :
				buffer = u.read(block_sz)

				if not buffer:
					break

				file_size_dl += len(buffer)
				f.write(buffer)

				status = r"  [%3.2f%%]" % (file_size_dl * 100. / file_size)
				status = baseText + status + chr(8)*(len(baseText + status)+1)

				sys.stdout.write('\r')
				sys.stdout.write(status)
				sys.stdout.flush()
				sleep(0.25)

				print(status, end="")

			except Exception as e:
				print(e)

		f.close()

		return file_size

	def get_valid_filename(self, value):
		"""
		copied fromDjango django/utils/text.py slugify() method
		"""
		value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
		value = re.sub('[^\w\s-]', '', value).strip().lower()
		return re.sub('[-\s]+', '-', value)


"""
Asynchronous Task controller. Executes task on given cron time

"""
import time
import logging as log
from croniter import croniter, CroniterBadCronError
from tornado.ioloop import IOLoop

class AsyncCrontabTask(object):
    """
    Asynchronous Task controller. Executes task on given cron time. No more than one at a time.

    If there is time to execute new task, but other task is already executing, the new task is ignored.
    If task is unfinishable, it would block executing new tasks.

    """
    def __init__(self, cron, func, event=None):
        self._cron = cron
        self._last_execute = None
        self._is_running = False
        self._started = False
        self._stop = False
        self.func = func
        self._event = event
        self._loop = IOLoop.current().instance()
        self._future = None

    @property
    def name(self):
        """
        Name of cron job

        Returns:
            str

        """
        return str(self.func)

    @property
    def cron(self):
        """
        Cron value of current task

        Returns:
            str

        """
        if callable(self._cron):
            return self._cron()

        return self._cron

    async def __call__(self, skip_cron=False, **kwargs):
        """
        Execute function. Do it only if it is no currently executing

        Returns:
            None

        """

        if self._is_running:
            log.error("AsyncCrontabTask[%s]: Already run. Skipping scan", self.name)
            return

        if self._future is not None:
            self._loop.remove_timeout(self._future)
            self._future = None

        if self._stop:
            log.error('AsyncCrontabTask[%s]: Is stopped. Skipping scan', self.name)
            self._started = False
            return

        self._is_running = True

        try:
            current_time = time.time()
            current_cron_time = int(current_time / 60) * 60
            try:
                current_cron = croniter(self.cron, current_time - 60).next()
            except CroniterBadCronError:
                log.error("AsyncCrontabTask[%s]: %s is invalid cron value. Skipping scan", self.name, self.cron)
                return

            if not skip_cron:
                # Check if time meet the cron time
                if current_cron != current_cron_time:
                    await self.func.update_scan_status()
                    return

                # Check if task was already executed in this cron minute
                if current_cron_time == self._last_execute:
                    return

            self._last_execute = current_cron_time

            log.debug("AsyncCrontabTask[%s]: Executing", self.name)
            if self._event is not None and self._event.is_set():
                await self._event.wait()
                if time.time() - current_cron_time > 60:
                    log.warning("Cannot run scan because similar scan is already scanning")
                    return

            if self._event is not None:
                self._event.set()
            try:
                await self.func(**kwargs)
            finally:
                if self._event is not None:
                    self._event.clear()

            log.debug("AsyncCrontabTask[%s]: Finished", self.name)
        except Exception:
            log.exception("AsyncCrontabTask[%s]: Exception", self.name)
        finally:
            self._prepare_next_iteration()

    def is_running(self):
        """
        Check if function is currently executing

        Returns:
            None

        """
        return self._is_running

    def start(self):
        """
        Start periodic function executing

        Returns:
            None

        """
        if not self._started:
            self._started = True
            self._loop.call_later(1, self)

    def stop(self):
        """
        Stop periodic function executing

        Returns:
            None

        """
        self._stop = True

    def _prepare_next_iteration(self):
        self._is_running = False
        if self._future is not None:
            log.error("AsyncCrontabTask[%s]: For some reason other scan is already working", self.name)
        self._future = self._loop.call_later(60 - int(time.time()) % 60, self)


#Connect to the dq database as the user dq.
#Write a SQL query that creates a table called users in the dq database, with the following columns and data types:
#id -- integer data type, and is a primary key.
#email -- text data type.
#name -- text data type.
#address -- text data type.
#Execute the query using the execute method.
#Use the commit method on the Connection object to apply the changes in the transaction to the database.
#Close the Connection.

import psycopg2
conn = psycopg2.connect("dbname = dq user = dq")
cur = conn.cursor()
cur.execute("CREATE TABLE users(id integer PRIMARY KEY, email text, name text, address text)")
conn.commit()
conn.close()


#!/usr/bin/env python3
"""Calculate the difference between geocodes and some truth data in a csv.

The input consistss of a truth csv (without a header) of imageId,lat,lng and some geocode_results.
The script checks to see how many of calculated lat, lngs are corrected in comparision to the
truth csv. Incorrect image ids get saved to a csv and correct image ids get printed to stderr.

Geocodes
    - Missing a location
    - Have a location where there should be none
    - Location is far (>250m?) from true location. Bullseye = within 25m?
    - Not Geocodable. geocode.py did not find any lat, lng.
"""
import argparse
import csv
import json
import sys

from calculate_metrics import diff_geocode

def check_against_truth_data(geocodes, truth):
    correct = []
    incorrect = []
    for row in truth:
        image_id = row['image_id']
        geocode_result = geocodes.get(image_id)
        if geocode_result:
            geocode_lat = geocode_result['lat']
            geocode_lng = geocode_result['lng']
            is_same, error_reason = diff_geocode((float(row['lng']), float(row['lat'])),
                                                 (geocode_lng, geocode_lat))
            if is_same:
                correct.append(image_id)
            else:
                incorrect.append((image_id, error_reason))
        else:
            incorrect.append((image_id, 'Not Geocodable'))
    return correct, incorrect

def main(geocodes_json_path, truth_csv_path, incorrect_file):
    geocodes = json.load(open(geocodes_json_path))
    with open(truth_csv_path) as csv_file:
        truth = csv.DictReader(csv_file, ['image_id', 'lat', 'lng'])
        corrected_ids, incorrect_geocodes = check_against_truth_data(geocodes, truth)
        with open(incorrect_file, 'w') as csv_file:
            writer = csv.writer(csv_file)
            for incorrect in incorrect_geocodes:
                writer.writerow(incorrect)
        for correct in corrected_ids:
            sys.stderr.write(f'{correct}\n')
        sys.stderr.write('\n\n\n\n\n')
        total_ids = len(corrected_ids) + len(incorrect_geocodes)
        sys.stderr.write(f'corrected ids: {len(corrected_ids)} / {total_ids}\n')
        sys.stderr.write(f'inncorrect ids: {len(incorrect_geocodes)} / {total_ids}\n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        'Check lat, lngs against a csv of truth data, print out matches.')
    parser.add_argument('--geocodes', type=str,
                        help='file calculated by geocode.results',
                        default='data/corrections.geocode_results.json')
    parser.add_argument('--truth_data', type=str,
                        help='CSV file (without a header line) of imageId,lat,lng',
                        default='data/corrections.csv')
    parser.add_argument('--incorrect_file', type=str,
                        help='file to write a csv of image_id,error_reason',
                        default='data/incorrect.csv')
    args = parser.parse_args()

    main(args.geocodes, args.truth_data, args.incorrect_file)