import json
from collections import defaultdict
from dataclasses import dataclass
from datetime import date, datetime, timedelta, timezone
from pprint import pprint
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Iterable,
    Iterator,
    List,
    Mapping,
    Optional,
    Sequence,
    Tuple,
    Union,
)
from uuid import uuid4 as uuid

import requests
import yaml
from dagster import (
    AssetCheckResult,
    AssetExecutionContext,
    AssetKey,
    AssetObservation,
    AssetsDefinition,
    AssetSelection,
    ConfigurableResource,
    HourlyPartitionsDefinition,
    Output,
    ScheduleDefinition,
    asset,
    build_schedule_from_partitioned_job,
    define_asset_job,
)
from pydantic import Field

if TYPE_CHECKING:
    from dagster_dbt import DbtCliInvocation
    from dagster_snowflake import SnowflakeConnection

try:
    pass
except ImportError:
    pass

@dataclass(frozen=True, eq=True)
class AssetMaterializationId:
    run_id: str
    asset_key: AssetKey
    partition: Optional[str]
    step_key: str

OPAQUE_ID_METADATA_KEY_PREFIX = "dagster_snowflake_opaque_id:"
OPAQUE_ID_SQL_SIGIL = "snowflake_dagster_dbt_v1_opaque_id"

def add_asset_context_to_sql_query(
    context: AssetExecutionContext,
    sql: str,
    comment_factory=lambda comment: f"\n-- {comment}\n",
    opaque_id=None,
):
    if opaque_id is None:
        opaque_id = str(uuid())
    context.add_output_metadata({f"{OPAQUE_ID_METADATA_KEY_PREFIX}{opaque_id}": True})
    return sql + comment_factory(f"{OPAQUE_ID_SQL_SIGIL}[[[{opaque_id}]]]")

def dbt_with_snowflake_insights(
    context: AssetExecutionContext,
    dbt_cli_invocation: "DbtCliInvocation",
    dagster_events: Optional[Iterable[Union[Output, AssetObservation, AssetCheckResult]]] = None,
    skip_config_check=False,
) -> Iterator[Union[Output, AssetObservation, AssetCheckResult]]:
    if not skip_config_check:
        is_snowflake = dbt_cli_invocation.manifest["metadata"]["adapter_type"] == "snowflake"
        dbt_project_config = yaml.safe_load(
            (dbt_cli_invocation.project_dir / "dbt_project.yml").open("r")
        )
        # sanity check that the sigil is present somewhere in the query comment
        query_comment = dbt_project_config.get("query-comment")
        if query_comment is None:
            raise RuntimeError("query-comment is required in dbt_project.yml but it was missing")
        comment = query_comment.get("comment")
        if comment is None:
            raise RuntimeError(
                "query-comment.comment is required in dbt_project.yml but it was missing"
            )
        if OPAQUE_ID_SQL_SIGIL not in comment:
            raise RuntimeError(
                "query-comment.comment in dbt_project.yml must contain the string"
                f" '{OPAQUE_ID_SQL_SIGIL}'. Read the Dagster Insights docs for more info."
            )
        if is_snowflake:
            # snowflake throws away prepended comments
            if not query_comment.get("append"):
                raise RuntimeError(
                    "query-comment.append must be true in dbt_project.yml when using the Snowflake"
                    " adapter"
                )

    if dagster_events is None:
        dagster_events = dbt_cli_invocation.stream()

    asset_and_partition_key_to_unique_id = {}
    for dagster_event in dagster_events:
        if isinstance(dagster_event, Output):
            unique_id = dagster_event.metadata["unique_id"].value
            asset_key = context.asset_key_for_output(dagster_event.output_name)
            if context._step_execution_context.has_asset_partitions_for_output(  # noqa: SLF001
                dagster_event.output_name
            ):
                partition_key = context.asset_partition_key_for_output(dagster_event.output_name)
            else:
                partition_key = None
            asset_and_partition_key_to_unique_id[(asset_key, partition_key)] = unique_id
        elif isinstance(dagster_event, AssetObservation):
            unique_id = dagster_event.metadata["unique_id"].value
            asset_key = dagster_event.asset_key
            partition_key = dagster_event.partition
            asset_and_partition_key_to_unique_id[(asset_key, partition_key)] = unique_id

        yield dagster_event

    run_results_json = dbt_cli_invocation.get_artifact("run_results.json")
    invocation_id = run_results_json["metadata"]["invocation_id"]

    for (asset_key, partition), unique_id in asset_and_partition_key_to_unique_id.items():
        # must match the query-comment in dbt_project.yml
        opaque_id = f"{unique_id}:{invocation_id}"
        context.log_event(
            AssetObservation(
                asset_key=asset_key,
                partition=partition,
                metadata={f"{OPAQUE_ID_METADATA_KEY_PREFIX}{opaque_id}": True},
            )
        )

class GraphqlClientResource(ConfigurableResource):
    url: str = Field(
        description=(
            "The URL of the Dagster Cloud GraphQL endpoint for your organization. For example,"
            " https://hooli.dagster.cloud/prod/graphql"
        )
    )
    cloud_token: Optional[str] = Field(
        description="A Dagster Cloud agent token to submit metrics to the Dagster Cloud API."
    )

    def execute(self, query_text: str, variables=None) -> Dict[str, Any]:
        headers = {}
        if self.cloud_token is not None:
            headers["Dagster-Cloud-API-Token"] = self.cloud_token
        return requests.post(
            self.url,
            json={"operationName": None, "variables": variables or dict(), "query": query_text},
            headers=headers,
        ).json()

def get_opaque_ids_to_assets(
    graphql_client: GraphqlClientResource, min_datetime: datetime, max_datetime: datetime
) -> Iterable[Tuple[AssetMaterializationId, str]]:
    query = """
{
	assetsOrError {
    ... on AssetConnection {
      nodes {
        key {
            path
        }
        assetObservations(afterTimestampMillis:"%d", beforeTimestampMillis:"%d") {
          runId
          partition
          stepKey
          metadataEntries {
            ... on BoolMetadataEntry {
              label
            }
          }
        }
      }
    }
  }
}
    """ % (
        min_datetime.timestamp() * 1000,
        max_datetime.timestamp() * 1000,
    )

    response = graphql_client.execute(
        #        "http://localhost:3000/graphql",
        query,
        variables={},
    )

    for node in response["data"]["assetsOrError"]["nodes"]:
        asset_key = AssetKey.from_graphql_input(node["key"])
        if asset_key is None:
            # should never happen, but we must appease the type checker
            raise RuntimeError("asset_key is None, which should never happen")
        for observation in node["assetObservations"]:
            run_id = observation["runId"]
            partition = observation["partition"]
            step_key = observation["stepKey"]
            opaque_id = None
            for metadata in observation["metadataEntries"]:
                if metadata.get("label", "").startswith(OPAQUE_ID_METADATA_KEY_PREFIX):
                    opaque_id = metadata["label"][len(OPAQUE_ID_METADATA_KEY_PREFIX) :]
                    break
            if opaque_id is not None:
                yield (
                    AssetMaterializationId(
                        run_id=run_id, asset_key=asset_key, partition=partition, step_key=step_key
                    ),
                    opaque_id,
                )

SNOWFLAKE_QUERY_HISTORY_LATENCY_SLA = timedelta(minutes=45)
QUERY_HISTORY_TIME_PADDING = timedelta(hours=1)  # deal with desynchronized clocks

@dataclass
class SnowflakeInsightsDefinitions:
    assets: Sequence[AssetsDefinition]
    schedule: ScheduleDefinition

def get_cost_data_for_hour(
    context: AssetExecutionContext,
    snowflake: "SnowflakeConnection",
    graphql_client: GraphqlClientResource,
    start_hour: datetime,
    end_hour: datetime,
) -> List[Tuple[AssetMaterializationId, Any]]:
    queries = list(get_opaque_ids_to_assets(graphql_client, start_hour, end_hour))
    print(
        f"Found {len(queries)} queries to check from {start_hour.isoformat()} to"
        f" {end_hour.isoformat()}"
    )
    for am, opaque_id in queries:
        print(opaque_id, am.run_id)
    if len(queries) == 0:
        return []

    opaque_id_to_asset_materialization_id: Dict[str, AssetMaterializationId] = {}
    for asset_materialization_id, opaque_id in queries:
        opaque_id_to_asset_materialization_id[opaque_id] = asset_materialization_id

    opaque_ids_sql = rf"""
    regexp_substr_all(query_text, '{OPAQUE_ID_SQL_SIGIL}\\[\\[\\[(.*?)\\]\\]\\]', 1, 1, 'ce', 1)
    """.strip()

    sql = f"""
WITH
warehouse_sizes AS (
    SELECT 'X-Small' AS warehouse_size, 1 AS credits_per_hour UNION ALL
    SELECT 'Small' AS warehouse_size, 2 AS credits_per_hour UNION ALL
    SELECT 'Medium'  AS warehouse_size, 4 AS credits_per_hour UNION ALL
    SELECT 'Large' AS warehouse_size, 8 AS credits_per_hour UNION ALL
    SELECT 'X-Large' AS warehouse_size, 16 AS credits_per_hour UNION ALL
    SELECT '2X-Large' AS warehouse_size, 32 AS credits_per_hour UNION ALL
    SELECT '3X-Large' AS warehouse_size, 64 AS credits_per_hour UNION ALL
    SELECT '4X-Large' AS warehouse_size, 128 AS credits_per_hour UNION ALL
    SELECT 'missing' as warehouse_size, 0 as credits_per_hour
)
SELECT
    {opaque_ids_sql} as opaque_ids,
    qh.execution_time/(1000*60*60)*wh.credits_per_hour AS query_cost,
    query_id
FROM snowflake.account_usage.query_history AS qh
INNER JOIN warehouse_sizes AS wh
    ON ifnull(qh.warehouse_size, 'missing')=wh.warehouse_size
WHERE
    start_time >= '{start_hour - QUERY_HISTORY_TIME_PADDING}'
    AND start_time <= '{end_hour + QUERY_HISTORY_TIME_PADDING}'
HAVING ARRAY_SIZE(opaque_ids) > 0
        """

    with snowflake.get_connection() as conn:
        with conn.cursor() as cursor:
            result = cursor.execute(sql)
            assert result
            results = result.fetchall()

    costs: List[Tuple[AssetMaterializationId, Any]] = []

    print(
        f"{len(results) if results else 0} annotated queries returned from snowflake query_history"
    )

    if not results:
        return []

    total = 0
    for result_opaque_ids, result_cost, query_id in results:
        opaque_ids = json.loads(result_opaque_ids)
        # TODO: is this cost splitting logic correct?
        cost = result_cost / len(opaque_ids)
        total += len(opaque_ids)
        for opaque_id in opaque_ids:
            print(opaque_id, query_id)
            if opaque_id in opaque_id_to_asset_materialization_id:
                costs.append((opaque_id_to_asset_materialization_id[opaque_id], cost))

    print(
        f"Reported costs for {len(costs)} of {total} asset materializations found in the"
        " query_history."
    )
    return costs

def create_snowflake_insights_asset_and_schedule(
    start_date: date,
    code_location_name: str,
    repository_name: str = "__repository__",
    name: Optional[str] = None,
    group_name: Optional[str] = None,
    job_name: str = "snowflake_insights_import",
    dry_run=True,
    allow_partial_partitions=False,
    snowflake_resource_key: str = "snowflake",
    graphql_client_resource_key: str = "graphql_client",
) -> SnowflakeInsightsDefinitions:
    @asset(
        name=name,
        group_name=group_name,
        partitions_def=HourlyPartitionsDefinition(start_date=start_date.strftime("%Y-%m-%d-%H:%M")),
        required_resource_keys={snowflake_resource_key, graphql_client_resource_key},
    )
    def poll_snowflake_query_history_hour(
        context: AssetExecutionContext,
    ) -> None:
        snowflake: "SnowflakeConnection" = getattr(context.resources, snowflake_resource_key)
        graphql_client: GraphqlClientResource = getattr(
            context.resources, graphql_client_resource_key
        )

        start_hour = context.partition_time_window.start
        end_hour = context.partition_time_window.end

        if not allow_partial_partitions:
            now = datetime.now().astimezone(timezone.utc)
            earliest_call_time = end_hour + SNOWFLAKE_QUERY_HISTORY_LATENCY_SLA
            if now < earliest_call_time:
                raise RuntimeError(
                    "This function was called before the Snowflake query_history table may be"
                    f" available. For hour starting {start_hour.isoformat()} you can call it"
                    f" starting at {earliest_call_time.isoformat()} (it is currently"
                    f" {now.isoformat()})"
                )

        costs = (
            get_cost_data_for_hour(context, snowflake, graphql_client, start_hour, end_hour) or []
        )
        if dry_run:
            pprint(costs)
        else:
            metrics_to_submit = []
            mat_by_step: Mapping[Tuple[str, str], List[Tuple[AssetMaterializationId, Any]]] = (
                defaultdict(list)
            )
            for mat, cost in costs:
                mat_by_step[(mat.run_id, mat.step_key)].append((mat, cost))

            for run_id, step_key in mat_by_step.keys():
                materializations = mat_by_step[(run_id, step_key)]
                for i in range(0, len(materializations), 5):
                    metrics_to_submit.append(
                        {
                            "repositoryName": repository_name,
                            "codeLocationName": code_location_name,
                            "stepKey": step_key,
                            "runId": run_id,
                            "assetMetricDefinitions": [
                                {
                                    "assetKey": mat.asset_key.to_string(),
                                    "assetGroup": "default",
                                    "metricValues": [
                                        {
                                            "metricName": "snowflake_credits",
                                            "metricValue": float(cost),
                                        }
                                    ],
                                }
                                for mat, cost in materializations[i : i + 5]
                            ],
                        }
                    )
                metrics_to_submit.append(
                    {
                        "repositoryName": repository_name,
                        "codeLocationName": code_location_name,
                        "stepKey": step_key,
                        "runId": run_id,
                        "jobMetricDefinitions": [
                            {
                                "metricValues": [
                                    {
                                        "metricName": "snowflake_credits",
                                        "metricValue": sum(
                                            [float(cost) for mat, cost in materializations]
                                        ),
                                    }
                                ],
                            }
                        ],
                    }
                )

            mutation = """
                mutation AddMetrics($metrics: [MetricInputs]) {
                    createOrUpdateMetrics(
                        metrics: $metrics
                    ) {
                        __typename
                        ... on CreateOrUpdateMetricsFailed {
                            message
                        }
                    }
                }
            """

            result = graphql_client.execute(
                query_text=mutation, variables={"metrics": metrics_to_submit}
            )

            assert (
                result["data"]["createOrUpdateMetrics"]["__typename"]
                == "CreateOrUpdateMetricsSuccess"
            ), result

    schedule = build_schedule_from_partitioned_job(
        job=define_asset_job(job_name, AssetSelection.assets(poll_snowflake_query_history_hour)),
        minute_of_hour=59,
    )
    # schedule may be a UnresolvedPartitionedAssetScheduleDefinition so we ignore the type check
    return SnowflakeInsightsDefinitions(assets=[poll_snowflake_query_history_hour], schedule=schedule)  # type: ignore


import requests
import json
from bs4 import BeautifulSoup
URL = 'https://www.indiabix.com/aptitude/permutation-and-combination/065002'
page = requests.get(URL)
soup = BeautifulSoup(page.content, 'html.parser')
#job_elems = soup.find_all('td', class_='bix-td-qtxt')
questions = []
job_elems = soup.find_all('div', class_='bix-div-container')
i = 0
for job_elem in job_elems:
    #print((job_elem.find('p')).text)
    questions.append({})
    #print(job_elem)
    q = job_elem.find('td',class_='bix-td-qtxt')
    questions[i]["question"] = (((q.find('p')).text).strip()).encode('ascii')
    options = job_elem.find_all('td',class_ = 'bix-td-option')
    #print(options)
    even = True
    opval = ""
    answer = job_elem.find('div',class_='bix-div-answer')
    #print(answer)
    ans = ((((answer.find('p')).find('span',class_='jq-hdnakqb')).text).strip()).encode('ascii')
    print(ans)
    first = 1
    '''for a in two :
        if first == 0:
            ans = ((a.text).strip()).encode('ascii')
        first = 0'''
    for option in options :
        if even :
            opt = (((option.text)[:-1]).strip()).encode('ascii')
            opval = "option" + opt
            even = False
        else :
            value = ((option.text).strip()).encode('ascii')
            questions[i][opval] = value
            if opt == ans :
                questions[i]["answer"] = value
            even = True

    i += 1

print(questions)
with open('data.txt', 'w') as outfile:
    json.dump(questions, outfile)


from sklearn import model_selection

class Model:

    NUM_FOLDS = 5

    def __init__(self, data_x, data_y, model, metric, randomSeed=42):
        self.randomSeed = randomSeed
        self.X = data_x
        self.y = data_y
        self.metric = metric
        self.kfold = model_selection.KFold(n_splits=self.NUM_FOLDS)
        self.model = model

    def __len__(self):
        return self.X.shape[1]

    def getMeanAccuracy(self, zeroOneList):
        # drop unselected features
        zeroIndices = [i for i, n in enumerate(zeroOneList) if n == 0]
        currentX = self.X.drop(self.X.columns[zeroIndices], axis=1)

        # k-fold validation
        cv_results = model_selection.cross_val_score(
            self.model,
            currentX,
            self.y,
            cv=self.kfold,
            scoring=self.metric
            )

        return cv_results.mean()


import torch
import torch.nn.functional as func
import torch.nn as nn
import settings
from torch.utils.data import DataLoader
from torchvision import transforms
import pytorch_lightning as pl
from argparse import ArgumentParser
from dataset_loader import MazeDataset, CollateFn, ToTensor, Normalise

def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=dilation, groups=groups, bias=False, dilation=dilation)

def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
                 base_width=64, dilation=1, norm_layer=None):
        super(BasicBlock, self).__init__()
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = norm_layer(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = norm_layer(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class Bottleneck(nn.Module):
    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
    # This variant is also known as ResNet V1.5 and improves accuracy according to
    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.

    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
                 base_width=64, dilation=1, norm_layer=None):
        super(Bottleneck, self).__init__()
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        width = int(planes * (base_width / 64.)) * groups
        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv1x1(inplanes, width)
        self.bn1 = norm_layer(width)
        self.conv2 = conv3x3(width, width, stride, groups, dilation)
        self.bn2 = norm_layer(width)
        self.conv3 = conv1x1(width, planes * self.expansion)
        self.bn3 = norm_layer(planes * self.expansion)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class ResNet(pl.LightningModule):

    def __init__(self, hparams, block=BasicBlock, layers=[2, 2, 2, 2], num_classes=20, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None,
                 norm_layer=None):
        super(ResNet, self).__init__()
        self.hparams = hparams
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        self._norm_layer = norm_layer

        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            # each element in the tuple indicates if we should replace
            # the 2x2 stride with a dilated convolution instead
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(1, self.inplanes, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = norm_layer(self.inplanes)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(512 * block.expansion, num_classes * settings.NUM_POSE_DIMS)
        if self.hparams.dropout > 0:
            self.fc = nn.Sequential(nn.Dropout(self.hparams.dropout),
                                    nn.Linear(512 * block.expansion, num_classes * settings.NUM_POSE_DIMS))

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        # Zero-initialize the last BN in each residual branch,
        # so that the residual branch starts with zeros, and each residual block behaves like an identity.
        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, Bottleneck):
                    nn.init.constant_(m.bn3.weight, 0)
                elif isinstance(m, BasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        norm_layer = self._norm_layer
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                norm_layer(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
                            self.base_width, previous_dilation, norm_layer))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(block(self.inplanes, planes, groups=self.groups,
                                base_width=self.base_width, dilation=self.dilation,
                                norm_layer=norm_layer))

        return nn.Sequential(*layers)

    def _forward_impl(self, x):
        x = x.unsqueeze_(1)

        # See note [TorchScript super()]
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.fc(x)
        # attempt at making this work for more than just dx (so dy, dth)
        x = x.view(-1, settings.NUM_POSE_DIMS, settings.MAX_ITERATIONS)

        return x

    def forward(self, x):
        return self._forward_impl(x.float())

    def training_step(self, batch, batch_nb):
        x, y = batch
        y_hat = self(x)
        loss = func.mse_loss(y_hat, y.float())
        tensorboard_logs = {'train_loss': loss}
        return {'loss': loss, 'log': tensorboard_logs}

    def validation_step(self, batch, batch_nb):
        x, y = batch
        y_hat = self(x)
        return {'val_loss': func.mse_loss(y_hat, y.float())}

    def validation_epoch_end(self, outputs):
        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
        tensorboard_logs = {'val_loss': avg_loss}
        return {'val_loss': avg_loss, 'log': tensorboard_logs}

    def test_step(self, batch, batch_nb):
        x, y = batch
        y_hat = self(x)
        return {'test_loss': func.mse_loss(y_hat, y.float())}

    def test_epoch_end(self, outputs):
        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
        logs = {'test_loss': avg_loss}
        return {'test_loss': avg_loss, 'log': logs, 'progress_bar': logs}

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate)

    def train_dataloader(self):
        data_transform_for_training = transforms.Compose([ToTensor(), Normalise()])
        train_dataset = MazeDataset(root_dir=settings.MAZE_IMAGE_DIR,
                                    data_subset_type=settings.TRAIN_SUBSET,
                                    transform=data_transform_for_training)
        return DataLoader(train_dataset, batch_size=self.hparams.batch_size,
                          shuffle=True, num_workers=settings.NUM_CPUS, collate_fn=CollateFn)

    def val_dataloader(self):
        data_transform_for_evaluation = transforms.Compose([ToTensor(), Normalise()])
        val_dataset = MazeDataset(root_dir=settings.MAZE_IMAGE_DIR,
                                  data_subset_type=settings.VAL_SUBSET,
                                  transform=data_transform_for_evaluation)
        return DataLoader(val_dataset, batch_size=self.hparams.batch_size,
                          shuffle=False, num_workers=settings.NUM_CPUS, collate_fn=CollateFn)

    def test_dataloader(self):
        data_transform_for_evaluation = transforms.Compose([ToTensor(), Normalise()])
        test_dataset = MazeDataset(root_dir=settings.MAZE_IMAGE_DIR,
                                   data_subset_type=settings.TEST_SUBSET,
                                   transform=data_transform_for_evaluation)
        return DataLoader(test_dataset, batch_size=self.hparams.batch_size,
                          shuffle=False, num_workers=settings.NUM_CPUS, collate_fn=CollateFn)

    @staticmethod
    def add_model_specific_args(parent_parser):
        """
        Specify the hyperparams for this LightningModule
        """
        # MODEL specific
        parser = ArgumentParser(parents=[parent_parser], add_help=False)
        parser.add_argument('--learning_rate', default=settings.LEARNING_RATE, type=float)
        parser.add_argument('--batch_size', default=settings.BATCH_SIZE, type=int)
        parser.add_argument('--dropout', default=0, type=float)

        # training specific (for this model)
        parser.add_argument('--max_num_epochs', default=settings.MAX_EPOCHS, type=int)

        # program specific
        # parser.add_argument('--default_root_dir', default=settings.MAZE_MODEL_DIR, type=str)

        return parser


"""
Class used to parse and update template when tracing is enabled
"""
import logging
from typing import Any

from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from ruamel.yaml.representer import RoundTripRepresenter

from samcli.lib.init.template_modifiers.cli_template_modifier import TemplateModifier

LOG = logging.getLogger(__name__)

class XRayTracingTemplateModifier(TemplateModifier):
    FIELD_NAME_FUNCTION_TRACING = "Tracing"
    FIELD_NAME_API_TRACING = "TracingEnabled"
    GLOBALS = "Globals"
    RESOURCE = "Resources"
    FUNCTION = "Function"
    TRACING_FUNCTION = "Tracing"
    ACTIVE_TRACING = "Active"
    API = "Api"
    TRACING_API = "TracingEnabled"
    TRACING_API_VALUE = True
    COMMENT = (
        "# More info about Globals: "
        "https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst\n"
    )

    # set ignore aliases to true. This configuration avoids usage yaml aliases which is not parsed by CloudFormation.
    class NonAliasingRTRepresenter(RoundTripRepresenter):
        def ignore_aliases(self, data):
            return True

    def __init__(self, location):
        self.yaml = YAML()
        self.yaml.Representer = XRayTracingTemplateModifier.NonAliasingRTRepresenter
        super().__init__(location)

    def _get_template(self) -> Any:
        with open(self.template_location) as file:
            return self.yaml.load(file)

    def _update_template_fields(self):
        """
        Add new field to SAM template
        """
        if self.template.get(self.GLOBALS):
            template_globals = self.template.get(self.GLOBALS)

            function_globals = template_globals.get(self.FUNCTION, {})
            if not function_globals:
                template_globals[self.FUNCTION] = {}
            template_globals[self.FUNCTION][self.TRACING_FUNCTION] = self.ACTIVE_TRACING

            api_globals = template_globals.get(self.API, {})
            if not api_globals:
                template_globals[self.API] = {}
            template_globals[self.API][self.TRACING_API] = self.TRACING_API_VALUE

        else:
            self._add_tracing_with_globals()

    def _add_tracing_with_globals(self):
        """Adds Globals and tracing fields"""
        global_section = {
            self.FUNCTION: {self.TRACING_FUNCTION: self.ACTIVE_TRACING},
            self.API: {self.TRACING_API: self.TRACING_API_VALUE},
        }

        self.template = CommentedMap(self.template)
        self.template[self.GLOBALS] = CommentedMap(global_section)
        self.template.yaml_set_comment_before_after_key(self.GLOBALS, before=self.COMMENT)

    def _print_sanity_check_error(self):
        link = (
            "https://docs.aws.amazon.com/serverless-application-model/latest"
            "/developerguide/sam-resource-function.html#sam-function-tracing"
        )
        message = f"Warning: Unable to add Tracing to the project. To learn more about Tracing visit {link}"
        LOG.warning(message)

    def _write(self, template: list):
        """
        write generated template into SAM template

        Parameters
        ----------
        template : list
            array with updated template data
        """
        with open(self.template_location, "w") as file:
            self.yaml.dump(self.template, file)


"""
actuators.py
Classes to control the motors and servos. These classes
are wrapped in a mixer class before being used in the drive loop.
"""

from abc import ABC, abstractmethod
import time
import logging
from typing import Tuple

import donkeycar as dk
from donkeycar import utils
from donkeycar.utils import clamp

logger = logging.getLogger(__name__)

try:
    import RPi.GPIO as GPIO
except ImportError as e:
    logger.warn(f"RPi.GPIO was not imported. {e}")
    globals()["GPIO"] = None

from donkeycar.parts.pins import OutputPin, PwmPin, PinState
from donkeycar.utilities.deprecated import deprecated

logger = logging.getLogger(__name__)

#
# pwm/duty-cycle/pulse
# - Standard RC servo pulses range from 1 millisecond (full reverse)
#   to 2 milliseconds (full forward) with 1.5 milliseconds being neutral (stopped).
# - These pulses are typically send at 50 hertz (every 20 milliseconds).
# - This means that, using the standard 50hz frequency, a 1 ms pulse
#   represents a 5% duty cycle and a 2 ms pulse represents a 10% duty cycle.
# - The important part is the length of the pulse;
#   it must be in the range of 1 ms to 2ms.
# - So this means that if a different frequency is used, then the duty cycle
#   must be adjusted in order to get the 1ms to 2ms pulse.
# - For instance, if a 60hz frequency is used, then a 1 ms pulse requires
#   a duty cycle of 0.05 * 60 / 50 = 0.06 (6%) duty cycle
# - We default the frequency of our PCA9685 to 60 hz, so pulses in
#   config are generally based on 60hz frequency and 12 bit values.
#   We use 12 bit values because the PCA9685 has 12 bit resolution.
#   So a 1 ms pulse is 0.06 * 4096 ~= 246, a neutral pulse of 0.09 duty cycle
#   is 0.09 * 4096 ~= 367 and full forward pulse of 0.12 duty cycles
#   is 0.12 * 4096 ~= 492
# - These are generalizations that are useful for understanding the underlying
#   api call arguments.  The final choice of duty-cycle/pulse length depends
#   on your hardware and perhaps your strategy (you may not want to go too fast,
#   and so you may choose is low max throttle pwm)
#

def duty_cycle(pulse_ms:float, frequency_hz:float) -> float:
    """
    Calculate the duty cycle, 0 to 1, of a pulse given
    the frequency and the pulse length

    :param pulse_ms:float the desired pulse length in milliseconds
    :param frequency_hz:float the pwm frequency in hertz
    :return:float duty cycle in range 0 to 1
    """
    ms_per_cycle = 1000 / frequency_hz
    duty = pulse_ms / ms_per_cycle
    return duty

def pulse_ms(pulse_bits:int) -> float:
    """
    Calculate pulse width in milliseconds given a
    12bit pulse (as a PCA9685 would use).
    Donkeycar throttle and steering PWM values are
    based on PCA9685 12bit pulse values, where
    0 is zero duty cycle and 4095 is 100% duty cycle.

    :param pulse_bits:int 12bit integer in range 0 to 4095
    :return:float pulse length in milliseconds
    """
    if pulse_bits < 0 or pulse_bits > 4095:
        raise ValueError("pulse_bits must be in range 0 to 4095 (12bit integer)")
    return pulse_bits / 4095

class PulseController:
    """
    Controller that provides a servo PWM pulse using the given PwmPin
    See pins.py for pin provider implementations.
    """

    def __init__(self, pwm_pin:PwmPin, pwm_scale:float = 1.0, pwm_inverted:bool = False) -> None:
        """
        :param pwm_pin:PwnPin pin that will emit the pulse.
        :param pwm_scale:float scaling the 12 bit pulse value to compensate
                        for non-standard pwm frequencies.
        :param pwm_inverted:bool True to invert the duty cycle
        """
        self.pwm_pin = pwm_pin
        self.scale = pwm_scale
        self.inverted = pwm_inverted
        self.started = pwm_pin.state() != PinState.NOT_STARTED

    def set_pulse(self, pulse:int) -> None:
        """
        Set the length of the pulse using a 12 bit integer (0..4095)
        :param pulse:int 12bit integer (0..4095)
        """
        if pulse < 0 or pulse > 4095:
            logging.error("pulse must be in range 0 to 4095")
            pulse = clamp(pulse, 0, 4095)

        if not self.started:
            self.pwm_pin.start()
            self.started = True
        if self.inverted:
            pulse = 4095 - pulse
        self.pwm_pin.duty_cycle(int(pulse * self.scale) / 4095)

    def run(self, pulse:int) -> None:
        """
        Set the length of the pulse using a 12 bit integer (0..4095)
        :param pulse:int 12bit integer (0..4095)
        """
        self.set_pulse(pulse)

@deprecated("Deprecated in favor or PulseController.  This will be removed in a future release")
class PCA9685:
    '''
    PWM motor controler using PCA9685 boards.
    This is used for most RC Cars
    '''
    def __init__(self, channel, address=0x40, frequency=60, busnum=None, init_delay=0.1):

        self.default_freq = 60
        self.pwm_scale = frequency / self.default_freq

        import Adafruit_PCA9685
        # Initialise the PCA9685 using the default address (0x40).
        if busnum is not None:
            from Adafruit_GPIO import I2C
            # replace the get_bus function with our own
            def get_bus():
                return busnum
            I2C.get_default_bus = get_bus
        self.pwm = Adafruit_PCA9685.PCA9685(address=address)
        self.pwm.set_pwm_freq(frequency)
        self.channel = channel
        time.sleep(init_delay) # "Tamiya TBLE-02" makes a little leap otherwise

    def set_high(self):
        self.pwm.set_pwm(self.channel, 4096, 0)

    def set_low(self):
        self.pwm.set_pwm(self.channel, 0, 4096)

    def set_duty_cycle(self, duty_cycle):
        if duty_cycle < 0 or duty_cycle > 1:
            logging.error("duty_cycle must be in range 0 to 1")
            duty_cycle = clamp(duty_cycle, 0, 1)

        if duty_cycle == 1:
            self.set_high()
        elif duty_cycle == 0:
            self.set_low()
        else:
            # duty cycle is fraction of the 12 bits
            pulse = int(4096 * duty_cycle)
            try:
                self.pwm.set_pwm(self.channel, 0, pulse)
            except:
                self.pwm.set_pwm(self.channel, 0, pulse)

    def set_pulse(self, pulse):
        try:
            self.pwm.set_pwm(self.channel, 0, int(pulse * self.pwm_scale))
        except:
            self.pwm.set_pwm(self.channel, 0, int(pulse * self.pwm_scale))

    def run(self, pulse):
        self.set_pulse(pulse)

class VESC:
    '''
    VESC Motor controler using pyvesc
    This is used for most electric scateboards.

    inputs: serial_port---- port used communicate with vesc. for linux should be something like /dev/ttyACM1
    has_sensor=False------- default value from pyvesc
    start_heartbeat=True----default value from pyvesc (I believe this sets up a heartbeat and kills speed if lost)
    baudrate=115200--------- baudrate used for communication with VESC
    timeout=0.05-------------time it will try before giving up on establishing connection

    percent=.2--------------max percentage of the dutycycle that the motor will be set to
    outputs: none

    uses the pyvesc library to open communication with the VESC and sets the servo to the angle (0-1) and the duty_cycle(speed of the car) to the throttle (mapped so that percentage will be max/min speed)

    Note that this depends on pyvesc, but using pip install pyvesc will create a pyvesc file that
    can only set the speed, but not set the servo angle.

    Instead please use:
    pip install git+https://github.com/LiamBindle/PyVESC.git@master
    to install the pyvesc library
    '''
    def __init__(self, serial_port, percent=.2, has_sensor=False, start_heartbeat=True, baudrate=115200, timeout=0.05, steering_scale = 1.0, steering_offset = 0.0 ):

        try:
            import pyvesc
        except Exception as err:
            print("\n\n\n\n", err, "\n")
            print("please use the following command to import pyvesc so that you can also set")
            print("the servo position:")
            print("pip install git+https://github.com/LiamBindle/PyVESC.git@master")
            print("\n\n\n")
            time.sleep(1)
            raise

        assert percent <= 1 and percent >= -1,'\n\nOnly percentages are allowed for MAX_VESC_SPEED (we recommend a value of about .2) (negative values flip direction of motor)'
        self.steering_scale = steering_scale
        self.steering_offset = steering_offset
        self.percent = percent

        try:
            self.v = pyvesc.VESC(serial_port, has_sensor, start_heartbeat, baudrate, timeout)
        except Exception as err:
            print("\n\n\n\n", err)
            print("\n\nto fix permission denied errors, try running the following command:")
            print("sudo chmod a+rw {}".format(serial_port), "\n\n\n\n")
            time.sleep(1)
            raise

    def run(self, angle, throttle):
        self.v.set_servo((angle * self.steering_scale) + self.steering_offset)
        self.v.set_duty_cycle(throttle*self.percent)

@deprecated("Deprecated in favor or PulseController.  This will be removed in a future release")
class PiGPIO_PWM():
    '''
    # Use the pigpio python module and daemon to get hardware pwm controls from
    # a raspberrypi gpio pins and no additional hardware. Can serve as a replacement
    # for PCA9685.
    #
    # Install and setup:
    # sudo apt update && sudo apt install pigpio python3-pigpio
    # sudo systemctl start pigpiod
    #
    # Note: the range of pulses will differ from those required for PCA9685
    # and can range from 12K to 170K
    #
    # If you use a control circuit that inverts the steering signal, set inverted to True
    # Default multipler for pulses from config etc is 100
    #
    #
    '''

    def __init__(self, pin, pgio=None, freq=75, inverted=False):
        import pigpio
        self.pin = pin
        self.pgio = pgio or pigpio.pi()
        self.freq = freq
        self.inverted = inverted
        self.pgio.set_mode(self.pin, pigpio.OUTPUT)
        self.dead_zone = 37000

    def __del__(self):
        self.pgio.stop()

    def set_pulse(self, pulse):
#
        self.output = pulse * 200
        if self.output > 0:
            self.pgio.hardware_PWM(self.pin, self.freq, int(self.output if self.inverted == False else 1e6 - self.output))

    def run(self, pulse):
        self.set_pulse(pulse)

class PWMSteering:
    """
    Wrapper over a PWM pulse controller to convert angles to PWM pulses.
    """
    LEFT_ANGLE = -1
    RIGHT_ANGLE = 1

    def __init__(self, controller, left_pulse, right_pulse):

        if controller is None:
            raise ValueError("PWMSteering requires a set_pulse controller to be passed")
        set_pulse = getattr(controller, "set_pulse", None)
        if set_pulse is None or not callable(set_pulse):
            raise ValueError("controller must have a set_pulse method")
        if not utils.is_number_type(left_pulse):
            raise ValueError("left_pulse must be a number")
        if not utils.is_number_type(right_pulse):
            raise ValueError("right_pulse must be a number")

        self.controller = controller
        self.left_pulse = left_pulse
        self.right_pulse = right_pulse
        self.pulse = dk.utils.map_range(0, self.LEFT_ANGLE, self.RIGHT_ANGLE,
                                        self.left_pulse, self.right_pulse)
        self.running = True
        logger.info('PWM Steering created')

    def update(self):
        while self.running:
            self.controller.set_pulse(self.pulse)

    def run_threaded(self, angle):
        # map absolute angle to angle that vehicle can implement.
        angle = utils.clamp(angle, self.LEFT_ANGLE, self.RIGHT_ANGLE)
        self.pulse = dk.utils.map_range(angle,
                                        self.LEFT_ANGLE, self.RIGHT_ANGLE,
                                        self.left_pulse, self.right_pulse)

    def run(self, angle):
        self.run_threaded(angle)
        self.controller.set_pulse(self.pulse)

    def shutdown(self):
        # set steering straight
        self.pulse = 0
        time.sleep(0.3)
        self.running = False

class PWMThrottle:
    """
    Wrapper over a PWM pulse controller to convert -1 to 1 throttle
    values to PWM pulses.
    """
    MIN_THROTTLE = -1
    MAX_THROTTLE = 1

    def __init__(self, controller, max_pulse, min_pulse, zero_pulse):

        if controller is None:
            raise ValueError("PWMThrottle requires a set_pulse controller to be passed")
        set_pulse = getattr(controller, "set_pulse", None)
        if set_pulse is None or not callable(set_pulse):
            raise ValueError("controller must have a set_pulse method")

        self.controller = controller
        self.max_pulse = max_pulse
        self.min_pulse = min_pulse
        self.zero_pulse = zero_pulse
        self.pulse = zero_pulse

        # send zero pulse to calibrate ESC
        logger.info("Init ESC")
        self.controller.set_pulse(self.max_pulse)
        time.sleep(0.01)
        self.controller.set_pulse(self.min_pulse)
        time.sleep(0.01)
        self.controller.set_pulse(self.zero_pulse)
        time.sleep(1)
        self.running = True
        logger.info('PWM Throttle created')

    def update(self):
        while self.running:
            self.controller.set_pulse(self.pulse)

    def run_threaded(self, throttle):
        throttle = utils.clamp(throttle, self.MIN_THROTTLE, self.MAX_THROTTLE)
        if throttle > 0:
            self.pulse = dk.utils.map_range(throttle, 0, self.MAX_THROTTLE,
                                            self.zero_pulse, self.max_pulse)
        else:
            self.pulse = dk.utils.map_range(throttle, self.MIN_THROTTLE, 0,
                                            self.min_pulse, self.zero_pulse)

    def run(self, throttle):
        self.run_threaded(throttle)
        self.controller.set_pulse(self.pulse)

    def shutdown(self):
        # stop vehicle
        self.run(0)
        self.running = False

#
# This seems redundant.  If it's really emulating and PCA9685, then
# why don't we just use that code?
# - this is not used in any templates
# - this is not documented in docs or elsewhere
# - this seems redundant; if it emultates a PCA9685 then we should be able to use that code.
# - there is an intention to implement a Firmata driver in pins.py.
#   Teensy can run Firmata, so that is a way to get Teensy support.
#   See https://www.pjrc.com/teensy/td_libs_Firmata.html
#
@deprecated("JHat is unsupported/undocumented in the framework.  It will be removed in a future release.")
class JHat:
    '''
    PWM motor controller using Teensy emulating PCA9685.
    '''
    def __init__(self, channel, address=0x40, frequency=60, busnum=None):
        logger.info("Firing up the Hat")
        import Adafruit_PCA9685
        LED0_OFF_L = 0x08
        # Initialise the PCA9685 using the default address (0x40).
        if busnum is not None:
            from Adafruit_GPIO import I2C
            # replace the get_bus function with our own
            def get_bus():
                return busnum
            I2C.get_default_bus = get_bus
        self.pwm = Adafruit_PCA9685.PCA9685(address=address)
        self.pwm.set_pwm_freq(frequency)
        self.channel = channel
        self.register = LED0_OFF_L+4*channel

        # we install our own write that is more efficient use of interrupts
        self.pwm.set_pwm = self.set_pwm

    def set_pulse(self, pulse):
        self.set_pwm(self.channel, 0, pulse)

    def set_pwm(self, channel, on, off):
        # sets a single PWM channel
        self.pwm._device.writeList(self.register, [off & 0xFF, off >> 8])

    def run(self, pulse):
        self.set_pulse(pulse)

#
# JHatReader is on the chopping block for removal
# - it is not integrated into any templates
# - it is not documented in docs or elsewhere
# This appears to be a way to read RC receiving input.  As such
# it would more appropriately be integrated into controllers.py.
# If that can be addressed then we would keep this,
# otherwise this should be removed.
#
@deprecated("JHatReader is unsupported/undocumented in the framework.  It may be removed in a future release.")
class JHatReader:
    '''
    Read RC controls from teensy
    '''
    def __init__(self, channel, address=0x40, frequency=60, busnum=None):
        import Adafruit_PCA9685
        self.pwm = Adafruit_PCA9685.PCA9685(address=address)
        self.pwm.set_pwm_freq(frequency)
        self.register = 0 #i2c read doesn't take an address
        self.steering = 0
        self.throttle = 0
        self.running = True
        #send a reset
        self.pwm._device.writeRaw8(0x06)

    def read_pwm(self):
        '''
        send read requests via i2c bus to Teensy to get
        pwm control values from last RC input
        '''
        h1 = self.pwm._device.readU8(self.register)
        # first byte of header must be 100, otherwize we might be reading
        # in the wrong byte offset
        while h1 != 100:
            logger.debug("skipping to start of header")
            h1 = self.pwm._device.readU8(self.register)

        h2 = self.pwm._device.readU8(self.register)
        # h2 ignored now

        val_a = self.pwm._device.readU8(self.register)
        val_b = self.pwm._device.readU8(self.register)
        self.steering = (val_b << 8) + val_a

        val_c = self.pwm._device.readU8(self.register)
        val_d = self.pwm._device.readU8(self.register)
        self.throttle = (val_d << 8) + val_c

        # scale the values from -1 to 1
        self.steering = (self.steering - 1500.0) / 500.0  + 0.158
        self.throttle = (self.throttle - 1500.0) / 500.0  + 0.136

    def update(self):
        while(self.running):
            self.read_pwm()

    def run_threaded(self):
        return self.steering, self.throttle

    def shutdown(self):
        self.running = False
        time.sleep(0.1)

#
# Adafruit_DCMotor_Hat support is on the block for removal
# - it is not integrated into any templates
# - It is not documented in the docs or otherwise
# A path forward would be to add a drive train option
# DRIVE_TRAIN_TYPE = "DC_TWO_WHEEL_ADAFRUIT"
# and integrate this into complete.py
#
@deprecated("This appears to be unsupported/undocumented in the framework. This may be removed in a future release")
class Adafruit_DCMotor_Hat:
    '''
    Adafruit DC Motor Controller
    Used for each motor on a differential drive car.
    '''
    def __init__(self, motor_num):
        from Adafruit_MotorHAT import Adafruit_MotorHAT, Adafruit_DCMotor
        import atexit

        self.FORWARD = Adafruit_MotorHAT.FORWARD
        self.BACKWARD = Adafruit_MotorHAT.BACKWARD
        self.mh = Adafruit_MotorHAT(addr=0x60)

        self.motor = self.mh.getMotor(motor_num)
        self.motor_num = motor_num

        atexit.register(self.turn_off_motors)
        self.speed = 0
        self.throttle = 0

    def run(self, speed):
        '''
        Update the speed of the motor where 1 is full forward and
        -1 is full backwards.
        '''
        if speed > 1 or speed < -1:
            raise ValueError( "Speed must be between 1(forward) and -1(reverse)")

        self.speed = speed
        self.throttle = int(dk.utils.map_range(abs(speed), -1, 1, -255, 255))

        if speed > 0:
            self.motor.run(self.FORWARD)
        else:
            self.motor.run(self.BACKWARD)

        self.motor.setSpeed(self.throttle)

    def shutdown(self):
        self.mh.getMotor(self.motor_num).run(Adafruit_MotorHAT.RELEASE)

#
# Maestro Servo Controller support is on the block for removal
# - it is not integrated into any templates
# - It is not documented in the docs or otherwise
# - It seems to require a separate AStar microcontroller for which there is no firmware included or referenced
# If that can be addressed, then we can add Maestro support to the pin provide api in pins.py
# so it can be used a source of TTL and PWM for the generalized motor drivers.
# Perhaps the AStar controller is not integral to using this as a source of servo pulses in which
# case it seems pretty straight forward to integrate this into pins.py and then delete this class.
#
@deprecated("This appears to be unsupported/undocumented in the framework. This may be removed in a future release")
class Maestro:
    '''
    Pololu Maestro Servo controller
    Use the MaestroControlCenter to set the speed & acceleration values to 0!
    See https://www.pololu.com/docs/0J40/all
    '''
    import threading

    maestro_device = None
    astar_device = None
    maestro_lock = threading.Lock()
    astar_lock = threading.Lock()

    def __init__(self, channel, frequency = 60):
        import serial

        if Maestro.maestro_device == None:
            Maestro.maestro_device = serial.Serial('/dev/ttyACM0', 115200)

        self.channel = channel
        self.frequency = frequency
        self.lturn = False
        self.rturn = False
        self.headlights = False
        self.brakelights = False

        if Maestro.astar_device == None:
            Maestro.astar_device = serial.Serial('/dev/ttyACM2', 115200, timeout= 0.01)

    def set_pulse(self, pulse):
        # Recalculate pulse width from the Adafruit values
        w = pulse * (1 / (self.frequency * 4096)) # in seconds
        w *= 1000 * 1000  # in microseconds
        w *= 4  # in quarter microsenconds the maestro wants
        w = int(w)

        with Maestro.maestro_lock:
            Maestro.maestro_device.write(bytearray([ 0x84,
                                                     self.channel,
                                                     (w & 0x7F),
                                                     ((w >> 7) & 0x7F)]))

    def set_turn_left(self, v):
        if self.lturn != v:
            self.lturn = v
            b = bytearray('L' if v else 'l', 'ascii')
            with Maestro.astar_lock:
                Maestro.astar_device.write(b)

    def set_turn_right(self, v):
        if self.rturn != v:
            self.rturn = v
            b = bytearray('R' if v else 'r', 'ascii')
            with Maestro.astar_lock:
                Maestro.astar_device.write(b)

    def set_headlight(self, v):
        if self.headlights != v:
            self.headlights = v
            b = bytearray('H' if v else 'h', 'ascii')
            with Maestro.astar_lock:
                Maestro.astar_device.write(b)

    def set_brake(self, v):
        if self.brakelights != v:
            self.brakelights = v
            b = bytearray('B' if v else 'b', 'ascii')
            with Maestro.astar_lock:
                Maestro.astar_device.write(b)

    def readline(self):
        ret = None
        with Maestro.astar_lock:
            # expecting lines like
            # E n nnn n
            if Maestro.astar_device.inWaiting() > 8:
                ret = Maestro.astar_device.readline()

        if ret is not None:
            ret = ret.rstrip()

        return ret

#
# Teensy support is on the chopping block.
# - It is not integrated into any template
# - It is not documented in the docs or otherwise
# - It presumably requires a firmware to be uploaded to the teensy, but no firmware is referenced.
# If that can be addressed, then we can add Teensy support to the pin provide api in pins.py
# so it can be used a source of TTL and PWM for the generalized motor drivers.
# Another route is to implement Firmata protocol (perhaps a version of the Arduino sketch,
# see ArduinoFirmata below)
#
@deprecated("This appears to be unsupported/undocumented in the framework. This may be removed in a future release")
class Teensy:
    '''
    Teensy Servo controller
    '''
    import threading

    teensy_device = None
    astar_device = None
    teensy_lock = threading.Lock()
    astar_lock = threading.Lock()

    def __init__(self, channel, frequency = 60):
        import serial

        if Teensy.teensy_device == None:
            Teensy.teensy_device = serial.Serial('/dev/teensy', 115200, timeout = 0.01)

        self.channel = channel
        self.frequency = frequency
        self.lturn = False
        self.rturn = False
        self.headlights = False
        self.brakelights = False

        if Teensy.astar_device == None:
            Teensy.astar_device = serial.Serial('/dev/astar', 115200, timeout = 0.01)

    def set_pulse(self, pulse):
        # Recalculate pulse width from the Adafruit values
        w = pulse * (1 / (self.frequency * 4096)) # in seconds
        w *= 1000 * 1000  # in microseconds

        with Teensy.teensy_lock:
            Teensy.teensy_device.write(("%c %.1f\n" % (self.channel, w)).encode('ascii'))

    def set_turn_left(self, v):
        if self.lturn != v:
            self.lturn = v
            b = bytearray('L' if v else 'l', 'ascii')
            with Teensy.astar_lock:
                Teensy.astar_device.write(b)

    def set_turn_right(self, v):
        if self.rturn != v:
            self.rturn = v
            b = bytearray('R' if v else 'r', 'ascii')
            with Teensy.astar_lock:
                Teensy.astar_device.write(b)

    def set_headlight(self, v):
        if self.headlights != v:
            self.headlights = v
            b = bytearray('H' if v else 'h', 'ascii')
            with Teensy.astar_lock:
                Teensy.astar_device.write(b)

    def set_brake(self, v):
        if self.brakelights != v:
            self.brakelights = v
            b = bytearray('B' if v else 'b', 'ascii')
            with Teensy.astar_lock:
                Teensy.astar_device.write(b)

    def teensy_readline(self):
        ret = None
        with Teensy.teensy_lock:
            # expecting lines like
            # E n nnn n
            if Teensy.teensy_device.inWaiting() > 8:
                ret = Teensy.teensy_device.readline()

        if ret != None:
            ret = ret.rstrip()

        return ret

    def astar_readline(self):
        ret = None
        with Teensy.astar_lock:
            # expecting lines like
            # E n nnn n
            if Teensy.astar_device.inWaiting() > 8:
                ret = Teensy.astar_device.readline()

        if ret != None:
            ret = ret.rstrip()

        return ret

class MockController(object):
    def __init__(self):
        pass

    def run(self, pulse):
        pass

    def shutdown(self):
        pass

class L298N_HBridge_3pin(object):
    """
    Motor controlled with an L298N hbridge,
    chosen with configuration DRIVETRAIN_TYPE=DC_TWO_WHEEL_L298N
    Uses two OutputPins to select direction and
    a PwmPin to control the power to the motor.
    See pins.py for pin provider implementations.

    See https://www.electronicshub.org/raspberry-pi-l298n-interface-tutorial-control-dc-motor-l298n-raspberry-pi/
    for a discussion of how the L298N hbridge module is wired in 3-pin mode.
    This also applies to the some other driver chips that emulate
    the L298N, such as the TB6612FNG motor driver.
    """

    def __init__(self, pin_forward:OutputPin, pin_backward:OutputPin, pwm_pin:PwmPin, zero_throttle:float=0, max_duty=0.9):
        """
        :param pin_forward:OutputPin when HIGH the motor will turn clockwise
                        using the output of the pwm_pin as a duty_cycle
        :param pin_backward:OutputPin when HIGH the motor will turn counter-clockwise
                            using the output of the pwm_pin as a duty_cycle
        :param pwm_pin:PwmPin takes a duty cycle in the range of 0 to 1,
                    where 0 is fully off and 1 is fully on.
        :param zero_throttle: values at or below zero_throttle are treated as zero.
        :param max_duty: the maximum duty cycle that will be send to the motors

        NOTE: if pin_forward and pin_backward are both LOW, then the motor is
            'detached' and will glide to a stop.
            if pin_forward and pin_backward are both HIGH, then the motor
            will be forcibly stopped (can be used for braking)
        """
        self.pin_forward = pin_forward
        self.pin_backward = pin_backward
        self.pwm_pin = pwm_pin
        self.zero_throttle = zero_throttle
        self.throttle = 0
        self.max_duty = max_duty
        self.pin_forward.start(PinState.LOW)
        self.pin_backward.start(PinState.LOW)
        self.pwm_pin.start(0)

    def run(self, throttle:float) -> None:
        """
        Update the speed of the motor
        :param throttle:float throttle value in range -1 to 1,
                        where 1 is full forward and -1 is full backwards.
        """
        if throttle is None:
            logger.warn("TwoWheelSteeringThrottle throttle is None")
            return
        if throttle > 1 or throttle < -1:
            logger.warn( f"TwoWheelSteeringThrottle throttle is {throttle}, but it must be between 1(forward) and -1(reverse)")
            throttle = clamp(throttle, -1, 1)

        self.speed = throttle
        self.throttle = dk.utils.map_range_float(throttle, -1, 1, -self.max_duty, self.max_duty)
        if self.throttle > self.zero_throttle:
            self.pwm_pin.duty_cycle(self.throttle)
            self.pin_backward.output(PinState.LOW)
            self.pin_forward.output(PinState.HIGH)
        elif self.throttle < -self.zero_throttle:
            self.pwm_pin.duty_cycle(-self.throttle)
            self.pin_forward.output(PinState.LOW)
            self.pin_backward.output(PinState.HIGH)
        else:
            self.pwm_pin.duty_cycle(0)
            self.pin_forward.output(PinState.LOW)
            self.pin_backward.output(PinState.LOW)

    def shutdown(self):
        self.pwm_pin.stop()
        self.pin_forward.stop()
        self.pin_backward.stop()

class TwoWheelSteeringThrottle(object):
    """
    Modify individual differential drive wheel throttles
    in order to implemeht steering.
    """

    def run(self, throttle:float, steering:float) -> Tuple[float, float]:
        """
        :param throttle:float throttle value in range -1 to 1,
                        where 1 is full forward and -1 is full backwards.
        :param steering:float steering value in range -1 to 1,
                        where -1 is full left and 1 is full right.
        :return: tuple of left motor and right motor throttle values in range -1 to 1
                 where 1 is full forward and -1 is full backwards.
        """
        if throttle is None:
            logger.warn("TwoWheelSteeringThrottle throttle is None")
            return
        if steering is None:
            logger.warn("TwoWheelSteeringThrottle steering is None")
            return
        if throttle > 1 or throttle < -1:
            logger.warn( f"TwoWheelSteeringThrottle throttle is {throttle}, but it must be between 1(forward) and -1(reverse)")
            throttle = clamp(throttle, -1, 1)
        if steering > 1 or steering < -1:
            logger.warn( f"TwoWheelSteeringThrottle steering is {steering}, but it must be between 1(right) and -1(left)")
            steering = clamp(steering, -1, 1)

        left_motor_speed = throttle
        right_motor_speed = throttle

        if steering < 0:
            left_motor_speed *= (1.0 - (-steering))
        elif steering > 0:
            right_motor_speed *= (1.0 - steering)

        return left_motor_speed, right_motor_speed

    def shutdown(self) -> None:
        pass

class L298N_HBridge_2pin(object):
    """
    Motor controlled with an 'mini' L298N hbridge using 2 PwmPins,
    one for forward pwm and for reverse pwm.
    Chosen with configuration DRIVETRAIN_TYPE=DC_TWO_WHEEL
    See pins.py for pin provider implementations.

    See https://www.instructables.com/Tutorial-for-Dual-Channel-DC-Motor-Driver-Board-PW/
    for how an L298N mini-hbridge modules is wired.
    This driver can also be used for an L9110S/HG7881 motor driver.  See
    https://electropeak.com/learn/interfacing-l9110s-dual-channel-h-bridge-motor-driver-module-with-arduino/
    for how an L9110S motor driver module is wired.
    """

    def __init__(self, pin_forward:PwmPin, pin_backward:PwmPin, zero_throttle:float=0, max_duty = 0.9):
        """
        pin_forward:PwmPin Takes a duty cycle in the range of 0 to 1,
                        where 0 is fully off and 1 is fully on.
                        When the duty_cycle > 0 the motor will turn clockwise
                        proportial to the duty_cycle
        pin_backward:PwmPin Takes a duty cycle in the range of 0 to 1,
                            where 0 is fully off and 1 is fully on.
                            When the duty_cycle > 0 the motor will turn counter-clockwise
                            proportial to the duty_cycle
        zero_throttle: values at or below zero_throttle are treated as zero.
        max_duty: the maximum duty cycle that will be send to the motors

        NOTE: if pin_forward and pin_backward are both at duty_cycle == 0,
            then the motor is 'detached' and will glide to a stop.
            if pin_forward and pin_backward are both at duty_cycle == 1,
            then the motor will be forcibly stopped (can be used for braking)
        max_duty is from 0 to 1 (fully off to fully on). I've read 0.9 is a good max.
        """
        self.pin_forward = pin_forward
        self.pin_backward = pin_backward
        self.zero_throttle = zero_throttle
        self.max_duty = max_duty

        self.throttle=0
        self.speed=0

        self.pin_forward.start(0)
        self.pin_backward.start(0)

    def run(self, throttle:float) -> None:
        """
        Update the speed of the motor
        :param throttle:float throttle value in range -1 to 1,
                        where 1 is full forward and -1 is full backwards.
        """
        if throttle is None:
            logger.warn("TwoWheelSteeringThrottle throttle is None")
            return
        if throttle > 1 or throttle < -1:
            logger.warn( f"TwoWheelSteeringThrottle throttle is {throttle}, but it must be between 1(forward) and -1(reverse)")
            throttle = clamp(throttle, -1, 1)

        self.speed = throttle
        self.throttle = dk.utils.map_range_float(throttle, -1, 1, -self.max_duty, self.max_duty)

        if self.throttle > self.zero_throttle:
            self.pin_backward.duty_cycle(0)
            self.pin_forward.duty_cycle(self.throttle)
        elif self.throttle < -self.zero_throttle:
            self.pin_forward.duty_cycle(0)
            self.pin_backward.duty_cycle(-self.throttle)
        else:
            self.pin_forward.duty_cycle(0)
            self.pin_backward.duty_cycle(0)

    def shutdown(self):
        self.pin_forward.stop()
        self.pin_backward.stop()

#
# This is being replaced by pins.py and PulseController.
# GPIO pins can be configured using RPi.GPIO or PIGPIO,
# so this is redundant
#
@deprecated("This will be removed in a future release in favor of PulseController")
class RPi_GPIO_Servo(object):
    '''
    Servo controlled from the gpio pins on Rpi
    '''
    def __init__(self, pin, pin_scheme=None, freq=50, min=5.0, max=7.8):
        self.pin = pin
        if pin_scheme is None:
            pin_scheme = GPIO.BCM
        GPIO.setmode(pin_scheme)
        GPIO.setup(self.pin, GPIO.OUT)

        self.throttle = 0
        self.pwm = GPIO.PWM(self.pin, freq)
        self.pwm.start(0)
        self.min = min
        self.max = max

    def run(self, pulse):
        '''
        Update the speed of the motor where 1 is full forward and
        -1 is full backwards.
        '''
        # I've read 90 is a good max
        self.throttle = dk.map_frange(pulse, -1.0, 1.0, self.min, self.max)
        # logger.debug(pulse, self.throttle)
        self.pwm.ChangeDutyCycle(self.throttle)

    def shutdown(self):
        self.pwm.stop()
        GPIO.cleanup()

#
# This is being replaced by pins.py.  GPIO pins can be
# configured using RPi.GPIO or PIGPIO, so ServoBlaster is redundant
#
@deprecated("This will be removed in a future release in favor of PulseController")
class ServoBlaster(object):
    '''
    Servo controlled from the gpio pins on Rpi
    This uses a user space service to generate more efficient PWM via DMA control blocks.
    Check readme and install here:
    https://github.com/richardghirst/PiBits/tree/master/ServoBlaster
    cd PiBits/ServoBlaster/user
    make
    sudo ./servod
    will start the daemon and create the needed device file:
    /dev/servoblaster

    to test this from the command line:
    echo P1-16=120 > /dev/servoblaster

    will send 1200us PWM pulse to physical pin 16 on the pi.

    If you want it to start on boot:
    sudo make install
    '''
    def __init__(self, pin):
        self.pin = pin
        self.servoblaster = open('/dev/servoblaster', 'w')
        self.min = min
        self.max = max

    def set_pulse(self, pulse):
        s = 'P1-%d=%d\n' % (self.pin, pulse)
        self.servoblaster.write(s)
        self.servoblaster.flush()

    def run(self, pulse):
        self.set_pulse(pulse)

    def shutdown(self):
        self.run((self.max + self.min) / 2)
        self.servoblaster.close()

#
# TODO: integrate ArduinoFirmata support into pin providers, then we can remove all of this code and use PulseController
#
# Arduino/Microcontroller PWM support.
# Firmata is a specification for configuring general purpose microcontrollers remotey.
# The implementatino for Arduino is used here.
#
# See https://docs.donkeycar.com/parts/actuators/#arduino for how to set this up.
# Firmata Protocol https://github.com/firmata/protocol
# Arduino implementation https://github.com/firmata/arduino
#
# NOTE: to create a general purpose InputPin/OutputPin/PwmPin with support for servo pulses between 1ms and 2ms
#       with good resolution, it is likely we will need to create our own sketch based on the Arduino Firmata
#       examples.  By default, analog write is not adequate to support servos; it is good for motor duty cycles
#       but is poor for servos because of the low resolution and poor control of frequency.  So we need to
#       use the Servo.h library and dynamically add a Servo instance to a pin when it is configured for
#       analog output.  Further, we should use writeMicroseconds for output and so interpret values
#       to the pin as microseconds for the on part of the pulse.  See the various flavors of examples
#       in the Arduino Firmata repo linked above.
#
@deprecated("This will be removed in a future release and Arduino support will be added to pins.py")
class ArduinoFirmata:
    '''
    PWM controller using Arduino board.
    This is particularly useful for boards like Latte Panda with built it Arduino.
    Standard Firmata sketch needs to be loaded on Arduino side.
    Refer to docs/parts/actuators.md for more details
    '''

    def __init__(self, servo_pin = 6, esc_pin = 5):
        from pymata_aio.pymata3 import PyMata3
        self.board = PyMata3()
        self.board.sleep(0.015)
        self.servo_pin = servo_pin
        self.esc_pin = esc_pin
        self.board.servo_config(servo_pin)
        self.board.servo_config(esc_pin)

    def set_pulse(self, pin, angle):
        try:
            self.board.analog_write(pin, int(angle))
        except:
            self.board.analog_write(pin, int(angle))

    def set_servo_pulse(self, angle):
        self.set_pulse(self.servo_pin, int(angle))

    def set_esc_pulse(self, angle):
        self.set_pulse(self.esc_pin, int(angle))

@deprecated("This will be removed in a future release and Arduino PWM support will be add to pins.py")
class ArdPWMSteering:
    """
    Wrapper over a Arduino Firmata controller to convert angles to PWM pulses.
    """
    LEFT_ANGLE = -1
    RIGHT_ANGLE = 1

    def __init__(self,
                 controller=None,
                 left_pulse=60,
                 right_pulse=120):

        self.controller = controller
        self.left_pulse = left_pulse
        self.right_pulse = right_pulse
        self.pulse = dk.utils.map_range(0, self.LEFT_ANGLE, self.RIGHT_ANGLE,
                                        self.left_pulse, self.right_pulse)
        self.running = True
        logger.info('Arduino PWM Steering created')

    def run(self, angle):
        # map absolute angle to angle that vehicle can implement.
        self.pulse = dk.utils.map_range(angle,
                                        self.LEFT_ANGLE, self.RIGHT_ANGLE,
                                        self.left_pulse, self.right_pulse)
        self.controller.set_servo_pulse(self.pulse)

    def shutdown(self):
        # set steering straight
        self.pulse = dk.utils.map_range(0, self.LEFT_ANGLE, self.RIGHT_ANGLE,
                                        self.left_pulse, self.right_pulse)
        time.sleep(0.3)
        self.running = False

@deprecated("This will be removed in a future release and Arduino PWM support will be add to pins.py")
class ArdPWMThrottle:

    """
    Wrapper over Arduino Firmata controller to convert -1 to 1 throttle
    values to PWM pulses.
    """
    MIN_THROTTLE = -1
    MAX_THROTTLE = 1

    def __init__(self,
                 controller=None,
                 max_pulse=105,
                 min_pulse=75,
                 zero_pulse=90):

        self.controller = controller
        self.max_pulse = max_pulse
        self.min_pulse = min_pulse
        self.zero_pulse = zero_pulse
        self.pulse = zero_pulse

        # send zero pulse to calibrate ESC
        logger.info("Init ESC")
        self.controller.set_esc_pulse(self.max_pulse)
        time.sleep(0.01)
        self.controller.set_esc_pulse(self.min_pulse)
        time.sleep(0.01)
        self.controller.set_esc_pulse(self.zero_pulse)
        time.sleep(1)
        self.running = True
        logger.info('Arduino PWM Throttle created')

    def run(self, throttle):
        if throttle > 0:
            self.pulse = dk.utils.map_range(throttle, 0, self.MAX_THROTTLE,
                                            self.zero_pulse, self.max_pulse)
        else:
            self.pulse = dk.utils.map_range(throttle, self.MIN_THROTTLE, 0,
                                            self.min_pulse, self.zero_pulse)
        self.controller.set_esc_pulse(self.pulse)

    def shutdown(self):
        # stop vehicle
        self.run(0)
        self.running = False

{'threading': 'threading', 'serial': 'serial'}.maestro_lock

{'threading': 'threading', 'serial': 'serial'}.maestro_device.write

{'threading': 'threading', 'serial': 'serial'}.astar_lock

{'threading': 'threading', 'serial': 'serial'}.astar_device.write

{'threading': 'threading', 'serial': 'serial'}.astar_device.inWaiting

{'threading': 'threading', 'serial': 'serial'}.astar_device.readline

{'threading': 'threading', 'serial': 'serial'}.teensy_lock

{'threading': 'threading', 'serial': 'serial'}.teensy_device.write

{'threading': 'threading', 'serial': 'serial'}.teensy_device.inWaiting

{'threading': 'threading', 'serial': 'serial'}.teensy_device.readline


import argparse
import asyncio
from datetime import datetime, timedelta
import json
import pytz

from dateutil.tz import tzlocal
import requests
import websockets

ARGS = argparse.ArgumentParser(description='Coinbase Exchange Data Collector')
ARGS.add_argument('--m', action='store', dest='minutes', default=5, help='Minutes to run')
args = ARGS.parse_args()

minutes = int(args.minutes)
begin = datetime.now(tzlocal())
end = begin + timedelta(minutes=minutes)

def get_beginning_level_3():
    global beginning_level_3
    beginning_level_3 = requests.get('https://api.pro.coinbase.com/products/BTC-USD/book?level=3').json()

def get_ending_level_3():
    global ending_level_3
    ending_level_3 = requests.get('https://api.pro.coinbase.com/products/BTC-USD/book?level=3').json()

@asyncio.coroutine
def get_websocket_data():
    coinbase_websocket = yield from websockets.connect("wss://ws-feed.pro.coinbase.com")
    yield from coinbase_websocket.send('{"type": "subscribe", "product_id": "BTC-USD"}')
    global messages
    global latencies
    messages = []
    latencies = {}
    while True:
        message = yield from coinbase_websocket.recv()
        message = json.loads(message)
        messages += [message]
        latencies[message['time']] = int((datetime.now(tzlocal()) -
                                              datetime.strptime(message['time'], '%Y-%m-%dT%H:%M:%S.%fZ')
                                              .replace(tzinfo=pytz.UTC)).microseconds)
        if datetime.now(tzlocal()) > end:
            return

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.call_later(10, get_beginning_level_3)
    loop.call_later(minutes*60-10, get_ending_level_3)
    loop.run_until_complete(get_websocket_data())
    global messages
    global beginning_level_3
    global ending_level_3
    global latencies

    first_sequence = beginning_level_3['sequence']
    last_sequence = ending_level_3['sequence']

    messages = [message for message in messages if first_sequence < message['sequence'] <= last_sequence]
    messages = sorted(messages, key=lambda k: k['sequence'])
    for message in messages:
        assert message['sequence'] == first_sequence + 1
        first_sequence = message['sequence']

    with open('latencies.json', 'w') as json_file:
        json.dump(latencies, json_file, indent=4, sort_keys=True)
    with open('messages.json', 'w') as json_file:
        json.dump(messages, json_file, indent=4, sort_keys=True)
    with open('beginning_level_3.json', 'w') as json_file:
        json.dump(beginning_level_3, json_file, indent=4, sort_keys=True)
    with open('ending_level_3.json', 'w') as json_file:
        json.dump(ending_level_3, json_file, indent=4, sort_keys=True)


#training model

import cv2                        #open cv library
import numpy as np                #for mathematical calculation
from os import listdir            #class of os module // when fetching data
from os.path import isfile,join

data_path ="E:/PROJECT/Face-Recognition-Project/Face-Recognition-Project-master/sample"
onlyfiles = [f for f in listdir(data_path) if isfile(join(data_path,f))]

Training_Data,Labels = [],[]

for i,files in enumerate(onlyfiles):
    image_path = data_path + onlyfiles[i]
    images = cv2.imread(image_path,cv2.IMREAD_GRAYSCALE)
    Training_Data.append(np.asarray(images,dtype=np.uint8))
    Labels.append(i)

Labels = np.asarray(Labels,dtype=np.int32)

model = cv2.face.LBPHFaceRecognizer_create()

model.train(np.asarray(Training_Data),np.asarray(Labels))

print("Congratulations model is TRAINED ... *_*...")

face_classifier = cv2.CascadeClassifier("C:/Users/AYUSH/AppData/Local/Programs/Python/Python310/Lib/site-packages/cv2/data/haarcascade_frontalface_default.xml")


import nltk
import inflect

from nltk.tokenize import word_tokenize

inflect = inflect.engine()

def is_noun(tag):
    return tag in ['NN', 'NNS', 'NNP', 'NNPS']

def is_verb(tag):
    return tag in ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']

def is_adverb(tag):
    return tag in ['RB', 'RBR', 'RBS']

def is_adjective(tag):
    return tag in ['JJ', 'JJR', 'JJS']

def is_pronoun(tag):
    return tag in ['PRP$', 'PRP', 'WP', 'WP$']

def is_numeral(tag):
    return tag in ['JJ', 'CD']

def get_tags(query):
    return nltk.pos_tag(word_tokenize(query))

def is_singular_noun(tag):
    return (is_noun(tag[1])) and (not inflect.singular_noun(tag[0]))

def is_plural_noun(tag):
    return (is_noun(tag[1])) and (inflect.singular_noun(tag[0]))

def check_to_be(word, tags, i):
    after_word = None
    after_word2 = None
    before_word = None
    after_tag = None
    if i > 0:
        before_word = tags[i - 1][0]
    if i + 1 < len(tags):
        after_word = tags[i + 1][0]
        after_tag = tags[i + 1][1]
    if i + 2 < len(tags):
        after_word2 = tags[i + 2][1]
    if word.lower() == 'i':
        if after_word in ['am', 'was', 'wasn\'t']:
            if is_verb(after_word2):
                return False
            else:
                return True
        elif is_verb(after_tag):
            return True
        else:
            if before_word in ['am', 'was', 'wasn\'t', 'not']:
                return True
            else:
                return False
    elif word.lower() in ['you', 'we', 'they']:
        if after_word in ['are', 'were', 'weren\'t']:
            if is_verb(after_word2):
                return False
            else:
                return True
        else:
            if before_word in ['are', 'were', 'weren\'t', 'not']:
                return True
            else:
                return False
    elif word.lower() in ['he', 'she', 'it']:
        if after_word in ['is', 'was', 'wasn\'t']:
            if is_verb(after_word2):
                return False
            else:
                return True
        else:
            if before_word in ['is', 'was', 'wasn\'t', 'not']:
                return True
            else:
                return False
    else:
        return True

def check_adverbs_frequency(i, tags):
    if is_adverb(tags[i][1]):
        if (tags[i][0] in ['always', 'usually', 'sometimes', 'never']) and (i == 0):
            return False
        elif tags[i][0] in ['always', 'usually', 'sometimes']:
            if (not is_noun(tags[i-1][1])) and (not is_pronoun(tags[i-1][1])):
                return False
            else:
                return is_verb(tags[i+1])
        elif tags[i][0] == 'never':
            if(not is_noun(tags[i - 1][1])) and (not is_pronoun(tags[i - 1][1])):
                return False
            else:
                if is_verb(tags[i+1][1]):
                    f = True
                    for x in tags:
                        if ('not' in x) or ('n\'t' in x):
                            f = False
                            break
                    return f
                else:
                    return False
    return True

def check_articles(i, tags):
    if is_plural_noun(tags[i]):
        if tags[i-1][0].lower() in ['a', 'an']:
            return False
        elif tags[i-1][0].lower() == 'the':
            return True
        elif is_adjective(tags[i-1][1]):
            if tags[i-2][0].lower() in ['a', 'an']:
                return False
            if tags[i-2][0].lower() == 'the':
                return True
        else:
            return False
    if is_singular_noun(tags[i]):
        if (not tags[i-1][0] in ['a', 'an', 'the']) and (not is_adjective(tags[i-1][1])):
            return False
        else:
            return True
    return True

def check_there(i, tags):
    if tags[i][0].lower() == 'there':
        if tags[i + 1][0] in ['is', 'isn\'t']:
            if (tags[i + 2][0] in ['a', 'an']) and (is_singular_noun(tags[i + 3])):
                return True
            elif (tags[i + 2][0] in ['a', 'an']) and (is_adjective(tags[i + 3][1])) and (is_singular_noun(tags[i + 4])):
                return True
            elif (tags[i + 2][0] in ['one']) and (is_singular_noun(tags[i + 3])):
                return True
            else:
                return False
        elif tags[i + 1][0] in ['are']:
            if is_plural_noun(tags[i + 2]):
                return True
            elif (tags[i + 2][0] == 'some') and (is_plural_noun(tags[i + 3])):
                return True
            elif (tags[i + 2][0] != 'one') and (is_numeral(tags[i + 2][1])) and (is_plural_noun(tags[i + 3])):
                return True
            else:
                return False
        elif tags[i + 1][0] in ['aren\'t']:
            if is_plural_noun(tags[i + 2]):
                return True
            elif (tags[i + 2][0] == 'any') and (is_plural_noun(tags[i + 3])):
                return True
            elif (tags[i + 2][0] != 'one') and (is_numeral(tags[i + 2][1])) and (is_plural_noun(tags[i + 3])):
                return True
            else:
                return False
        elif tags[i - 1][0].lower() == 'is':
            if (tags[i + 1][0] in ['a', 'an']) and (is_singular_noun(tags[i + 2])):
                return True
            elif (tags[i + 1][0] in ['a', 'an']) and (is_adjective(tags[i + 2][1])) and (is_singular_noun(tags[i + 3])):
                return True
            else:
                return False
        elif tags[i - 1][0].lower() == 'are':
            if (tags[i + 1][0] == 'any') and (is_plural_noun(tags[i + 2])):
                return True
            else:
                return False
    elif tags[i][0].lower() == 'there\'s':
        if (tags[i + 1][0] in ['a', 'an']) and (is_singular_noun(tags[i + 2])):
            return True
        elif (tags[i + 1][0] in ['a', 'an']) and (is_adjective(tags[i + 2][1])) and (is_singular_noun(tags[i + 3])):
            return True
        elif (tags[i + 1][0] != 'one') and (is_numeral(tags[i + 1][1])) and (is_plural_noun(tags[i + 2])):
            return True
        else:
            return False
    else:
        return True

def check_object_pronouns(word, before_word, before_prep):
    if (word.lower() in ['you', 'it']) and (before_word is None):
        return True
    if word in ['me', 'him', 'her', 'us', 'them', 'you', 'it']:
        if before_word[1] == 'IN':
            if is_verb(before_prep[1]):
                return True
            else:
                return False
        elif is_verb(before_word[1]):
            return True
        else:
            return False
    else:
        return True

def check_plural_form_noun(word):
    if inflect.singular_noun(word) is False:
        return True
    else:
        sing_word = inflect.singular_noun(word)
        plural_word = inflect.plural_noun(sing_word)
        if plural_word == word:
            return True
        else:
            return plural_word

def check_can_permission(before_word, after_word):
    if (before_word is None) or (after_word is None):
        return True
    elif after_word in ['can\'t', 'can', 'cannot']:
        return True
    elif before_word not in ['can\'t', 'can', 'cannot', 'not']:
        return False
    else:
        return True

def get_errors_plural_forn_nouns(message):
    errors = []
    position = -1
    tags = nltk.pos_tag(word_tokenize(message))
    print(tags)
    for i, tag in enumerate(tags):
        position = message.find(tag[0], position + 1)
        answer = check_to_be(tag[0], tags, i)
        if not answer:
            errors.append({'ErrorType': 'error4 (to be)', 'Position': position, 'Correct': ''})
        if is_noun(tag[1]):
            answer = check_plural_form_noun(tag[0])
            if answer != True:
                errors.append({'ErrorType': 'error1', 'Position': position, 'Correct': answer})
        if is_pronoun(tag[1]):
            before_word = None
            after_word = None
            if i > 0:
                before_word = tags[i - 1][0]
            if i + 1 < len(tags):
                after_word = tags[i + 1][0]
            answer = check_can_permission(before_word, after_word)
            if not answer:
                errors.append({'ErrorType': 'error2 (can/cant - permission)', 'Position': position, 'Correct': ''})
            before_word = None
            before_prep = None
            if i > 0:
                before_word = tags[i - 1]
            if i - 1 > 0:
                before_prep = tags[i - 2]
            answer = check_object_pronouns(tag[0], before_word, before_prep)
            if not answer:
                errors.append({'ErrorType': 'error3 (Object Pronouns)', 'Position': position, 'Correct': ''})
        try:
            answer = check_there(i, tags)
            if not answer:
                errors.append({'ErrorType': 'error5 (there is/are)', 'Position': position, 'Correct': ''})
        except:
            print('error5')

        try:
            answer = check_articles(i, tags)
            if not answer:
                errors.append({'ErrorType': 'error6 (articles)', 'Position': position, 'Correct': ''})
        except:
            print('error6')
        try:
            answer = check_adverbs_frequency(i, tags)
            if not answer:
                errors.append({'ErrorType': 'error7 (adverb of frequency)', 'Position': position, 'Correct': ''})
        except:
            print('error7')
    return errors


import copy
from typing import Generic, Iterable, TypeVar, Optional, Union,List

T = TypeVar('T')

class BSTNode(Generic[T]):
    """
    Your node should permit at least the following
    node.left: get the left child
    node.right: get the right child
    """

    def __init__(self, value: T, children: Optional[List["BSTNode[T]"]] = None,
                 parent: Optional["BSTNode[T]"] = None) -> None:
        # children[1] is the right node, children[0] is the left node.
        self.value = value
        if children == None:
            self.left: Union[None, "BSTNode"] = None
            self.right: Union[None, "BSTNode"] = None
        else:
            self.left = children[0]
            self.right = children[1]
        self.parent: Union[None, "BSTNode"] = parent

    def __iter__(self) -> Iterable["BSTNode[T]"]:
        if self.left is not None:
            yield self.left
        if self.right is not None:
            yield self.right

    # when I do for x in BST node what I return here will be iterated through

    def __deepcopy__(self, memodict) -> "BSTNode[T]":
        """
        I noticed that for some tests deepcopying didn't
        work correctly until I implemented this method so here
        it is for you
        :param memodict:
        :return:
        """
        copy_node = BSTNode(copy.deepcopy(self.value, memodict))
        copy_node.left = copy.deepcopy(self.left, memodict)
        copy_node.right = copy.deepcopy(self.right, memodict)
        copy_node.parent = copy.deepcopy(self.parent, memodict)
        return copy_node

    def __str__(self):
        return self.__dict__

    def __eq__(self, other:"BSTNode")->bool:
        if type(self)!=type(other):
            return False
        elif self.value==other.value:
            return True
        else:
            return False


import sys
import socket
from _socket import gethostbyname, gethostname

import psutil
import platform

host = socket.gethostname()

import socket

def serveur():
    msg = ""
    conn = None
    server_socket = None
    while msg != "kill" :
        msg = ""
        server_socket = socket.socket()
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(("0.0.0.0", 10111))

        server_socket.listen(1)
        print('Serveur en attente de connexion')
        while msg != "kill" and msg != "reset":
            msg = ""
            msgsrv = ""
            try :
                conn, addr = server_socket.accept()
                print (addr)
            except ConnectionError:
                print ("erreur de connection")
                break
            else :
                while msg != "kill" and msg != "reset" and msg != "disconnect":
                    msg = conn.recv(1024).decode()
                    print ("réception du client: ", msg)
                    if msg == "cpu":
                        msg = f"utilisation actuelle du processeur en pourcentage : {str(psutil.cpu_times_percent())}"
                        conn.send(msg.encode())
                    if msg == "name":
                        msg = socket.gethostname()
                        conn.send(msg.encode())
                    if msg == "ip":
                        msg = gethostbyname(gethostname())
                        conn.send(msg.encode())
                    if msg == "ram":
                        msg = f"Pourcentage de la RAM utilisé : {psutil.virtual_memory().percent}%"
                        conn.send(msg.encode())
                    if msg == "os":
                        msg = f"système d'exploitation: {platform.system()}"
                        conn.send(msg.encode())
                    if msg == "python":
                        print("version de python: ", sys.version)

                    """msg_srv = str(input("Serveur:"))
                    conn.send(msg_srv.encode())"""

                conn.close()
        print ("Connection closed")
        server_socket.close()
        print ("Server closed")

# Coder les commande ici

if __name__ == '__main__':
    serveur()


import sys

from PyQt5 import QtWidgets,QtGui

def Pencere():
    app = QtWidgets.QApplication(sys.argv)
    pencere = QtWidgets.QWidget()
    etiket  = QtWidgets.QLabel(pencere)
    image = QtWidgets.QLabel(pencere)
    buton = QtWidgets.QPushButton(pencere)

    pencere.setGeometry(200,200,500,800)
    pencere.setWindowTitle("Merhaba Dünya")

    etiket.setText("Mehaba Dünya")
    etiket.move(50,100)

    image.move(75,210)
    image.setPixmap(QtGui.QPixmap("Images/logo.png"))

    buton.setText("Click It!!!")
    buton.move(300,100)
    pencere.show()
    sys.exit(app.exec())

Pencere()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import datetime

class Migration(migrations.Migration):

    dependencies = [
        ('landing', '0003_auto_20150115_2009'),
    ]

    operations = [
        migrations.CreateModel(
            name='Video',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('file_name', models.CharField(verbose_name='Название видеофайла', max_length=400)),
                ('enabled', models.BooleanField(default=False)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.AlterField(
            model_name='metric',
            name='visible',
            field=models.BooleanField(default=True),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='value',
            name='time',
            field=models.DateTimeField(default=datetime.datetime(2015, 1, 17, 6, 47, 39, 403884)),
            preserve_default=True,
        ),
    ]


import time
from functools import wraps

def timethis(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        start = time.perf_counter()
        r = func(*args, **kwargs)
        end = time.perf_counter()
        print('{}.{} : {}'.format(func.__module__,
                                  func.__name__,
                                  end-start))
        return r
    return wrapper


from collections import deque

def bfs(i, j):
    #초기값
    q = deque([(i, j)])
    while q:
        si, sj = q.popleft()
        for di, dj in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
            ni = si+di
            nj = sj+dj

            if 0<=ni<N and 0<=nj<M and arr[ni][nj] == 1: #이동가능한칸 + 범위내
                q.append((ni, nj))
                arr[ni][nj] += arr[si][sj]

    return arr[N-1][M-1]

N, M = map(int, input().split())
arr = [list(map(int, list(input()))) for _ in range(N)]
print(bfs(0, 0))