"""
    This file is apdated from https://github.com/sniklaus/pytorch-liteflownet
"""
import torch
import UnFlowNet.correlation as correlation
import math
Backward_tensorGrid = {}

device = 'cpu'
if torch.cuda.device_count() > 0 and torch.cuda.is_available():
    print("Cuda installed! Running on GPU!")
    device = 'cuda'
else:
    print("No GPU available!")

def Backward(tensorInput, tensorFlow):
    if str(tensorFlow.size()) not in Backward_tensorGrid:
        tensorHorizontal = torch.linspace(-1.0, 1.0, tensorFlow.size(3)).view(
            1, 1, 1, tensorFlow.size(3)).expand(tensorFlow.size(0), -1,
                                                tensorFlow.size(2), -1)
        tensorVertical = torch.linspace(-1.0, 1.0, tensorFlow.size(2)).view(
            1, 1, tensorFlow.size(2), 1).expand(tensorFlow.size(0), -1, -1,
                                                tensorFlow.size(3))

        Backward_tensorGrid[str(tensorFlow.size())] = torch.cat(
            [tensorHorizontal, tensorVertical], 1).to(device)
    # end
    tensorFlow = torch.cat([
        tensorFlow[:, 0:1, :, :] / ((tensorInput.size(3) - 1.0) / 2.0),
        tensorFlow[:, 1:2, :, :] / ((tensorInput.size(2) - 1.0) / 2.0)
    ], 1)
    return torch.nn.functional.grid_sample(
        input=tensorInput,
        grid=(Backward_tensorGrid[str(tensorFlow.size())] +
              tensorFlow).permute(0, 2, 3, 1),
        mode='bilinear',
        padding_mode='zeros',
        align_corners=True)

# end

# LiteFlowNet model
class Network(torch.nn.Module):
    def __init__(self):
        super(Network, self).__init__()

        class Features(torch.nn.Module):
            def __init__(self):
                super(Features, self).__init__()

                self.moduleOne = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=1,
                                    out_channels=32,
                                    kernel_size=7,
                                    stride=1,
                                    padding=3),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                self.moduleTwo = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=2,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                self.moduleThr = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=64,
                                    kernel_size=3,
                                    stride=2,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=64,
                                    out_channels=64,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                self.moduleFou = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=64,
                                    out_channels=96,
                                    kernel_size=3,
                                    stride=2,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=96,
                                    out_channels=96,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                self.moduleFiv = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=96,
                                    out_channels=128,
                                    kernel_size=3,
                                    stride=2,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                self.moduleSix = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=128,
                                    out_channels=192,
                                    kernel_size=3,
                                    stride=2,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

            # end

            def forward(self, tensorInput):
                tensorOne = self.moduleOne(tensorInput)
                tensorTwo = self.moduleTwo(tensorOne)
                tensorThr = self.moduleThr(tensorTwo)
                tensorFou = self.moduleFou(tensorThr)
                tensorFiv = self.moduleFiv(tensorFou)
                tensorSix = self.moduleSix(tensorFiv)

                return [
                    tensorOne, tensorTwo, tensorThr, tensorFou, tensorFiv,
                    tensorSix
                ]

            # end

        # end

        class Matching(torch.nn.Module):
            def __init__(self, intLevel):
                super(Matching, self).__init__()

                self.dblBackward = [0.0, 0.0, 10.0, 5.0, 2.5, 1.25,
                                    0.625][intLevel]

                if intLevel != 2:
                    self.moduleFeat = torch.nn.Sequential()

                elif intLevel == 2:
                    self.moduleFeat = torch.nn.Sequential(
                        torch.nn.Conv2d(in_channels=32,
                                        out_channels=64,
                                        kernel_size=1,
                                        stride=1,
                                        padding=0),
                        torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                # end

                if intLevel == 6:
                    self.moduleUpflow = None

                elif intLevel != 6:
                    self.moduleUpflow = torch.nn.ConvTranspose2d(
                        in_channels=2,
                        out_channels=2,
                        kernel_size=4,
                        stride=2,
                        padding=1,
                        bias=False,
                        groups=2)

                # end

                if intLevel >= 4:
                    self.moduleUpcorr = None

                elif intLevel < 4:
                    self.moduleUpcorr = torch.nn.ConvTranspose2d(
                        in_channels=49,
                        out_channels=49,
                        kernel_size=4,
                        stride=2,
                        padding=1,
                        bias=False,
                        groups=49)

                # end

                self.moduleMain = torch.nn.Sequential(
                    torch.nn.Conv2d(in_channels=49,
                                    out_channels=128,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=128,
                                    out_channels=64,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=64,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=2,
                                    kernel_size=[0, 0, 7, 5, 5, 3,
                                                 3][intLevel],
                                    stride=1,
                                    padding=[0, 0, 3, 2, 2, 1, 1][intLevel]))

            # end

            def forward(self, tensorFirst, tensorSecond, tensorFeaturesFirst,
                        tensorFeaturesSecond, tensorFlow):
                tensorFeaturesFirst = self.moduleFeat(tensorFeaturesFirst)
                tensorFeaturesSecond = self.moduleFeat(tensorFeaturesSecond)

                if tensorFlow is not None:
                    tensorFlow = self.moduleUpflow(tensorFlow)
                # end

                if tensorFlow is not None:
                    tensorFeaturesSecond = Backward(
                        tensorInput=tensorFeaturesSecond,
                        tensorFlow=tensorFlow * self.dblBackward)
                # end

                if self.moduleUpcorr is None:
                    tensorCorrelation = torch.nn.functional.leaky_relu(
                        input=correlation.FunctionCorrelation(
                            tensorFirst=tensorFeaturesFirst,
                            tensorSecond=tensorFeaturesSecond,
                            intStride=1),
                        negative_slope=0.1,
                        inplace=False)

                elif self.moduleUpcorr is not None:
                    tensorCorrelation = self.moduleUpcorr(
                        torch.nn.functional.leaky_relu(
                            input=correlation.FunctionCorrelation(
                                tensorFirst=tensorFeaturesFirst,
                                tensorSecond=tensorFeaturesSecond,
                                intStride=2),
                            negative_slope=0.1,
                            inplace=False))

                # end

                return (tensorFlow if tensorFlow is not None else
                        0.0) + self.moduleMain(tensorCorrelation)

            # end

        # end

        class Subpixel(torch.nn.Module):
            def __init__(self, intLevel):
                super(Subpixel, self).__init__()

                self.dblBackward = [0.0, 0.0, 10.0, 5.0, 2.5, 1.25,
                                    0.625][intLevel]

                if intLevel != 2:
                    self.moduleFeat = torch.nn.Sequential()

                elif intLevel == 2:
                    self.moduleFeat = torch.nn.Sequential(
                        torch.nn.Conv2d(in_channels=32,
                                        out_channels=64,
                                        kernel_size=1,
                                        stride=1,
                                        padding=0),
                        torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                # end

                self.moduleMain = torch.nn.Sequential(
                    torch.nn.Conv2d(
                        in_channels=[0, 0, 130, 130, 194, 258, 386][intLevel],
                        out_channels=128,
                        kernel_size=3,
                        stride=1,
                        padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=128,
                                    out_channels=64,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=64,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=2,
                                    kernel_size=[0, 0, 7, 5, 5, 3,
                                                 3][intLevel],
                                    stride=1,
                                    padding=[0, 0, 3, 2, 2, 1, 1][intLevel]))

            # end

            def forward(self, tensorFirst, tensorSecond, tensorFeaturesFirst,
                        tensorFeaturesSecond, tensorFlow):
                tensorFeaturesFirst = self.moduleFeat(tensorFeaturesFirst)
                tensorFeaturesSecond = self.moduleFeat(tensorFeaturesSecond)

                if tensorFlow is not None:
                    tensorFeaturesSecond = Backward(
                        tensorInput=tensorFeaturesSecond,
                        tensorFlow=tensorFlow * self.dblBackward)
                # end

                return (tensorFlow
                        if tensorFlow is not None else 0.0) + self.moduleMain(
                            torch.cat([
                                tensorFeaturesFirst, tensorFeaturesSecond,
                                tensorFlow
                            ], 1))

            # end

        # end

        class Regularization(torch.nn.Module):
            def __init__(self, intLevel):
                super(Regularization, self).__init__()

                self.dblBackward = [0.0, 0.0, 10.0, 5.0, 2.5, 1.25,
                                    0.625][intLevel]

                self.intUnfold = [0, 0, 7, 5, 5, 3, 3][intLevel]

                if intLevel >= 5:
                    self.moduleFeat = torch.nn.Sequential()

                elif intLevel < 5:
                    self.moduleFeat = torch.nn.Sequential(
                        torch.nn.Conv2d(
                            in_channels=[0, 0, 32, 64, 96, 128, 192][intLevel],
                            out_channels=128,
                            kernel_size=1,
                            stride=1,
                            padding=0),
                        torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                # end

                self.moduleMain = torch.nn.Sequential(
                    torch.nn.Conv2d(
                        in_channels=[0, 0, 131, 131, 131, 131, 195][intLevel],
                        out_channels=128,
                        kernel_size=3,
                        stride=1,
                        padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=128,
                                    out_channels=128,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=128,
                                    out_channels=64,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=64,
                                    out_channels=64,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=64,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1),
                    torch.nn.Conv2d(in_channels=32,
                                    out_channels=32,
                                    kernel_size=3,
                                    stride=1,
                                    padding=1),
                    torch.nn.LeakyReLU(inplace=False, negative_slope=0.1))

                if intLevel >= 5:
                    self.moduleDist = torch.nn.Sequential(
                        torch.nn.Conv2d(in_channels=32,
                                        out_channels=[0, 0, 49, 25, 25, 9,
                                                      9][intLevel],
                                        kernel_size=[0, 0, 7, 5, 5, 3,
                                                     3][intLevel],
                                        stride=1,
                                        padding=[0, 0, 3, 2, 2, 1,
                                                 1][intLevel]))

                elif intLevel < 5:
                    self.moduleDist = torch.nn.Sequential(
                        torch.nn.Conv2d(in_channels=32,
                                        out_channels=[0, 0, 49, 25, 25, 9,
                                                      9][intLevel],
                                        kernel_size=([0, 0, 7, 5, 5, 3,
                                                      3][intLevel], 1),
                                        stride=1,
                                        padding=([0, 0, 3, 2, 2, 1,
                                                  1][intLevel], 0)),
                        torch.nn.Conv2d(in_channels=[0, 0, 49, 25, 25, 9,
                                                     9][intLevel],
                                        out_channels=[0, 0, 49, 25, 25, 9,
                                                      9][intLevel],
                                        kernel_size=(1, [0, 0, 7, 5, 5, 3,
                                                         3][intLevel]),
                                        stride=1,
                                        padding=(0, [0, 0, 3, 2, 2, 1,
                                                     1][intLevel])))

                # end

                self.moduleScaleX = torch.nn.Conv2d(
                    in_channels=[0, 0, 49, 25, 25, 9, 9][intLevel],
                    out_channels=1,
                    kernel_size=1,
                    stride=1,
                    padding=0)
                self.moduleScaleY = torch.nn.Conv2d(
                    in_channels=[0, 0, 49, 25, 25, 9, 9][intLevel],
                    out_channels=1,
                    kernel_size=1,
                    stride=1,
                    padding=0)

            # eny

            def forward(self, tensorFirst, tensorSecond, tensorFeaturesFirst,
                        tensorFeaturesSecond, tensorFlow):
                tensorDifference = (tensorFirst - Backward(
                    tensorInput=tensorSecond,
                    tensorFlow=tensorFlow * self.dblBackward)).pow(2.0).sum(
                        1, True).sqrt().detach()

                tensorDist = self.moduleDist(
                    self.moduleMain(
                        torch.cat([
                            tensorDifference, tensorFlow -
                            tensorFlow.view(tensorFlow.size(0), 2, -1).mean(
                                2, True).view(tensorFlow.size(0), 2, 1, 1),
                            self.moduleFeat(tensorFeaturesFirst)
                        ], 1)))
                tensorDist = tensorDist.pow(2.0).neg()
                tensorDist = (tensorDist - tensorDist.max(1, True)[0]).exp()

                tensorDivisor = tensorDist.sum(1, True).reciprocal()

                tensorScaleX = self.moduleScaleX(
                    tensorDist * torch.nn.functional.unfold(
                        input=tensorFlow[:, 0:1, :, :],
                        kernel_size=self.intUnfold,
                        stride=1,
                        padding=int((self.intUnfold - 1) /
                                    2)).view_as(tensorDist)) * tensorDivisor
                tensorScaleY = self.moduleScaleY(
                    tensorDist * torch.nn.functional.unfold(
                        input=tensorFlow[:, 1:2, :, :],
                        kernel_size=self.intUnfold,
                        stride=1,
                        padding=int((self.intUnfold - 1) /
                                    2)).view_as(tensorDist)) * tensorDivisor

                return torch.cat([tensorScaleX, tensorScaleY], 1)

            # end

        # end

        self.moduleFeatures = Features()
        self.moduleMatching = torch.nn.ModuleList(
            [Matching(intLevel) for intLevel in [2, 3, 4, 5, 6]])
        self.moduleSubpixel = torch.nn.ModuleList(
            [Subpixel(intLevel) for intLevel in [2, 3, 4, 5, 6]])
        self.moduleRegularization = torch.nn.ModuleList(
            [Regularization(intLevel) for intLevel in [2, 3, 4, 5, 6]])

    # end

    def forward(self, tensorFirst, tensorSecond):
        tensorFirst[:, 0, :, :] = tensorFirst[:, 0, :, :] - 0.411618
        tensorSecond[:, 0, :, :] = tensorSecond[:, 0, :, :] - 0.410782

        tensorFeaturesFirst = self.moduleFeatures(tensorFirst)
        tensorFeaturesSecond = self.moduleFeatures(tensorSecond)

        tensorFirst = [tensorFirst]
        tensorSecond = [tensorSecond]

        for intLevel in [1, 2, 3, 4, 5]:
            tensorFirst.append(
                torch.nn.functional.interpolate(
                    input=tensorFirst[-1],
                    size=(tensorFeaturesFirst[intLevel].size(2),
                          tensorFeaturesFirst[intLevel].size(3)),
                    mode='bilinear',
                    align_corners=False))
            tensorSecond.append(
                torch.nn.functional.interpolate(
                    input=tensorSecond[-1],
                    size=(tensorFeaturesSecond[intLevel].size(2),
                          tensorFeaturesSecond[intLevel].size(3)),
                    mode='bilinear',
                    align_corners=False))
        # end

        tensorFlow = None
        flowCollection = []

        for intLevel in [-1, -2, -3, -4, -5]:
            tensorFlow = self.moduleMatching[intLevel](
                tensorFirst[intLevel], tensorSecond[intLevel],
                tensorFeaturesFirst[intLevel], tensorFeaturesSecond[intLevel],
                tensorFlow)
            tensorFlow = self.moduleSubpixel[intLevel](
                tensorFirst[intLevel], tensorSecond[intLevel],
                tensorFeaturesFirst[intLevel], tensorFeaturesSecond[intLevel],
                tensorFlow)
            tensorFlow = self.moduleRegularization[intLevel](
                tensorFirst[intLevel], tensorSecond[intLevel],
                tensorFeaturesFirst[intLevel], tensorFeaturesSecond[intLevel],
                tensorFlow)
            flowCollection.append(tensorFlow)
        # end
        self.training = True
        if self.training:
            flowCollection[-1] *= 4.0  #Final flow scale 20.0, others 5.0
            return [flow * 5.0 for flow in flowCollection]
        else:
            return tensorFlow * 20.0

    # end

# end

# Estimate the flow
def estimate(tensorFirst, tensorSecond, model, train=False):
    assert (tensorFirst.size(3) == tensorSecond.size(3))
    assert (tensorFirst.size(2) == tensorSecond.size(2))

    intWidth = tensorFirst.size(3)
    intHeight = tensorFirst.size(2)

    tensorPreprocessedFirst = tensorFirst.view(-1, 1, intHeight, intWidth)
    tensorPreprocessedSecond = tensorSecond.view(-1, 1, intHeight, intWidth)

    intPreprocessedWidth = int(math.floor(math.ceil(intWidth / 32.0) * 32.0))
    intPreprocessedHeight = int(math.floor(math.ceil(intHeight / 32.0) * 32.0))

    tensorPreprocessedFirst = torch.nn.functional.interpolate(
        input=tensorPreprocessedFirst,
        size=(intPreprocessedHeight, intPreprocessedWidth),
        mode='bilinear',
        align_corners=False)
    tensorPreprocessedSecond = torch.nn.functional.interpolate(
        input=tensorPreprocessedSecond,
        size=(intPreprocessedHeight, intPreprocessedWidth),
        mode='bilinear',
        align_corners=False)
    raw_output = model(tensorPreprocessedFirst, tensorPreprocessedSecond)

    tensorFlow = torch.nn.functional.interpolate(input=raw_output[-1],
                                                 size=(intHeight, intWidth),
                                                 mode='bilinear',
                                                 align_corners=False)

    tensorFlow[:, 0, :, :] *= float(intWidth) / float(intPreprocessedWidth)
    tensorFlow[:, 1, :, :] *= float(intHeight) / float(intPreprocessedHeight)
    if train:
        return raw_output + [tensorFlow]
    else:
        return tensorFlow[:, :, :, :].cpu()

# end


# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Wrapper for Resource Manager API client."""

from googleapiclient.errors import HttpError
from httplib2 import HttpLib2Error
from ratelimiter import RateLimiter

from google.cloud.security.common.gcp_api._base_client import _BaseClient
from google.cloud.security.common.gcp_api._base_client import ApiExecutionError
from google.cloud.security.common.util.log_util import LogUtil

LOGGER = LogUtil.setup_logging(__name__)

class CloudResourceManagerClient(_BaseClient):
    """Resource Manager Client."""

    API_NAME = 'cloudresourcemanager'
    DEFAULT_MAX_QUERIES = 400

    def __init__(self, credentials=None, rate_limiter=None):
        super(CloudResourceManagerClient, self).__init__(
            credentials=credentials, api_name=self.API_NAME)
        if rate_limiter:
            self.rate_limiter = rate_limiter
        else:
            self.rate_limiter = RateLimiter(self.DEFAULT_MAX_QUERIES, 100)

    def get_project(self, project_id):
        """Get all the projects from organization.

        Args:
            project_id: The string project id.

        Returns:
            The project response object.

        Raises:
            ApiExecutionError: An error has occurred when executing the API.
        """
        projects_stub = self.service.projects()

        try:
            with self.rate_limiter:
                request = projects_stub.get(projectId=project_id)
                response = self._execute(request)
                return response
        except (HttpError, HttpLib2Error) as e:
            LOGGER.error(ApiExecutionError(project_id, e))
        return None

    def get_projects(self, resource_name, organization_id):
        """Get all the projects from organization.

        Args:
            resource_name: String of the resource's name.
            organization_id: String of the organization id
                in Google Cloud Platform

        Yields:
            An iterable of resource manager project list response.
            https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#response-body

        Raises:
            ApiExecutionError: An error has occurred when executing the API.
        """
        projects_stub = self.service.projects()
        # TODO: The filter may break once folders are implemented.
        # pylint: disable=redefined-builtin
        # TODO: Stop redefining a built-in to remove pylint disable.
        filter = 'parent.type:organization parent.id:%s' % organization_id
        request = projects_stub.list(filter=filter)

        try:
            with self.rate_limiter:
                while request is not None:
                    response = self._execute(request)
                    yield response
                    request = projects_stub.list_next(
                        previous_request=request,
                        previous_response=response)
        except (HttpError, HttpLib2Error) as e:
            raise ApiExecutionError(resource_name, e)

    def get_project_iam_policies(self, resource_name, project_numbers):
        """Get all the iam policies of given project numbers.

        Args:
            resource_name: String of the resource's name.
            project_numbers: MySQLdb cursor object of project numbers.

        Yields:
            An iterable of iam policies as per-project dictionary.
            Example: {project_number: policy}
            https://cloud.google.com/resource-manager/reference/rest/Shared.Types/Policy

        Raises:
            ApiExecutionError: An error has occurred when executing the API.
        """
        projects_stub = self.service.projects()

        try:
            for project_number in project_numbers:
                with self.rate_limiter:
                    request = projects_stub.getIamPolicy(
                        resource=project_number, body={})
                    response = self._execute(request)
                    yield {'project_number': project_number,
                           'iam_policy': response}
        except (HttpError, HttpLib2Error) as e:
            raise ApiExecutionError(resource_name, e)

    def get_organization(self, org_name):
        """Get organizations.

        Args:
            org_name: The string org name with format "organizations/$ORG_ID"

        Returns:
            The org response object if found, otherwise False.

        Raises:
            ApiExecutionError: An error has occurred when executing the API.
        """
        orgs_stub = self.service.organizations()

        try:
            with self.rate_limiter:
                request = orgs_stub.get(name=org_name)
                response = self._execute(request)
                return response
        except (HttpError, HttpLib2Error) as e:
            LOGGER.error(ApiExecutionError(org_name, e))
        return None

    def get_org_iam_policies(self, resource_name, org_id):
        """Get all the iam policies of an org.

        Args:
            resource_name: String of the resource's name.
            org_id: Integer of the org id.

        Yields:
            An iterable of iam policies as per-org dictionary.
            Example: {org_id: org_id,
                      iam_policy: iam_policy}
            https://cloud.google.com/resource-manager/reference/rest/Shared.Types/Policy

        Raises:
            ApiExecutionError: An error has occurred when executing the API.
        """
        orgs_stub = self.service.organizations()
        resource_id = 'organizations/' + str(org_id)

        try:
            with self.rate_limiter:
                request = orgs_stub.getIamPolicy(
                    resource=resource_id, body={})
                response = self._execute(request)
                yield {'org_id': org_id,
                       'iam_policy': response}
        except (HttpError, HttpLib2Error) as e:
            raise ApiExecutionError(resource_name, e)


import torch
import torch.nn as nn
import torch.nn.functional as F

NOISE_DIM = 256

class Generator(nn.Module):
    def __init__(self, z_dim, act_func = F.relu):
        super(Generator, self).__init__()
        self.z_dim = z_dim
        self.activation = act_func

        # 128 + 5 -> (+reshape) 128 x 2 x 2
        self.fc1 = nn.Linear(self.z_dim + 5, self.z_dim * 2 * 2)

        # Z x 2 x 2
        self.conv1 = nn.ConvTranspose2d(self.z_dim, 128, 3, stride=2, padding=1, output_padding=1)
        # 128 x 8 x 8
        self.conv2 = nn.ConvTranspose2d(128, 64, 3, stride=2, padding=1, output_padding=1)
        # 64 x 16 x 16
        self.conv3 = nn.ConvTranspose2d(64, 32,  3, stride=2, padding=1, output_padding=1)
        # 32 x 32 x 32
        self.conv4 = nn.ConvTranspose2d(32, 1,  3, stride=2, padding=1, output_padding=1)
        # 1 x 32 x 32
        # crop
        # 1 x 30 x 30

        self.bn1 = nn.BatchNorm2d(128)
        self.bn2 = nn.BatchNorm2d(64)
        self.bn3 = nn.BatchNorm2d(32)

    def forward(self, z, ParticleMomentum_ParticlePoint):
        x = torch.cat([z, ParticleMomentum_ParticlePoint], dim=1)
        #print(x.shape)
        x = F.leaky_relu(self.fc1(x))

        #print(x.shape)
        EnergyDeposit = x.view(-1, self.z_dim, 2, 2)

        #print(EnergyDeposit.shape)
        EnergyDeposit = self.activation(self.bn1(self.conv1(EnergyDeposit)))
        #print(EnergyDeposit.shape)
        EnergyDeposit = self.activation(self.bn2(self.conv2(EnergyDeposit)))
        #print(EnergyDeposit.shape)
        EnergyDeposit = self.activation(self.bn3(self.conv3(EnergyDeposit)))
        #print(EnergyDeposit.shape)
        EnergyDeposit = self.activation(self.conv4(EnergyDeposit))
        #print(EnergyDeposit.shape)
        EnergyDeposit = EnergyDeposit[:,:,1:31,1:31]
        #print(EnergyDeposit.shape)

        return EnergyDeposit


from __future__ import unicode_literals, absolute_import, print_function, division
import configparser
import json
import mwclient
from mwclient import errors
import requests
import re
import time
from sopel.module import rule, commands, example

pages = ''

def save_wrap(site, request, bot, trigger):
    pagename = 'User:' + request[0] + '/Status'
    bot.say(trigger.nick + " updating " + pagename + "!", trigger.sender)
    page = site.Pages[pagename]
    save_edit(page, request[1], bot, trigger)

def save_edit(page, status, bot, trigger):
    time.sleep(5)
    edit_summary = "BOT: Setting Status to: " + status + " per " + trigger.hostmask
    times = 0
    while True:
        if times > 1:
            break
        try:
            page.save(status, summary=edit_summary, bot=True, minor=True)
            bot.say(trigger.nick + ": Done!", trigger.sender)
        except errors.ProtectedPageError:
            print('Could not edit ' + page + ' due to protection')
            bot.say(trigger.nick + ": Error: Page Protected", trigger.sender)
            times += 1
        except errors.EditError:
            print("Error")
            bot.say(trigger.nick + ": An Error Occured :(", trigger.sender)
            times += 1
            time.sleep(5)  # sleep for 5 seconds before trying again
            continue
        except errors.UserBlocked:
            bot.say(trigger.nick + ": StatusBot is currently unavaiable for that wiki. Our team are working on it!", trigger.sender)
            bot.say("ERR: The bot is blocked on " + page, '#ZppixBot')
        break

def main(bot, trigger, options):
    cont = 0
    if len(options) == 2:
        wiki = options[0]
        status = options[1]
        host = trigger.host
        host = host.split('/')
        cont = 1
    elif len(options) > 2:
        wiki = options[0]
        host = trigger.host
        host = host.split('/')
        x = 1
        status = ''
        while x < len(options):
            status = status + options[x]
            x = x + 1
        cont = 1
    else:
        bot.say(trigger.nick + ": Syntax: .mh wikicode status", trigger.sender)
        cont = 0
    if cont == 1:
        cloakfile = open('/data/project/zppixbot/.sopel/modules/config/cloaks.csv', 'r')
        for line in cloakfile:
            auth = line.split(',')
        if host[0] == auth[0]:
            user = host[1]
            sulgroup = auth[1]
            wiki = [wiki, sulgroup]
            request = [user, status]
            cont = 1
        else:
            bot.say(trigger.nick + ":This service is only avaiable to users with a Miraheze/Wikimedia Cloaks. "
                    + "See phabricator.wikimedia.org/T234716 for updates.", trigger.sender)
            cont = 0
    if cont == 1:
        wikiurl = 'example.org'
        file = open('/data/project/zppixbot/.sopel/modules/config/statuswikis.csv', 'r')
        for line in file:
            data = line.split(',')
            if data[1] == wiki[0] and wiki[1] == data[2]:
                wikiurl = data[0]
        site = mwclient.Site(('https', wikiurl), '/w/')
        config = configparser.RawConfigParser()
        config.read('/data/project/zppixbot/.sopel/credentials.txt')
        try:
            site.login(config.get('zppixbot_status', 'username'), config.get('zppixbot_status', 'password'))
        except errors.LoginError as e:
            print(e)
            raise ValueError("Login failed.")
        save_wrap(site, request, bot, trigger)

@commands('status')
@example('.status mhtest offline')
def status(bot, trigger):
    options = trigger.group(2).split(" ")
    main(bot, trigger, options)


"""
旋转数组 只能写出两种 剩下的参考题解
"""
from typing import List

class Solution:
    def rotate(self, nums: List[int], k: int) -> None:
        """
        Do not return anything, modify nums in-place instead.
        """
        if not nums:
            return []
        # 第一种 切片法
        # nums[:] = nums[len(nums) - k:] + nums[:len(nums) - k]

        # 第二种 插入法
        # for i in range(k):
        #     nums.insert(0, nums[-1])
        #     nums.pop()

        # 第三种 三次反转
        # 防止移动的k的值大于列表的长度, 需要进行取余操作
        # 将数据分为两段 [0: n-k-1] [n-k: n-1]
        # 其实这个reverse可以使用列表自带的reversed进行操作
        """
        先翻转第一段
        翻转第二段
        反转整个列表
        """
        def reverse(left, right):
            while left < right:
                nums[left], nums[right] = nums[right], nums[left]
                left += 1
                right -= 1

        x = len(nums)
        k = k % x

        reverse(0, x-k-1)
        reverse(x-k, x-1)
        reverse(0, x-1)


"""this class build and run a trainer by a configuration"""
import datetime
import os
import traceback

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim.lr_scheduler import ReduceLROnPlateau
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm

from utils.generals import make_batch
from utils.metrics.metrics import accuracy
from utils.radam import RAdam

# from torch.optim import Adam as RAdam
# from torch.optim import SGD as RAdam

WORLD_SIZE = 1  # number of nodes for distributed training.
RANK = 0  # node rank for distributed training

"""
--multiprocessing-distributed
    Use multi-processing distributed training to launch N
    processes per node, which has N GPUs. This is the
    fastest way to use PyTorch for either single node or
    multi node data parallel training
"""

EMO_DICT = {0: "ne", 1: "an", 2: "di", 3: "fe", 4: "ha", 5: "sa", 6: "su"}

class Trainer(object):
    """base class for trainers"""

    def __init__(self):
        pass

class FER2013Trainer(Trainer):
    """for classification task"""

    def __init__(self, model, train_set, val_set, test_set, configs):
        super().__init__()
        print("Start trainer..")
        print(configs)

        # load config
        self._configs = configs
        self._lr = self._configs["lr"]
        self._batch_size = self._configs["batch_size"]
        self._momentum = self._configs["momentum"]
        self._weight_decay = self._configs["weight_decay"]
        self._distributed = self._configs["distributed"]
        self._num_workers = self._configs["num_workers"]
        self._device = torch.device(self._configs["device"])
        self._max_epoch_num = self._configs["max_epoch_num"]
        self._max_plateau_count = self._configs["max_plateau_count"]

        # load dataloader and model
        self._train_set = train_set
        self._val_set = val_set
        self._test_set = test_set
        self._model = model(
            in_channels=configs["in_channels"],
            num_classes=configs["num_classes"],
        )

        # self._model.fc = nn.Linear(512, 7)
        # self._model.fc = nn.Linear(256, 7)

        # for dist
        self._ngpus_per_node = torch.cuda.device_count()
        if self._distributed == 1:  # DISTINGUST WITH MULTIPROCESSING DISTRIBUTED
            self._world_size = self._ngpus_per_node * WORLD_SIZE

            # torch.distributed.init_process_group(backend='nccl')
            # self._model = nn.parallel.DistributedDataParallel(self._model)

            train_sampler = torch.utils.data.distributed.DistributedSampler(
                self._train_set
            )
            self._train_loader = DataLoader(
                self._train_set,
                batch_size=self._batch_size,
                num_workers=self._num_workers,
                pin_memory=True,
                shuffle=True,
                worker_init_fn=lambda x: np.random.seed(x),
                sampler=train_sampler,
            )
            self._val_loader = DataLoader(
                self._val_set,
                batch_size=self._batch_size,
                num_workers=self._num_workers,
                pin_memory=True,
                shuffle=False,
                worker_init_fn=lambda x: np.random.seed(x),
            )

            self._test_loader = DataLoader(
                self._test_set,
                batch_size=1,
                num_workers=self._num_workers,
                pin_memory=True,
                shuffle=False,
                worker_init_fn=lambda x: np.random.seed(x),
            )
        else:
            self._model = self._model.to(self._device)
            self._train_loader = DataLoader(
                self._train_set,
                batch_size=self._batch_size,
                num_workers=self._num_workers,
                pin_memory=True,
                shuffle=True,
            )
            self._val_loader = DataLoader(
                self._val_set,
                batch_size=self._batch_size,
                num_workers=self._num_workers,
                pin_memory=True,
                shuffle=False,
            )
            self._test_loader = DataLoader(
                self._test_set,
                batch_size=1,
                num_workers=self._num_workers,
                pin_memory=True,
                shuffle=False,
            )

        # define loss function (criterion) and optimizer
        class_weights = [
            1.02660468,
            9.40661861,
            1.00104606,
            0.56843877,
            0.84912748,
            1.29337298,
            0.82603942,
        ]
        class_weights = torch.FloatTensor(np.array(class_weights))
        self._criterion = nn.CrossEntropyLoss(class_weights).to(self._device)
        # self._criterion = nn.CrossEntropyLoss().to(self._device)

        self._optimizer = RAdam(
            params=self._model.parameters(),
            lr=self._lr,
            weight_decay=self._weight_decay,
        )

        self._scheduler = ReduceLROnPlateau(
            self._optimizer,
            patience=self._configs["plateau_patience"],
            min_lr=1e-6,
            verbose=True,
        )

        """ TODO set step size equal to configs
        self._scheduler = StepLR(
            self._optimizer,
            step_size=self._configs['steplr']
        )
        """

        # training info
        self._start_time = datetime.datetime.now()
        self._start_time = self._start_time.replace(microsecond=0)

        log_dir = os.path.join(
            self._configs["cwd"],
            self._configs["log_dir"],
            "{}_{}_{}".format(
                self._configs["arch"],
                self._configs["model_name"],
                self._start_time.strftime("%Y%b%d_%H.%M"),
            ),
        )
        self._writer = SummaryWriter(log_dir)
        self._train_loss_list = []
        self._train_acc_list = []
        self._val_loss_list = []
        self._val_acc_list = []
        self._best_val_loss = 1e9
        self._best_val_acc = 0
        self._best_train_loss = 1e9
        self._best_train_acc = 0
        self._test_acc = 0.0
        self._plateau_count = 0
        self._current_epoch_num = 0

        # for checkpoints
        self._checkpoint_dir = os.path.join(self._configs["cwd"], "saved/checkpoints")
        if not os.path.exists(self._checkpoint_dir):
            os.makedirs(self._checkpoint_dir, exist_ok=True)

        self._checkpoint_path = os.path.join(
            self._checkpoint_dir,
            "{}_{}_{}".format(
                self._configs["arch"],
                self._configs["model_name"],
                self._start_time.strftime("%Y%b%d_%H.%M"),
            ),
        )

    def _train(self):
        self._model.train()
        train_loss = 0.0
        train_acc = 0.0

        for i, (images, targets) in tqdm(
            enumerate(self._train_loader), total=len(self._train_loader), leave=False
        ):
            images = images.cuda(non_blocking=True)
            targets = targets.cuda(non_blocking=True)

            # compute output, measure accuracy and record loss
            outputs = self._model(images)

            loss = self._criterion(outputs, targets)
            acc = accuracy(outputs, targets)[0]
            # acc = eval_metrics(targets, outputs, 2)[0]

            train_loss += loss.item()
            train_acc += acc.item()

            # compute gradient and do SGD step
            self._optimizer.zero_grad()
            loss.backward()
            self._optimizer.step()

        i += 1
        self._train_loss_list.append(train_loss / i)
        self._train_acc_list.append(train_acc / i)

    def _val(self):
        self._model.eval()
        val_loss = 0.0
        val_acc = 0.0

        with torch.no_grad():
            for i, (images, targets) in tqdm(
                enumerate(self._val_loader), total=len(self._val_loader), leave=False
            ):
                images = images.cuda(non_blocking=True)
                targets = targets.cuda(non_blocking=True)

                # compute output, measure accuracy and record loss
                outputs = self._model(images)

                loss = self._criterion(outputs, targets)
                acc = accuracy(outputs, targets)[0]

                val_loss += loss.item()
                val_acc += acc.item()

            i += 1
            self._val_loss_list.append(val_loss / i)
            self._val_acc_list.append(val_acc / i)

    def _calc_acc_on_private_test(self):
        self._model.eval()
        test_acc = 0.0
        print("Calc acc on private test..")
        f = open("private_test_log.txt", "w")
        with torch.no_grad():
            for i, (images, targets) in tqdm(
                enumerate(self._test_loader), total=len(self._test_loader), leave=False
            ):

                images = images.cuda(non_blocking=True)
                targets = targets.cuda(non_blocking=True)

                outputs = self._model(images)
                print(outputs.shape, outputs)
                acc = accuracy(outputs, targets)[0]
                test_acc += acc.item()
                f.writelines("{}_{}\n".format(i, acc.item()))

            test_acc = test_acc / (i + 1)
        print("Accuracy on private test: {:.3f}".format(test_acc))
        f.close()
        return test_acc

    def _calc_acc_on_private_test_with_tta(self):
        self._model.eval()
        test_acc = 0.0
        print("Calc acc on private test with tta..")
        f = open(
            "private_test_log_{}_{}.txt".format(
                self._configs["arch"], self._configs["model_name"]
            ),
            "w",
        )

        with torch.no_grad():
            for idx in tqdm(
                range(len(self._test_set)), total=len(self._test_set), leave=False
            ):
                images, targets = self._test_set[idx]
                targets = torch.LongTensor([targets])

                images = make_batch(images)
                images = images.cuda(non_blocking=True)
                targets = targets.cuda(non_blocking=True)

                outputs = self._model(images)
                outputs = F.softmax(outputs, 1)

                # outputs.shape [tta_size, 7]
                outputs = torch.sum(outputs, 0)

                outputs = torch.unsqueeze(outputs, 0)
                # print(outputs.shape)
                # TODO: try with softmax first and see the change
                acc = accuracy(outputs, targets)[0]
                test_acc += acc.item()
                f.writelines("{}_{}\n".format(idx, acc.item()))

            test_acc = test_acc / (idx + 1)
        print("Accuracy on private test with tta: {:.3f}".format(test_acc))
        f.close()
        return test_acc

    def train(self, gpu, ngpus_per_node, args):
        """make a training job"""
        # print(self._model)

        # freeze the model

        """
        print('=' * 10)
        for idx, child in enumerate(self._model.children()):
            if idx < 6:
                print(child)
                print('=' * 10)

                for m in child.parameters():
                    m.requires_grad = False
        """

        # exit(0)
        if self._distributed:
            torch.distributed.init_process_group(
                backend="nccl", world_size=torch.cuda.device_count(), rank=1
            )
            self._model.cuda()
            self._model = torch.nn.parallel.DistributedDataParallel(self._model)

        try:
            while not self._is_stop():
                self._increase_epoch_num()
                self._train()
                self._val()

                self._update_training_state()
                self._logging()
        except KeyboardInterrupt:
            traceback.print_exc()

        # training stop
        try:
            # state = torch.load('saved/checkpoints/resatt18_rot30_2019Nov06_18.56')
            state = torch.load(self._checkpoint_path)
            if self._distributed:
                self._model.module.load_state_dict(state["net"])
            else:
                self._model.load_state_dict(state["net"])

            if not self._test_set.is_tta():
                self._test_acc = self._calc_acc_on_private_test()
            else:
                self._test_acc = self._calc_acc_on_private_test_with_tta()

            # self._test_acc = self._calc_acc_on_private_test()
            self._save_weights()
        except Exception:
            traceback.print_exc()

        consume_time = str(datetime.datetime.now() - self._start_time)
        self._writer.add_text(
            "Summary",
            "Converged after {} epochs, consume {}".format(
                self._current_epoch_num, consume_time[:-7]
            ),
        )
        self._writer.add_text(
            "Results", "Best validation accuracy: {:.3f}".format(self._best_val_acc)
        )
        self._writer.add_text(
            "Results", "Best training accuracy: {:.3f}".format(self._best_train_acc)
        )
        self._writer.add_text(
            "Results", "Private test accuracy: {:.3f}".format(self._test_acc)
        )
        self._writer.close()

    def _update_training_state(self):
        if self._val_acc_list[-1] > self._best_val_acc:
            self._save_weights()
            self._plateau_count = 0
            self._best_val_acc = self._val_acc_list[-1]
            self._best_val_loss = self._val_loss_list[-1]
            self._best_train_acc = self._train_acc_list[-1]
            self._best_train_loss = self._train_loss_list[-1]
        else:
            self._plateau_count += 1

        # self._scheduler.step(self._train_loss_list[-1])
        self._scheduler.step(100 - self._val_acc_list[-1])
        # self._scheduler.step()

    def _logging(self):
        consume_time = str(datetime.datetime.now() - self._start_time)

        message = "\nE{:03d}  {:.3f}/{:.3f}/{:.3f} {:.3f}/{:.3f}/{:.3f} | p{:02d}  Time {}\n".format(
            self._current_epoch_num,
            self._train_loss_list[-1],
            self._val_loss_list[-1],
            self._best_val_loss,
            self._train_acc_list[-1],
            self._val_acc_list[-1],
            self._best_val_acc,
            self._plateau_count,
            consume_time[:-7],
        )

        self._writer.add_scalar(
            "Accuracy/Train", self._train_acc_list[-1], self._current_epoch_num
        )
        self._writer.add_scalar(
            "Accuracy/Val", self._val_acc_list[-1], self._current_epoch_num
        )
        self._writer.add_scalar(
            "Loss/Train", self._train_loss_list[-1], self._current_epoch_num
        )
        self._writer.add_scalar(
            "Loss/Val", self._val_loss_list[-1], self._current_epoch_num
        )

        print(message)

    def _is_stop(self):
        """check stop condition"""
        return (
            self._plateau_count > self._max_plateau_count
            or self._current_epoch_num > self._max_epoch_num
        )

    def _increase_epoch_num(self):
        self._current_epoch_num += 1

    def _save_weights(self, test_acc=0.0):
        if self._distributed == 0:
            state_dict = self._model.state_dict()
        else:
            state_dict = self._model.module.state_dict()

        state = {
            **self._configs,
            "net": state_dict,
            "best_val_loss": self._best_val_loss,
            "best_val_acc": self._best_val_acc,
            "best_train_loss": self._best_train_loss,
            "best_train_acc": self._best_train_acc,
            "train_losses": self._train_loss_list,
            "val_loss_list": self._val_loss_list,
            "train_acc_list": self._train_acc_list,
            "val_acc_list": self._val_acc_list,
            "test_acc": self._test_acc,
        }

        torch.save(state, self._checkpoint_path)


import sys
sys.path.append('/scratch/ppcode/HOS')
import importlib as imp
import numpy as np
import math
from numpy import fft
import pickle
import matplotlib.pyplot as plt

import statistics as stt
import mathtool as mt
import waveClass as wvc

prjname = "regular"
jobname = "ka01"
# suffix = [".256_64.3.7", ".512_128.3.7", ".256_64.2.7", ".256_64.1.7", ".128_32.3.7", ".256_64.3.8", ".256_64.3.4"]
suffix = [""]
sffNum = len(suffix)

data = []
for i in range(sffNum):
    readDir = '/scratch/HOSdata/pp/' + prjname + '/' + jobname + '/data/'
    readName = "VP_card_fitted_Dict" + suffix[i]
    fr = open(readDir + readName, 'rb')
    data.append(pickle.load(fr))
    fr.close()

# print('time step: ', data[0]['time'], ' time-size: ', data[0]['time'].size)
# print('x-grids: ', data[0]['x'], ' x-size: ', data[0]['x'].size)
# print('y-grids: ', data[0]['y'], ' y-size: ', data[0]['y'].size)
# print('z-grids: ', data[0]['z'], ' z-size: ', data[0]['z'].size)

### plot time seris of eta at certain point
suffixNo = 0
xind = 25
yind = 5
x = data[suffixNo]['time']
y = data[suffixNo]['eta'][:,0,yind,xind]

fig, axs = plt.subplots(1,1)
fig.set_figwidth(8)
fig.set_figheight(6)

axs.plot(x, y, linewidth=1.0, color='k', marker=None, linestyle='-', label=None)
axs.set_ylim(-0.15,0.15)
axs.set_ylabel(r'$\eta$ (m)', fontsize=12)
axs.grid()
axs.set_xlabel('time (s)', fontsize=12)
# plt.legend(bbox_to_anchor=(1.05,0.5), loc=6, borderaxespad=0) # (1.05,0.5) is the relative position of legend to the origin, loc is the reference point of the legend
fig.tight_layout() # adjust the layout
saveSuffix = 'fs25'
saveDir = '/scratch/HOSdata/pp/' + prjname + '/' + jobname + '/'
saveName = 'eta_ts' + '_' + saveSuffix + '.png'
plt.savefig(saveDir + saveName)
plt.show()

### plot time seris of u at certain point
suffixNo = 0
xind = 25
yind = 5
x = data[suffixNo]['time']
y = data[suffixNo]['u'][:,0,yind,xind]

fig, axs = plt.subplots(1,1)
fig.set_figwidth(8)
fig.set_figheight(6)

axs.plot(x, y, linewidth=1.0, color='k', marker=None, linestyle='-', label=None)
axs.set_ylim(-0.01,0.01)
axs.set_ylabel('u (m/s)', fontsize=12)
axs.grid()
axs.set_xlabel('time (s)', fontsize=12)
# plt.legend(bbox_to_anchor=(1.05,0.5), loc=6, borderaxespad=0) # (1.05,0.5) is the relative position of legend to the origin, loc is the reference point of the legend
fig.tight_layout() # adjust the layout
saveSuffix = 'fs10'
saveDir = '/scratch/HOSdata/pp/' + prjname + '/' + jobname + '/'
saveName = 'u_ts' + '_' + saveSuffix + '.png'
plt.savefig(saveDir + saveName)
plt.show()

### plot the surface elevation of y = constant at a certain time
tind = -3
yind = 16

x = []
y = []
for i in range(sffNum):
    x.append(data[i]['x'])
    y.append(data[i]['se'][tind][yind,:])

fig, axs = plt.subplots(1,1)
fig.set_figwidth(8)
fig.set_figheight(6)

colors = plt.cm.jet(np.linspace(0,1,sffNum))
# colors = ['k']
# linestyles = ['-']
labels = ['case' + str(i) for i in range(sffNum)]

for i in range(sffNum):
    axs.plot(x[i], y[i], linewidth=1.0, color=colors[i], label=labels[i])
# axs.set_ylim(ylims[0])
axs.set_ylabel('SE (m)', fontsize=12)
axs.grid()
axs.set_xlabel('x (m)', fontsize=12)
axs.set_title('Surface Elevation y =' + str(data[0]['y'][yind]))
plt.legend(bbox_to_anchor=(1.05,0.5), loc=6, borderaxespad=0) # (1.05,0.5) is the relative position of legend to the origin, loc is the reference point of the legend
fig.tight_layout() # adjust the layout
saveDir = '/scratch/HOSdata/pp/' + prjname + '/' + jobname + '/'
saveName = 'SE_x' + '.png'
plt.savefig(saveDir + saveName)
plt.show()

### plot the surface elevation time series of x, y = constant
xind = 16
yind = 16

x = []
y = []
for i in range(sffNum):
    x.append(data[i]['time'])
    y.append(data[i]['se'][:,yind,xind])

fig, axs = plt.subplots(1,1)
fig.set_figwidth(12)
fig.set_figheight(6)

colors = plt.cm.jet(np.linspace(0,1,sffNum))
# colors = ['k']
# linestyles = ['-']
labels = ['case' + str(i) for i in range(sffNum)]

# ylims = [(1.26,1.27)]
for i in range(sffNum):
    axs.plot(x[i], y[i], linewidth=1.0, color=colors[i], label=labels[i])
# axs.set_ylim(ylims[0])
axs.set_ylabel('SE (m)', fontsize=12)
axs.grid()
axs.set_xlabel('t (s)', fontsize=12)
axs.set_title('Surface Elevation Time Series')
plt.legend(bbox_to_anchor=(1.05,0.5), loc=6, borderaxespad=0)
fig.tight_layout() # adjust the layout
saveDir = '/scratch/HOSdata/pp/' + prjname + '/' + jobname + '/'
saveName = 'SE_ts' + ".png"
plt.savefig(saveDir + saveName)
plt.show()

### check wave data of a certain timestep
caseNo = 0
tind = -3
yind = 16
x = data[caseNo]['x']
y = data[caseNo]['se'][tind][yind,:]
wave = wvc.count_wave(x, y)
print("lambda = ", np.mean(wave[0]))
print("k = ", 2*np.pi / np.mean(wave[0]))

### check wave data of a certain position
xind = 16
yind = 16
x = data[caseNo]['time']
y = data[caseNo]['se'][:,yind,xind]
wave = wvc.count_wave(x, y)
print("T = ", np.mean(wave[0]))
print("var_T = ", np.var(wave[0]))
print("H = ", np.mean(wave[1]))
print("var_H = ", np.var(wave[1]))
print("omega = ", 2*np.pi / np.mean(wave[0]))


# -*- coding: utf-8 -*-
import json
from random import randint
from typing import Tuple

from openeo_grass_gis_driver.models.process_graph_schemas import \
     ProcessGraphNode, ProcessGraph

from openeo_grass_gis_driver.actinia_processing.base import \
     check_node_parents, DataObject, GrassDataType, \
     create_output_name
from openeo_grass_gis_driver.models.process_schemas import \
     Parameter, ProcessDescription, ReturnValue, ProcessExample
from .base import PROCESS_DICT, PROCESS_DESCRIPTION_DICT, Node

__license__ = "Apache License, Version 2.0"
__author__ = "Markus Metz"
__copyright__ = "Copyright 2018, Sören Gebbert, mundialis"
__maintainer__ = "Soeren Gebbert"
__email__ = "soerengebbert@googlemail.com"

# not in the official list
PROCESS_NAME = "mask_invalid_values"

def create_process_description():
    p_data = Parameter(
        description="Any openEO process object that returns raster datasets "
        "or space-time raster dataset",
        schema={
            "type": "object",
            "subtype": "raster-cube"},
        optional=False)
    p_min = Parameter(description="Minimum allowed value",
                      schema={"type": "object", "subtype": "float"},
                      optional=False)
    p_max = Parameter(description="Maximum allowed value",
                      schema={"type": "object", "subtype": "float"},
                      optional=False)

    rv = ReturnValue(description="Processed EO data.",
                     schema={"type": "object", "subtype": "raster-cube"})

    examples = {
        "mask_invalid_values_1": {
            "process_id": PROCESS_NAME,
            "arguments": {
                "data": {"from_node": "get_data_1"},
                "min": 1,
                "max": 150,
            }
        }
    }

    # Example
    arguments = {
        "data": {"from_node": "get_data_1"},
        "mask": {"from_node": "get_data_2"},
        "value": "null",
    }
    node = ProcessGraphNode(process_id=PROCESS_NAME, arguments=arguments)
    graph = ProcessGraph(
        title="title",
        description="description",
        process_graph={
            "mask_1": node})
    examples = [
        ProcessExample(
            title="Simple example",
            description="Simple example",
            process_graph=graph)]

    pd = ProcessDescription(
        id=PROCESS_NAME,
        description="Drops observations from raster data or raster time series data "
        " that are outside of the specified interval.",
        summary="Filter raster based data on the specified interval",
        parameters={
            "data": p_data,
            "min": p_min,
            "max": p_max},
        returns=rv,
        examples=[examples])

    return json.loads(pd.to_json())

PROCESS_DESCRIPTION_DICT[PROCESS_NAME] = create_process_description()

def create_process_chain_entry(
        input_object: DataObject,
        vmin: float,
        vmax: float,
        output_object: DataObject):
    """Create a Actinia command of the process chain that uses t.rast.mapcalc
    to filter raster values by the specified interval

    :param input_object: The input time series object
    :param min: smallest allowed value
    :param max: largest allowed value
    :param output_object: The output time series object
    :return: A Actinia process chain description
    """

    rn = randint(0, 1000000)

    pc = {"id": "t_rast_mapcalc_%i" % rn,
          "module": "t.rast.mapcalc",
          "inputs": [{"param": "expression",
                     "value": "%(result)s = if(%(raw)s < %(min)s || "
                              "%(raw)s > %(max)s), null(), %(raw)s)" % {"result": output_object.grass_name(),
                                                                        "raw": input_object.grass_name(),
                                                                        "min": str(vmin),
                                                                        "max": str(vmax)}},
                     {"param": "basename",
                     "value": output_object.name},
                     {"param": "output",
                     "value": output_object.grass_name()},
                     ]}

    return pc

def get_process_list(node: Node) -> Tuple[list, list]:
    """Analyse the process node and return the Actinia process chain and the name of the processing result

    :param node: The process node
    :return: (output_objects, actinia_process_list)
    """

    input_objects, process_list = check_node_parents(node=node)
    output_objects = []

    if "data" not in node.arguments or \
            "min" not in node.arguments or \
            "max" not in node.arguments:
        raise Exception(
            "Process %s requires parameter data, min, max" %
            PROCESS_NAME)

    vmin = node.arguments["min"]
    vmax = node.arguments["max"]

    for data_object in input_objects:

        output_object = DataObject(
            name=create_output_name(data_object.name, node),
            datatype=GrassDataType.STRDS)
        output_objects.append(output_object)
        node.add_output(output_object=output_object)

        pc = create_process_chain_entry(data_object, vmin, vmax, output_object)
        process_list.append(pc)

    return output_objects, process_list

PROCESS_DICT[PROCESS_NAME] = get_process_list


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer

from sklearn.impute import SimpleImputer

class dataset:
    def __init__(self, dataTrainfile, dataTestFile):
        self.dataTrain = pd.read_csv(dataTrainfile)
        self.dataTest = pd.read_csv(dataTestFile)

    def getColumns(self, dataFrame, columns):
        return dataFrame.loc[: ,columns].values

    def fillMissingData(self, dataFrame, indexColumns):
        imputer = SimpleImputer()
        imputer = imputer.fit(dataFrame[:,indexColumns])
        dataFrame[:,indexColumns] = imputer.transform(dataFrame[:,indexColumns])

    def numerizeData(self, dataFrame, indexColumns):
        labelencoder = LabelEncoder()
        for i in indexColumns:
            dataFrame[:,i] = labelencoder.fit_transform(dataFrame[:,i]) #Cette ligne permet de numériser les données de la colonne i

    def encodeData(self,dataFrame, indexColumns):
        ct = ColumnTransformer(
            [('one_hot_encoder', OneHotEncoder(categories='auto'), [indexColumns])],   # The column numbers to be transformed (here is [0] but can be [0, 1, 3])
            remainder='passthrough')                                         # Leave the rest of the columns untouched

        dataFrame = ct.fit_transform(dataFrame)

dataset = dataset("train.csv","train.csv")

ExtractedData = dataset.getColumns(dataset.dataTrain, ['Pclass' ,'Sex' ,'Age' ,'SibSp' ,'Parch' ,'Fare' ,'Embarked'])

dataset.fillMissingData(ExtractedData,[0,2,3,4,5] )

ExtractedData[:,2] = ExtractedData[:,2].astype(int)

dataset.numerizeData(ExtractedData, [1,-1])
ExtractedData = pd.DataFrame(data=ExtractedData, columns=["Pclass", "Sex", "Age", "Sibsp", "Parch", "Fare", "Embareked"])


import sparkutils
import traceback

import datetime as dt
from net_balance.abt_netbalance.loading.table_loader import TableLoader
from net_balance.abt_netbalance.process.net_balance import NetBalance
from net_balance import NetBalanceException
from net_balance.conf.module import Module

class OpheliaBaseControl:
    """
        This class generates the net balance of a specific date
    """
    __logger = sparkutils.get_logger('NetBalanceProcess')

    def __init__(self):
        self.__logger = sparkutils.get_logger(NetBalanceProcess.__qualname__)

    def save_table(self, path, dataframe):
        self.__logger.info("Writing in HDFS: " + str(dt.datetime.now()))
        dataframe.write.mode('overwrite').parquet(path)
        self.__logger.info("Saved in: " + path)

    @staticmethod
    def make_net_balance_tables(params, spark_session, activation_date):
        try:
            time_frame = Module.TIME_FRAME
            fide_time_frame = Module.FIDE_TIME_FRAME
            months_back = Module.USER_ACTIVE_MONTHS
            update_day = Module.MONTHLY_UPDATE_DATE
            table_loader = TableLoader()
            table_loader.load_params(params)
            net_balance = NetBalance(spark_session)
            (tmlara14_data, tla018_data, tla1872_data,
             tla1199_data, tla026_data,
             latest_tla1872_date, latest_tla026_date,
             customer_liabilities_tables, tla1705_data,
             tla938_data, latest_tla938_date, tla539_data) = (table_loader.load_net_balance_tables(spark_session,
                                                                                activation_date,
                                                                                time_frame,
                                                                                fide_time_frame,
                                                                                update_day))

            customers_net_balance = net_balance.make_calendar(tmlara14_data, tla018_data,
                                                              tla1872_data, tla1199_data,
                                                              tla026_data,
                                                              latest_tla1872_date,
                                                              latest_tla026_date,
                                                              customer_liabilities_tables, tla1705_data,
                                                              tla938_data, latest_tla938_date, tla539_data,
                                                              months_back)
            return customers_net_balance
        except Exception as e:
            traceback.print_exc()
            NetBalanceProcess.__logger.error("Error in make_net_balance_tables method: " + str(e))
            raise NetBalanceException('Error in make_net_balance_tables method: ' + str(e))

    def run(self, spark, app_config):
        try:
            param_date = str(app_config.getString("net_balance.target_date"))
            year, month, day = map(int, param_date.split("-"))
            activation_date = dt.date(year, month, day)
            self.__logger.info("*** "+str(activation_date)+" *** ")

            self.__logger.info("Start process: " + str(dt.datetime.now()))
            net_balance_path = Module.OUTPUT_MODULE_GOV_PATH + "/" + "information_date=" + str(activation_date)

            self.__logger.info("Loading tables from HDFS: " + str(dt.datetime.now()))
            net_balance_table = self.make_net_balance_tables(app_config, spark, activation_date)
            self.save_table(net_balance_path, net_balance_table)

            self.__logger.info("End process: " + str(dt.datetime.now()))
        except NetBalanceException as chcex:
            self.__logger.error('Error creating NetBalance ABT: ' + str(chcex))
            return -1

        except Exception as e:
            traceback.print_exc()
            self.__logger.error('Error creating NetBalance ABT: ' + str(e))
            return -1

        return 0


import tensorflow as tf
import numpy as np
import os
import sqlite3
import logging

from mnist_gan import GAN

logging.basicConfig()
logger = logging.getLogger('train_mnist_gan')
logger.setLevel(logging.INFO)

FLAGS = tf.app.flags.FLAGS
# hyper parameters
tf.app.flags.DEFINE_float('learning_rate', 1e-3, 'learning rate to train')
tf.app.flags.DEFINE_integer('batch_size', 32, 'batch size to train')
tf.app.flags.DEFINE_integer('max_epoch', 100000, 'max epoches to train')
tf.app.flags.DEFINE_integer('decay_epoch', 10000,
  'epoches to decay learning rate')
tf.app.flags.DEFINE_float('keep_prob', 0.9, 'keep prob for dropout')
tf.app.flags.DEFINE_integer('tau', 10, 'training ratio')

tf.app.flags.DEFINE_integer('display_epoch', 10, 'epoches to display result')
tf.app.flags.DEFINE_integer('summary_epoch', 10, 'epoches to save summary')
tf.app.flags.DEFINE_integer('save_epoch', 1000, 'epoches to save model')
tf.app.flags.DEFINE_string('db_path', './mnist.sqlite3', 'db to load')
tf.app.flags.DEFINE_bool('saving', False, 'saving model')

def load_data(db_path):
  if os.path.isfile(db_path):
    connection = sqlite3.connect(db_path)
    cursor = connection.cursor()
    cursor.execute("""SELECT * FROM train;""")
    train_data = cursor.fetchall()

    cursor.execute("""SELECT * FROM test;""")
    test_data = cursor.fetchall()
    connection.close()
    return train_data, test_data

def parse_data(raw_data):
  data = []
  label = []
  for entry in raw_data:
    img = np.frombuffer(entry[0], dtype=np.uint8) / 255.0
    data.append(np.reshape(img, [28, 28, 1]))
    label.append(entry[1])
  return np.array(data), np.array(label)

def prepare_folder():
  index = 0
  folder = 'mnist_gan_%d' % index
  while os.path.isdir(folder):
    index += 1
    folder = 'mnist_gan_%d' % index

  logger.info('creating folder %s...', folder)
  os.mkdir(folder)
  return folder

def main():
  logger.info('loading data from %s...', FLAGS.db_path)
  train_data, test_data = load_data(FLAGS.db_path)

  logger.info('parsing data...')
  train_images, train_labels = parse_data(train_data)
  test_images, test_labels = parse_data(test_data)

  train_data_size = len(train_images)

  logger.info('setting up model...')
  gan = GAN(FLAGS.learning_rate)

  if FLAGS.saving:
    logger.info('prepare saver...')
    saver = tf.train.Saver()

    folder = prepare_folder()
    checkpoint = os.path.join(folder, 'mnist_gan')
    summary_dir = os.path.join(folder, 'summary')

    logger.info('prepare summary writer...')
    summary_writer = tf.summary.FileWriter(summary_dir,
      tf.get_default_graph())

  logger.info('setting up session...')
  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True
  with tf.Session(config=config) as sess:
    logger.info('initializing variables...')
    sess.run(tf.global_variables_initializer())

    index = 0
    try:
      for epoch in range(FLAGS.max_epoch + 1):
        # get batch data
        start = index
        end = min(index + FLAGS.batch_size, train_data_size)

        index += FLAGS.batch_size
        if index >= train_data_size: index = 0

        batch_images = train_images[start:end, ...]
        batch_labels = train_labels[start:end, ...]

        if epoch % FLAGS.display_epoch == 0:
          losses = sess.run([gan.g_loss, gan.d_loss1, gan.d_loss2], feed_dict={
            gan.target_images: batch_images,
            gan.keep_prob: 1.0,
            gan.target_labels: batch_labels,
            gan.input_noise: np.random.randn(FLAGS.batch_size, 28, 28, 1),
          })
          accu = sess.run(gan.accuracy, feed_dict={
            gan.target_images: batch_images,
            gan.keep_prob: 1.0,
            gan.target_labels: batch_labels,
            gan.input_noise: np.random.randn(FLAGS.batch_size, 28, 28, 1),
          })
          logger.info('%d. loss: %s, train: %f', epoch, str(losses), accu)

        # training
        if epoch % FLAGS.tau == 0:
          sess.run(gan.train_g, feed_dict={
            gan.target_images: batch_images,
            gan.keep_prob: FLAGS.keep_prob,
            gan.target_labels: batch_labels,
            gan.input_noise: np.random.randn(FLAGS.batch_size, 28, 28, 1),
          })

        sess.run(gan.train_d, feed_dict={
          gan.target_images: batch_images,
          gan.keep_prob: FLAGS.keep_prob,
          gan.target_labels: batch_labels,
          gan.input_noise: np.random.randn(FLAGS.batch_size, 28, 28, 1),
        })

        if FLAGS.saving and epoch % FLAGS.save_epoch == 0 and epoch != 0:
          logger.info('saving session...')
          saver.save(sess, checkpoint, global_step=epoch)

        if FLAGS.saving and epoch % FLAGS.summary_epoch == 0 and epoch != 0:
          summary_log = sess.run(gan.summary, feed_dict={
            gan.target_images: batch_images,
            gan.keep_prob: 1.0,
            gan.target_labels: batch_labels,
            gan.input_noise: np.random.randn(FLAGS.batch_size, 28, 28, 1),
          })
          summary_writer.add_summary(summary_log, global_step=epoch)
    except KeyboardInterrupt:
      logger.info('stopping...')

    if FLAGS.saving and epoch % FLAGS.save_epoch == 0 and epoch != 0:
      logger.info('saving last session...')
      saver.save(sess, checkpoint, global_step=epoch)

if __name__ == '__main__':
  main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('club', '0002_auto_20160527_1633'),
    ]

    operations = [
        migrations.AlterField(
            model_name='administration_doc',
            name='doc_file',
            field=models.FileField(upload_to=b'club/templates/images/docs'),
        ),
        migrations.AlterField(
            model_name='medical_doc',
            name='doc_file',
            field=models.FileField(upload_to=b'club/templates/images/docs'),
        ),
    ]


import logging
from pathlib import Path

import numpy as np
import pytest
import tfs
from omc3.correction.constants import (BETA, DISP, NORM_DISP, F1001, F1010, TUNE, PHASE, VALUE, ERROR,
                                       ERR, WEIGHT, DELTA)
from omc3.correction.handler import get_measurement_data, _rms
from omc3.correction.model_appenders import add_coupling_to_model
from omc3.correction.model_diff import diff_twiss_parameters
from omc3.global_correction import global_correction_entrypoint as global_correction, OPTICS_PARAMS_CHOICES
from omc3.optics_measurements.constants import NAME, AMPLITUDE, IMAG, REAL
from omc3.scripts.fake_measurement_from_model import VALUES, ERRORS
from omc3.scripts.fake_measurement_from_model import generate as fake_measurement
from omc3.utils import logging_tools

LOG = logging_tools.get_logger(__name__)
# LOG = logging_tools.get_logger('__main__', level_console=logging_tools.MADX)

# Paths ---
INPUTS = Path(__file__).parent.parent / 'inputs'
CORRECTION_INPUTS = INPUTS / "correction"

# Correction Input Parameters ---

RMS_TOL_DICT = {
    f"{PHASE}X": 0.001,
    f"{PHASE}Y": 0.001,
    f"{BETA}X": 0.01,
    f"{BETA}Y": 0.01,
    f"{DISP}X": 0.0015,
    f"{DISP}Y": 0.0015,
    f"{NORM_DISP}X": 0.001,
    f"{TUNE}": 0.01,
    f"{F1001}R": 0.0015,
    f"{F1001}I": 0.0015,
    f"{F1010}R": 0.002,
    f"{F1010}I": 0.002,
}

def get_skew_params(beam):
    twiss = CORRECTION_INPUTS / f"inj_beam{beam}" / f"twiss_skew_quadrupole_error.dat"
    optics_params = OPTICS_PARAMS_CHOICES[8:]
    variables = ["MQSl"]
    fullresponse = "fullresponse_MQSl.h5"
    seed = 2234
    return twiss, optics_params, variables, fullresponse, seed

def get_normal_params(beam):
    twiss = CORRECTION_INPUTS / f"inj_beam{beam}" / f"twiss_quadrupole_error.dat"
    optics_params = OPTICS_PARAMS_CHOICES[:6]
    variables = ["MQY"]
    fullresponse = "fullresponse_MQY.h5"
    seed = 12368
    return twiss, optics_params, variables, fullresponse, seed

@pytest.mark.basic
@pytest.mark.parametrize('orientation', ('skew', 'normal'))
def test_lhc_global_correct(tmp_path, model_inj_beams, orientation):
    """Creates a fake measurement from a modfied model-twiss with (skew)
    quadrupole errors and runs global correction on this measurement.
    It is asserted that the resulting model approaches the modified twiss.
    Hint: the `model_inj_beam1` fixture is defined in `conftest.py`."""
    beam = model_inj_beams.beam
    twiss_path, optics_params, variables, fullresponse, seed = get_skew_params(beam) if orientation == 'skew' else get_normal_params(beam)
    iterations = 2

    # create and load fake measurement
    error_val = 0.1
    twiss_df, model_df, meas_dict = _create_fake_measurement(
        tmp_path, model_inj_beams.model_dir, twiss_path, error_val, optics_params, seed
    )

    # Perform global correction
    global_correction(
        **model_inj_beams,
        # correction params
        meas_dir=tmp_path,
        variable_categories=variables,
        fullresponse_path=model_inj_beams.model_dir / fullresponse,
        optics_params=list(optics_params),
        output_dir=tmp_path,
        weights=[1.0] * len(optics_params),
        svd_cut=0.01,
        max_iter=iterations,
    )

    # Test if corrected model is closer to model used to create measurement
    for iter_step in range(iterations+1):
        if iter_step == 0:
            model_iter_df = model_df
        else:
            model_iter_df = tfs.read(tmp_path / f"twiss_{iter_step}.tfs", index=NAME)
            model_iter_df = add_coupling_to_model(model_iter_df)

        diff_df = diff_twiss_parameters(model_iter_df, twiss_df, optics_params)
        if TUNE in optics_params:
            diff_df.headers[f"{DELTA}{TUNE}"] = np.array([diff_df[f"{DELTA}{TUNE}1"], diff_df[f"{DELTA}{TUNE}2"]])
        diff_rms = {param: _rms(diff_df[f"{DELTA}{param}"]) for param in optics_params}

        # ############ FOR DEBUGGING #############
        # print(f"ITERATION {iter_step}")
        # for param in optics_params:
        #     print(f"{param}: {diff_rms[param]}")
        # print(f"Sum: {sum(diff_rms.values())}")
        # print()
        # continue
        # ########################################

        if iter_step > 0:
            # assert RMS after correction smaller than tolerances
            for param in optics_params:
                assert diff_rms[param] < RMS_TOL_DICT[param], (
                    f"RMS for {param} in iteration {iter_step} larger than tolerance: "
                    f"{diff_rms[param]} >= {RMS_TOL_DICT[param]}."
                    )

            # assert total RMS decreases between steps
            # ('skew' is converged after one step, still works with seed 2234)
            assert sum(diff_rms_prev.values()) > sum(diff_rms.values()), (
                f"Total RMS in iteration {iter_step} larger than in previous iteration."
                f"{sum(diff_rms.values())} >= {sum(diff_rms_prev.values())}."
            )

        diff_rms_prev = diff_rms

# Helper -----------------------------------------------------------------------

def _create_fake_measurement(tmp_path, model_path, twiss_path, error_val, optics_params, seed):
    model_df = tfs.read(model_path / "twiss.dat", index=NAME)
    model_df = add_coupling_to_model(model_df)

    twiss_df = tfs.read(twiss_path, index=NAME)
    twiss_df = add_coupling_to_model(twiss_df)

    # create fake measurement data
    fake_measurement(
        model=model_df,
        twiss=twiss_df,
        randomize=[VALUES, ERRORS],
        relative_errors=[error_val],
        seed=seed,
        outputdir=tmp_path,
    )

    # load the fake data into a dict
    _, meas_dict = get_measurement_data(
        optics_params,
        meas_dir=tmp_path,
        beta_file_name='beta_phase_',
    )

    # map to VALUE, ERROR and WEIGHT, similar to filter_measurement
    # but without the filtering
    for col, meas in meas_dict.items():
        if col[:-1] in (F1010, F1001):
            col = {c[0]: c for c in (REAL, IMAG, PHASE, AMPLITUDE)}[col[-1]]

        if col != TUNE:
            meas[VALUE] = meas.loc[:, col].to_numpy()
            meas[ERROR] = meas.loc[:, f"{ERR}{col}"].to_numpy()
        meas[WEIGHT] = 1.
    return twiss_df, model_df, meas_dict


import json
import os

from flask import Flask, jsonify, request
from connecttion import get_database_by_name
from pprint import pprint
import csv
from decisiontree import my_main

import pandas as pd
import numpy as np
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.tree import export_text

from sklearn import tree
from matplotlib import pyplot as plt
import graphviz
import joblib

app = Flask(__name__)

def create_csv_train(num):
    db = get_database_by_name('Roomsystem')
    col_static = db['static']
    # sum = num
    # print("จำนวน record ==> ",sum)
    statics = col_static.find().limit(num)
    header = ['_id', 'datetime', 'luminance', 'motion', 'temperature', 'status']
    with open('data_train.csv', 'w', encoding='UTF8') as f:
        writer = csv.writer(f)

        # write the header
        writer.writerow(header)

        # write the data
        for static in statics:
            # print(static['_id'])
            writer.writerow(
                [static['_id'],
                 static['datetime'],
                 static['luminance'],
                 static['motion'],
                 static['temperature'],
                 'Full' if static['label'] == 'กำลังถูกใช้' else 'Empty'
                 ]
            )

@app.route('/trainnig/<int:num>')
def train_ning(num):
    # sum = num
    # print("จำนวน record ==> ",sum)
    [confusion, accuracy, report] = my_main(num)
    create_csv_train(num)
    json_str = json.dumps(
        {'message': 'OK','Record data':num ,'confusion': confusion, 'accuracy': accuracy, 'report': report},
        ensure_ascii=False, default=str)
    return app.response_class(json_str, mimetype='application/json')

@app.route('/prediction/<int:number>')
def hello_world(number):
    db = get_database_by_name('Roomsystem')
    col_infomation1 = db['information']
    informotions1 = col_infomation1.find()
    # print("==>",informotions1)
    count_data = informotions1.count()
    # count_data = col_infomation1.count_documents({})
    print(count_data)
    datasensor = np.zeros((count_data,3))
    # print(datasensor)
    data = []

    # #เรียกโมเดล
    if number == 5:
        path_model = 'model_500.ph5'
    elif number == 4:
        path_model = 'model_400.ph5'
    elif number == 3:
        path_model = 'model_300.ph5'
    elif number == 2:
        path_model = 'model_200.ph5'
    # path_model = 'model_500.ph5'
    loaded_model = joblib.load(path_model)

    for i,data in enumerate(informotions1):
        datasensor[i][0] = data['luminance']
        datasensor[i][1] = data['motion']
        datasensor[i][2] = data['temperature']

    currentId = data['_id']
    currentId1 = data['luminance']
    currentId2 = data['motion']
    currentId3 = data['temperature']
    print(currentId)
    print(currentId1)
    print(currentId2)
    print(currentId3)
    #last index
    value = datasensor[count_data-1]
    # print(value)

    re = ""
    status = 0
    va = value.reshape(1,-1)
    pred = loaded_model.predict(va)
    listToStr = ' '.join([str(elem) for elem in pred])
    print(listToStr)

    #update MongoDB
    col_infomation1.update_many({'_id': currentId},{"$set":{"label": listToStr}})
    re = listToStr
    if re == "Empty":
        status = 0
    else:
        status = 1

    json_str = json.dumps(
        {'message': 'OK','Model':number,'Result':listToStr, 'status': status})
    return app.response_class(json_str, mimetype='application/json')

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=20000)


import gym
import highway_env
import numpy as np

import gym
from d3rlpy.algos import DQN
from d3rlpy.online.buffers import ReplayBuffer
from d3rlpy.online.explorers import LinearDecayEpsilonGreedy

from d3rlpy.wrappers.sb3 import to_mdp_dataset
import torch.nn as nn
import torch
from d3rlpy.algos import DiscreteCQL
from d3rlpy.metrics.scorer import evaluate_on_environment
from d3rlpy.metrics.scorer import td_error_scorer
from d3rlpy.metrics.scorer import discounted_sum_of_advantage_scorer
from d3rlpy.metrics.scorer import average_value_estimation_scorer
from sklearn.model_selection import train_test_split
from d3rlpy.dataset import MDPDataset
from d3rlpy.models.q_functions import QRQFunctionFactory
import d3rlpy
import gym.spaces as spaces
from gym import ObservationWrapper

class FlattenObservation(ObservationWrapper):
    r"""Observation wrapper that flattens the observation."""
    def __init__(self, env):
        super(FlattenObservation, self).__init__(env)
        self.observation_space = spaces.flatten_space(env.observation_space)

    def observation(self, observation):
        return spaces.flatten(self.env.observation_space, observation)

params = {
    "environment": "two-way-v0",
    "model_name": "CQL-QRDQN",
    "train_steps": 200000,
    "buffer_size": 100000,
    "batch_size": 128,
    "gamma": 0.9,
    "target_update_interval": 2000,
    "train_freq": 128,
    "gradient_steps": -1,
    "exploration_fraction": 0.496277345160653,
    "exploration_final_eps": 0.168354028699081,
    "learning_rate": 0.000784942289604,
    "learning_starts": 1000,
    "policy": "MlpPolicy",
    "test_episodes": 10000,
    "n_quantiles": 131
}

env = gym.make(params.get("environment"))
eval_env = gym.make(params.get("environment"))
env = FlattenObservation(env)
eval_env = FlattenObservation(env)
exp_name = params.get("model_name") + "_online_" + params.get("environment")
log_dir = '../../../logs/' + exp_name
pretrain = True

def train(params):
    # setup algorithm
    # setup algorithm
    if pretrain:

        dqn = DQN(batch_size=params.get("batch_size"),
                  learning_rate=params.get("learning_rate"),
                  target_update_interval=params.get("target_update_interval"),
                  q_func_factory=QRQFunctionFactory(n_quantiles=params.get("n_quantiles")),
                  n_steps= params.get("train_freq"),
                  gamma= params.get("gamma"),
                  n_critics= 1,
                  target_reduction_type= "min",
                  use_gpu=True)

        # setup replay buffer
        buffer = ReplayBuffer(maxlen=params.get("buffer_size"), env=env)

        # setup explorers
        explorer = LinearDecayEpsilonGreedy(start_epsilon=1.0,
                                            end_epsilon=params.get("exploration_final_eps"),
                                            duration=100000)

        # start training
        dqn.fit_online(env,
                       buffer,
                       n_steps=params.get("train_steps"),
                       explorer=explorer, # you don't need this with probablistic policy algorithms
                       tensorboard_dir= log_dir,
                       eval_env=eval_env)

        print("Saving Model")
        dqn.save_model(exp_name)

        print("convert buffer to dataset")
        dataset = buffer.to_mdp_dataset()
        # save MDPDataset
        dataset.dump('{0}.h5'.format(exp_name))

    print("Loading Dataset for Offline Training")
    dataset = d3rlpy.dataset.MDPDataset.load('{0}.h5'.format(exp_name))
    train_episodes, test_episodes = train_test_split(dataset, test_size=0.2)
    # The dataset can then be used to train a d3rlpy model

    cql = DiscreteCQL(
        learning_rate=6.25e-05,
        encoder_factory='default',
        q_func_factory='mean',
        batch_size=32,
        n_frames=1,
        n_steps=1,
        gamma=0.99,
        n_critics=1,
        bootstrap=False,
        share_encoder=False,
        target_reduction_type='min',
        target_update_interval=8000,
        use_gpu=True,
        scaler=None,
        augmentation=None,
        generator=None,
        impl=None)

    cql_exp = params.get("model_name") + "_offline_" + params.get("environment")
    cql_log = '../../../logs/' + cql_exp

    cql.fit(dataset.episodes,
            eval_episodes=test_episodes,
            n_epochs=1000,
            scorers={
                'environment': evaluate_on_environment(env, epsilon=0.05),
                'td_error': td_error_scorer,
                'discounted_advantage': discounted_sum_of_advantage_scorer,
                'value_scale': average_value_estimation_scorer,

            },
            tensorboard_dir= cql_log)

    cql.save_model(cql_exp)

train(params)


import pytest
import json

from pyramid.httpexceptions import HTTPBadRequest
from prodnet.models import *
from prodnet.tests.util import get_user, get_project, get_config, get_group, \
    is_now, make_request, get_catalog_item

get = get_catalog_item

class TestBuilderAPI():
    def _view(self, request=None, user=None):
        from prodnet.views.admin import CatalogBuilderView
        req = request or make_request(user=user)
        return CatalogBuilderView(req)

    def update(self, ud):
        req = make_request()
        req.method = 'POST'
        req.params = {'update': json.dumps(ud)}
        self._view(req).view_update_catalog()
        DBSession.flush()

    # simulate a bulk create update
    update_create = [{
        'data': {'placeholder_id': 'ph1','parent_id': 1, 'name': 'category1', 'type': 'Category'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph2','parent_id': 1, 'name': 'category2', 'type': 'Category'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph6','parent_id': 'ph1', 'name': 'subcategory', 'takes_projects': True, 'type': 'Category'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph3','parent_id': 'ph1', 'name': 'topgroup', 'type': 'NodeGroup',},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph4','parent_id': 'ph6', 'name': 'group', 'type': 'NodeGroup'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph5','parent_id': 'ph6', 'name': 'select', 'multiple': True, 'type': 'Select'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph12','parent_id': 'ph4', 'name': 'restrictedSelect', 'multiple': True, 'type': 'Select'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph7','parent_id': 'ph6', 'name': 'bool', 'default': True, 'type': 'Bool'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph8','parent_id': 'ph6', 'name': 'real', 'minimum': 10.3, 'maximum': 100.3, 'type': 'Real'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph9','parent_id': 'ph6', 'name': 'date', 'type': 'Date'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph16','parent_id': 'ph6', 'name': 'str', 'dynamic': True, 'type': 'Str'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph17','parent_id': 'ph6', 'name': 'int', 'inherit_group_id': 'ph3', 'type': 'Int'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph10','parent_id': 'ph5', 'name': 'selectopt1', 'type': 'Option'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph11','parent_id': 'ph10', 'name': 'optsub', 'type': 'Real'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph13','parent_id': 'ph12', 'name': 'restrictedOpt', 'type': 'Option'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph14','parent_id': 'ph12', 'name': 'allowedOpt', 'type': 'Option'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph15', 'node_id': 'ph12', 'option_id': 'ph10', 'allowed_options': ['ph14', 'ph13'], 'type': 'Restriction'},
        'op': 'create'
    }, {
        'data': {'id': 'ph15', 'allowed_options': ['ph14'], 'type': 'Restriction'},
        'op': 'update'
    }]
    def test_update_create(self):
        self.update(self.update_create)
        self.compare()

    def compare(self):
        # structure
        assert set(Category.get_root().children) == {get('category1'), get('category2'), get('TC1')}
        assert set(get('category1').children) == {get('subcategory'), get('topgroup')}
        assert not get('category1').takes_projects
        assert get('subcategory').takes_projects
        assert get('subcategory').children == [get('group'),
            get('select'), get('bool'), get('real'), get('date'),
            get('str'), get('int')]
        assert get('group').children == [get('restrictedSelect')]
        assert set(get('restrictedSelect').children) == {get('restrictedOpt'), get('allowedOpt')}
        assert get('select').children == [get('selectopt1')]

        # types and attrs
        assert get('category1').type == 'Category'
        assert get('int').type == 'Int'
        assert get('bool').type == 'Bool'
        assert get('str').type == 'Str'
        assert get('real').type == 'Real'
        assert get('optsub').type == 'Real'
        assert get('select').type == 'Select'
        restriction = get('restrictedSelect').restrictions[0]
        assert restriction.node == get('restrictedSelect')
        assert restriction.option == get('selectopt1')
        assert restriction.allowed_options == [get('allowedOpt')]
        assert get('real').maximum == 100.3
        assert get('bool').default
        assert get('int')._inherit_group == get('topgroup')

    update_delete = [{
        'data': {'placeholder_id': 'ph1','parent_id': 1, 'name': 'category1', 'type': 'Category'},
        'op': 'create'
    }, {
        'data': {'id': 'ph1', 'name': 'bla', 'type': 'Category'},
        'op': 'update'
    }, {
        'data': {'id': 'ph1', 'type': 'Category'},
        'op': 'delete'
    }]
    def test_update_delete(self):
        self.update(self.update_delete)
        assert len(Category.get_root().children) == 1

    update_delete_with_children = [{
        'data': {'placeholder_id': 'ph1','parent_id': 1, 'name': 'cat', 'type': 'Category'},
        'op': 'create'
    }, {
        'data': {'placeholder_id': 'ph2','parent_id': 'ph1', 'name': 'grp', 'type': 'NodeGroup'},
        'op': 'create'
    }, {
        'data': {'id': 'ph1', 'type': 'Category'},
        'op': 'delete'
    }]
    def test_update_delete_with_children(self):
        with pytest.raises(HTTPBadRequest):
            self.update(self.update_delete_with_children)

    update_delete_root = [{
        'data': {'id': 1, 'type': 'Category'},
        'op': 'delete'
    }]
    def test_update_delete_root(self):
        with pytest.raises(HTTPBadRequest):
            self.update(self.update_delete_root)

    json_catalog = {
        'data': {'id': 1, 'type': 'Category'},
        'children': [{
            'data': {'id': 2, 'type': 'NodeGroup'},
            'children': []
        }, {
            'data': {'id': 3, 'type': 'Category'},
            'children': []
        }]
    }
    def test_restore_from_json(self):
        # inital state to be deleted now
        self.update(self.update_create)
        req = make_request()
        req.method = 'POST'
        req.json_body = json.dumps(self.json_catalog)
        self._view(req).view_restore_catalog_from_json()

    def test_store_and_restore_from_json(self):
        # create
        self.update(self.update_create)

        # "store"
        req = make_request()
        req.method = 'GET'
        data = self._view(req).view_get_catalog_json()

        # restore
        req.method = 'POST'
        req.json_body = data
        self._view(req).view_restore_catalog_from_json()
        self.compare()